From 8fca5898da15fe1b3f4f87fc35bed34c236e74cb Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 11 Apr 2022 09:29:41 -0400 Subject: Suppress warning for missing MSVC compiler for initial call to msvc_setup_env with undefined MSVC_VERSION (default tools). --- SCons/Tool/MSCommon/vc.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index fe31cb3..3373301 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -714,7 +714,9 @@ def get_installed_vcs(env=None): def reset_installed_vcs(): """Make it try again to find VC. This is just for the tests.""" global __INSTALLED_VCS_RUN + global __MSVC_SETUP_ENV_DEFAULT __INSTALLED_VCS_RUN = None + __MSVC_SETUP_ENV_DEFAULT = None # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" @@ -920,14 +922,18 @@ def msvc_find_valid_batch_script(env, version): return d +__MSVC_SETUP_ENV_DEFAULT = None def msvc_setup_env(env): + global __MSVC_SETUP_ENV_DEFAULT debug('called') version = get_default_version(env) if version is None: - warn_msg = "No version of Visual Studio compiler found - C/C++ " \ - "compilers most likely not set correctly" - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + if __MSVC_SETUP_ENV_DEFAULT: + warn_msg = "No version of Visual Studio compiler found - C/C++ " \ + "compilers most likely not set correctly" + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + __MSVC_SETUP_ENV_DEFAULT = True return None # XXX: we set-up both MSVS version for backward -- cgit v0.12 From f338fee76045f94e78f25f15fea1d4bf33786d6e Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 12 Apr 2022 10:20:49 -0400 Subject: Raise MSVCVersionNotFound exception instead of returning an empty/undefined dictionary when attempting to find a valid msvc batch script. Raise an MSVCVersionNotFound exception when the default msvc version is requested and there are no msvc versions installed. Suppress raising an MSVCVersionNotFound exception during default msvc tool initialization. Add additional tests. --- SCons/Tool/MSCommon/vc.py | 36 ++++++++++--- test/MSVC/msvc_badversion.py | 59 ++++++++++++++++++++++ test/MSVC/no_msvc.py | 26 ++++++++-- test/fixture/no_msvc/no_msvcs_sconstruct.py | 1 - test/fixture/no_msvc/no_msvcs_sconstruct_tools.py | 14 +++++ .../fixture/no_msvc/no_msvcs_sconstruct_version.py | 14 +++++ 6 files changed, 138 insertions(+), 12 deletions(-) create mode 100644 test/MSVC/msvc_badversion.py create mode 100644 test/fixture/no_msvc/no_msvcs_sconstruct_tools.py create mode 100644 test/fixture/no_msvc/no_msvcs_sconstruct_version.py diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 3373301..d9a6e50 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -79,6 +79,9 @@ class BatchFileExecutionError(VisualCException): class MSVCScriptNotFound(VisualCException): pass +class MSVCVersionNotFound(VisualCException): + pass + # Dict to 'canonalize' the arch _ARCH_TO_CANONICAL = { "amd64" : "amd64", @@ -853,6 +856,7 @@ def msvc_find_valid_batch_script(env, version): debug("host_platform: %s, try_target_archs: %s", host_platform, try_target_archs) d = None + version_installed = False for tp in try_target_archs: # Set to current arch. env['TARGET_ARCH'] = tp @@ -869,14 +873,11 @@ def msvc_find_valid_batch_script(env, version): try: (vc_script, use_arg, sdk_script) = find_batch_file(env, version, host_platform, tp) debug('vc_script:%s sdk_script:%s', vc_script, sdk_script) + version_installed = True except VisualCException as e: msg = str(e) debug('Caught exception while looking for batch file (%s)', msg) - warn_msg = "VC version %s not installed. " + \ - "C/C++ compilers are most likely not set correctly.\n" + \ - " Installed versions are: %s" - warn_msg = warn_msg % (version, get_installed_vcs(env)) - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + version_installed = False continue # Try to use the located batch file for this host/target platform combo @@ -919,6 +920,25 @@ def msvc_find_valid_batch_script(env, version): # To it's initial value if not d: env['TARGET_ARCH']=req_target_platform + installed_vcs = get_installed_vcs(env) + if version_installed: + err_msg = "MSVC version {} working host/target script was not found.\n" \ + " Host = {}, Target = {}\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format( + version, host_platform, target_platform + ) + elif version and installed_vcs: + err_msg = "MSVC version {} was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(version, installed_vcs) + elif version: + err_msg = "MSVC version {} was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format(version) + else: + err_msg = "No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly" + raise MSVCVersionNotFound(err_msg) return d @@ -930,9 +950,9 @@ def msvc_setup_env(env): version = get_default_version(env) if version is None: if __MSVC_SETUP_ENV_DEFAULT: - warn_msg = "No version of Visual Studio compiler found - C/C++ " \ - "compilers most likely not set correctly" - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + err_msg = "No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly" + raise MSVCVersionNotFound(err_msg) __MSVC_SETUP_ENV_DEFAULT = True return None diff --git a/test/MSVC/msvc_badversion.py b/test/MSVC/msvc_badversion.py new file mode 100644 index 0000000..7af8ca5 --- /dev/null +++ b/test/MSVC/msvc_badversion.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test scons with an invalid MSVC version when at least one MSVC is present. +""" + +import sys + +import TestSCons +import SCons.Tool.MSCommon.vc as msvc + +test = TestSCons.TestSCons() + +if sys.platform != 'win32': + test.skip_test("Not win32 platform. Skipping test\n") + +test.skip_if_not_msvc() + +installed_msvc_versions = msvc.get_installed_vcs() +# MSVC guaranteed to be at least one version on the system or else +# skip_if_not_msvc() function would have skipped the test + +test.write('SConstruct', """\ +env = Environment(MSVC_VERSION='12.9') +""") + +test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/MSVC/no_msvc.py b/test/MSVC/no_msvc.py index d1161c6..b6442e8 100644 --- a/test/MSVC/no_msvc.py +++ b/test/MSVC/no_msvc.py @@ -43,9 +43,29 @@ test.run(arguments='-Q -s', stdout='') # test no msvc's test.file_fixture('no_msvc/no_msvcs_sconstruct.py', 'SConstruct') +test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) + +# test msvc version number request with no msvc's +test.file_fixture('no_msvc/no_msvcs_sconstruct_version.py', 'SConstruct') +test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) + +# test that MSVCVersionNotFound is not raised for default msvc tools +# when a non-msvc tool list is used +test.subdir('site_scons', ['site_scons', 'site_tools']) + +test.write(['site_scons', 'site_tools', 'myignoredefaultmsvctool.py'], """ +import SCons.Tool +def generate(env): + env['MYIGNOREDEFAULTMSVCTOOL']='myignoredefaultmsvctool' +def exists(env): + return 1 +""") + +test.file_fixture('no_msvc/no_msvcs_sconstruct_tools.py', 'SConstruct') +test.run(arguments='-Q -s') + +test.file_fixture('no_msvc/no_msvcs_sconstruct_tools.py', 'SConstruct') test.run(arguments='-Q -s') -if 'MSVC_VERSION=None' not in test.stdout(): - test.fail_test() +test.pass_test() -test.pass_test() \ No newline at end of file diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct.py b/test/fixture/no_msvc/no_msvcs_sconstruct.py index 18366d8..590142b 100644 --- a/test/fixture/no_msvc/no_msvcs_sconstruct.py +++ b/test/fixture/no_msvc/no_msvcs_sconstruct.py @@ -12,4 +12,3 @@ SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere env = SCons.Environment.Environment() -print('MSVC_VERSION='+str(env.get('MSVC_VERSION'))) \ No newline at end of file diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py b/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py new file mode 100644 index 0000000..ca9b699 --- /dev/null +++ b/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py @@ -0,0 +1,14 @@ +import SCons +import SCons.Tool.MSCommon + +def DummyVsWhere(msvc_version, env): + # not testing versions with vswhere, so return none + return None + +for key in SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR: + SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR[key]=[(SCons.Util.HKEY_LOCAL_MACHINE, r'')] + +SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere + +env = SCons.Environment.Environment(tools=['MYIGNOREDEFAULTMSVCTOOL']) + diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct_version.py b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py new file mode 100644 index 0000000..81d47cf --- /dev/null +++ b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py @@ -0,0 +1,14 @@ +import SCons +import SCons.Tool.MSCommon + +def DummyVsWhere(msvc_version, env): + # not testing versions with vswhere, so return none + return None + +for key in SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR: + SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR[key]=[(SCons.Util.HKEY_LOCAL_MACHINE, r'')] + +SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere + +env = SCons.Environment.Environment(MSVC_VERSION='14.3') + -- cgit v0.12 From b19e130873dbf44d4e3edb0d0cc47df12f99b865 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 24 Apr 2022 06:12:26 -0400 Subject: Rework exception message construction when a specified msvc version is not found. Remove exception for default msvc version and adjust test accordingly. --- SCons/Tool/MSCommon/vc.py | 43 ++++++++++++++++++++++--------------------- test/MSVC/no_msvc.py | 2 +- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index d9a6e50..cac1744 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -717,9 +717,7 @@ def get_installed_vcs(env=None): def reset_installed_vcs(): """Make it try again to find VC. This is just for the tests.""" global __INSTALLED_VCS_RUN - global __MSVC_SETUP_ENV_DEFAULT __INSTALLED_VCS_RUN = None - __MSVC_SETUP_ENV_DEFAULT = None # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" @@ -920,40 +918,43 @@ def msvc_find_valid_batch_script(env, version): # To it's initial value if not d: env['TARGET_ARCH']=req_target_platform - installed_vcs = get_installed_vcs(env) + if version_installed: err_msg = "MSVC version {} working host/target script was not found.\n" \ " Host = {}, Target = {}\n" \ " Visual Studio C/C++ compilers may not be set correctly".format( version, host_platform, target_platform ) - elif version and installed_vcs: - err_msg = "MSVC version {} was not found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly.\n" \ - " Installed versions are: {}".format(version, installed_vcs) - elif version: - err_msg = "MSVC version {} was not found.\n" \ - " No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format(version) else: - err_msg = "No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly" + installed_vcs = get_installed_vcs(env) + if version is not None: + if not installed_vcs: + err_msg = "MSVC version {} was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format(version) + else: + err_msg = "MSVC version {} was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(version, installed_vcs) + else: + # should never get here due to early exit in msvc_setup_env + if not installed_vcs: + err_msg = "MSVC default version was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly" + else: + # should be impossible: version is None and len(installed_vcs) > 0 + err_msg = "MSVC default version was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(installed_vcs) raise MSVCVersionNotFound(err_msg) return d -__MSVC_SETUP_ENV_DEFAULT = None - def msvc_setup_env(env): - global __MSVC_SETUP_ENV_DEFAULT debug('called') version = get_default_version(env) if version is None: - if __MSVC_SETUP_ENV_DEFAULT: - err_msg = "No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly" - raise MSVCVersionNotFound(err_msg) - __MSVC_SETUP_ENV_DEFAULT = True return None # XXX: we set-up both MSVS version for backward diff --git a/test/MSVC/no_msvc.py b/test/MSVC/no_msvc.py index b6442e8..683951e 100644 --- a/test/MSVC/no_msvc.py +++ b/test/MSVC/no_msvc.py @@ -43,7 +43,7 @@ test.run(arguments='-Q -s', stdout='') # test no msvc's test.file_fixture('no_msvc/no_msvcs_sconstruct.py', 'SConstruct') -test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) +test.run(arguments='-Q -s', stdout='') # test msvc version number request with no msvc's test.file_fixture('no_msvc/no_msvcs_sconstruct_version.py', 'SConstruct') -- cgit v0.12 From 6f68a4fc1646be9ec4b1b6c5e8fbda031d239a0b Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 24 Apr 2022 07:19:26 -0400 Subject: Simplify msvc version not found exception messages by removing impossible combinations. Update exception message formatting. --- SCons/Tool/MSCommon/vc.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index cac1744..0b25ed5 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -920,33 +920,21 @@ def msvc_find_valid_batch_script(env, version): env['TARGET_ARCH']=req_target_platform if version_installed: - err_msg = "MSVC version {} working host/target script was not found.\n" \ - " Host = {}, Target = {}\n" \ + err_msg = "MSVC version '{}' working host/target script was not found.\n" \ + " Host = '{}', Target = '{}'\n" \ " Visual Studio C/C++ compilers may not be set correctly".format( version, host_platform, target_platform ) else: installed_vcs = get_installed_vcs(env) - if version is not None: - if not installed_vcs: - err_msg = "MSVC version {} was not found.\n" \ - " No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format(version) - else: - err_msg = "MSVC version {} was not found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly.\n" \ - " Installed versions are: {}".format(version, installed_vcs) + if installed_vcs: + err_msg = "MSVC version '{}' was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(version, installed_vcs) else: - # should never get here due to early exit in msvc_setup_env - if not installed_vcs: - err_msg = "MSVC default version was not found.\n" \ - " No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly" - else: - # should be impossible: version is None and len(installed_vcs) > 0 - err_msg = "MSVC default version was not found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly.\n" \ - " Installed versions are: {}".format(installed_vcs) + err_msg = "MSVC version '{}' was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format(version) raise MSVCVersionNotFound(err_msg) return d -- cgit v0.12 From 35d0a4621711902e9d8ee1ddb625fc782dff5de3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 26 Apr 2022 12:39:54 -0400 Subject: Replace msvc_exists in tools exists functions for msvc tools with msvc_setup_env_tool that also registers the tool name. The new function may indicate that the tool should be included even when no instances of msvc are installed. This is necessary for some error checking. Add a tool name argument to msvc_setup_env_once as well. Add default msvc version detection. Add a global and environment local policy variable for handling warnings and/or exceptions: error, warn, ignore. By default warnings are produced. Update tests accordingly. --- SCons/Tool/MSCommon/__init__.py | 4 +- SCons/Tool/MSCommon/vc.py | 312 ++++++++++++++++++++- SCons/Tool/midl.py | 8 +- SCons/Tool/mslib.py | 8 +- SCons/Tool/mslink.py | 8 +- SCons/Tool/msvc.py | 8 +- SCons/Tool/msvs.py | 8 +- test/MSVC/msvc_badversion.py | 14 + test/MSVC/no_msvc.py | 8 +- test/fixture/no_msvc/no_msvcs_sconstruct.py | 1 + .../fixture/no_msvc/no_msvcs_sconstruct_version.py | 2 + 11 files changed, 349 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/MSCommon/__init__.py b/SCons/Tool/MSCommon/__init__.py index 50eed73..9d8a8ff 100644 --- a/SCons/Tool/MSCommon/__init__.py +++ b/SCons/Tool/MSCommon/__init__.py @@ -34,10 +34,12 @@ from SCons.Tool.MSCommon.sdk import mssdk_exists, mssdk_setup_env from SCons.Tool.MSCommon.vc import ( msvc_exists, - msvc_setup_env, + msvc_setup_env_tool, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere, + set_msvc_notfound_policy, + get_msvc_notfound_policy, ) from SCons.Tool.MSCommon.vs import ( diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 0b25ed5..c100663 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -45,6 +45,7 @@ import os import platform from string import digits as string_digits from subprocess import PIPE +import re import SCons.Util import SCons.Warnings @@ -82,6 +83,27 @@ class MSVCScriptNotFound(VisualCException): class MSVCVersionNotFound(VisualCException): pass +# MSVC_NOTFOUND_POLICY: +# error: raise exception +# warn: issue warning and continue +# ignore: continue +_MSVC_NOTFOUND_POLICY_DEFAULT = False +_MSVC_NOTFOUND_POLICY = _MSVC_NOTFOUND_POLICY_DEFAULT + +_MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC = [] +_MSVC_NOTFOUND_POLICY_SYMBOLS_DICT = {} + +for value, symbol_list in [ + (True, ['Error', 'Exception']), + (False, ['Warn', 'Warning']), + (None, ['Ignore', 'Suppress']), +]: + for symbol in symbol_list: + _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC.append(symbol.lower()) + _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] = value + _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol.lower()] = value + _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol.upper()] = value + # Dict to 'canonalize' the arch _ARCH_TO_CANONICAL = { "amd64" : "amd64", @@ -718,6 +740,7 @@ def reset_installed_vcs(): """Make it try again to find VC. This is just for the tests.""" global __INSTALLED_VCS_RUN __INSTALLED_VCS_RUN = None + _MSVCSetupEnvDefault.reset() # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" @@ -762,6 +785,248 @@ def script_env(script, args=None): return cache_data +def _msvc_notfound_policy_lookup(symbol): + + try: + notfound_policy = _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] + except KeyError: + err_msg = "Value specified for MSVC_NOTFOUND_POLICY is not supported: {}.\n" \ + " Valid values are: {}".format( + repr(symbol), + ', '.join([repr(s) for s in _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC]) + ) + raise ValueError(err_msg) + + return notfound_policy + +def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): + global _MSVC_NOTFOUND_POLICY + + if MSVC_NOTFOUND_POLICY is not None: + _MSVC_NOTFOUND_POLICY = _msvc_notfound_policy_lookup(MSVC_NOTFOUND_POLICY) + + return _MSVC_NOTFOUND_POLICY + +def get_msvc_notfound_policy(): + return _MSVC_NOTFOUND_POLICY + +def _msvc_notfound_policy_handler(env, msg): + + if env and 'MSVC_NOTFOUND_POLICY' in env: + # use environment setting + notfound_policy = _msvc_notfound_policy_lookup(env['MSVC_NOTFOUND_POLICY']) + else: + # use active global setting + notfound_policy = _MSVC_NOTFOUND_POLICY + + if notfound_policy is None: + debug('notfound policy: ignore') + elif notfound_policy: + debug('notfound policy: exception') + raise MSVCVersionNotFound(msg) + else: + debug('notfound policy: warning') + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) + +class _MSVCSetupEnvDefault: + + # Determine if and/or when an error/warning should be issued when there + # are no versions of msvc installed. If there is at least one version of + # msvc installed, these routines do (almost) nothing. + + # Notes: + # * When msvc is the default compiler because there are no compilers + # installed, a build may fail due to the cl.exe command not being + # recognized. Currently, there is no easy way to detect during + # msvc initialization if the default environment will be used later + # to build a program and/or library. There is no error/warning + # as there are legitimate SCons uses that do not require a c compiler. + # * As implemented, the default is that a warning is issued. This can + # be changed globally via the function set_msvc_notfound_policy and/or + # through the environment via the MSVC_NOTFOUND_POLICY variable. + + separator = r';' + + need_init = True + + @classmethod + def reset(cls): + cls.n_setup = 0 # number of calls to msvc_setup_env_once + cls.default_ismsvc = False # is msvc the default compiler + cls.default_tools_re_list = [] # list of default tools regular expressions + cls.msvc_tools_init = set() # tools registered via msvc_exists + cls.msvc_tools = None # tools registered via msvc_setup_env_once + cls.msvc_installed = False # is msvc installed (vcs_installed > 0) + cls.msvc_nodefault = False # is there a default version of msvc + cls.need_init = False # clear initialization indicator + + @classmethod + def _initialize(cls, env): + if cls.need_init: + cls.reset() + vcs = get_installed_vcs(env) + cls.msvc_installed = len(vcs) > 0 + debug('msvc default:initialize:msvc_installed=%s', cls.msvc_installed) + + @classmethod + def register_tool(cls, env, tool): + if cls.need_init: + cls._initialize(env) + if cls.msvc_installed: + return None + if not tool: + return None + if cls.n_setup == 0: + if tool not in cls.msvc_tools_init: + cls.msvc_tools_init.add(tool) + debug('msvc default:register tool:tool=%s msvc_tools_init=%s', tool, cls.msvc_tools_init) + return None + if tool not in cls.msvc_tools: + cls.msvc_tools.add(tool) + debug('msvc default:register tool:tool=%s msvc_tools=%s', tool, cls.msvc_tools) + + @classmethod + def register_setup(cls, env): + if cls.need_init: + cls._initialize(env) + cls.n_setup += 1 + if not cls.msvc_installed: + cls.msvc_tools = set(cls.msvc_tools_init) + if cls.n_setup == 1: + tool_list = env.get('TOOLS', None) + if tool_list and tool_list[0] == 'default': + if len(tool_list) > 1 and tool_list[1] in cls.msvc_tools: + # msvc tools are the default compiler + cls.default_ismsvc = True + cls.msvc_nodefault = False + debug( + 'msvc default:register setup:n=%d msvc_installed=%s default_ismsvc=%s', + cls.n_setup, cls.msvc_installed, cls.default_ismsvc + ) + + @classmethod + def set_nodefault(cls): + # default msvc version, msvc not installed + cls.msvc_nodefault = True + debug('msvc default:set nodefault:msvc_nodefault=%s', cls.msvc_nodefault) + + @classmethod + def register_iserror(cls, env, tool): + + cls.register_tool(env, tool) + + if cls.msvc_installed: + # msvc installed + return None + + if not cls.msvc_nodefault: + # msvc version specified + return None + + tool_list = env.get('TOOLS', None) + if not tool_list: + # tool list is empty + return None + + debug( + 'msvc default:register iserror:n=%s default_ismsvc=%s msvc_tools=%s tool_list=%s', + cls.n_setup, cls.default_ismsvc, cls.msvc_tools, tool_list + ) + + if not cls.default_ismsvc: + + # Summary: + # * msvc is not installed + # * msvc version not specified (default) + # * msvc is not the default compiler + + # construct tools set + tools_set = set(tool_list) + + else: + + if cls.n_setup == 1: + # first setup and msvc is default compiler: + # build default tools regex for current tool state + tools = cls.separator.join(tool_list) + tools_nchar = len(tools) + debug('msvc default:register iserror:add regex nchar=%d, tools=%s', tools_nchar, tools) + re_default_tools = re.compile(re.escape(tools)) + cls.default_tools_re_list.insert(0, (tools_nchar, re_default_tools)) + # early exit: no error for default environment when msvc is not installed + return None + + # Summary: + # * msvc is not installed + # * msvc version not specified (default) + # * environment tools list is not empty + # * default tools regex list constructed + # * msvc tools set constructed + # + # Algorithm using tools string and sets: + # * convert environment tools list to a string + # * iteratively remove default tools sequences via regex + # substition list built from longest sequence (first) + # to shortest sequence (last) + # * build environment tools set with remaining tools + # * compute intersection of environment tools and msvc tools sets + # * if the intersection is: + # empty - no error: default tools and/or no additional msvc tools + # not empty - error: user specified one or more msvc tool(s) + # + # This will not produce an error or warning when there are no + # msvc installed instances nor any other recognized compilers + # and the default environment is needed for a build. The msvc + # compiler is forcibly added to the environment tools list when + # there are no compilers installed on win32. In this case, cl.exe + # will not be found on the path resulting in a failed build. + + # construct tools string + tools = cls.separator.join(tool_list) + tools_nchar = len(tools) + + debug('msvc default:register iserror:nchar=%d tools=%s', tools_nchar, tools) + + # iteratively remove default tool sequences (longest to shortest) + re_nchar_min, re_tools_min = cls.default_tools_re_list[-1] + if tools_nchar >= re_nchar_min and re_tools_min.search(tools): + # minimum characters satisfied and minimum pattern exists + for re_nchar, re_default_tool in cls.default_tools_re_list: + if tools_nchar < re_nchar: + # not enough characters for pattern + continue + tools = re_default_tool.sub('', tools).strip(cls.separator) + tools_nchar = len(tools) + debug('msvc default:register iserror:nchar=%d tools=%s', tools_nchar, tools) + if tools_nchar < re_nchar_min or not re_tools_min.search(tools): + # less than minimum characters or minimum pattern does not exist + break + + # construct non-default list(s) tools set + tools_set = {msvc_tool for msvc_tool in tools.split(cls.separator) if msvc_tool} + + debug('msvc default:register iserror:tools=%s', tools_set) + if not tools_set: + return None + + # compute intersection of remaining tools set and msvc tools set + tools_found = cls.msvc_tools.intersection(tools_set) + debug('msvc default:register iserror:tools_exist=%s', tools_found) + if not tools_found: + return None + + # construct in same order as tools list + tools_found_list = [] + seen_tool = set() + for tool in tool_list: + if tool not in seen_tool: + seen_tool.add(tool) + if tool in tools_found: + tools_found_list.append(tool) + + # return tool list in order presented + return tools_found_list + def get_default_version(env): msvc_version = env.get('MSVC_VERSION') msvs_version = env.get('MSVS_VERSION') @@ -796,16 +1061,24 @@ def get_default_version(env): return msvc_version -def msvc_setup_env_once(env): +def msvc_setup_env_once(env, tool=None): try: has_run = env["MSVC_SETUP_RUN"] except KeyError: has_run = False if not has_run: + _MSVCSetupEnvDefault.register_setup(env) msvc_setup_env(env) env["MSVC_SETUP_RUN"] = True + req_tools = _MSVCSetupEnvDefault.register_iserror(env, tool) + if req_tools: + msg = "No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Requested tool(s) are: {}".format(req_tools) + _msvc_notfound_policy_handler(env, msg) + def msvc_find_valid_batch_script(env, version): """Find and execute appropriate batch script to set up build env. @@ -920,22 +1193,23 @@ def msvc_find_valid_batch_script(env, version): env['TARGET_ARCH']=req_target_platform if version_installed: - err_msg = "MSVC version '{}' working host/target script was not found.\n" \ - " Host = '{}', Target = '{}'\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format( - version, host_platform, target_platform - ) + msg = "MSVC version '{}' working host/target script was not found.\n" \ + " Host = '{}', Target = '{}'\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format( + version, host_platform, target_platform + ) else: installed_vcs = get_installed_vcs(env) if installed_vcs: - err_msg = "MSVC version '{}' was not found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly.\n" \ - " Installed versions are: {}".format(version, installed_vcs) + msg = "MSVC version '{}' was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(version, installed_vcs) else: - err_msg = "MSVC version '{}' was not found.\n" \ - " No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format(version) - raise MSVCVersionNotFound(err_msg) + msg = "MSVC version '{}' was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format(version) + + _msvc_notfound_policy_handler(env, msg) return d @@ -943,6 +1217,7 @@ def msvc_setup_env(env): debug('called') version = get_default_version(env) if version is None: + _MSVCSetupEnvDefault.set_nodefault() return None # XXX: we set-up both MSVS version for backward @@ -991,3 +1266,14 @@ def msvc_exists(env=None, version=None): if version is None: return len(vcs) > 0 return version in vcs + +def msvc_setup_env_tool(env=None, version=None, tool=None): + _MSVCSetupEnvDefault.register_tool(env, tool) + if msvc_exists(env, version): + return True + if env: + for key in ('MSVC_VERSION', 'MSVS_VERSION'): + if key in env: + return True + return False + diff --git a/SCons/Tool/midl.py b/SCons/Tool/midl.py index 2757c34..22cdd85 100644 --- a/SCons/Tool/midl.py +++ b/SCons/Tool/midl.py @@ -33,13 +33,17 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os + import SCons.Action import SCons.Builder import SCons.Defaults import SCons.Scanner.IDL import SCons.Util -from .MSCommon import msvc_exists +from .MSCommon import msvc_setup_env_tool + +tool_name = os.path.splitext(os.path.basename(__file__))[0] def midl_emitter(target, source, env): """Produces a list of outputs from the MIDL compiler""" @@ -79,7 +83,7 @@ def generate(env): env['BUILDERS']['TypeLibrary'] = midl_builder def exists(env): - return msvc_exists(env) + return msvc_setup_env_tool(env, tool=tool_name) # Local Variables: # tab-width:4 diff --git a/SCons/Tool/mslib.py b/SCons/Tool/mslib.py index 354f5cf..cbeaa7f 100644 --- a/SCons/Tool/mslib.py +++ b/SCons/Tool/mslib.py @@ -41,14 +41,16 @@ import SCons.Tool.msvs import SCons.Tool.msvc import SCons.Util -from .MSCommon import msvc_exists, msvc_setup_env_once +from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once + +tool_name = os.path.splitext(os.path.basename(__file__))[0] def generate(env): """Add Builders and construction variables for lib to an Environment.""" SCons.Tool.createStaticLibBuilder(env) # Set-up ms tools paths - msvc_setup_env_once(env) + msvc_setup_env_once(env, tool=tool_name) env['AR'] = 'lib' env['ARFLAGS'] = SCons.Util.CLVar('/nologo') @@ -64,7 +66,7 @@ def generate(env): def exists(env): - return msvc_exists(env) + return msvc_setup_env_tool(env, tool=tool_name) # Local Variables: # tab-width:4 diff --git a/SCons/Tool/mslink.py b/SCons/Tool/mslink.py index 3dac7f0..1376020 100644 --- a/SCons/Tool/mslink.py +++ b/SCons/Tool/mslink.py @@ -43,9 +43,11 @@ import SCons.Tool.msvc import SCons.Tool.msvs import SCons.Util -from .MSCommon import msvc_setup_env_once, msvc_exists +from .MSCommon import msvc_setup_env_once, msvc_setup_env_tool from .MSCommon.common import get_pch_node +tool_name = os.path.splitext(os.path.basename(__file__))[0] + def pdbGenerator(env, target, source, for_signature): try: return ['/PDB:%s' % target[0].attributes.pdb, '/DEBUG'] @@ -307,7 +309,7 @@ def generate(env): env['_MANIFEST_SOURCES'] = None # _windowsManifestSources # Set-up ms tools paths - msvc_setup_env_once(env) + msvc_setup_env_once(env, tool=tool_name) # Loadable modules are on Windows the same as shared libraries, but they # are subject to different build parameters (LDMODULE* variables). @@ -330,7 +332,7 @@ def generate(env): env['TEMPFILEARGJOIN'] = os.linesep def exists(env): - return msvc_exists(env) + return msvc_setup_env_tool(env, tool=tool_name) # Local Variables: # tab-width:4 diff --git a/SCons/Tool/msvc.py b/SCons/Tool/msvc.py index f2cd418..ee876ac 100644 --- a/SCons/Tool/msvc.py +++ b/SCons/Tool/msvc.py @@ -44,9 +44,11 @@ import SCons.Util import SCons.Warnings import SCons.Scanner.RC -from .MSCommon import msvc_exists, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere +from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere from .MSCommon.common import get_pch_node +tool_name = os.path.splitext(os.path.basename(__file__))[0] + CSuffixes = ['.c', '.C'] CXXSuffixes = ['.cc', '.cpp', '.cxx', '.c++', '.C++'] @@ -293,7 +295,7 @@ def generate(env): env['VSWHERE'] = env.get('VSWHERE', msvc_find_vswhere()) # Set-up ms tools paths - msvc_setup_env_once(env) + msvc_setup_env_once(env, tool=tool_name) env['CFILESUFFIX'] = '.c' env['CXXFILESUFFIX'] = '.cc' @@ -319,7 +321,7 @@ def generate(env): def exists(env): - return msvc_exists(env) + return msvc_setup_env_tool(env, tool=tool_name) # Local Variables: # tab-width:4 diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 887cb59..2eae8ee 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -45,7 +45,9 @@ import SCons.Util import SCons.Warnings from SCons.Defaults import processDefines from SCons.compat import PICKLE_PROTOCOL -from .MSCommon import msvc_exists, msvc_setup_env_once +from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once + +tool_name = os.path.splitext(os.path.basename(__file__))[0] ############################################################################## # Below here are the classes and functions for generation of @@ -2077,7 +2079,7 @@ def generate(env): env['MSVSCLEANCOM'] = '$MSVSSCONSCOM -c "$MSVSBUILDTARGET"' # Set-up ms tools paths for default version - msvc_setup_env_once(env) + msvc_setup_env_once(env, tool=tool_name) if 'MSVS_VERSION' in env: version_num, suite = msvs_parse_version(env['MSVS_VERSION']) @@ -2107,7 +2109,7 @@ def generate(env): env['SCONS_HOME'] = os.environ.get('SCONS_HOME') def exists(env): - return msvc_exists(env) + return msvc_setup_env_tool(env, tool=tool_name) # Local Variables: # tab-width:4 diff --git a/test/MSVC/msvc_badversion.py b/test/MSVC/msvc_badversion.py index 7af8ca5..ce419a8 100644 --- a/test/MSVC/msvc_badversion.py +++ b/test/MSVC/msvc_badversion.py @@ -47,7 +47,21 @@ installed_msvc_versions = msvc.get_installed_vcs() test.write('SConstruct', """\ env = Environment(MSVC_VERSION='12.9') """) +test.run(arguments='-Q -s', stdout='') +test.write('SConstruct', """\ +env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='ignore') +""") +test.run(arguments='-Q -s', stdout='') + +test.write('SConstruct', """\ +env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='warning') +""") +test.run(arguments='-Q -s', stdout='') + +test.write('SConstruct', """\ +env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='error') +""") test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) test.pass_test() diff --git a/test/MSVC/no_msvc.py b/test/MSVC/no_msvc.py index 683951e..4ab7dd8 100644 --- a/test/MSVC/no_msvc.py +++ b/test/MSVC/no_msvc.py @@ -43,7 +43,10 @@ test.run(arguments='-Q -s', stdout='') # test no msvc's test.file_fixture('no_msvc/no_msvcs_sconstruct.py', 'SConstruct') -test.run(arguments='-Q -s', stdout='') +test.run(arguments='-Q -s') + +if 'MSVC_VERSION=None' not in test.stdout(): + test.fail_test() # test msvc version number request with no msvc's test.file_fixture('no_msvc/no_msvcs_sconstruct_version.py', 'SConstruct') @@ -64,8 +67,5 @@ def exists(env): test.file_fixture('no_msvc/no_msvcs_sconstruct_tools.py', 'SConstruct') test.run(arguments='-Q -s') -test.file_fixture('no_msvc/no_msvcs_sconstruct_tools.py', 'SConstruct') -test.run(arguments='-Q -s') - test.pass_test() diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct.py b/test/fixture/no_msvc/no_msvcs_sconstruct.py index 590142b..18366d8 100644 --- a/test/fixture/no_msvc/no_msvcs_sconstruct.py +++ b/test/fixture/no_msvc/no_msvcs_sconstruct.py @@ -12,3 +12,4 @@ SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere env = SCons.Environment.Environment() +print('MSVC_VERSION='+str(env.get('MSVC_VERSION'))) \ No newline at end of file diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct_version.py b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py index 81d47cf..f5cabf7 100644 --- a/test/fixture/no_msvc/no_msvcs_sconstruct_version.py +++ b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py @@ -10,5 +10,7 @@ for key in SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR: SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere +SCons.Tool.MSCommon.set_msvc_notfound_policy('error') + env = SCons.Environment.Environment(MSVC_VERSION='14.3') -- cgit v0.12 From 51aadabc68d29c0f5ef2c782c3573092e2606ec0 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 29 Apr 2022 12:38:35 -0400 Subject: Return previous policy when setting msvc notfound policy --- SCons/Tool/MSCommon/vc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index c100663..57e44ee 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -802,10 +802,12 @@ def _msvc_notfound_policy_lookup(symbol): def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): global _MSVC_NOTFOUND_POLICY + prev_policy = _MSVC_NOTFOUND_POLICY + if MSVC_NOTFOUND_POLICY is not None: _MSVC_NOTFOUND_POLICY = _msvc_notfound_policy_lookup(MSVC_NOTFOUND_POLICY) - return _MSVC_NOTFOUND_POLICY + return prev_policy def get_msvc_notfound_policy(): return _MSVC_NOTFOUND_POLICY -- cgit v0.12 From 7572fe5e0a721c7e85d0d3681c8f50bfcb09b944 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 2 May 2022 09:29:54 -0400 Subject: Fix set/get notfound policy to return policy symbol instead of internal policy, fix initialization state, rework debug statements. --- SCons/Tool/MSCommon/vc.py | 90 +++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 31 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 57e44ee..4c417f2 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -90,14 +90,16 @@ class MSVCVersionNotFound(VisualCException): _MSVC_NOTFOUND_POLICY_DEFAULT = False _MSVC_NOTFOUND_POLICY = _MSVC_NOTFOUND_POLICY_DEFAULT +_MSVC_NOTFOUND_POLICY_REVERSE_DICT = {} _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC = [] _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT = {} for value, symbol_list in [ - (True, ['Error', 'Exception']), - (False, ['Warn', 'Warning']), - (None, ['Ignore', 'Suppress']), + (True, ['Error', 'Exception']), + (False, ['Warning', 'Warn']), + (None, ['Ignore', 'Suppress']), ]: + _MSVC_NOTFOUND_POLICY_REVERSE_DICT[value] = symbol_list[0].lower() for symbol in symbol_list: _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC.append(symbol.lower()) _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] = value @@ -802,15 +804,19 @@ def _msvc_notfound_policy_lookup(symbol): def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): global _MSVC_NOTFOUND_POLICY - prev_policy = _MSVC_NOTFOUND_POLICY + prev_policy = _MSVC_NOTFOUND_POLICY_REVERSE_DICT[_MSVC_NOTFOUND_POLICY] - if MSVC_NOTFOUND_POLICY is not None: - _MSVC_NOTFOUND_POLICY = _msvc_notfound_policy_lookup(MSVC_NOTFOUND_POLICY) + policy = MSVC_NOTFOUND_POLICY + if policy is not None: + _MSVC_NOTFOUND_POLICY = _msvc_notfound_policy_lookup(policy) + debug('prev_policy=%s, policy=%s, internal_policy=%s', repr(prev_policy), repr(policy), _MSVC_NOTFOUND_POLICY) return prev_policy def get_msvc_notfound_policy(): - return _MSVC_NOTFOUND_POLICY + policy = _MSVC_NOTFOUND_POLICY_REVERSE_DICT[_MSVC_NOTFOUND_POLICY] + debug('policy=%s, internal_policy=%s', repr(policy), _MSVC_NOTFOUND_POLICY) + return policy def _msvc_notfound_policy_handler(env, msg): @@ -821,13 +827,14 @@ def _msvc_notfound_policy_handler(env, msg): # use active global setting notfound_policy = _MSVC_NOTFOUND_POLICY + debug('policy=%s, internal_policy=%s', _MSVC_NOTFOUND_POLICY_REVERSE_DICT[notfound_policy], repr(notfound_policy)) + if notfound_policy is None: - debug('notfound policy: ignore') + # ignore + pass elif notfound_policy: - debug('notfound policy: exception') raise MSVCVersionNotFound(msg) else: - debug('notfound policy: warning') SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) class _MSVCSetupEnvDefault: @@ -853,6 +860,7 @@ class _MSVCSetupEnvDefault: @classmethod def reset(cls): + debug('msvc default:init') cls.n_setup = 0 # number of calls to msvc_setup_env_once cls.default_ismsvc = False # is msvc the default compiler cls.default_tools_re_list = [] # list of default tools regular expressions @@ -860,18 +868,20 @@ class _MSVCSetupEnvDefault: cls.msvc_tools = None # tools registered via msvc_setup_env_once cls.msvc_installed = False # is msvc installed (vcs_installed > 0) cls.msvc_nodefault = False # is there a default version of msvc - cls.need_init = False # clear initialization indicator + cls.need_init = True # reset initialization indicator @classmethod def _initialize(cls, env): if cls.need_init: cls.reset() + cls.need_init = False vcs = get_installed_vcs(env) cls.msvc_installed = len(vcs) > 0 - debug('msvc default:initialize:msvc_installed=%s', cls.msvc_installed) + debug('msvc default:msvc_installed=%s', cls.msvc_installed) @classmethod def register_tool(cls, env, tool): + debug('msvc default:tool=%s', tool) if cls.need_init: cls._initialize(env) if cls.msvc_installed: @@ -881,14 +891,15 @@ class _MSVCSetupEnvDefault: if cls.n_setup == 0: if tool not in cls.msvc_tools_init: cls.msvc_tools_init.add(tool) - debug('msvc default:register tool:tool=%s msvc_tools_init=%s', tool, cls.msvc_tools_init) + debug('msvc default:tool=%s, msvc_tools_init=%s', tool, cls.msvc_tools_init) return None if tool not in cls.msvc_tools: cls.msvc_tools.add(tool) - debug('msvc default:register tool:tool=%s msvc_tools=%s', tool, cls.msvc_tools) + debug('msvc default:tool=%s, msvc_tools=%s', tool, cls.msvc_tools) @classmethod def register_setup(cls, env): + debug('msvc default') if cls.need_init: cls._initialize(env) cls.n_setup += 1 @@ -902,7 +913,7 @@ class _MSVCSetupEnvDefault: cls.default_ismsvc = True cls.msvc_nodefault = False debug( - 'msvc default:register setup:n=%d msvc_installed=%s default_ismsvc=%s', + 'msvc default:n_setup=%d, msvc_installed=%s, default_ismsvc=%s', cls.n_setup, cls.msvc_installed, cls.default_ismsvc ) @@ -910,7 +921,7 @@ class _MSVCSetupEnvDefault: def set_nodefault(cls): # default msvc version, msvc not installed cls.msvc_nodefault = True - debug('msvc default:set nodefault:msvc_nodefault=%s', cls.msvc_nodefault) + debug('msvc default:msvc_nodefault=%s', cls.msvc_nodefault) @classmethod def register_iserror(cls, env, tool): @@ -931,7 +942,7 @@ class _MSVCSetupEnvDefault: return None debug( - 'msvc default:register iserror:n=%s default_ismsvc=%s msvc_tools=%s tool_list=%s', + 'msvc default:n_setup=%s, default_ismsvc=%s, msvc_tools=%s, tool_list=%s', cls.n_setup, cls.default_ismsvc, cls.msvc_tools, tool_list ) @@ -952,7 +963,7 @@ class _MSVCSetupEnvDefault: # build default tools regex for current tool state tools = cls.separator.join(tool_list) tools_nchar = len(tools) - debug('msvc default:register iserror:add regex nchar=%d, tools=%s', tools_nchar, tools) + debug('msvc default:add regex:nchar=%d, tools=%s', tools_nchar, tools) re_default_tools = re.compile(re.escape(tools)) cls.default_tools_re_list.insert(0, (tools_nchar, re_default_tools)) # early exit: no error for default environment when msvc is not installed @@ -987,7 +998,7 @@ class _MSVCSetupEnvDefault: tools = cls.separator.join(tool_list) tools_nchar = len(tools) - debug('msvc default:register iserror:nchar=%d tools=%s', tools_nchar, tools) + debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) # iteratively remove default tool sequences (longest to shortest) re_nchar_min, re_tools_min = cls.default_tools_re_list[-1] @@ -999,7 +1010,7 @@ class _MSVCSetupEnvDefault: continue tools = re_default_tool.sub('', tools).strip(cls.separator) tools_nchar = len(tools) - debug('msvc default:register iserror:nchar=%d tools=%s', tools_nchar, tools) + debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) if tools_nchar < re_nchar_min or not re_tools_min.search(tools): # less than minimum characters or minimum pattern does not exist break @@ -1007,13 +1018,13 @@ class _MSVCSetupEnvDefault: # construct non-default list(s) tools set tools_set = {msvc_tool for msvc_tool in tools.split(cls.separator) if msvc_tool} - debug('msvc default:register iserror:tools=%s', tools_set) + debug('msvc default:tools=%s', tools_set) if not tools_set: return None # compute intersection of remaining tools set and msvc tools set tools_found = cls.msvc_tools.intersection(tools_set) - debug('msvc default:register iserror:tools_exist=%s', tools_found) + debug('msvc default:tools_exist=%s', tools_found) if not tools_found: return None @@ -1070,6 +1081,7 @@ def msvc_setup_env_once(env, tool=None): has_run = False if not has_run: + debug('tool=%s', repr(tool)) _MSVCSetupEnvDefault.register_setup(env) msvc_setup_env(env) env["MSVC_SETUP_RUN"] = True @@ -1264,18 +1276,34 @@ def msvc_setup_env(env): SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) def msvc_exists(env=None, version=None): + debug('version=%s', repr(version)) vcs = get_installed_vcs(env) if version is None: - return len(vcs) > 0 - return version in vcs + rval = len(vcs) > 0 + else: + rval = version in vcs + debug('version=%s, return=%s', repr(version), rval) + return rval -def msvc_setup_env_tool(env=None, version=None, tool=None): - _MSVCSetupEnvDefault.register_tool(env, tool) - if msvc_exists(env, version): - return True +def msvc_setup_env_user(env=None): + rval = False if env: - for key in ('MSVC_VERSION', 'MSVS_VERSION'): + for key in ('MSVC_VERSION', 'MSVS_VERSION', 'MSVC_USE_SCRIPT'): if key in env: - return True - return False + rval = True + debug('key=%s, return=%s', repr(key), rval) + return rval + debug('return=%s', rval) + return rval + +def msvc_setup_env_tool(env=None, version=None, tool=None): + debug('tool=%s, version=%s', repr(tool), repr(version)) + _MSVCSetupEnvDefault.register_tool(env, tool) + rval = False + if not rval and msvc_exists(env, version): + rval = True + if not rval and msvc_setup_env_user(env): + rval = True + debug('tool=%s, version=%s, return=%s', repr(tool), repr(version), rval) + return rval -- cgit v0.12 From 6d7bf1ec0efac43c2158ed21653a8070fba91f35 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 2 May 2022 11:29:41 -0400 Subject: Rename internal notfound policy dictionary. --- SCons/Tool/MSCommon/vc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 4c417f2..ec81eaf 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -90,7 +90,7 @@ class MSVCVersionNotFound(VisualCException): _MSVC_NOTFOUND_POLICY_DEFAULT = False _MSVC_NOTFOUND_POLICY = _MSVC_NOTFOUND_POLICY_DEFAULT -_MSVC_NOTFOUND_POLICY_REVERSE_DICT = {} +_MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL = {} _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC = [] _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT = {} @@ -99,7 +99,7 @@ for value, symbol_list in [ (False, ['Warning', 'Warn']), (None, ['Ignore', 'Suppress']), ]: - _MSVC_NOTFOUND_POLICY_REVERSE_DICT[value] = symbol_list[0].lower() + _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[value] = symbol_list[0].lower() for symbol in symbol_list: _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC.append(symbol.lower()) _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] = value @@ -804,7 +804,7 @@ def _msvc_notfound_policy_lookup(symbol): def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): global _MSVC_NOTFOUND_POLICY - prev_policy = _MSVC_NOTFOUND_POLICY_REVERSE_DICT[_MSVC_NOTFOUND_POLICY] + prev_policy = _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[_MSVC_NOTFOUND_POLICY] policy = MSVC_NOTFOUND_POLICY if policy is not None: @@ -814,7 +814,7 @@ def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): return prev_policy def get_msvc_notfound_policy(): - policy = _MSVC_NOTFOUND_POLICY_REVERSE_DICT[_MSVC_NOTFOUND_POLICY] + policy = _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[_MSVC_NOTFOUND_POLICY] debug('policy=%s, internal_policy=%s', repr(policy), _MSVC_NOTFOUND_POLICY) return policy @@ -827,7 +827,7 @@ def _msvc_notfound_policy_handler(env, msg): # use active global setting notfound_policy = _MSVC_NOTFOUND_POLICY - debug('policy=%s, internal_policy=%s', _MSVC_NOTFOUND_POLICY_REVERSE_DICT[notfound_policy], repr(notfound_policy)) + debug('policy=%s, internal_policy=%s', _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[notfound_policy], repr(notfound_policy)) if notfound_policy is None: # ignore -- cgit v0.12 From ba3dcaab98cf8c5156c7d5f80589a2b57162d02b Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Fri, 6 May 2022 09:19:24 -0500 Subject: improved ninja default targets and for ninja file as only target when generating. --- CHANGES.txt | 3 + RELEASE.txt | 4 + SCons/Tool/ninja/Globals.py | 1 + SCons/Tool/ninja/NinjaState.py | 93 +++++++++++++++------- SCons/Tool/ninja/__init__.py | 22 ++++- SCons/Tool/ninja/ninja_daemon_build.py | 9 ++- SCons/Tool/ninja/ninja_scons_daemon.py | 29 ++++--- test/ninja/default_targets.py | 83 +++++++++++++++++++ test/ninja/force_scons_callback.py | 2 +- .../sconstruct_default_targets | 14 ++++ 10 files changed, 215 insertions(+), 45 deletions(-) create mode 100644 test/ninja/default_targets.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_default_targets diff --git a/CHANGES.txt b/CHANGES.txt index 42d20d7..eee3fd2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -80,6 +80,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected format. Compiler tools will also configure the variable automatically. + - Made ninja tool force the ninja file as the only target. Also improved the default + targets setup and made sure there is always a default target for + the ninja file, which excludes targets that start and stop the daemon. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 6bb1546..3f8a11e 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -49,6 +49,10 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY output format written to stdout to include more information about the source for each message of MSVC initialization debugging output. A single space was added before the message for all debugging output records written to stdout and to files. +- Made ninja tool force the ninja file as the only target. Also improved the default + targets setup and made sure there is always a default target for + the ninja file, which excludes targets that start and stop the daemon. + FIXES ----- diff --git a/SCons/Tool/ninja/Globals.py b/SCons/Tool/ninja/Globals.py index 4d1d38b..d84cae6 100644 --- a/SCons/Tool/ninja/Globals.py +++ b/SCons/Tool/ninja/Globals.py @@ -29,6 +29,7 @@ NINJA_CUSTOM_HANDLERS = "__NINJA_CUSTOM_HANDLERS" NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} +NINJA_DEFAULT_TARGETS = [] __NINJA_RULE_MAPPING = {} # These are the types that get_command can do something with diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 63ea3a1..5f8d5c7 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -29,6 +29,7 @@ import signal import tempfile import shutil import sys +import time from os.path import splitext from tempfile import NamedTemporaryFile import ninja @@ -39,7 +40,7 @@ from SCons.Script import COMMAND_LINE_TARGETS from SCons.Util import is_List from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ - NINJA_CUSTOM_HANDLERS + NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ get_outputs, get_inputs, get_dependencies, get_rule, get_command_env, to_escaped_list @@ -68,7 +69,7 @@ class NinjaState: # couldn't find it, just give the bin name and hope # its in the path later self.ninja_bin_path = ninja_bin - + self.ninja_syntax = ninja_syntax self.writer_class = ninja_syntax.Writer self.__generated = False self.translator = SConsToNinjaTranslator(env) @@ -77,11 +78,6 @@ class NinjaState: # List of generated builds that will be written at a later stage self.builds = dict() - # List of targets for which we have generated a build. This - # allows us to take multiple Alias nodes as sources and to not - # fail to build if they have overlapping targets. - self.built = set() - # SCons sets this variable to a function which knows how to do # shell quoting on whatever platform it's run on. Here we use it # to make the SCONS_INVOCATION variable properly quoted for things @@ -96,6 +92,11 @@ class NinjaState: python_bin = ninja_syntax.escape(scons_escape(sys.executable)) self.variables = { "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + 'PORT': scons_daemon_port, + 'NINJA_DIR_PATH': env.get('NINJA_DIR').abspath, + 'PYTHON_BIN': sys.executable, + 'NINJA_TOOL_DIR': pathlib.Path(__file__).parent, + 'NINJA_SCONS_DAEMON_KEEP_ALIVE': str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE')), "SCONS_INVOCATION": '{} {} --disable-ninja __NINJA_NO=1 $out'.format( python_bin, " ".join( @@ -209,7 +210,7 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} $out", + "command": "$PYTHON_BIN $NINJA_TOOL_DIR/ninja_daemon_build.py $PORT $NINJA_DIR_PATH $out", "description": "Defer to SCons to build $out", "pool": "local_pool", "restat": 1 @@ -238,7 +239,7 @@ class NinjaState: }, "SCONS_DAEMON": { - "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {env.get('NINJA_DIR').abspath} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "command": "$PYTHON_BIN $NINJA_TOOL_DIR/ninja_run_daemon.py $PORT $NINJA_DIR_PATH $NINJA_SCONS_DAEMON_KEEP_ALIVE $SCONS_INVOCATION", "description": "Starting scons daemon...", "pool": "local_pool", # restat @@ -317,7 +318,6 @@ class NinjaState: else: raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build - self.built.update(build["outputs"]) return True # TODO: rely on SCons to tell us what is generated source @@ -361,8 +361,7 @@ class NinjaState: self.rules[rule]["depfile"] = "$out.d" else: raise Exception(f"Unknown 'NINJA_DEPFILE_PARSE_FORMAT'={self.env['NINJA_DEPFILE_PARSE_FORMAT']}, use 'mvsc', 'gcc', or 'clang'.") - - + for key, rule in self.env.get(NINJA_RULES, {}).items(): # make a non response file rule for users custom response file rules. if rule.get('rspfile') is not None: @@ -374,7 +373,6 @@ class NinjaState: else: self.rules.update({key: rule}) - self.rules.update(self.env.get(NINJA_RULES, {})) self.pools.update(self.env.get(NINJA_POOLS, {})) content = io.StringIO() @@ -451,10 +449,20 @@ class NinjaState: template_builders = [] scons_compiledb = False + if SCons.Script._Get_Default_Targets == SCons.Script._Set_Default_Targets_Has_Not_Been_Called: + all_targets = set() + else: + all_targets = None + for build in [self.builds[key] for key in sorted(self.builds.keys())]: if "compile_commands.json" in build["outputs"]: scons_compiledb = True + # this is for the no command line targets, no SCons default case. We want this default + # to just be all real files in the build. + if all_targets is not None and build['rule'] != 'phony': + all_targets = all_targets | set(build["outputs"]) + if build["rule"] == "TEMPLATE": template_builders.append(build) continue @@ -604,6 +612,22 @@ class NinjaState: ) + if all_targets is None: + # Look in SCons's list of DEFAULT_TARGETS, find the ones that + # we generated a ninja build rule for. + all_targets = [str(node) for node in NINJA_DEFAULT_TARGETS] + else: + all_targets = list(all_targets) + + if len(all_targets) == 0: + all_targets = ["phony_default"] + ninja.build( + outputs=all_targets, + rule="phony", + ) + + ninja.default([self.ninja_syntax.escape_path(path) for path in sorted(all_targets)]) + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) pidfile = None if os.path.exists(scons_daemon_dirty): @@ -619,24 +643,37 @@ class NinjaState: except OSError: pass + # wait for the server process to fully killed + try: + import psutil + while True: + if pid not in [proc.pid for proc in psutil.process_iter()]: + break + else: + time.sleep(0.1) + except ImportError: + # if psutil is not installed we can do this the hard way + while True: + if sys.platform == 'win32': + import ctypes + PROCESS_QUERY_INFORMATION = 0x1000 + processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) + if processHandle == 0: + break + else: + ctypes.windll.kernel32.CloseHandle(processHandle) + time.sleep(0.1) + else: + try: + os.kill(pid, 0) + except OSError: + break + else: + time.sleep(0.1) + if os.path.exists(scons_daemon_dirty): os.unlink(scons_daemon_dirty) - - # Look in SCons's list of DEFAULT_TARGETS, find the ones that - # we generated a ninja build rule for. - scons_default_targets = [ - get_path(tgt) - for tgt in SCons.Script.DEFAULT_TARGETS - if get_path(tgt) in self.built - ] - - # If we found an overlap between SCons's list of default - # targets and the targets we created ninja builds for then use - # those as ninja's default as well. - if scons_default_targets: - ninja.default(" ".join(scons_default_targets)) - with NamedTemporaryFile(delete=False, mode='w') as temp_ninja_file: temp_ninja_file.write(content.getvalue()) shutil.move(temp_ninja_file.name, ninja_file_path) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 526db8a..4429217 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -34,7 +34,7 @@ import SCons import SCons.Tool.ninja.Globals from SCons.Script import GetOption -from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ gen_get_response_file_command @@ -118,6 +118,7 @@ def ninja_builder(env, target, source): # leaving warnings and other output, seems a bit # prone to failure with such a simple check erase_previous = output.startswith('[') + sys.stdout.write("\n") def exists(env): @@ -200,7 +201,7 @@ def generate(env): ninja_file = env.Ninja() env['NINJA_FILE'] = ninja_file[0] env.AlwaysBuild(ninja_file) - env.Alias("$NINJA_ALIAS_NAME", ninja_file) + SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file)) else: if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") @@ -445,6 +446,23 @@ def generate(env): # date-ness. SCons.Script.Main.BuildTask.needs_execute = lambda x: True + + def ninja_Set_Default_Targets(env, tlist): + """ + Record the default targets if they were ever set by the user. Ninja + will need to write the default targets and make sure not to include + the scons daemon shutdown target. + """ + SCons.Script._Get_Default_Targets = SCons.Script._Set_Default_Targets_Has_Been_Called + SCons.Script.DEFAULT_TARGETS = ninja_file + for t in tlist: + if isinstance(t, SCons.Node.Node): + NINJA_DEFAULT_TARGETS.append(t) + else: + nodes = env.arg2nodes(t, env.fs.Entry) + NINJA_DEFAULT_TARGETS.extend(nodes) + SCons.Script._Set_Default_Targets = ninja_Set_Default_Targets + # We will eventually need to overwrite TempFileMunge to make it # handle persistent tempfiles or get an upstreamed change to add # some configurability to it's behavior in regards to tempfiles. diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index f34935b..62749b0 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -39,6 +39,7 @@ import pathlib import tempfile import hashlib import traceback +import socket ninja_builddir = pathlib.Path(sys.argv[2]) daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( @@ -65,8 +66,8 @@ while True: while not response: try: response = conn.getresponse() - except (http.client.RemoteDisconnected, http.client.ResponseNotReady): - time.sleep(0.01) + except (http.client.RemoteDisconnected, http.client.ResponseNotReady, socket.timeout): + time.sleep(0.1) except http.client.HTTPException: logging.debug(f"Error: {traceback.format_exc()}") exit(1) @@ -79,6 +80,10 @@ while True: logging.debug(f"Request Done: {sys.argv[3]}") exit(0) + except ConnectionRefusedError: + logging.debug(f"Server refused connection to build {sys.argv[3]}, maybe it was too busy, tring again: {traceback.format_exc()}") + time.sleep(0.1) + except Exception: logging.debug(f"Failed to send command: {traceback.format_exc()}") exit(1) diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index f900343..a50a478 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -50,6 +50,7 @@ from timeit import default_timer as timer import traceback import tempfile import hashlib +import signal port = int(sys.argv[1]) ninja_builddir = pathlib.Path(sys.argv[2]) @@ -134,6 +135,15 @@ building_cv = Condition() error_cv = Condition() thread_error = False +httpd = None +daemon_needs_to_shutdown = False + +def sigint_func(signum, frame): + global httpd, daemon_needs_to_shutdown + daemon_needs_to_shutdown = True + + +signal.signal(signal.SIGINT, sigint_func) def daemon_thread_func(): @@ -213,6 +223,8 @@ def daemon_thread_func(): finished_building += [building_node] daemon_ready = False + if daemon_needs_to_shutdown: + break time.sleep(0.01) except Exception: thread_error = True @@ -283,6 +295,7 @@ def server_thread_func(): def log_message(self, format, *args): return + socketserver.TCPServer.allow_reuse_address = True httpd = socketserver.TCPServer(("127.0.0.1", port), S) httpd.serve_forever() @@ -291,31 +304,23 @@ server_thread = threading.Thread(target=server_thread_func) server_thread.daemon = True server_thread.start() -while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: +while timer() - keep_alive_timer < daemon_keep_alive and not thread_error and not daemon_needs_to_shutdown: time.sleep(1) if thread_error: daemon_log(f"Shutting server on port {port} down because thread error.") +elif daemon_needs_to_shutdown: + daemon_log("Server shutting down upon request.") else: daemon_log( f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" ) - -# if there are errors, don't immediately shut down the daemon -# the process which started the server is attempt to connect to -# the daemon before allowing jobs to start being sent. If the daemon -# shuts down too fast, the launch script will think it has not -# started yet and sit and wait. If the launch script is able to connect -# and then the connection is dropped, it will immediately exit with fail. -time.sleep(5) - +httpd.shutdown() if os.path.exists(ninja_builddir / "scons_daemon_dirty"): os.unlink(ninja_builddir / "scons_daemon_dirty") if os.path.exists(daemon_dir / "pidfile"): os.unlink(daemon_dir / "pidfile") -httpd.shutdown() -server_thread.join() # Local Variables: diff --git a/test/ninja/default_targets.py b/test/ninja/default_targets.py new file mode 100644 index 0000000..dc7c7c1 --- /dev/null +++ b/test/ninja/default_targets.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_default_targets', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_not_exist([test.workpath('out1.txt')]) +test.must_exist([test.workpath('out2.txt')]) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed out2.txt', + 'Removed build.ninja']) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('out1.txt')) +test.must_not_exist(test.workpath('out2.txt')) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.must_not_exist([test.workpath('out1.txt')]) +test.must_exist(test.workpath('out2.txt')) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index c99ed58..e0da0dd 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -80,7 +80,7 @@ test.must_match("out2.txt", "test2.cpp" + os.linesep) # only generate the ninja file with specific NINJA_SCONS_DAEMON_PORT test.run(arguments="PORT=9999 --disable-execute-ninja", stdout=None) # Verify that port # propagates to call to ninja_run_daemon.py -test.must_contain(test.workpath("build.ninja"), "ninja_run_daemon.py 9999") +test.must_contain(test.workpath("build.ninja"), "PORT = 9999") test.pass_test() diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_default_targets b/test/ninja/ninja_test_sconscripts/sconstruct_default_targets new file mode 100644 index 0000000..8963270 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_default_targets @@ -0,0 +1,14 @@ +import SCons + +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=[]) + +env.Tool('ninja') + +env.Command('out1.txt', 'foo.c', 'echo test > $TARGET' ) +out2_node = env.Command('out2.txt', 'foo.c', 'echo test > $TARGET', NINJA_FORCE_SCONS_BUILD=True) +alias = env.Alias('def', out2_node) + +env.Default(alias) -- cgit v0.12 From 75b79e01089b5b835fd30fb5777044459144dc89 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 10 May 2022 10:42:40 -0500 Subject: added target propagation for ninja execution --- CHANGES.txt | 5 ++ RELEASE.txt | 8 ++- SCons/Tool/ninja/Globals.py | 1 + SCons/Tool/ninja/Utils.py | 3 + SCons/Tool/ninja/__init__.py | 23 ++++++-- SCons/Tool/ninja/ninja_run_daemon.py | 1 + SCons/Tool/ninja/ninja_scons_daemon.py | 101 ++++++++++++++++++++++----------- test/ninja/command_line_targets.py | 95 +++++++++++++++++++++++++++++++ 8 files changed, 196 insertions(+), 41 deletions(-) create mode 100644 test/ninja/command_line_targets.py diff --git a/CHANGES.txt b/CHANGES.txt index eee3fd2..a40bd1c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -83,6 +83,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Made ninja tool force the ninja file as the only target. Also improved the default targets setup and made sure there is always a default target for the ninja file, which excludes targets that start and stop the daemon. + - Update ninja tool so targets passed to SCons are propgated to ninja when scons + automatically executes ninja. + - Small refactor of scons daemons using a shared StateInfo class for communication + between the scons interactive thread and the http server thread. Added error handling + for scons interactive failing to startup. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index 3f8a11e..7c10e27 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -52,8 +52,9 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - Made ninja tool force the ninja file as the only target. Also improved the default targets setup and made sure there is always a default target for the ninja file, which excludes targets that start and stop the daemon. - - +- Update ninja tool so targets passed to SCons are propgated to ninja when scons + automatically executes ninja. + FIXES ----- @@ -104,6 +105,9 @@ IMPROVEMENTS and msvc batch file determination when configuring the build environment. Simplify the msvc code by eliminating special case handling primarily due to the differences between the full versions and express versions of visual studio. +- Small refactor of scons daemons using a shared StateInfo class for communication + between the scons interactive thread and the http server thread. Added error handling + for scons interactive failing to startup. PACKAGING --------- diff --git a/SCons/Tool/ninja/Globals.py b/SCons/Tool/ninja/Globals.py index d84cae6..6b079ef 100644 --- a/SCons/Tool/ninja/Globals.py +++ b/SCons/Tool/ninja/Globals.py @@ -30,6 +30,7 @@ NINJA_BUILD = "NINJA_BUILD" NINJA_WHEREIS_MEMO = {} NINJA_STAT_MEMO = {} NINJA_DEFAULT_TARGETS = [] +NINJA_CMDLINE_TARGETS = [] __NINJA_RULE_MAPPING = {} # These are the types that get_command can do something with diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 888218d..c5f7ee4 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -29,6 +29,9 @@ from SCons.Action import get_default_ENV, _string_from_cmd_list from SCons.Script import AddOption from SCons.Util import is_List, flatten_sequence +class NinjaExperimentalWarning(SCons.Warnings.WarningOnByDefault): + pass + def ninja_add_command_line_options(): """ diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 4429217..bf6345d 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -31,16 +31,17 @@ import subprocess import sys import SCons +import SCons.Script import SCons.Tool.ninja.Globals from SCons.Script import GetOption -from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS +from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS, NINJA_CMDLINE_TARGETS from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ set_build_node_callback, get_generic_shell_command, CheckNinjaCompdbExpand, get_command, \ gen_get_response_file_command from .Overrides import ninja_hack_linkcom, ninja_hack_arcom, NinjaNoResponseFiles, ninja_always_serial, AlwaysExecAction from .Utils import ninja_add_command_line_options, \ - ninja_noop, ninja_print_conf_log, ninja_csig, ninja_contents, ninja_stat, ninja_whereis + ninja_noop, ninja_print_conf_log, ninja_csig, ninja_contents, ninja_stat, ninja_whereis, NinjaExperimentalWarning try: import ninja @@ -84,7 +85,10 @@ def ninja_builder(env, target, source): else: cmd = [NINJA_STATE.ninja_bin_path, '-f', generated_build_ninja] - if not env.get("NINJA_DISABLE_AUTO_RUN"): + if str(env.get("NINJA_DISABLE_AUTO_RUN")).lower() not in ['1', 'true']: + num_jobs = env.get('NINJA_MAX_JOBS', env.GetOption("num_jobs")) + cmd += ['-j' + str(num_jobs)] + NINJA_CMDLINE_TARGETS + print(f"ninja will be run with command line targets: {' '.join(NINJA_CMDLINE_TARGETS)}") print("Executing:", str(' '.join(cmd))) # execute the ninja build at the end of SCons, trying to @@ -165,7 +169,7 @@ def ninja_emitter(target, source, env): def generate(env): """Generate the NINJA builders.""" - global NINJA_STATE + global NINJA_STATE, NINJA_CMDLINE_TARGETS if 'ninja' not in GetOption('experimental'): return @@ -201,6 +205,13 @@ def generate(env): ninja_file = env.Ninja() env['NINJA_FILE'] = ninja_file[0] env.AlwaysBuild(ninja_file) + + # We need to force SCons to only build the ninja target when ninja tool is loaded. + # The ninja tool is going to 'rip the guts out' of scons and make it basically unable + # to do anything in terms of building, so any targets besides the ninja target will + # end up doing nothing besides causing confusion. We save the targets however, so that + # SCons and invoke ninja to build them in lieu of the user. + NINJA_CMDLINE_TARGETS = SCons.Script.BUILD_TARGETS SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file)) else: if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: @@ -306,8 +317,8 @@ def generate(env): if GetOption('disable_ninja'): return env - SCons.Warnings.SConsWarning("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") - + print("Initializing ninja tool... this feature is experimental. SCons internals and all environments will be affected.") + print(f"SCons running in ninja mode. {env['NINJA_FILE']} will be generated.") # This is the point of no return, anything after this comment # makes changes to SCons that are irreversible and incompatible # with a normal SCons build. We return early if __NINJA_NO=1 has diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index e78b549..3ed9264 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -114,6 +114,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): msg = response.read() status = response.status if status != 200: + print("ERROR: SCons daemon failed to start:") print(msg.decode("utf-8")) exit(1) logging.debug("Server Responded it was ready!") diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index a50a478..1bf3979 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -128,28 +128,30 @@ input_q = queue.Queue() output_q = queue.Queue() error_q = queue.Queue() -finished_building = [] -error_nodes = [] - building_cv = Condition() error_cv = Condition() -thread_error = False -httpd = None -daemon_needs_to_shutdown = False +class StateInfo: + def __init__(self) -> None: + self.thread_error = False + self.finished_building = [] + self.error_nodes = [] + self.startup_failed = False + self.startup_output = '' + self.daemon_needs_to_shutdown = False + self.httpd = None -def sigint_func(signum, frame): - global httpd, daemon_needs_to_shutdown - daemon_needs_to_shutdown = True +shared_state = StateInfo() +def sigint_func(signum, frame): + global shared_state + shared_state.daemon_needs_to_shutdown = True signal.signal(signal.SIGINT, sigint_func) def daemon_thread_func(): - global thread_error - global finished_building - global error_nodes + global shared_state try: args_list = args + ["--interactive"] daemon_log(f"Starting daemon with args: {' '.join(args_list)}") @@ -167,16 +169,22 @@ def daemon_thread_func(): daemon_ready = False building_node = None + startup_complete = False + # While scons interactive process is stil running... while p.poll() is None: + # while there is scons output to process while True: try: line = output_q.get(block=False, timeout=0.01) except queue.Empty: + # breaks out of the output processing loop break else: daemon_log("output: " + line.strip()) + if not startup_complete: + shared_state.startup_output += line if "scons: building terminated because of errors." in line: error_output = "" @@ -185,13 +193,16 @@ def daemon_thread_func(): error_output += error_q.get(block=False, timeout=0.01) except queue.Empty: break - error_nodes += [{"node": building_node, "error": error_output}] + shared_state.error_nodes += [{"node": building_node, "error": error_output}] daemon_ready = True building_node = None with building_cv: building_cv.notify() elif line == "scons>>>": + shared_state.startup_output = '' + startup_complete = True + with error_q.mutex: error_q.queue.clear() daemon_ready = True @@ -199,6 +210,7 @@ def daemon_thread_func(): building_cv.notify() building_node = None + # while there is input to process... while daemon_ready and not input_q.empty(): try: @@ -209,7 +221,7 @@ def daemon_thread_func(): p.stdin.write("exit\n".encode("utf-8")) p.stdin.flush() with building_cv: - finished_building += [building_node] + shared_state.finished_building += [building_node] daemon_ready = False raise @@ -220,14 +232,21 @@ def daemon_thread_func(): p.stdin.write(input_command.encode("utf-8")) p.stdin.flush() with building_cv: - finished_building += [building_node] + shared_state.finished_building += [building_node] daemon_ready = False - if daemon_needs_to_shutdown: + if shared_state.daemon_needs_to_shutdown: break time.sleep(0.01) + + # our scons process is done, make sure we are shutting down in this case + if not shared_state.daemon_needs_to_shutdown: + if not startup_complete: + shared_state.startup_failed = True + shared_state.daemon_needs_to_shutdown = True + except Exception: - thread_error = True + shared_state.thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -241,18 +260,20 @@ logging.debug( ) keep_alive_timer = timer() -httpd = None - def server_thread_func(): + global shared_state class S(http.server.BaseHTTPRequestHandler): def do_GET(self): - global thread_error + global shared_state global keep_alive_timer - global error_nodes - try: gets = parse_qs(urlparse(self.path).query) + + # process a request from ninja for a node for scons to build. + # Currently this is a serial process because scons interactive is serial + # is it was originally meant for a real human user to be providing input + # parallel input was never implemented. build = gets.get("build") if build: keep_alive_timer = timer() @@ -261,12 +282,12 @@ def server_thread_func(): input_q.put(build[0]) def pred(): - return build[0] in finished_building + return build[0] in shared_state.finished_building with building_cv: building_cv.wait_for(pred) - for error_node in error_nodes: + for error_node in shared_state.error_nodes: if error_node["node"] == build[0]: self.send_response(500) self.send_header("Content-type", "text/html") @@ -279,6 +300,19 @@ def server_thread_func(): self.end_headers() return + # this message is used in server startup, to make sure the server launched + # successfully. If SCons interactive got to a input prompt (scons>>>), then + # the server is ready to start processing commands. Otherwise the server will + # send an error response back to ninja and shut itself down. + ready = gets.get("ready") + if ready: + if shared_state.startup_failed: + self.send_response(500) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(shared_state.startup_output.encode()) + return + exitbuild = gets.get("exit") if exitbuild: input_q.put("exit") @@ -288,7 +322,7 @@ def server_thread_func(): self.end_headers() except Exception: - thread_error = True + shared_state.thread_error = True daemon_log("SERVER ERROR: " + traceback.format_exc()) raise @@ -296,33 +330,34 @@ def server_thread_func(): return socketserver.TCPServer.allow_reuse_address = True - httpd = socketserver.TCPServer(("127.0.0.1", port), S) - httpd.serve_forever() + shared_state.httpd = socketserver.TCPServer(("127.0.0.1", port), S) + shared_state.httpd.serve_forever() server_thread = threading.Thread(target=server_thread_func) server_thread.daemon = True server_thread.start() -while timer() - keep_alive_timer < daemon_keep_alive and not thread_error and not daemon_needs_to_shutdown: +while ( + timer() - keep_alive_timer < daemon_keep_alive + and not shared_state.thread_error + and not shared_state.daemon_needs_to_shutdown): time.sleep(1) -if thread_error: +if shared_state.thread_error: daemon_log(f"Shutting server on port {port} down because thread error.") -elif daemon_needs_to_shutdown: +elif shared_state.daemon_needs_to_shutdown: daemon_log("Server shutting down upon request.") else: daemon_log( f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" ) -httpd.shutdown() +shared_state.httpd.shutdown() if os.path.exists(ninja_builddir / "scons_daemon_dirty"): os.unlink(ninja_builddir / "scons_daemon_dirty") if os.path.exists(daemon_dir / "pidfile"): os.unlink(daemon_dir / "pidfile") - - # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/test/ninja/command_line_targets.py b/test/ninja/command_line_targets.py new file mode 100644 index 0000000..ad61846 --- /dev/null +++ b/test/ninja/command_line_targets.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_default_targets', 'SConstruct') + +# generate simple build +test.run(arguments=['out1.txt'], stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_contain_all(test.stdout(), 'build.ninja -j1 out1.txt') +test.must_exist([test.workpath('out1.txt')]) +test.must_not_exist([test.workpath('out2.txt')]) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed out1.txt', + 'Removed build.ninja']) + +# generate simple build +test.run(arguments=['out2.txt'], stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_contain_all(test.stdout(), 'build.ninja -j1 out2.txt') +test.must_not_exist([test.workpath('out1.txt')]) +test.must_exist([test.workpath('out2.txt')]) + +# clean build and ninja files +test.run(arguments='-c', stdout=None) +test.must_contain_all_lines(test.stdout(), [ + 'Removed out2.txt', + 'Removed build.ninja']) + +test.run(arguments=['out1.txt', 'out2.txt'], stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_contain_all(test.stdout(), 'build.ninja -j1 out1.txt out2.txt') +test.must_exist([test.workpath('out1.txt')]) +test.must_exist([test.workpath('out2.txt')]) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 880109b6e0bef087b93fef93fda0178bd7defed4 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 10 May 2022 11:56:14 -0500 Subject: fix sider issues --- SCons/Tool/ninja/__init__.py | 2 +- SCons/Tool/ninja/ninja_scons_daemon.py | 7 +++---- test/ninja/command_line_targets.py | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index bf6345d..04a7abb 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -211,7 +211,7 @@ def generate(env): # to do anything in terms of building, so any targets besides the ninja target will # end up doing nothing besides causing confusion. We save the targets however, so that # SCons and invoke ninja to build them in lieu of the user. - NINJA_CMDLINE_TARGETS = SCons.Script.BUILD_TARGETS + NINJA_CMDLINE_TARGETS = SCons.Script.BUILD_TARGETS SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file)) else: if str(NINJA_STATE.ninja_file) != env["NINJA_FILE_NAME"]: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 1bf3979..c4a1d11 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -338,10 +338,9 @@ server_thread = threading.Thread(target=server_thread_func) server_thread.daemon = True server_thread.start() -while ( - timer() - keep_alive_timer < daemon_keep_alive - and not shared_state.thread_error - and not shared_state.daemon_needs_to_shutdown): +while (timer() - keep_alive_timer < daemon_keep_alive + and not shared_state.thread_error + and not shared_state.daemon_needs_to_shutdown): time.sleep(1) if shared_state.thread_error: diff --git a/test/ninja/command_line_targets.py b/test/ninja/command_line_targets.py index ad61846..9fa77ea 100644 --- a/test/ninja/command_line_targets.py +++ b/test/ninja/command_line_targets.py @@ -25,7 +25,6 @@ import os import TestSCons -from TestCmd import IS_WINDOWS test = TestSCons.TestSCons() -- cgit v0.12 From 50df0c523b69b1ec832e1b39f6fd16fcd139f36e Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Wed, 11 May 2022 23:15:18 -0500 Subject: add another exception which should be handled by retry --- SCons/Tool/ninja/ninja_run_daemon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 297bcf4..08029a2 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -46,6 +46,7 @@ import logging import time import http.client import traceback +import socket ninja_builddir = pathlib.Path(sys.argv[2]) daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( @@ -108,7 +109,7 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): try: response = conn.getresponse() - except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + except (http.client.RemoteDisconnected, http.client.ResponseNotReady, socket.timeout): time.sleep(0.01) except http.client.HTTPException: log_error(f"Error: {traceback.format_exc()}") -- cgit v0.12 From cc81a44b0095ad2e9df68e20aa9d3e4ce6f4dff9 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 May 2022 16:08:38 -0400 Subject: Hard-code tool name instead of deriving from file name --- SCons/Tool/midl.py | 2 +- SCons/Tool/mslib.py | 2 +- SCons/Tool/mslink.py | 2 +- SCons/Tool/msvc.py | 2 +- SCons/Tool/msvs.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/midl.py b/SCons/Tool/midl.py index 22cdd85..72d7696 100644 --- a/SCons/Tool/midl.py +++ b/SCons/Tool/midl.py @@ -43,7 +43,7 @@ import SCons.Util from .MSCommon import msvc_setup_env_tool -tool_name = os.path.splitext(os.path.basename(__file__))[0] +tool_name = 'midl' def midl_emitter(target, source, env): """Produces a list of outputs from the MIDL compiler""" diff --git a/SCons/Tool/mslib.py b/SCons/Tool/mslib.py index cbeaa7f..be4088b 100644 --- a/SCons/Tool/mslib.py +++ b/SCons/Tool/mslib.py @@ -43,7 +43,7 @@ import SCons.Util from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once -tool_name = os.path.splitext(os.path.basename(__file__))[0] +tool_name = 'mslib' def generate(env): """Add Builders and construction variables for lib to an Environment.""" diff --git a/SCons/Tool/mslink.py b/SCons/Tool/mslink.py index 1376020..2a90e17 100644 --- a/SCons/Tool/mslink.py +++ b/SCons/Tool/mslink.py @@ -46,7 +46,7 @@ import SCons.Util from .MSCommon import msvc_setup_env_once, msvc_setup_env_tool from .MSCommon.common import get_pch_node -tool_name = os.path.splitext(os.path.basename(__file__))[0] +tool_name = 'mslink' def pdbGenerator(env, target, source, for_signature): try: diff --git a/SCons/Tool/msvc.py b/SCons/Tool/msvc.py index ae47fdd..191d2cc 100644 --- a/SCons/Tool/msvc.py +++ b/SCons/Tool/msvc.py @@ -47,7 +47,7 @@ import SCons.Scanner.RC from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere from .MSCommon.common import get_pch_node -tool_name = os.path.splitext(os.path.basename(__file__))[0] +tool_name = 'msvc' CSuffixes = ['.c', '.C'] CXXSuffixes = ['.cc', '.cpp', '.cxx', '.c++', '.C++'] diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 2eae8ee..86df1ef 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -47,7 +47,7 @@ from SCons.Defaults import processDefines from SCons.compat import PICKLE_PROTOCOL from .MSCommon import msvc_setup_env_tool, msvc_setup_env_once -tool_name = os.path.splitext(os.path.basename(__file__))[0] +tool_name = 'msvs' ############################################################################## # Below here are the classes and functions for generation of -- cgit v0.12 From a90c14216220dcdf1112da7c3aff52b795c3b8a1 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 May 2022 16:12:46 -0400 Subject: Update test script header with template/test.py header --- test/MSVC/msvc_badversion.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/MSVC/msvc_badversion.py b/test/MSVC/msvc_badversion.py index ce419a8..a36bd2b 100644 --- a/test/MSVC/msvc_badversion.py +++ b/test/MSVC/msvc_badversion.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test scons with an invalid MSVC version when at least one MSVC is present. -- cgit v0.12 From 053f220e789a42e104faa29b467fb224fc1468e5 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 May 2022 20:29:20 -0400 Subject: Fix tool name in test fixture --- test/fixture/no_msvc/no_msvcs_sconstruct_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py b/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py index ca9b699..9aa924b 100644 --- a/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py +++ b/test/fixture/no_msvc/no_msvcs_sconstruct_tools.py @@ -10,5 +10,5 @@ for key in SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR: SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere -env = SCons.Environment.Environment(tools=['MYIGNOREDEFAULTMSVCTOOL']) +env = SCons.Environment.Environment(tools=['myignoredefaultmsvctool']) -- cgit v0.12 From 85159b73248da753504440d0fc39e868096ce770 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 15 May 2022 20:56:04 -0400 Subject: Remove unnecessary import --- SCons/Tool/midl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/SCons/Tool/midl.py b/SCons/Tool/midl.py index 72d7696..3def928 100644 --- a/SCons/Tool/midl.py +++ b/SCons/Tool/midl.py @@ -33,8 +33,6 @@ selection method. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" -import os - import SCons.Action import SCons.Builder import SCons.Defaults -- cgit v0.12 From a506af6e1eb8bede7cc4e227e7887d39001bb53e Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 17 May 2022 19:06:49 -0400 Subject: Raise exception for unexpected cache file format --- SCons/Tool/MSCommon/common.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index f542e02..577ec48 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -110,7 +110,15 @@ def read_script_env_cache(): # json to the cache dictionary. Reconstruct the cache key # tuple from the key list written to json. envcache_list = json.load(f) - envcache = {tuple(d['key']): d['data'] for d in envcache_list} + if isinstance(envcache_list, list): + envcache = {tuple(d['key']): d['data'] for d in envcache_list} + else: + raise TypeError( + 'SCONS_CACHE_MSVC_CONFIG cache file read error: expected type {}, found type {}.\n' \ + ' Remove cache file {} and try again'.format( + repr('list'), repr(type(envcache_list).__name__), repr(CONFIG_CACHE) + ) + ) except FileNotFoundError: # don't fail if no cache file, just proceed without it pass -- cgit v0.12 From 2efe7d108823f528bdfc162ea9aae572914ae383 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 18 May 2022 16:10:03 -0700 Subject: [ci skip] fix sider complaint. Reformat. Update file header to current standard --- SCons/Tool/midl.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/SCons/Tool/midl.py b/SCons/Tool/midl.py index 72d7696..c25ce69 100644 --- a/SCons/Tool/midl.py +++ b/SCons/Tool/midl.py @@ -1,15 +1,6 @@ -"""SCons.Tool.midl - -Tool-specific initialization for midl (Microsoft IDL compiler). - -There normally shouldn't be any need to import this module directly. -It will usually be imported through the generic SCons.Tool.Tool() -selection method. - -""" - +# MIT License # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -29,11 +20,16 @@ selection method. # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +"""SCons.Tool.midl -import os +Tool-specific initialization for midl (Microsoft IDL compiler). + +There normally shouldn't be any need to import this module directly. +It will usually be imported through the generic SCons.Tool.Tool() +selection method. + +""" import SCons.Action import SCons.Builder @@ -45,6 +41,7 @@ from .MSCommon import msvc_setup_env_tool tool_name = 'midl' + def midl_emitter(target, source, env): """Produces a list of outputs from the MIDL compiler""" base, _ = SCons.Util.splitext(str(target[0])) @@ -62,26 +59,29 @@ def midl_emitter(target, source, env): dlldata = base + '_data.c' targets.append(dlldata) - return (targets, source) + return targets, source + idl_scanner = SCons.Scanner.IDL.IDLScan() midl_action = SCons.Action.Action('$MIDLCOM', '$MIDLCOMSTR') -midl_builder = SCons.Builder.Builder(action = midl_action, - src_suffix = '.idl', +midl_builder = SCons.Builder.Builder(action=midl_action, + src_suffix='.idl', suffix='.tlb', - emitter = midl_emitter, - source_scanner = idl_scanner) + emitter=midl_emitter, + source_scanner=idl_scanner) + def generate(env): """Add Builders and construction variables for midl to an Environment.""" - env['MIDL'] = 'MIDL.EXE' - env['MIDLFLAGS'] = SCons.Util.CLVar('/nologo') - env['MIDLCOM'] = '$MIDL $MIDLFLAGS /tlb ${TARGETS[0]} /h ${TARGETS[1]} /iid ${TARGETS[2]} /proxy ${TARGETS[3]} /dlldata ${TARGETS[4]} $SOURCE 2> NUL' + env['MIDL'] = 'MIDL.EXE' + env['MIDLFLAGS'] = SCons.Util.CLVar('/nologo') + env['MIDLCOM'] = '$MIDL $MIDLFLAGS /tlb ${TARGETS[0]} /h ${TARGETS[1]} /iid ${TARGETS[2]} /proxy ${TARGETS[3]} /dlldata ${TARGETS[4]} $SOURCE 2> NUL' env['BUILDERS']['TypeLibrary'] = midl_builder + def exists(env): return msvc_setup_env_tool(env, tool=tool_name) -- cgit v0.12 From 0e51fe4467417350bc969c498698999b6f3b946e Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 19 May 2022 08:46:16 -0400 Subject: Change notfound policy parallel variables to namedtuple. Rework debug statement contents and formatting for notfound policy. Make comment for internal class a docstring per request. --- SCons/Tool/MSCommon/vc.py | 117 +++++++++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 2dd28de..33faca2 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -91,28 +91,36 @@ class MSVCUseSettingsError(VisualCException): class MSVCVersionNotFound(VisualCException): pass -# MSVC_NOTFOUND_POLICY: -# error: raise exception -# warn: issue warning and continue -# ignore: continue -_MSVC_NOTFOUND_POLICY_DEFAULT = False -_MSVC_NOTFOUND_POLICY = _MSVC_NOTFOUND_POLICY_DEFAULT - -_MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL = {} -_MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC = [] -_MSVC_NOTFOUND_POLICY_SYMBOLS_DICT = {} - -for value, symbol_list in [ +# MSVC_NOTFOUND_POLICY definition: +# error: raise exception +# warning: issue warning and continue +# ignore: continue + +_MSVC_NOTFOUND_POLICY_DEFINITION = namedtuple('MSVCNotFoundPolicyDefinition', [ + 'value', + 'symbol', +]) + +_MSVC_NOTFOUND_POLICY_INTERNAL = {} +_MSVC_NOTFOUND_POLICY_EXTERNAL = {} + +for policy_value, policy_symbol_list in [ (True, ['Error', 'Exception']), (False, ['Warning', 'Warn']), (None, ['Ignore', 'Suppress']), ]: - _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[value] = symbol_list[0].lower() - for symbol in symbol_list: - _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC.append(symbol.lower()) - _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] = value - _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol.lower()] = value - _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol.upper()] = value + + policy_symbol = policy_symbol_list[0].lower() + policy_def = _MSVC_NOTFOUND_POLICY_DEFINITION(policy_value, policy_symbol) + + _MSVC_NOTFOUND_POLICY_INTERNAL[policy_symbol] = policy_def + + for policy_symbol in policy_symbol_list: + _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.lower()] = policy_def + _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol] = policy_def + _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.upper()] = policy_def + +_MSVC_NOTFOUND_POLICY_DEF = _MSVC_NOTFOUND_POLICY_INTERNAL['warning'] # Dict to 'canonalize' the arch _ARCH_TO_CANONICAL = { @@ -1050,69 +1058,82 @@ def script_env(script, args=None): def _msvc_notfound_policy_lookup(symbol): try: - notfound_policy = _MSVC_NOTFOUND_POLICY_SYMBOLS_DICT[symbol] + notfound_policy_def = _MSVC_NOTFOUND_POLICY_EXTERNAL[symbol] except KeyError: err_msg = "Value specified for MSVC_NOTFOUND_POLICY is not supported: {}.\n" \ " Valid values are: {}".format( repr(symbol), - ', '.join([repr(s) for s in _MSVC_NOTFOUND_POLICY_SYMBOLS_PUBLIC]) + ', '.join([repr(s) for s in _MSVC_NOTFOUND_POLICY_EXTERNAL.keys()]) ) raise ValueError(err_msg) - return notfound_policy + return notfound_policy_def def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): - global _MSVC_NOTFOUND_POLICY + global _MSVC_NOTFOUND_POLICY_DEF - prev_policy = _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[_MSVC_NOTFOUND_POLICY] + prev_policy = _MSVC_NOTFOUND_POLICY_DEF.symbol policy = MSVC_NOTFOUND_POLICY if policy is not None: - _MSVC_NOTFOUND_POLICY = _msvc_notfound_policy_lookup(policy) + _MSVC_NOTFOUND_POLICY_DEF = _msvc_notfound_policy_lookup(policy) + + debug( + 'prev_policy=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', + repr(prev_policy), repr(policy), + repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) + ) - debug('prev_policy=%s, policy=%s, internal_policy=%s', repr(prev_policy), repr(policy), _MSVC_NOTFOUND_POLICY) return prev_policy def get_msvc_notfound_policy(): - policy = _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[_MSVC_NOTFOUND_POLICY] - debug('policy=%s, internal_policy=%s', repr(policy), _MSVC_NOTFOUND_POLICY) - return policy + debug( + 'policy.symbol=%s, policy.value=%s', + repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) + ) + return _MSVC_NOTFOUND_POLICY_DEF.symbol def _msvc_notfound_policy_handler(env, msg): if env and 'MSVC_NOTFOUND_POLICY' in env: # use environment setting - notfound_policy = _msvc_notfound_policy_lookup(env['MSVC_NOTFOUND_POLICY']) + notfound_policy_def = _msvc_notfound_policy_lookup(env['MSVC_NOTFOUND_POLICY']) + notfound_policy_src = 'environment' else: # use active global setting - notfound_policy = _MSVC_NOTFOUND_POLICY + notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF + notfound_policy_src = 'default' - debug('policy=%s, internal_policy=%s', _MSVC_NOTFOUND_POLICY_INTERNAL_SYMBOL[notfound_policy], repr(notfound_policy)) + debug( + 'source=%s, policy.symbol=%s, policy.value=%s', + notfound_policy_src, repr(notfound_policy_def.symbol), repr(notfound_policy_def.value) + ) - if notfound_policy is None: + if notfound_policy_def.value is None: # ignore pass - elif notfound_policy: + elif notfound_policy_def.value: raise MSVCVersionNotFound(msg) else: SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) class _MSVCSetupEnvDefault: - - # Determine if and/or when an error/warning should be issued when there - # are no versions of msvc installed. If there is at least one version of - # msvc installed, these routines do (almost) nothing. - - # Notes: - # * When msvc is the default compiler because there are no compilers - # installed, a build may fail due to the cl.exe command not being - # recognized. Currently, there is no easy way to detect during - # msvc initialization if the default environment will be used later - # to build a program and/or library. There is no error/warning - # as there are legitimate SCons uses that do not require a c compiler. - # * As implemented, the default is that a warning is issued. This can - # be changed globally via the function set_msvc_notfound_policy and/or - # through the environment via the MSVC_NOTFOUND_POLICY variable. + """ + Determine if and/or when an error/warning should be issued when there + are no versions of msvc installed. If there is at least one version of + msvc installed, these routines do (almost) nothing. + + Notes: + * When msvc is the default compiler because there are no compilers + installed, a build may fail due to the cl.exe command not being + recognized. Currently, there is no easy way to detect during + msvc initialization if the default environment will be used later + to build a program and/or library. There is no error/warning + as there are legitimate SCons uses that do not require a c compiler. + * As implemented, the default is that a warning is issued. This can + be changed globally via the function set_msvc_notfound_policy and/or + through the environment via the MSVC_NOTFOUND_POLICY variable. + """ separator = r';' -- cgit v0.12 From 479b123c6177fca85990b0d6890d9737a49016e2 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 19 May 2022 10:25:42 -0400 Subject: Treat environment notfound policy set to None the same as if undefined (default global setting) --- SCons/Tool/MSCommon/vc.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 33faca2..32d0e61 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1096,17 +1096,24 @@ def get_msvc_notfound_policy(): def _msvc_notfound_policy_handler(env, msg): if env and 'MSVC_NOTFOUND_POLICY' in env: - # use environment setting - notfound_policy_def = _msvc_notfound_policy_lookup(env['MSVC_NOTFOUND_POLICY']) + # environment setting notfound_policy_src = 'environment' + policy = env['MSVC_NOTFOUND_POLICY'] + if policy is not None: + # user policy request + notfound_policy_def = _msvc_notfound_policy_lookup(policy) + else: + # active global setting + notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF else: - # use active global setting - notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF + # active global setting notfound_policy_src = 'default' + policy = None + notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF debug( - 'source=%s, policy.symbol=%s, policy.value=%s', - notfound_policy_src, repr(notfound_policy_def.symbol), repr(notfound_policy_def.value) + 'source=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', + notfound_policy_src, repr(policy), repr(notfound_policy_def.symbol), repr(notfound_policy_def.value) ) if notfound_policy_def.value is None: -- cgit v0.12 From 64a652ee2aa8714322c1d07f0d2fc3056d610212 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 19 May 2022 16:46:03 -0400 Subject: Issue warning for incompatible cache format and continue; file possibly overwritten. --- SCons/Tool/MSCommon/common.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index 577ec48..c9f07f5 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -35,6 +35,10 @@ from contextlib import suppress from pathlib import Path import SCons.Util +import SCons.Warnings + +class MSVCCacheInvalidWarning(SCons.Warnings.WarningOnByDefault): + pass # SCONS_MSCOMMON_DEBUG is internal-use so undocumented: # set to '-' to print to console, else set to filename to log to @@ -113,12 +117,12 @@ def read_script_env_cache(): if isinstance(envcache_list, list): envcache = {tuple(d['key']): d['data'] for d in envcache_list} else: - raise TypeError( - 'SCONS_CACHE_MSVC_CONFIG cache file read error: expected type {}, found type {}.\n' \ - ' Remove cache file {} and try again'.format( - repr('list'), repr(type(envcache_list).__name__), repr(CONFIG_CACHE) - ) + # don't fail if incompatible format, just proceed without it + warn_msg = "Incompatible format for msvc cache file {}: file may be overwritten.".format( + repr(CONFIG_CACHE) ) + SCons.Warnings.warn(MSVCCacheInvalidWarning, warn_msg) + debug(warn_msg) except FileNotFoundError: # don't fail if no cache file, just proceed without it pass -- cgit v0.12 From 6848757227a6b9e605be699d6f0b28445b3deede Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 21 May 2022 11:08:03 -0400 Subject: Update CHANGES.txt and RELEASE.txt. [ci skip] --- CHANGES.txt | 19 +++++++++++++++++++ RELEASE.txt | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 17121f4..3978b5b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -48,6 +48,25 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER require delayed expansion to be enabled which is currently not supported and is typically not enabled by default on the host system. The batch files may also require environment variables that are not included by default in the msvc environment. + - Suppress issuing a warning when there are no installed Visual Studio instances for the default + tools configuration (issue #2813). When msvc is the default compiler because there are no + compilers installed, a build may fail due to the cl.exe command not being recognized. At + present, there is no easy way to detect during msvc initialization if the default environment + will be used later to build a program and/or library. There is no error/warning issued for the + default tools as there are legitimate SCons uses that do not require a c compiler. + - Added a global policy setting and an environment policy variable for specifying the action to + be taken when an msvc request cannot be satisfied. The available options are "error", + "exception", "warning", "warn", "ignore", and "suppress". The global policy variable may be + set and retrieved via the functions set_msvc_notfound_policy and get_msvc_notfound_policy, + respectively. These two methods may be imported from SCons.Tool.MSCommon. The environment + policy variable introduced is MSVC_NOTFOUND_POLICY. When defined, the environment policy + variable overrides the global policy setting for a given environment. When the active policy + is "error" or "exception", an MSVCVersionNotFound exception is raised. When the active policy + is "warning" or "warn", a VisualCMissingWarning warning is issued and the constructed + environment is likely incomplete. When the active policy is "ignore" or "suppress", no action + is taken and the constructed environment is likely incomplete. As implemented, the default + global policy is "warning". The ability to set the global policy via an SCons command-line + option may be added in a future enhancement. From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message diff --git a/RELEASE.txt b/RELEASE.txt index 1a00d02..94bedaf 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -21,6 +21,19 @@ NEW FUNCTIONALITY be used in the shell command of some Action. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. +- Added a global policy setting and an environment policy variable for specifying the action to + be taken when an msvc request cannot be satisfied. The available options are "error", + "exception", "warning", "warn", "ignore", and "suppress". The global policy variable may be + set and retrieved via the functions set_msvc_notfound_policy and get_msvc_notfound_policy, + respectively. These two methods may be imported from SCons.Tool.MSCommon. The environment + policy variable introduced is MSVC_NOTFOUND_POLICY. When defined, the environment policy + variable overrides the global policy setting for a given environment. When the active policy + is "error" or "exception", an MSVCVersionNotFound exception is raised. When the active policy + is "warning" or "warn", a VisualCMissingWarning warning is issued and the constructed + environment is likely incomplete. When the active policy is "ignore" or "suppress", no action + is taken and the constructed environment is likely incomplete. As implemented, the default + global policy is "warning". The ability to set the global policy via an SCons command-line + option may be added in a future enhancement. DEPRECATED FUNCTIONALITY @@ -120,6 +133,12 @@ FIXES - The system environment variable names imported for MSVC 7.0 and 6.0 were updated to be consistent with the variables names defined by their respective installers. This fixes an error caused when bypassing MSVC detection by specifying the MSVC 7.0 batch file directly. +- Suppress issuing a warning when there are no installed Visual Studio instances for the default + tools configuration (issue #2813). When msvc is the default compiler because there are no + compilers installed, a build may fail due to the cl.exe command not being recognized. At + present, there is no easy way to detect during msvc initialization if the default environment + will be used later to build a program and/or library. There is no error/warning issued for the + default tools as there are legitimate SCons uses that do not require a c compiler. IMPROVEMENTS ------------ -- cgit v0.12 From 2e80fc1c2796a705b9049aba686de8e35bb878cd Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 23 May 2022 22:55:33 -0500 Subject: Updated SHELL_ENV_GENERATOR to be list of functions now called SHELL_ENV_GENERATORS --- CHANGES.txt | 5 +++- RELEASE.txt | 3 +++ SCons/Action.py | 22 +++++++++++++--- SCons/Action.xml | 35 +++++++++++++++++++------ SCons/Tool/ninja/Methods.py | 4 +-- SCons/Tool/ninja/NinjaState.py | 11 +++++++- SCons/Tool/ninja/Utils.py | 4 +-- test/Actions/subst_shell_env-fixture/SConstruct | 24 ++++++++++++----- test/Actions/subst_shell_env.py | 4 +-- 9 files changed, 87 insertions(+), 25 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 519ced9..60e934f 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -99,9 +99,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected format. Compiler tools will also configure the variable automatically. - - Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define + - Added SHELL_ENV_GENERATOR construction variable. This variable allows the user to Define a function which will be called to generate or alter the execution environment which will be used in the shell command of some Action. + - Updated SHELL_ENV_GENERATOR construction variable to SHELL_ENV_GENERATORS. This variable + is now an iterable which will contain functions which each are called and each can customize + the execution environment. - Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. - Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry to connect to the server during start up. diff --git a/RELEASE.txt b/RELEASE.txt index 5bfee35..38fbbd7 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -19,6 +19,9 @@ NEW FUNCTIONALITY - Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define a function which will be called to generate or alter the execution environment which will be used in the shell command of some Action. +- Updated SHELL_ENV_GENERATOR construction variable to SHELL_ENV_GENERATORS. This variable + is now an iterable which will contain functions which each are called and each can customize + the execution environment. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. diff --git a/SCons/Action.py b/SCons/Action.py index 0849178..4dd0543 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -732,7 +732,7 @@ def _string_from_cmd_list(cmd_list): default_ENV = None -def get_default_ENV(env, target=None, source=None): +def get_default_ENV(env): """ A fiddlin' little function that has an 'import SCons.Environment' which can't be moved to the top level without creating an import loop. Since @@ -755,6 +755,23 @@ def get_default_ENV(env, target=None, source=None): return default_ENV +def _resolve_shell_env(env, target, source): + ENV = get_default_ENV(env) + shell_gen = env.get('SHELL_ENV_GENERATORS') + if shell_gen is not None: + ENV = ENV.copy() + try: + shell_gens = iter(shell_gen) + except: + raise SCons.Errors.UserError("SHELL_ENV_GENERATORS must be iteratable.") + else: + for generator in shell_gens: + ENV = generator(env, target, source, ENV) + if not isinstance(ENV, dict): + raise SCons.Errors.UserError("SHELL_ENV_GENERATORS function: {generator} must return a dict.") + return ENV + + def _subproc(scons_env, cmd, error='ignore', **kw): """Wrapper for subprocess which pulls from construction env. @@ -924,10 +941,9 @@ class CommandAction(_ActionAction): escape = env.get('ESCAPE', lambda x: x) - ENV = env.get('SHELL_ENV_GENERATOR', get_default_ENV)(env, target, source) + ENV = _resolve_shell_env(env, target, source) # Ensure that the ENV values are all strings: - for key, value in ENV.items(): if not is_String(value): if is_List(value): diff --git a/SCons/Action.xml b/SCons/Action.xml index 2c18d55..8994adb 100644 --- a/SCons/Action.xml +++ b/SCons/Action.xml @@ -200,18 +200,32 @@ in which the command should be executed. - + -A function to generate or alter the environment dictionary which will be used -when executing the &cv-link-SPAWN; function. This primarily give the -user a chance to customize the execution environment for particular Actions. -It must return a dictionary containing the environment variables as -keys and the values as values. +Must be an iterable containing functions where each function generates or +alters the environment dictionary which will be used +when executing the &cv-link-SPAWN; function. The functions will initially +be passed a reference of the current execution environment (e.g. env['ENV']), +and each called while iterating the list. Each function must return a dictionary +which will then be passed to the next function iterated. The return dictionary +should contain keys which represent the environment variables and their respective +values. + +This primary purpose of this construction variable is to give the user the ability +to substitute execution environment variables based on env, targets, and sources. +If desired, the user can completly customize the execution environment for particular +targets. -def custom_shell_env(env, target, source): +def custom_shell_env(env, target, source, shell_env): + # customize shell_env if desired + if str(target[0]) == 'special_target'is: + shell_env['SPECIAL_VAR'] = env.subst('SOME_VAR', target=target, source=source) + return shell_env + +env["SHELL_ENV_GENERATORS"] = [custom_shell_env] @@ -223,10 +237,15 @@ execution environment can be derived from. target The list of targets associated with this action. - + source The list of sources associated with this action. + + shell_env +The current shell_env after iterating other SHELL_ENV_GENERATORS functions. This can be compared +to the passed env['ENV'] to detect any changes. + diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 2be576f..c0afab8 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -81,7 +81,7 @@ def get_generic_shell_command(env, node, action, targets, sources, executor=None "GENERATED_CMD", { "cmd": generate_command(env, node, action, targets, sources, executor=executor), - "env": get_command_env(env), + "env": get_command_env(env, targets, sources), }, # Since this function is a rule mapping provider, it must return a list of dependencies, # and usually this would be the path to a tool, such as a compiler, used for this rule. @@ -266,7 +266,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom variables = {"rspc": rsp_content, rule: cmd} if use_command_env: - variables["env"] = get_command_env(env) + variables["env"] = get_command_env(env, targets, sources) for key, value in custom_env.items(): variables["env"] += env.subst( diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 63ea3a1..ac10b5e 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -809,6 +809,15 @@ class SConsToNinjaTranslator: # Remove all preceding and proceeding whitespace cmdline = cmdline.strip() + env = node.env if node.env else self.env + executor = node.get_executor() + if executor is not None: + targets = executor.get_all_targets() + else: + if hasattr(node, "target_peers"): + targets = node.target_peers + else: + targets = [node] # Make sure we didn't generate an empty cmdline if cmdline: @@ -817,7 +826,7 @@ class SConsToNinjaTranslator: "rule": get_rule(node, "GENERATED_CMD"), "variables": { "cmd": cmdline, - "env": get_command_env(node.env if node.env else self.env), + "env": get_command_env(env, targets, node.sources), }, "implicit": dependencies, } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 888218d..2bd2263 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -259,7 +259,7 @@ def ninja_noop(*_args, **_kwargs): return None -def get_command_env(env): +def get_command_env(env, target, source): """ Return a string that sets the environment for any environment variables that differ between the OS environment and the SCons command ENV. @@ -275,7 +275,7 @@ def get_command_env(env): # os.environ or differ from it. We assume if it's a new or # differing key from the process environment then it's # important to pass down to commands in the Ninja file. - ENV = get_default_ENV(env) + ENV = SCons.Action._resolve_shell_env(env, target, source) scons_specified_env = { key: value for key, value in ENV.items() diff --git a/test/Actions/subst_shell_env-fixture/SConstruct b/test/Actions/subst_shell_env-fixture/SConstruct index 6e48add..5ba822e 100644 --- a/test/Actions/subst_shell_env-fixture/SConstruct +++ b/test/Actions/subst_shell_env-fixture/SConstruct @@ -1,24 +1,36 @@ import sys -def custom_environment_expansion(env, target, source): - ENV = env['ENV'].copy() - ENV['EXPANDED_SHELL_VAR'] = env.subst(env['ENV']['EXPANDED_SHELL_VAR'], target=target, source=source) +def custom_environment_expansion1(env, target, source, shell_env): + ENV = shell_env.copy() + ENV['EXPANDED_SHELL_VAR1'] = env.subst(env['ENV']['EXPANDED_SHELL_VAR1'], target=target, source=source) + return ENV + +def custom_environment_expansion2(env, target, source, shell_env): + ENV = shell_env.copy() + ENV['EXPANDED_SHELL_VAR2'] = env.subst(env['ENV']['EXPANDED_SHELL_VAR2'], target=target, source=source) return ENV def expand_this_generator(env, target, source, for_signature): return "I_got_expanded_to_" + str(target[0]) +def expand_that_generator(env, target, source, for_signature): + return str(target[0]) + "_is_from_expansion" + env = Environment(tools=['textfile']) -env['SHELL_ENV_GENERATOR'] = custom_environment_expansion +env['SHELL_ENV_GENERATORS'] = [custom_environment_expansion1, custom_environment_expansion2] env['EXPAND_THIS'] = expand_this_generator -env['ENV']['EXPANDED_SHELL_VAR'] = "$EXPAND_THIS" +env['EXPAND_THAT'] = expand_that_generator + +env['ENV']['EXPANDED_SHELL_VAR1'] = "$EXPAND_THIS" +env['ENV']['EXPANDED_SHELL_VAR2'] = "$EXPAND_THAT" env['ENV']['NON_EXPANDED_SHELL_VAR'] = "$EXPAND_THIS" env.Textfile('expand_script.py', [ 'import os', - 'print(os.environ["EXPANDED_SHELL_VAR"])', + 'print(os.environ["EXPANDED_SHELL_VAR1"])', + 'print(os.environ["EXPANDED_SHELL_VAR2"])', 'print(os.environ["NON_EXPANDED_SHELL_VAR"])', ]) env.Command('out.txt', 'expand_script.py', fr'{sys.executable} $SOURCE > $TARGET') diff --git a/test/Actions/subst_shell_env.py b/test/Actions/subst_shell_env.py index 9f5c5db..d26f0ae 100644 --- a/test/Actions/subst_shell_env.py +++ b/test/Actions/subst_shell_env.py @@ -36,8 +36,8 @@ test = TestSCons.TestSCons() test.dir_fixture('subst_shell_env-fixture') test.run(arguments = ['-Q']) -test.must_match('out.txt', f"I_got_expanded_to_out.txt{os.linesep}$EXPAND_THIS{os.linesep}") -test.must_match('out2.txt', f"I_got_expanded_to_out2.txt{os.linesep}$EXPAND_THIS{os.linesep}") +test.must_match('out.txt', f"I_got_expanded_to_out.txt{os.linesep}out.txt_is_from_expansion{os.linesep}$EXPAND_THIS{os.linesep}") +test.must_match('out2.txt', f"I_got_expanded_to_out2.txt{os.linesep}out2.txt_is_from_expansion{os.linesep}$EXPAND_THIS{os.linesep}") test.pass_test() -- cgit v0.12 From c02b362bcb03bb74bfba8fb26636397f357a8cfa Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 23 May 2022 23:11:50 -0500 Subject: fix sider complaint --- SCons/Action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SCons/Action.py b/SCons/Action.py index 4dd0543..7de8454 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -108,6 +108,7 @@ import subprocess from subprocess import DEVNULL import inspect from collections import OrderedDict +from typing import Type import SCons.Debug from SCons.Debug import logInstanceCreation @@ -762,7 +763,7 @@ def _resolve_shell_env(env, target, source): ENV = ENV.copy() try: shell_gens = iter(shell_gen) - except: + except TypeError: raise SCons.Errors.UserError("SHELL_ENV_GENERATORS must be iteratable.") else: for generator in shell_gens: -- cgit v0.12 From cc5a1aa155a3bb95a0f573e01e5cf09fe7f30e90 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 23 May 2022 23:14:04 -0500 Subject: removed auto type --- SCons/Action.py | 1 - 1 file changed, 1 deletion(-) diff --git a/SCons/Action.py b/SCons/Action.py index 7de8454..172e74e 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -108,7 +108,6 @@ import subprocess from subprocess import DEVNULL import inspect from collections import OrderedDict -from typing import Type import SCons.Debug from SCons.Debug import logInstanceCreation -- cgit v0.12 From 4b9a5534225b93fc7038c8d801bb6652699bb97c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 23 May 2022 23:15:37 -0500 Subject: move ENV copy to more optimal location --- SCons/Action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Action.py b/SCons/Action.py index 172e74e..38570ce 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -759,12 +759,12 @@ def _resolve_shell_env(env, target, source): ENV = get_default_ENV(env) shell_gen = env.get('SHELL_ENV_GENERATORS') if shell_gen is not None: - ENV = ENV.copy() try: shell_gens = iter(shell_gen) except TypeError: raise SCons.Errors.UserError("SHELL_ENV_GENERATORS must be iteratable.") else: + ENV = ENV.copy() for generator in shell_gens: ENV = generator(env, target, source, ENV) if not isinstance(ENV, dict): -- cgit v0.12 From cdb1d2163a927981e9dab9f977094537e7b0d959 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 15 May 2022 09:44:59 -0600 Subject: Try to improve variantdir docs [skip appveyor] Reorder some bits, add some explantaions, include example of passing different environments to subsidiary SConscript. Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 64 ++++++++++++++++++++++----------------------- SCons/Script/SConscript.xml | 56 ++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 63 deletions(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 3a6df97..79289c3 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -3416,42 +3416,35 @@ env.UpdateValue(target = Value(output), source = Value(input)) -Sets up an alternate build location. -When building in the variant_dir, -&SCons; backfills as needed with files from src_dir -to create a complete build directory. +Sets up a mapping to define a variant build directory +variant_dir, +which must be in or underneath the project top directory. +src_dir may not be underneath +variant_dir. +A &f-VariantDir; mapping is global, even if called using the +the &f-env-VariantDir; form. &f-VariantDir; can be called multiple times with the same src_dir -to set up multiple builds with different options -(variants). - - - -The -variant -location must be in or underneath the project top directory, -and src_dir -may not be underneath -variant_dir. +to set up multiple variant builds with different options. +When files in variant_dir are referenced, +&SCons; backfills as needed with files from src_dir +to create a complete build directory. By default, &SCons; -physically duplicates the source files and SConscript files -as needed into the variant tree. -Thus, a build performed in the variant tree is guaranteed to be identical -to a build performed in the source tree even if +physically duplicates the source files, SConscript files, +and directory structure as needed into the variant directory. +Thus, a build performed in the variant directory is guaranteed to be identical +to a build performed in the source directory even if intermediate source files are generated during the build, or if preprocessors or other scanners search for included files -relative to the source file, +using paths relative to the source file, or if individual compilers or other invoked tools are hard-coded to put derived files in the same directory as source files. Only the files &SCons; calculates are needed for the build are duplicated into variant_dir. - - - If possible on the platform, the duplication is performed by linking rather than copying. This behavior is affected by the @@ -3470,38 +3463,43 @@ to invoke Builders using the path names of source files in src_dir and the path names of derived files within variant_dir. -This is more efficient than -duplicate=True, +This is more efficient than duplicating, and is safe for most builds; -revert to True +revert to duplicate=True if it causes problems. &f-VariantDir; works most naturally with used with a subsidiary SConscript file. -The subsidiary SConscript file is called as if it -were in +The subsidiary SConscript file must be called as if it were in variant_dir, regardless of the value of duplicate. +When calling an SConscript file, +you can pass an appropriately set up environment +using the exports keyword +argument so the SConscript can pick up the right settings +for that variant build. This is how you tell &scons; -which variant of a source tree to build: +which variant of a source directory to build: +env1 = Environment(...settings for variant1...) +env2 = Environment(...settings for variant2...) + # run src/SConscript in two variant directories VariantDir('build/variant1', 'src') -SConscript('build/variant1/SConscript') +SConscript('build/variant1/SConscript', exports={"env": env1}) VariantDir('build/variant2', 'src') -SConscript('build/variant2/SConscript') +SConscript('build/variant2/SConscript', exports={"env": env2}) See also the -&f-link-SConscript; -function, described above, +&f-link-SConscript; function for another way to specify a variant directory in conjunction with calling a subsidiary SConscript file. diff --git a/SCons/Script/SConscript.xml b/SCons/Script/SConscript.xml index 3c5b907..0710ea5 100644 --- a/SCons/Script/SConscript.xml +++ b/SCons/Script/SConscript.xml @@ -368,7 +368,7 @@ Return('val1 val2') -Execute one or more subsidiary SConscript (configuration) files. +Executes one or more subsidiary SConscript (configuration) files. There are two ways to call the &f-SConscript; function. @@ -395,8 +395,8 @@ config = SConscript('MyConfig.py') The second way to call &f-SConscript; -is to specify a list of (sub)directory names -as a +is to specify a list of directory names +using the dirs=subdirs keyword argument. In this case, @@ -448,38 +448,24 @@ SConscript(dirs=['one', 'two', 'three'], exports='shared_info') If the optional variant_dir argument is present, it causes an effect equivalent to the -&f-link-VariantDir; function. +&f-link-VariantDir; function, +but in effect only during the execution of the SConscript file. The variant_dir -argument is interpreted relative to the directory of the calling -SConscript file. -The optional -duplicate argument is -interpreted as for &f-link-VariantDir;. -If variant_dir -is omitted, the duplicate argument is ignored. -See the description of -&f-link-VariantDir; -below for additional details and restrictions. - - - -If -variant_dir -is present, -the source directory is the directory in which the -SConscript -file resides and the -SConscript +argument is interpreted relative to the directory of the +calling SConscript file. +The source directory is the directory in which the +called SConscript +file resides and the SConscript file is evaluated as if it were in the variant_dir -directory: +directory. Thus: SConscript('src/SConscript', variant_dir='build') -is equivalent to +is equivalent to: @@ -488,9 +474,8 @@ SConscript('build/SConscript') -This later paradigm is often used when the sources are -in the same directory as the -&SConstruct;: +If the sources are in the same directory as the +&SConstruct;, @@ -498,7 +483,7 @@ SConscript('SConscript', variant_dir='build') -is equivalent to +is equivalent to: @@ -507,6 +492,17 @@ SConscript('build/SConscript') +The optional +duplicate argument is +interpreted as for &f-link-VariantDir;. +If the variant_dir argument +is omitted, the duplicate argument is ignored. +See the description of +&f-link-VariantDir; +for additional details and restrictions. + + + -(dirs=subdirs, [name=script, exports, variant_dir, duplicate, must_exist]) - +(dirs=subdirs, [name=scriptname, exports, variant_dir, duplicate, must_exist]) + @@ -374,31 +374,27 @@ There are two ways to call the -The first calling style -is to explicitly specify one or more -scripts -as the first argument. +The first calling style is to supply +one or more SConscript file names +as the first (positional) argument. A single script may be specified as a string; -multiple scripts must be specified as a list +multiple scripts must be specified as a list of strings (either explicitly or as created by a function like &f-link-Split;). Examples: -SConscript('SConscript') # run SConscript in the current directory +SConscript('SConscript') # run SConscript in the current directory SConscript('src/SConscript') # run SConscript in the src directory SConscript(['src/SConscript', 'doc/SConscript']) config = SConscript('MyConfig.py') -The second way to call -&f-SConscript; -is to specify a list of directory names -using the -dirs=subdirs -keyword argument. +The other calling style is to omit the positional argument naming +scripts and instead specify a list of directory names using the +dirs keyword argument. In this case, &scons; will @@ -408,14 +404,13 @@ in each of the specified directories. You may specify a name other than &SConscript; by supplying an optional -name=script -keyword argument. +name keyword argument. The first three examples below have the same effect as the first three examples above: -SConscript(dirs='.') # run SConscript in the current directory -SConscript(dirs='src') # run SConscript in the src directory +SConscript(dirs='.') # run SConscript in the current directory +SConscript(dirs='src') # run SConscript in the src directory SConscript(dirs=['src', 'doc']) SConscript(dirs=['sub1', 'sub2'], name='MySConscript') @@ -423,8 +418,12 @@ SConscript(dirs=['sub1', 'sub2'], name='MySConscript') The optional exports -argument provides a string or list of strings representing +keyword argument provides a string or list of strings representing variable names, or a dictionary of named values, to export. +For the first calling style only, a second positional argument +will be interpreted as exports; the +second calling style must use the keyword argument form +for exports. These variables are locally exported only to the called SConscript file(s) and do not affect the global pool of variables managed by the @@ -585,11 +584,11 @@ SConscript('src/SConscript', variant_dir='build/ppc', duplicate=0) &f-SConscript; returns the values of any variables -named by the executed SConscript(s) in arguments -to the &f-link-Return; function (see above for details). +named by the executed SConscript file(s) in arguments +to the &f-link-Return; function. If a single &f-SConscript; call causes multiple scripts to be executed, the return value is a tuple containing -the returns of all of the scripts. If an executed +the returns of each of the scripts. If an executed script does not explicitly call &Return;, it returns None. -- cgit v0.12 From 45e9eec5a86adcd815a9bebfba536a44c7f9a756 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 May 2022 09:01:11 -0600 Subject: Fix grammar error ("the the") [ci skip] Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 3700a44..346fc75 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -3421,7 +3421,7 @@ Sets up a mapping to define a variant build directory in src_dir may not be underneath variant_dir. A &f-VariantDir; mapping is global, even if called using the -the &f-env-VariantDir; form. +&f-env-VariantDir; form. &f-VariantDir; can be called multiple times with the same src_dir -- cgit v0.12 From b14c20e353712291e0321c0fa1e7557bf385ea04 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 23 May 2022 08:18:51 -0600 Subject: Further tweak of VariantDir doc [skip appveyor] Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 346fc75..294562d 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -3482,14 +3482,11 @@ The subsidiary SConscript file must be called as if it were in variant_dir, regardless of the value of duplicate. -When calling an SConscript file, -you can pass an appropriately set up environment -using the exports keyword -argument so the SConscript can pick up the right settings -for that variant build. -This is how you tell -&scons; -which variant of a source directory to build: +When calling an SConscript file, you can use the +exports keyword argument +to pass parameters (individually or as an appropriately set up environment) +so the SConscript can pick up the right settings for that variant build. +The SConscript must &f-link-Import; these to use them. Example: @@ -3511,7 +3508,7 @@ in conjunction with calling a subsidiary SConscript file. -Examples: +More examples: -- cgit v0.12 From cf2641d678eb6142f5f00e686f8e4d46959267ee Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 25 May 2022 13:31:38 -0600 Subject: Typo fix [ci skip] Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 294562d..5c4326c 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -318,7 +318,7 @@ Added methods propagate through &f-env-Clone; calls. -Examples: +More examples: @@ -3477,7 +3477,7 @@ if it causes problems. &f-VariantDir; -works most naturally with used with a subsidiary SConscript file. +works most naturally when used with a subsidiary SConscript file. The subsidiary SConscript file must be called as if it were in variant_dir, regardless of the value of -- cgit v0.12 From a4daaa6d49f7f5972e8d7d083d492068b1a5d362 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 26 May 2022 06:41:09 -0600 Subject: SConscript doc tweaks per #4150 review comments [skip appveyor] Signed-off-by: Mats Wichmann --- SCons/Script/SConscript.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SCons/Script/SConscript.xml b/SCons/Script/SConscript.xml index 17c9236..eb52acc 100644 --- a/SCons/Script/SConscript.xml +++ b/SCons/Script/SConscript.xml @@ -404,7 +404,8 @@ in each of the specified directories. You may specify a name other than &SConscript; by supplying an optional -name keyword argument. +name=scriptname +keyword argument. The first three examples below have the same effect as the first three examples above: @@ -448,7 +449,7 @@ If the optional variant_dir argument is present, it causes an effect equivalent to the &f-link-VariantDir; function, -but in effect only during the execution of the SConscript file. +but in effect only within the scope of the &f-SConscript; call. The variant_dir argument is interpreted relative to the directory of the calling SConscript file. -- cgit v0.12 From 9f6cfbdd5dc35a9711e5c5ab5ef3414b4a05e0ef Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 26 May 2022 22:58:55 -0500 Subject: collapsed related CHANGES comments --- CHANGES.txt | 9 +++------ RELEASE.txt | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 60e934f..d5aba9f 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -99,12 +99,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected format. Compiler tools will also configure the variable automatically. - - Added SHELL_ENV_GENERATOR construction variable. This variable allows the user to Define - a function which will be called to generate or alter the execution environment which will - be used in the shell command of some Action. - - Updated SHELL_ENV_GENERATOR construction variable to SHELL_ENV_GENERATORS. This variable - is now an iterable which will contain functions which each are called and each can customize - the execution environment. + - Added SHELL_ENV_GENERATORS construction variable. This variable + is an iterable which will contain functions in which each are called and each can allow + the user a method to customize the execution environment. - Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. - Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry to connect to the server during start up. diff --git a/RELEASE.txt b/RELEASE.txt index 38fbbd7..195e325 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -16,12 +16,9 @@ NEW FUNCTIONALITY - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. - Added Configure.CheckMember() checker to check if struct/class has the specified member. -- Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define - a function which will be called to generate or alter the execution environment which will - be used in the shell command of some Action. -- Updated SHELL_ENV_GENERATOR construction variable to SHELL_ENV_GENERATORS. This variable - is now an iterable which will contain functions which each are called and each can customize - the execution environment. +- Added SHELL_ENV_GENERATORS construction variable. This variable + is an iterable which will contain functions in which each are called and each can allow + the user a method to customize the execution environment. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. -- cgit v0.12 From f262fbd6bacf1c0d7f26d747d853b9ce909f4442 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Thu, 26 May 2022 23:03:13 -0500 Subject: fix missing f string --- SCons/Action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Action.py b/SCons/Action.py index 38570ce..e29c4e9 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -768,7 +768,7 @@ def _resolve_shell_env(env, target, source): for generator in shell_gens: ENV = generator(env, target, source, ENV) if not isinstance(ENV, dict): - raise SCons.Errors.UserError("SHELL_ENV_GENERATORS function: {generator} must return a dict.") + raise SCons.Errors.UserError(f"SHELL_ENV_GENERATORS function: {generator} must return a dict.") return ENV -- cgit v0.12 From 150197f2273fa639aa4815a68f9bcd38d3068a8d Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 30 May 2022 12:28:57 -0600 Subject: Fix some Py 3.11 depr warns in tests A couple of unittest methods that a few SCons tests use have been marked deprecated in Python 3.11, with replacements provided. Partial fix for #4162, do not close just off this PR. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ SCons/Scanner/ScannerTests.py | 4 +++- SCons/Tool/ToolTests.py | 4 +++- SCons/Tool/msvsTests.py | 4 +++- SCons/cppTests.py | 4 +++- testing/framework/TestCmdTests.py | 6 ++++-- testing/framework/TestCommonTests.py | 6 ++++-- 7 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9a6a26f..7d3eadb 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -93,6 +93,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER "dict" (avoid redefining builtin) - Fix an old use-before-set bug in tex tool (issue #2888) - Fix a test harness exception returning stderr if a wait_for timed out. + - Modernize a few tests that use now-deprecated unittest.getTestCaseNames + and unittest.makeSuite - Python itself suggests the replacements. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Scanner/ScannerTests.py b/SCons/Scanner/ScannerTests.py index 68332a0..b9cb209 100644 --- a/SCons/Scanner/ScannerTests.py +++ b/SCons/Scanner/ScannerTests.py @@ -621,7 +621,9 @@ def suite(): ClassicCPPTestCase, ] for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + names = loader.getTestCaseNames(tclass) suite.addTests(list(map(tclass, names))) return suite diff --git a/SCons/Tool/ToolTests.py b/SCons/Tool/ToolTests.py index 7cff7c8..9338c2d 100644 --- a/SCons/Tool/ToolTests.py +++ b/SCons/Tool/ToolTests.py @@ -116,7 +116,9 @@ class ToolTestCase(unittest.TestCase): if __name__ == "__main__": - suite = unittest.makeSuite(ToolTestCase, 'test_') + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + suite = loader.loadTestsFromTestCase(ToolTestCase) TestUnit.run(suite) # Local Variables: diff --git a/SCons/Tool/msvsTests.py b/SCons/Tool/msvsTests.py index 4cbaf0e..c4d2e98 100644 --- a/SCons/Tool/msvsTests.py +++ b/SCons/Tool/msvsTests.py @@ -982,7 +982,9 @@ if __name__ == "__main__": if k in os.environ: del os.environ[k] - suite = unittest.makeSuite(test_class, 'test_') + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + suite = loader.loadTestsFromTestCase(test_class) if not TestUnit.cli.get_runner()().run(suite).wasSuccessful(): exit_val = 1 finally: diff --git a/SCons/cppTests.py b/SCons/cppTests.py index a9aef9d..f20c302 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -876,7 +876,9 @@ if __name__ == '__main__': fileTestCase, ] for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + names = loader.getTestCaseNames(tclass) try: names = sorted(set(names)) except NameError: diff --git a/testing/framework/TestCmdTests.py b/testing/framework/TestCmdTests.py index 212c59a..3b29091 100644 --- a/testing/framework/TestCmdTests.py +++ b/testing/framework/TestCmdTests.py @@ -3389,8 +3389,10 @@ if __name__ == "__main__": ]) suite = unittest.TestSuite() for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests([ tclass(n) for n in names ]) + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + names = loader.getTestCaseNames(tclass) + suite.addTests([tclass(n) for n in names]) if not unittest.TextTestRunner().run(suite).wasSuccessful(): sys.exit(1) diff --git a/testing/framework/TestCommonTests.py b/testing/framework/TestCommonTests.py index 03a5508..dff7a50 100644 --- a/testing/framework/TestCommonTests.py +++ b/testing/framework/TestCommonTests.py @@ -2429,8 +2429,10 @@ if __name__ == "__main__": ] suite = unittest.TestSuite() for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests([ tclass(n) for n in names ]) + loader = unittest.TestLoader() + loader.testMethodPrefix = 'test_' + names = loader.getTestCaseNames(tclass) + suite.addTests([tclass(n) for n in names]) if not unittest.TextTestRunner().run(suite).wasSuccessful(): sys.exit(1) -- cgit v0.12 From c7272c7367f1c3f819645d996c6d93449cf14f5b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 13 May 2022 08:18:22 -0600 Subject: Normalize use of test.sleep() Some tests used time.sleep while others used the harness's test.sleep, the ones that are done for file timestamp delay management now all use test.sleep - with a consistent comment, so it's easier to spot. There are still tests that use time.sleep with smaller intervals that were left alone. Affected tests were reformatted, in the spirit of gradually improving tests that you otherwise touch... Signed-off-by: Mats Wichmann --- test/CacheDir/timestamp-match.py | 25 ++-- test/CacheDir/timestamp-newer.py | 24 ++-- test/Copy-Action.py | 70 +++++----- test/Decider/MD5-timestamp-Repository.py | 20 +-- test/Decider/MD5-timestamp.py | 21 +-- test/Decider/timestamp.py | 23 +--- test/Dir/source.py | 85 +++++++------ test/Libs/LIBPATH.py | 71 +++++------ test/PharLap.py | 19 ++- test/Program.py | 87 ++++++------- test/Repository/no-SConsignFile.py | 14 +- test/Repository/variants.py | 212 +++++++++++++++---------------- test/Touch.py | 35 +++-- test/chained-build.py | 62 +++++---- test/sconsign/script/Signatures.py | 26 ++-- test/sconsign/script/dblite.py | 45 +++---- 16 files changed, 378 insertions(+), 461 deletions(-) diff --git a/test/CacheDir/timestamp-match.py b/test/CacheDir/timestamp-match.py index 4b64137..fd9c659 100644 --- a/test/CacheDir/timestamp-match.py +++ b/test/CacheDir/timestamp-match.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that CAcheDir() works when using 'timestamp-match' decisions. @@ -41,21 +40,15 @@ Command('file.out', 'file.in', Copy('$TARGET', '$SOURCE')) test.write('file.in', "file.in\n") -test.run(arguments = '--cache-show --debug=explain .') - +test.run(arguments='--cache-show --debug=explain .') test.must_match('file.out', "file.in\n") +test.up_to_date(options='--cache-show --debug=explain', arguments='.') -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.sleep() - +test.sleep() # delay for timestamps test.touch('file.in') - -test.not_up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') +test.not_up_to_date(options='--cache-show --debug=explain', arguments='.') +test.up_to_date(options='--cache-show --debug=explain', arguments='.') +test.up_to_date(options='--cache-show --debug=explain', arguments='.') test.pass_test() diff --git a/test/CacheDir/timestamp-newer.py b/test/CacheDir/timestamp-newer.py index 618f467..567078e 100644 --- a/test/CacheDir/timestamp-newer.py +++ b/test/CacheDir/timestamp-newer.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that CAcheDir() works when using 'timestamp-newer' decisions. @@ -41,21 +40,16 @@ Command('file.out', 'file.in', Copy('$TARGET', '$SOURCE')) test.write('file.in', "file.in\n") -test.run(arguments = '--cache-show --debug=explain .') - +test.run(arguments='--cache-show --debug=explain .') test.must_match('file.out', "file.in\n") +test.up_to_date(options='--cache-show --debug=explain', arguments='.') -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.sleep() - +test.sleep() # delay for timestamps test.touch('file.in') -test.not_up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') - -test.up_to_date(options = '--cache-show --debug=explain', arguments = '.') +test.not_up_to_date(options='--cache-show --debug=explain', arguments='.') +test.up_to_date(options='--cache-show --debug=explain', arguments='.') +test.up_to_date(options='--cache-show --debug=explain', arguments='.') test.pass_test() diff --git a/test/Copy-Action.py b/test/Copy-Action.py index 4bfa0da..2179744 100644 --- a/test/Copy-Action.py +++ b/test/Copy-Action.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that the Copy() Action works, and preserves file modification @@ -37,29 +36,32 @@ import TestSCons test = TestSCons.TestSCons() -test.write('SConstruct', """ +test.write('SConstruct', """\ Execute(Copy('f1.out', 'f1.in')) Execute(Copy(File('d2.out'), 'd2.in')) Execute(Copy('d3.out', File('f3.in'))) + def cat(env, source, target): target = str(target[0]) with open(target, "w") as f: for src in source: with open(str(src), "r") as ifp: f.write(ifp.read()) + Cat = Action(cat) env = Environment() -env.Command('bar.out', 'bar.in', [Cat, - Copy("f4.out", "f4.in"), - Copy("d5.out", "d5.in"), - Copy("d6.out", "f6.in")]) -env = Environment(OUTPUT = 'f7.out', INPUT = 'f7.in') +env.Command( + 'bar.out', + 'bar.in', + [Cat, Copy("f4.out", "f4.in"), Copy("d5.out", "d5.in"), Copy("d6.out", "f6.in")], +) +env = Environment(OUTPUT='f7.out', INPUT='f7.in') env.Command('f8.out', 'f8.in', [Copy('$OUTPUT', '$INPUT'), Cat]) env.Command('f9.out', 'f9.in', [Cat, Copy('${TARGET}-Copy', '$SOURCE')]) -env.CopyTo( 'd4', 'f10.in' ) -env.CopyAs( 'd4/f11.out', 'f11.in') -env.CopyAs( 'd4/f12.out', 'd5/f12.in') +env.CopyTo('d4', 'f10.in') +env.CopyAs('d4/f11.out', 'f11.in') +env.CopyAs('d4/f12.out', 'd5/f12.in') env.Command('f 13.out', 'f 13.in', Copy('$TARGET', '$SOURCE')) """) @@ -87,19 +89,20 @@ test.write('f 13.in', "f 13.in\n") os.chmod('f1.in', 0o646) os.chmod('f4.in', 0o644) -test.sleep() +test.sleep() # delay for timestamps d4_f10_in = os.path.join('d4', 'f10.in') d4_f11_out = os.path.join('d4', 'f11.out') d4_f12_out = os.path.join('d4', 'f12.out') d5_f12_in = os.path.join('d5', 'f12.in') -expect = test.wrap_stdout(read_str = """\ +expect = test.wrap_stdout( + read_str="""\ Copy("f1.out", "f1.in") Copy("d2.out", "d2.in") Copy("d3.out", "f3.in") """, - build_str = """\ + build_str="""\ cat(["bar.out"], ["bar.in"]) Copy("f4.out", "f4.in") Copy("d5.out", "d5.in") @@ -112,9 +115,10 @@ Copy("f7.out", "f7.in") cat(["f8.out"], ["f8.in"]) cat(["f9.out"], ["f9.in"]) Copy("f9.out-Copy", "f9.in") -""" % locals()) +""" % locals(), +) -test.run(options = '-n', arguments = '.', stdout = expect) +test.run(options='-n', arguments='.', stdout=expect) test.must_not_exist('f1.out') test.must_not_exist('d2.out') @@ -162,23 +166,21 @@ def must_be_same(f1, f2): for value in ['ST_MODE', 'ST_MTIME']: v = getattr(stat, value) if s1[v] != s2[v]: - msg = '%s[%s] %s != %s[%s] %s\n' % \ - (repr(f1), value, s1[v], - repr(f2), value, s2[v],) + msg = f"{f1!r}[{value}] {s1[v1]} != {f2!r}[{value}] {s2[v]}\n" sys.stderr.write(msg) - errors = errors + 1 - -must_be_same('f1.out', 'f1.in') -must_be_same(['d2.out', 'file'], ['d2.in', 'file']) -must_be_same(['d3.out', 'f3.in'], 'f3.in') -must_be_same('f4.out', 'f4.in') -must_be_same(['d5.out', 'file'], ['d5.in', 'file']) -must_be_same(['d6.out', 'f6.in'], 'f6.in') -must_be_same('f7.out', 'f7.in') -must_be_same(['d4', 'f10.in'], 'f10.in') -must_be_same(['d4', 'f11.out'], 'f11.in') -must_be_same(['d4', 'f12.out'], ['d5', 'f12.in']) -must_be_same('f 13.out', 'f 13.in') + errors += 1 + +must_be_same('f1.out', 'f1.in') +must_be_same(['d2.out', 'file'], ['d2.in', 'file']) +must_be_same(['d3.out', 'f3.in'], 'f3.in') +must_be_same('f4.out', 'f4.in') +must_be_same(['d5.out', 'file'], ['d5.in', 'file']) +must_be_same(['d6.out', 'f6.in'], 'f6.in') +must_be_same('f7.out', 'f7.in') +must_be_same(['d4', 'f10.in'], 'f10.in') +must_be_same(['d4', 'f11.out'], 'f11.in') +must_be_same(['d4', 'f12.out'], ['d5', 'f12.in']) +must_be_same('f 13.out', 'f 13.in') if errors: test.fail_test() diff --git a/test/Decider/MD5-timestamp-Repository.py b/test/Decider/MD5-timestamp-Repository.py index 1826f68..201bdfe 100644 --- a/test/Decider/MD5-timestamp-Repository.py +++ b/test/Decider/MD5-timestamp-Repository.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify behavior of the MD5-timestamp Decider() setting when combined with Repository() usage @@ -38,13 +37,11 @@ test = TestSCons.TestSCons() test.subdir('Repository', 'work') repository = test.workpath('Repository') - test.write(['Repository','content1.in'], "content1.in 1\n") test.write(['Repository','content2.in'], "content2.in 1\n") test.write(['Repository','content3.in'], "content3.in 1\n") # test.writable('Repository', 0) - test.write(['work','SConstruct'], """\ Repository(r'%s') DefaultEnvironment(tools=[]) @@ -53,18 +50,14 @@ m.Decider('MD5-timestamp') m.Command('content1.out', 'content1.in', Copy('$TARGET', '$SOURCE')) m.Command('content2.out', 'content2.in', Copy('$TARGET', '$SOURCE')) m.Command('content3.out', 'content3.in', Copy('$TARGET', '$SOURCE')) -"""%repository) +""" % repository) test.run(chdir='work',arguments='.') - test.up_to_date(chdir='work',arguments='.') -test.sleep() - +test.sleep() # delay for timestamps test.write(['Repository','content1.in'], "content1.in 2\n") - test.touch(['Repository','content2.in']) - time_content = os.stat(os.path.join(repository,'content3.in'))[stat.ST_MTIME] test.write(['Repository','content3.in'], "content3.in 2\n") test.touch(['Repository','content3.in'], time_content) @@ -76,10 +69,9 @@ test.touch(['Repository','content3.in'], time_content) expect = test.wrap_stdout("""\ Copy("content1.out", "%s") -"""%os.path.join(repository,'content1.in')) +""" % os.path.join(repository, 'content1.in')) test.run(chdir='work', arguments='.', stdout=expect) - test.up_to_date(chdir='work', arguments='.') test.pass_test() diff --git a/test/Decider/MD5-timestamp.py b/test/Decider/MD5-timestamp.py index 6fcdb42..3815639 100644 --- a/test/Decider/MD5-timestamp.py +++ b/test/Decider/MD5-timestamp.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify behavior of the MD5-timestamp Decider() setting. @@ -49,15 +48,10 @@ test.write('content2.in', "content2.in 1\n") test.write('content3.in', "content3.in 1\n") test.run(arguments = '.') - test.up_to_date(arguments = '.') - - -test.sleep() - +test.sleep() # delay for timestamps test.write('content1.in', "content1.in 2\n") - test.touch('content2.in') time_content = os.stat('content3.in')[stat.ST_MTIME] @@ -73,11 +67,8 @@ expect = test.wrap_stdout("""\ Copy("content1.out", "content1.in") """) -test.run(arguments = '.', stdout=expect) - -test.up_to_date(arguments = '.') - - +test.run(arguments='.', stdout=expect) +test.up_to_date(arguments='.') test.pass_test() diff --git a/test/Decider/timestamp.py b/test/Decider/timestamp.py index e528d77..d713a62 100644 --- a/test/Decider/timestamp.py +++ b/test/Decider/timestamp.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify various interactions of the timestamp-match and timestamp-newer @@ -54,27 +53,21 @@ test.write('newer1.in', "newer1.in\n") test.write('newer2.in', "newer2.in\n") test.run(arguments = '.') - test.up_to_date(arguments = '.') time_match = os.stat('match2.out')[stat.ST_MTIME] time_newer = os.stat('newer2.out')[stat.ST_MTIME] - - # Now make all the source files newer than (different timestamps from) # the last time the targets were built, and touch the target files # of match1.out and newer1.out to see the different effects. - -test.sleep() - +test.sleep() # delay for timestamps test.touch('match1.in') test.touch('newer1.in') test.touch('match2.in') test.touch('newer2.in') -test.sleep() - +test.sleep() # delay for timestamps test.touch('match1.out') test.touch('newer1.out') @@ -90,7 +83,7 @@ Copy("match2.out", "match2.in") Copy("newer2.out", "newer2.in") """) -test.run(arguments = '.', stdout=expect) +test.run(arguments='.', stdout=expect) # Now, for the somewhat pathological case, reset the match2.out and # newer2.out timestamps to the older timestamp when the targets were @@ -107,9 +100,7 @@ expect = test.wrap_stdout("""\ Copy("newer2.out", "newer2.in") """) -test.run(arguments = '.', stdout=expect) - - +test.run(arguments='.', stdout=expect) test.pass_test() diff --git a/test/Dir/source.py b/test/Dir/source.py index c35d169..a0ba987 100644 --- a/test/Dir/source.py +++ b/test/Dir/source.py @@ -42,15 +42,16 @@ test.subdir('tstamp', [ 'tstamp', 'subdir' ], test.write('SConstruct', """\ DefaultEnvironment(tools=[]) + def writeTarget(target, source, env): - f=open(str(target[0]), 'w') + f = open(str(target[0]), 'w') f.write("stuff\\n") f.close() return 0 -test_bld_dir = Builder(action=writeTarget, - source_factory=Dir, - source_scanner=DirScanner) +test_bld_dir = Builder( + action=writeTarget, source_factory=Dir, source_scanner=DirScanner +) test_bld_file = Builder(action=writeTarget) env = Environment(tools=[]) env['BUILDERS']['TestDir'] = test_bld_dir @@ -61,34 +62,36 @@ env_tstamp.Decider('timestamp-newer') env_tstamp.TestFile(source='junk.txt', target='tstamp/junk.out') env_tstamp.TestDir(source='tstamp', target='tstamp.out') env_tstamp.Command('cmd-tstamp-noscan.out', 'cmd-tstamp', writeTarget) -env_tstamp.Command('cmd-tstamp.out', 'cmd-tstamp', writeTarget, - source_scanner=DirScanner) +env_tstamp.Command( + 'cmd-tstamp.out', 'cmd-tstamp', writeTarget, source_scanner=DirScanner +) env_content = env.Clone() env_content.Decider('content') env_content.TestFile(source='junk.txt', target='content/junk.out') env_content.TestDir(source='content', target='content.out') env_content.Command('cmd-content-noscan.out', 'cmd-content', writeTarget) -env_content.Command('cmd-content.out', 'cmd-content', writeTarget, - source_scanner=DirScanner) +env_content.Command( + 'cmd-content.out', 'cmd-content', writeTarget, source_scanner=DirScanner +) """) -test.write([ 'tstamp', 'foo.txt' ], 'foo.txt 1\n') -test.write([ 'tstamp', '#hash.txt' ], 'hash.txt 1\n') -test.write([ 'tstamp', 'subdir', 'bar.txt'], 'bar.txt 1\n') -test.write([ 'tstamp', 'subdir', '#hash.txt'], 'hash.txt 1\n') -test.write([ 'content', 'foo.txt' ], 'foo.txt 1\n') -test.write([ 'content', '#hash.txt' ], 'hash.txt 1\n') -test.write([ 'content', 'subdir', 'bar.txt' ], 'bar.txt 1\n') -test.write([ 'content', 'subdir', '#hash.txt' ], 'hash.txt 1\n') -test.write([ 'cmd-tstamp', 'foo.txt' ], 'foo.txt 1\n') -test.write([ 'cmd-tstamp', '#hash.txt' ], 'hash.txt 1\n') -test.write([ 'cmd-tstamp', 'subdir', 'bar.txt' ], 'bar.txt 1\n') -test.write([ 'cmd-tstamp', 'subdir', '#hash.txt' ], 'hash.txt 1\n') -test.write([ 'cmd-content', 'foo.txt' ], 'foo.txt 1\n') -test.write([ 'cmd-content', '#hash.txt' ], '#hash.txt 1\n') -test.write([ 'cmd-content', 'subdir', 'bar.txt' ], 'bar.txt 1\n') -test.write([ 'cmd-content', 'subdir', '#hash.txt' ], 'hash.txt 1\n') +test.write(['tstamp', 'foo.txt'], 'foo.txt 1\n') +test.write(['tstamp', '#hash.txt'], 'hash.txt 1\n') +test.write(['tstamp', 'subdir', 'bar.txt'], 'bar.txt 1\n') +test.write(['tstamp', 'subdir', '#hash.txt'], 'hash.txt 1\n') +test.write(['content', 'foo.txt'], 'foo.txt 1\n') +test.write(['content', '#hash.txt'], 'hash.txt 1\n') +test.write(['content', 'subdir', 'bar.txt'], 'bar.txt 1\n') +test.write(['content', 'subdir', '#hash.txt'], 'hash.txt 1\n') +test.write(['cmd-tstamp', 'foo.txt'], 'foo.txt 1\n') +test.write(['cmd-tstamp', '#hash.txt'], 'hash.txt 1\n') +test.write(['cmd-tstamp', 'subdir', 'bar.txt'], 'bar.txt 1\n') +test.write(['cmd-tstamp', 'subdir', '#hash.txt'], 'hash.txt 1\n') +test.write(['cmd-content', 'foo.txt'], 'foo.txt 1\n') +test.write(['cmd-content', '#hash.txt'], '#hash.txt 1\n') +test.write(['cmd-content', 'subdir', 'bar.txt'], 'bar.txt 1\n') +test.write(['cmd-content', 'subdir', '#hash.txt'], 'hash.txt 1\n') test.write('junk.txt', 'junk.txt\n') test.run(arguments=".", stderr=None) @@ -106,61 +109,61 @@ test.up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') test.up_to_date(arguments='cmd-content-noscan.out') -test.sleep() +test.sleep() # delay for timestamps -test.write([ 'tstamp', 'foo.txt' ], 'foo.txt 2\n') +test.write(['tstamp', 'foo.txt'], 'foo.txt 2\n') test.not_up_to_date(arguments='tstamp.out') -test.write([ 'tstamp', 'new.txt' ], 'new.txt\n') +test.write(['tstamp', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='tstamp.out') -test.write([ 'content', 'foo.txt' ], 'foo.txt 2\n') +test.write(['content', 'foo.txt'], 'foo.txt 2\n') test.not_up_to_date(arguments='content.out') -test.write([ 'content', 'new.txt' ], 'new.txt\n') +test.write(['content', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='content.out') -test.write([ 'cmd-tstamp', 'foo.txt' ], 'foo.txt 2\n') +test.write(['cmd-tstamp', 'foo.txt'], 'foo.txt 2\n') test.not_up_to_date(arguments='cmd-tstamp.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') -test.write([ 'cmd-tstamp', 'new.txt' ], 'new.txt\n') +test.write(['cmd-tstamp', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='cmd-tstamp.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') -test.write([ 'cmd-content', 'foo.txt' ], 'foo.txt 2\n') +test.write(['cmd-content', 'foo.txt'], 'foo.txt 2\n') test.not_up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-content-noscan.out') -test.write([ 'cmd-content', 'new.txt' ], 'new.txt\n') +test.write(['cmd-content', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-content-noscan.out') -test.write([ 'tstamp', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.write(['tstamp', 'subdir', 'bar.txt'], 'bar.txt 2\n') test.not_up_to_date(arguments='tstamp.out') -test.write([ 'tstamp', 'subdir', 'new.txt' ], 'new.txt\n') +test.write(['tstamp', 'subdir', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='tstamp.out') -test.write([ 'content', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.write(['content', 'subdir', 'bar.txt'], 'bar.txt 2\n') test.not_up_to_date(arguments='content.out') -test.write([ 'content', 'subdir', 'new.txt' ], 'new.txt\n') +test.write(['content', 'subdir', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='content.out') -test.write([ 'cmd-tstamp', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.write(['cmd-tstamp', 'subdir', 'bar.txt'], 'bar.txt 2\n') test.not_up_to_date(arguments='cmd-tstamp.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') -test.write([ 'cmd-tstamp', 'subdir', 'new.txt' ], 'new.txt\n') +test.write(['cmd-tstamp', 'subdir', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='cmd-tstamp.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') -test.write([ 'cmd-content', 'subdir', 'bar.txt' ], 'bar.txt 2\n') +test.write(['cmd-content', 'subdir', 'bar.txt'], 'bar.txt 2\n') test.not_up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-content-noscan.out') -test.write([ 'cmd-content', 'subdir', 'new.txt' ], 'new.txt\n') +test.write(['cmd-content', 'subdir', 'new.txt'], 'new.txt\n') test.not_up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-content-noscan.out') diff --git a/test/Libs/LIBPATH.py b/test/Libs/LIBPATH.py index b5a1b54..d663e56 100644 --- a/test/Libs/LIBPATH.py +++ b/test/Libs/LIBPATH.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path import time @@ -40,20 +40,17 @@ test.subdir('lib1', 'lib2') prog1 = test.workpath('prog') + _exe prog2 = test.workpath(dll_ + 'shlib') + _dll -test.write('SConstruct', """ -env1 = Environment(LIBS = [ 'foo1' ], - LIBPATH = [ '$FOO' ], - FOO='./lib1') +test.write('SConstruct', """\ +env1 = Environment(LIBS=['foo1'], LIBPATH=['$FOO'], FOO='./lib1') f1 = env1.SharedObject('f1', 'f1.c') -env1.Program(target = 'prog', source = 'prog.c') -env1.Library(target = './lib1/foo1', source = f1) +env1.Program(target='prog', source='prog.c') +env1.Library(target='./lib1/foo1', source=f1) -env2 = Environment(LIBS = 'foo2', - LIBPATH = '.') -env2.SharedLibrary(target = 'shlib', source = 'shlib.c', no_import_lib = 1) -env2.Library(target = 'foo2', source = f1) +env2 = Environment(LIBS='foo2', LIBPATH='.') +env2.SharedLibrary(target='shlib', source='shlib.c', no_import_lib=1) +env2.Library(target='foo2', source=f1) """) test.write('f1.c', r""" @@ -99,8 +96,8 @@ test.run(program = prog1, oldtime1 = os.path.getmtime(prog1) oldtime2 = os.path.getmtime(prog2) -time.sleep(2) -test.run(arguments = '.') +test.sleep() # delay for timestamps +test.run(arguments='.') test.fail_test(oldtime1 != os.path.getmtime(prog1)) test.fail_test(oldtime2 != os.path.getmtime(prog2)) @@ -115,30 +112,25 @@ f1(void) } """) -test.run(arguments = '.', - stderr=TestSCons.noisy_ar, - match=TestSCons.match_re_dotall) -test.run(program = prog1, - stdout = "f1.c 1\nprog.c\n") +test.run(arguments='.', stderr=TestSCons.noisy_ar, match=TestSCons.match_re_dotall) +test.run(program=prog1, stdout="f1.c 1\nprog.c\n") test.fail_test(oldtime2 == os.path.getmtime(prog2)) #test.up_to_date(arguments = '.') # Change LIBPATH and make sure we don't rebuild because of it. -test.write('SConstruct', """ -env1 = Environment(LIBS = [ 'foo1' ], - LIBPATH = [ './lib1', './lib2' ]) +test.write('SConstruct', """\ +env1 = Environment(LIBS=['foo1'], LIBPATH=['./lib1', './lib2']) f1 = env1.SharedObject('f1', 'f1.c') -env1.Program(target = 'prog', source = 'prog.c') -env1.Library(target = './lib1/foo1', source = f1) +env1.Program(target='prog', source='prog.c') +env1.Library(target='./lib1/foo1', source=f1) -env2 = Environment(LIBS = 'foo2', - LIBPATH = Split('. ./lib2')) -env2.SharedLibrary(target = 'shlib', source = 'shlib.c', no_import_lib = 1) -env2.Library(target = 'foo2', source = f1) +env2 = Environment(LIBS='foo2', LIBPATH=Split('. ./lib2')) +env2.SharedLibrary(target='shlib', source='shlib.c', no_import_lib=1) +env2.Library(target='foo2', source=f1) """) -test.up_to_date(arguments = '.', stderr=None) +test.up_to_date(arguments='.', stderr=None) test.write('f1.c', r""" #include @@ -150,27 +142,22 @@ f1(void) } """) -test.run(arguments = '.', - stderr=TestSCons.noisy_ar, - match=TestSCons.match_re_dotall) -test.run(program = prog1, - stdout = "f1.c 2\nprog.c\n") +test.run(arguments='.', stderr=TestSCons.noisy_ar, match=TestSCons.match_re_dotall) +test.run(program=prog1, stdout="f1.c 2\nprog.c\n") -test.up_to_date(arguments = '.') +test.up_to_date(arguments='.') # We need at least one file for some implementations of the Library # builder, notably the SGI one. test.write('empty.c', 'int a=0;\n') # Check that a null-string LIBPATH doesn't blow up. -test.write('SConstruct', """ -env = Environment(LIBPATH = '') -env.Library('foo', source = 'empty.c') +test.write('SConstruct', """\ +env = Environment(LIBPATH='') +env.Library('foo', source='empty.c') """) -test.run(arguments = '.', - stderr=TestSCons.noisy_ar, - match=TestSCons.match_re_dotall) +test.run(arguments='.', stderr=TestSCons.noisy_ar, match=TestSCons.match_re_dotall) test.pass_test() diff --git a/test/PharLap.py b/test/PharLap.py index 8e56f21..0a54151 100644 --- a/test/PharLap.py +++ b/test/PharLap.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os import sys @@ -284,10 +283,10 @@ test.write([ "baz", "bar.lnk"],""" @asm.emb """) -test.write("SConstruct", """ -env=Environment(tools = [ 'linkloc', '386asm' ], - ASFLAGS='-twocase -cvsym', - LINKFLAGS='@foo.lnk') +test.write("SConstruct", """\ +env = Environment( + tools=['linkloc', '386asm'], ASFLAGS='-twocase -cvsym', LINKFLAGS='@foo.lnk' +) env.Program(target='minasm', source='minasm.asm') """) @@ -304,8 +303,8 @@ test.write([ "baz", "bar.lnk"],""" """) oldtime = os.path.getmtime(test.workpath('minasm.exe')) -time.sleep(2) # Give the time stamp time to change -test.run(arguments = '.') +test.sleep() # delay for timestamps +test.run(arguments='.') test.fail_test(oldtime == os.path.getmtime(test.workpath('minasm.exe'))) test.pass_test() diff --git a/test/Program.py b/test/Program.py index 15fd0c3..640787f 100644 --- a/test/Program.py +++ b/test/Program.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path import time @@ -40,13 +39,13 @@ foo4 = test.workpath('foo4' + _exe) foo5 = test.workpath('foo5' + _exe) foo_args = 'foo1%s foo2%s foo3%s foo4%s foo5%s' % (_exe, _exe, _exe, _exe, _exe) -test.write('SConstruct', """ +test.write('SConstruct', """\ env = Environment() -env.Program(target = 'foo1', source = 'f1.c') -env.Program(target = 'foo2', source = Split('f2a.c f2b.c f2c.c')) +env.Program(target='foo1', source='f1.c') +env.Program(target='foo2', source=Split('f2a.c f2b.c f2c.c')) f3a = File('f3a.c') f3b = File('f3b.c') -Program(target = 'foo3', source = [f3a, [f3b, 'f3c.c']]) +Program(target='foo3', source=[f3a, [f3b, 'f3c.c']]) env.Program('foo4', 'f4.c') env.Program('foo5.c') """) @@ -156,15 +155,14 @@ main(int argc, char *argv[]) } """) -test.run(arguments = '.') +test.run(arguments='.') -test.run(program = foo1, stdout = "f1.c\n") -test.run(program = foo2, stdout = "f2a.c\nf2b.c\nf2c.c\n") -test.run(program = foo3, stdout = "f3a.c\nf3b.c\nf3c.c\n") -test.run(program = foo4, stdout = "f4.c\n") -test.run(program = foo5, stdout = "foo5.c\n") - -test.up_to_date(arguments = '.') +test.run(program=foo1, stdout="f1.c\n") +test.run(program=foo2, stdout="f2a.c\nf2b.c\nf2c.c\n") +test.run(program=foo3, stdout="f3a.c\nf3b.c\nf3c.c\n") +test.run(program=foo4, stdout="f4.c\n") +test.run(program=foo5, stdout="foo5.c\n") +test.up_to_date(arguments='.') test.write('f1.c', r""" #include @@ -211,15 +209,14 @@ main(int argc, char *argv[]) } """) -test.run(arguments = '.') - -test.run(program = foo1, stdout = "f1.c X\n") -test.run(program = foo2, stdout = "f2a.c\nf2b.c\nf2c.c\n") -test.run(program = foo3, stdout = "f3a.c\nf3b.c X\nf3c.c\n") -test.run(program = foo4, stdout = "f4.c X\n") -test.run(program = foo5, stdout = "foo5.c X\n") +test.run(arguments='.') -test.up_to_date(arguments = '.') +test.run(program=foo1, stdout="f1.c X\n") +test.run(program=foo2, stdout="f2a.c\nf2b.c\nf2c.c\n") +test.run(program=foo3, stdout="f3a.c\nf3b.c X\nf3c.c\n") +test.run(program=foo4, stdout="f4.c X\n") +test.run(program=foo5, stdout="foo5.c X\n") +test.up_to_date(arguments='.') # make sure the programs didn't get rebuilt, because nothing changed: oldtime1 = os.path.getmtime(foo1) @@ -228,10 +225,8 @@ oldtime3 = os.path.getmtime(foo3) oldtime4 = os.path.getmtime(foo4) oldtime5 = os.path.getmtime(foo5) -time.sleep(2) # introduce a small delay, to make the test valid - -test.run(arguments = foo_args) - +test.sleep() # delay for timestamps +test.run(arguments=foo_args) test.fail_test(oldtime1 != os.path.getmtime(foo1)) test.fail_test(oldtime2 != os.path.getmtime(foo2)) test.fail_test(oldtime3 != os.path.getmtime(foo3)) @@ -284,15 +279,14 @@ main(int argc, char *argv[]) } """) -test.run(arguments = foo_args) - -test.run(program = foo1, stdout = "f1.c Y\n") -test.run(program = foo2, stdout = "f2a.c\nf2b.c\nf2c.c\n") -test.run(program = foo3, stdout = "f3a.c\nf3b.c Y\nf3c.c\n") -test.run(program = foo4, stdout = "f4.c Y\n") -test.run(program = foo5, stdout = "foo5.c Y\n") +test.run(arguments=foo_args) -test.up_to_date(arguments = foo_args) +test.run(program=foo1, stdout="f1.c Y\n") +test.run(program=foo2, stdout="f2a.c\nf2b.c\nf2c.c\n") +test.run(program=foo3, stdout="f3a.c\nf3b.c Y\nf3c.c\n") +test.run(program=foo4, stdout="f4.c Y\n") +test.run(program=foo5, stdout="foo5.c Y\n") +test.up_to_date(arguments=foo_args) test.write('f1.c', r""" #include @@ -340,15 +334,14 @@ main(int argc, char *argv[]) } """) -test.run(arguments = foo_args) - -test.run(program = foo1, stdout = "f1.c Z\n") -test.run(program = foo2, stdout = "f2a.c\nf2b.c\nf2c.c\n") -test.run(program = foo3, stdout = "f3a.c\nf3b.c Z\nf3c.c\n") -test.run(program = foo4, stdout = "f4.c Z\n") -test.run(program = foo5, stdout = "foo5.c Z\n") +test.run(arguments=foo_args) -test.up_to_date(arguments = foo_args) +test.run(program=foo1, stdout="f1.c Z\n") +test.run(program=foo2, stdout="f2a.c\nf2b.c\nf2c.c\n") +test.run(program=foo3, stdout="f3a.c\nf3b.c Z\nf3c.c\n") +test.run(program=foo4, stdout="f4.c Z\n") +test.run(program=foo5, stdout="foo5.c Z\n") +test.up_to_date(arguments=foo_args) # make sure the programs didn't get rebuilt, because nothing changed: oldtime1 = os.path.getmtime(foo1) @@ -357,10 +350,8 @@ oldtime3 = os.path.getmtime(foo3) oldtime4 = os.path.getmtime(foo4) oldtime5 = os.path.getmtime(foo5) -time.sleep(2) # introduce a small delay, to make the test valid - -test.run(arguments = foo_args) - +test.sleep() # delay for timestamps +test.run(arguments=foo_args) test.fail_test(not (oldtime1 == os.path.getmtime(foo1))) test.fail_test(not (oldtime2 == os.path.getmtime(foo2))) test.fail_test(not (oldtime3 == os.path.getmtime(foo3))) diff --git a/test/Repository/no-SConsignFile.py b/test/Repository/no-SConsignFile.py index d7f570c..6b5b3e2 100644 --- a/test/Repository/no-SConsignFile.py +++ b/test/Repository/no-SConsignFile.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test that using Repository() works even when the Repository has no @@ -63,13 +62,10 @@ test.write(['src', 'foo.h'], """\ # Make sure it's past the max_drift time, # so the source file signatures get saved. -test.sleep(2) +test.sleep() # delay for timestamps test.run(chdir='build', arguments='.') - -test.run(program=test.workpath('build', 'foo'), - stdout="src/foo.h\nsrc/foo.c\n") - +test.run(program=test.workpath('build', 'foo'), stdout="src/foo.h\nsrc/foo.c\n") test.up_to_date(chdir='build', arguments='.') test.pass_test() diff --git a/test/Repository/variants.py b/test/Repository/variants.py index c95e853..a07d7a0 100644 --- a/test/Repository/variants.py +++ b/test/Repository/variants.py @@ -31,20 +31,22 @@ from TestSCons import TestSCons, _exe, _obj test = TestSCons() -test.subdir('repository', - ['repository', 'src1'], - ['repository', 'src2'], - ['repository', 'src2', 'include'], - ['repository', 'src2', 'xxx'], - ['repository', 'build2'], - ['repository', 'build2', 'foo'], - ['repository', 'build2', 'bar'], - 'work1', - ['work1', 'src1'], - 'work2', - ['work2', 'src2'], - ['work2', 'src2', 'include'], - ['work2', 'src2', 'xxx']) +test.subdir( + 'repository', + ['repository', 'src1'], + ['repository', 'src2'], + ['repository', 'src2', 'include'], + ['repository', 'src2', 'xxx'], + ['repository', 'build2'], + ['repository', 'build2', 'foo'], + ['repository', 'build2', 'bar'], + 'work1', + ['work1', 'src1'], + 'work2', + ['work2', 'src2'], + ['work2', 'src2', 'include'], + ['work2', 'src2', 'xxx'], +) aaa_obj = 'aaa' + _obj bbb_obj = 'bbb' + _obj @@ -56,14 +58,18 @@ repository_build1_foo_xxx = test.workpath('repository', 'build1', 'foo', 'xxx') work1_build1_foo_xxx = test.workpath('work1', 'build1', 'foo', 'xxx') work1_build1_bar_xxx = test.workpath('work1', 'build1', 'bar', 'xxx') -repository_build2_foo_src2_xxx_xxx = test.workpath('repository', 'build2', - 'foo', 'src2', 'xxx', 'xxx') -repository_build2_bar_src2_xxx_xxx = test.workpath('repository', 'build2', - 'bar', 'src2', 'xxx', 'xxx') -work2_build2_foo_src2_xxx_xxx = test.workpath('work2', 'build2', - 'foo', 'src2', 'xxx', 'xxx') -work2_build2_bar_src2_xxx_xxx = test.workpath('work2', 'build2', - 'bar', 'src2', 'xxx', 'xxx') +repository_build2_foo_src2_xxx_xxx = test.workpath( + 'repository', 'build2', 'foo', 'src2', 'xxx', 'xxx' +) +repository_build2_bar_src2_xxx_xxx = test.workpath( + 'repository', 'build2', 'bar', 'src2', 'xxx', 'xxx' +) +work2_build2_foo_src2_xxx_xxx = test.workpath( + 'work2', 'build2', 'foo', 'src2', 'xxx', 'xxx' +) +work2_build2_bar_src2_xxx_xxx = test.workpath( + 'work2', 'build2', 'bar', 'src2', 'xxx', 'xxx' +) opts = "-Y " + test.workpath('repository') @@ -73,12 +79,13 @@ OS = ARGUMENTS.get('OS', '') build1_os = "#build1/" + OS default = Environment() ccflags = { - '' : '', - 'foo' : '-DFOO', - 'bar' : '-DBAR', + '': '', + 'foo': '-DFOO', + 'bar': '-DBAR', } -env1 = Environment(CCFLAGS = default.subst('$CCFLAGS %s' % ccflags[OS]), - CPPPATH = build1_os) +env1 = Environment( + CCFLAGS=default.subst('$CCFLAGS %s' % ccflags[OS]), CPPPATH=build1_os +) VariantDir(build1_os, 'src1') SConscript(build1_os + '/SConscript', "env1") @@ -95,8 +102,9 @@ test.write(['repository', 'build2', 'foo', 'SConscript'], r""" VariantDir('src2', '#src2') default = Environment() -env2 = Environment(CCFLAGS = default.subst('$CCFLAGS -DFOO'), - CPPPATH = ['#src2/xxx', '#src2/include']) +env2 = Environment( + CCFLAGS=default.subst('$CCFLAGS -DFOO'), CPPPATH=['#src2/xxx', '#src2/include'] +) SConscript('src2/xxx/SConscript', "env2") """) @@ -105,8 +113,9 @@ test.write(['repository', 'build2', 'bar', 'SConscript'], r""" VariantDir('src2', '#src2') default = Environment() -env2 = Environment(CCFLAGS = default.subst('$CCFLAGS -DBAR'), - CPPPATH = ['#src2/xxx', '#src2/include']) +env2 = Environment( + CCFLAGS=default.subst('$CCFLAGS -DBAR'), CPPPATH=['#src2/xxx', '#src2/include'] +) SConscript('src2/xxx/SConscript', "env2") """) @@ -216,12 +225,9 @@ repository/src1/bbb.c: REPOSITORY_FOO repository/src1/main.c: REPOSITORY_FOO """) -database_name=test.get_sconsignname() - -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +database_name = test.get_sconsignname() +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) @@ -237,10 +243,8 @@ repository/src2/xxx/include.h: BAR repository/src2/xxx/main.c: BAR """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) @@ -251,15 +255,10 @@ test.writable('repository', 0) # test.up_to_date(chdir='work1', options=opts + " OS=foo", arguments='build1') -test.fail_test(os.path.exists( - test.workpath('work1', 'build1', 'foo', aaa_obj))) -test.fail_test(os.path.exists( - test.workpath('work1', 'build1', 'foo', bbb_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work1', 'build1', 'foo', main_obj))) - -test.fail_test(os.path.exists( - test.workpath('work1', 'build1', 'foo', xxx_exe))) +test.fail_test(os.path.exists(test.workpath('work1', 'build1', 'foo', aaa_obj))) +test.fail_test(os.path.exists(test.workpath('work1', 'build1', 'foo', bbb_obj))) +test.fail_test(os.path.exists(test.workpath('work1', 'build1', 'foo', main_obj))) +test.fail_test(os.path.exists(test.workpath('work1', 'build1', 'foo', xxx_exe))) # test.run(chdir='work1', options=opts, arguments='OS=bar .') @@ -271,18 +270,13 @@ repository/src1/bbb.c: REPOSITORY_BAR repository/src1/main.c: REPOSITORY_BAR """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) - test.up_to_date(chdir='work1', options=opts + " OS=bar", arguments='build1') -# Ensure file time stamps will be newer. -time.sleep(2) - +test.sleep() # delay for timestamps test.write(['work1', 'src1', 'iii.h'], r""" #ifdef FOO #define STRING "WORK_FOO" @@ -302,13 +296,10 @@ repository/src1/bbb.c: WORK_BAR repository/src1/main.c: WORK_BAR """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) - test.up_to_date(chdir='work1', options=opts + " OS=bar", arguments='build1') # @@ -320,38 +311,39 @@ repository/src1/bbb.c: WORK_FOO repository/src1/main.c: WORK_FOO """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) - test.up_to_date(chdir='work1', options=opts + " OS=foo", arguments='build1') - -# test.up_to_date(chdir='work2', options=opts, arguments='build2') -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'foo', 'src2', 'xxx', aaa_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'foo', 'src2', 'xxx', bbb_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'foo', 'src2', 'xxx', main_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'foo', 'src2', 'xxx', xxx_exe))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'bar', 'src2', 'xxx', aaa_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'bar', 'src2', 'xxx', bbb_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'bar', 'src2', 'xxx', main_obj))) -test.fail_test(os.path.exists(test.workpath( - 'work2', 'build2', 'bar', 'src2', 'xxx', xxx_exe))) - -# Ensure file time stamps will be newer. -time.sleep(2) - +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'foo', 'src2', 'xxx', aaa_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'foo', 'src2', 'xxx', bbb_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'foo', 'src2', 'xxx', main_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'foo', 'src2', 'xxx', xxx_exe)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'bar', 'src2', 'xxx', aaa_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'bar', 'src2', 'xxx', bbb_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'bar', 'src2', 'xxx', main_obj)) +) +test.fail_test( + os.path.exists(test.workpath('work2', 'build2', 'bar', 'src2', 'xxx', xxx_exe)) +) + +test.sleep() # delay for timestamps test.write(['work2', 'src2', 'include', 'my_string.h'], r""" #ifdef FOO #define INCLUDE_OS "FOO" @@ -362,7 +354,6 @@ test.write(['work2', 'src2', 'include', 'my_string.h'], r""" #define INCLUDE_STRING "work2/src2/include/my_string.h: %s\n" """) -# test.run(chdir='work2', options=opts, arguments='build2') test.run(program=work2_build2_foo_src2_xxx_xxx, stdout="""\ @@ -377,16 +368,12 @@ repository/src2/xxx/include.h: BAR repository/src2/xxx/main.c: BAR """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) -# Ensure file time stamps will be newer. -time.sleep(2) - +test.sleep() # delay for timestamps test.write(['work2', 'src2', 'xxx', 'include.h'], r""" #include #ifdef FOO @@ -412,38 +399,37 @@ work2/src2/xxx/include.h: BAR repository/src2/xxx/main.c: BAR """) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) -# test.unlink(['work2', 'src2', 'include', 'my_string.h']) - test.run(chdir='work2', options=opts, arguments='build2') -test.run(program=work2_build2_foo_src2_xxx_xxx, stdout="""\ +test.run( + program=work2_build2_foo_src2_xxx_xxx, + stdout="""\ repository/src2/include/my_string.h: FOO work2/src2/xxx/include.h: FOO repository/src2/xxx/main.c: FOO -""") +""", +) -test.run(program=work2_build2_bar_src2_xxx_xxx, stdout="""\ +test.run( + program=work2_build2_bar_src2_xxx_xxx, + stdout="""\ repository/src2/include/my_string.h: BAR work2/src2/xxx/include.h: BAR repository/src2/xxx/main.c: BAR -""") +""", +) -test.fail_test(os.path.exists( - test.workpath('repository', 'src1', database_name))) -test.fail_test(os.path.exists( - test.workpath('repository', 'src2', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src1', database_name))) +test.fail_test(os.path.exists(test.workpath('repository', 'src2', database_name))) test.fail_test(os.path.exists(test.workpath('work1', 'src1', database_name))) test.fail_test(os.path.exists(test.workpath('work2', 'src2', database_name))) -# test.pass_test() # Local Variables: diff --git a/test/Touch.py b/test/Touch.py index 431cd6c..3538c7d 100644 --- a/test/Touch.py +++ b/test/Touch.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that the Touch() Action works. @@ -34,30 +33,29 @@ import TestSCons test = TestSCons.TestSCons() -test.write('SConstruct', """ +test.write('SConstruct', """\ Execute(Touch('f1')) Execute(Touch(File('f1-File'))) + def cat(env, source, target): target = str(target[0]) with open(target, "wb") as f: for src in source: with open(str(src), "rb") as ifp: f.write(ifp.read()) + Cat = Action(cat) env = Environment() env.Command('f2.out', 'f2.in', [Cat, Touch("f3")]) env = Environment(FILE='f4') env.Command('f5.out', 'f5.in', [Touch("$FILE"), Cat]) -env.Command('f6.out', 'f6.in', [Cat, - Touch("Touch-$SOURCE"), - Touch("$TARGET-Touch")]) +env.Command('f6.out', 'f6.in', [Cat, Touch("Touch-$SOURCE"), Touch("$TARGET-Touch")]) # Make sure Touch works with a list of arguments env = Environment() -env.Command('f7.out', 'f7.in', [Cat, - Touch(["Touch-$SOURCE", - "$TARGET-Touch", - File("f8")])]) +env.Command( + 'f7.out', 'f7.in', [Cat, Touch(["Touch-$SOURCE", "$TARGET-Touch", File("f8")])] +) """) test.write('f1', "f1\n") @@ -70,11 +68,12 @@ test.write('f7.in', "f7.in\n") old_f1_time = os.path.getmtime(test.workpath('f1')) old_f1_File_time = os.path.getmtime(test.workpath('f1-File')) -expect = test.wrap_stdout(read_str = """\ +expect = test.wrap_stdout( + read_str="""\ Touch("f1") Touch("f1-File") """, - build_str = """\ + build_str="""\ cat(["f2.out"], ["f2.in"]) Touch("f3") Touch("f4") @@ -84,11 +83,11 @@ Touch("Touch-f6.in") Touch("f6.out-Touch") cat(["f7.out"], ["f7.in"]) Touch(["Touch-f7.in", "f7.out-Touch", "f8"]) -""") -test.run(options = '-n', arguments = '.', stdout = expect) - -test.sleep(2) +""", +) +test.run(options='-n', arguments='.', stdout=expect) +test.sleep() # delay for timestamps new_f1_time = os.path.getmtime(test.workpath('f1')) test.fail_test(old_f1_time != new_f1_time) new_f1_File_time = os.path.getmtime(test.workpath('f1-File')) diff --git a/test/chained-build.py b/test/chained-build.py index 871a593..10d0b46 100644 --- a/test/chained-build.py +++ b/test/chained-build.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -58,23 +57,20 @@ test.write(['w1', 'SConstruct1'], SConstruct1_contents) test.write(['w1', 'SConstruct2'], SConstruct2_contents) test.write(['w1', 'foo.in'], "foo.in 1") -test.run(chdir='w1', - arguments="--max-drift=0 -f SConstruct1 foo.mid", - stdout = test.wrap_stdout('build(["foo.mid"], ["foo.in"])\n')) - -test.run(chdir='w1', - arguments="--max-drift=0 -f SConstruct2 foo.out", - stdout = test.wrap_stdout('build(["foo.out"], ["foo.mid"])\n')) - -test.up_to_date(chdir='w1', - options="--max-drift=0 -f SConstruct1", - arguments="foo.mid") - -test.up_to_date(chdir='w1', - options="--max-drift=0 -f SConstruct2", - arguments="foo.out") - -test.sleep() # make sure foo.in rewrite has new mod-time +test.run( + chdir='w1', + arguments="--max-drift=0 -f SConstruct1 foo.mid", + stdout=test.wrap_stdout('build(["foo.mid"], ["foo.in"])\n'), +) +test.run( + chdir='w1', + arguments="--max-drift=0 -f SConstruct2 foo.out", + stdout=test.wrap_stdout('build(["foo.out"], ["foo.mid"])\n'), +) +test.up_to_date(chdir='w1', options="--max-drift=0 -f SConstruct1", arguments="foo.mid") +test.up_to_date(chdir='w1', options="--max-drift=0 -f SConstruct2", arguments="foo.out") + +test.sleep() # delay for timestamps test.write(['w1', 'foo.in'], "foo.in 2") # Because we're using --max-drift=0, we use the cached csig value @@ -86,17 +82,19 @@ test.up_to_date(chdir='w1', # Now try with --max-drift disabled. The build of foo.out should still # be considered up-to-date, but the build of foo.mid now detects the # change and rebuilds, too, which then causes a rebuild of foo.out. -test.up_to_date(chdir='w1', - options="--max-drift=-1 -f SConstruct2", - arguments="foo.out") - -test.run(chdir='w1', - arguments="--max-drift=-1 -f SConstruct1 foo.mid", - stdout = test.wrap_stdout('build(["foo.mid"], ["foo.in"])\n')) - -test.run(chdir='w1', - arguments="--max-drift=-1 -f SConstruct2 foo.out", - stdout = test.wrap_stdout('build(["foo.out"], ["foo.mid"])\n')) +test.up_to_date( + chdir='w1', options="--max-drift=-1 -f SConstruct2", arguments="foo.out" +) +test.run( + chdir='w1', + arguments="--max-drift=-1 -f SConstruct1 foo.mid", + stdout=test.wrap_stdout('build(["foo.mid"], ["foo.in"])\n'), +) +test.run( + chdir='w1', + arguments="--max-drift=-1 -f SConstruct2 foo.out", + stdout=test.wrap_stdout('build(["foo.out"], ["foo.mid"])\n'), +) test.pass_test() diff --git a/test/sconsign/script/Signatures.py b/test/sconsign/script/Signatures.py index ef8bac0..2225f83 100644 --- a/test/sconsign/script/Signatures.py +++ b/test/sconsign/script/Signatures.py @@ -44,8 +44,8 @@ if NEED_HELPER: # in the expected output because paths in the .sconsign files are # canonicalized to use / as the separator. -sub1_hello_c = 'sub1/hello.c' -sub1_hello_obj = 'sub1/hello.obj' +sub1_hello_c = 'sub1/hello.c' +sub1_hello_obj = 'sub1/hello.obj' test.subdir('sub1', 'sub2') @@ -135,7 +135,7 @@ env2.Program('sub2/hello.c') """, ) # TODO in the above, we would normally want to run a python program -# using "our python" like this: +# using "our Python" like this: # CCCOM=[[r'{_python_}', r'{fake_cc_py}', 'sub2', '$TARGET', '$SOURCE']], # LINKCOM=[[r'{_python_}', r'{fake_link_py}', '$TARGET', '$SOURCE']], # however we're looking at dependencies with sconsign, so that breaks things. @@ -160,17 +160,16 @@ test.write(['sub2', 'inc2.h'], r"""\ #define STRING2 "inc2.h" """) -test.sleep() - +test.sleep() # delay for timestamps test.run(arguments = '. --max-drift=1') sig_re = r'[0-9a-fA-F]{32,64}' date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d' - database_name = test.get_sconsignname() -test.run_sconsign(arguments = f"-e hello.exe -e hello.obj sub1/{database_name}", - stdout = r"""hello.exe: %(sig_re)s \d+ \d+ +test.run_sconsign( + arguments=f"-e hello.exe -e hello.obj sub1/{database_name}", + stdout=r"""hello.exe: %(sig_re)s \d+ \d+ %(sub1_hello_obj)s: %(sig_re)s \d+ \d+ fake_link\.py: None \d+ \d+ %(sig_re)s \[.*\] @@ -178,10 +177,12 @@ hello.obj: %(sig_re)s \d+ \d+ %(sub1_hello_c)s: None \d+ \d+ fake_cc\.py: None \d+ \d+ %(sig_re)s \[.*\] -""" % locals()) +""" % locals(), +) -test.run_sconsign(arguments = f"-e hello.exe -e hello.obj -r sub1/{database_name}", - stdout = r"""hello.exe: %(sig_re)s '%(date_re)s' \d+ +test.run_sconsign( + arguments=f"-e hello.exe -e hello.obj -r sub1/{database_name}", + stdout=r"""hello.exe: %(sig_re)s '%(date_re)s' \d+ %(sub1_hello_obj)s: %(sig_re)s '%(date_re)s' \d+ fake_link\.py: None '%(date_re)s' \d+ %(sig_re)s \[.*\] @@ -189,7 +190,8 @@ hello.obj: %(sig_re)s '%(date_re)s' \d+ %(sub1_hello_c)s: None '%(date_re)s' \d+ fake_cc\.py: None '%(date_re)s' \d+ %(sig_re)s \[.*\] -""" % locals()) +""" % locals(), +) test.pass_test() diff --git a/test/sconsign/script/dblite.py b/test/sconsign/script/dblite.py index 1fcf8c0..24d8403 100644 --- a/test/sconsign/script/dblite.py +++ b/test/sconsign/script/dblite.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that various ways of getting at a an sconsign file written with @@ -97,17 +96,11 @@ main(int argc, char *argv[]) } """) -test.write(['sub2', 'inc1.h'], r"""\ -#define STRING1 "inc1.h" -""") - -test.write(['sub2', 'inc2.h'], r"""\ -#define STRING2 "inc2.h" -""") - -test.sleep() +test.write(['sub2', 'inc1.h'], r'#define STRING1 "inc1.h"') +test.write(['sub2', 'inc2.h'], r'#define STRING2 "inc2.h"') -test.run(arguments = '. --max-drift=1') +test.sleep() # delay for timestamps +test.run(arguments='. --max-drift=1') sig_re = r'[0-9a-fA-F]{32,64}' date_re = r'\S+ \S+ [ \d]\d \d\d:\d\d:\d\d \d\d\d\d' @@ -146,23 +139,23 @@ hello%(_obj)s: %(sig_re)s '%(date_re)s' \d+ common_flags = '-e hello%(_exe)s -e hello%(_obj)s -d sub1' % locals() -test.run_sconsign(arguments = "%s my_sconsign" % common_flags, - stdout = expect) +test.run_sconsign(arguments="%s my_sconsign" % common_flags, stdout=expect) -test.run_sconsign(arguments = "%s my_sconsign.dblite" % common_flags, - stdout = expect) +test.run_sconsign(arguments="%s my_sconsign.dblite" % common_flags, stdout=expect) -test.run_sconsign(arguments = "%s -f dblite my_sconsign" % common_flags, - stdout = expect) +test.run_sconsign(arguments="%s -f dblite my_sconsign" % common_flags, stdout=expect) -test.run_sconsign(arguments = "%s -f dblite my_sconsign.dblite" % common_flags, - stdout = expect) +test.run_sconsign( + arguments="%s -f dblite my_sconsign.dblite" % common_flags, stdout=expect +) -test.run_sconsign(arguments = "%s -r -f dblite my_sconsign" % common_flags, - stdout = expect_r) +test.run_sconsign( + arguments="%s -r -f dblite my_sconsign" % common_flags, stdout=expect_r +) -test.run_sconsign(arguments = "%s -r -f dblite my_sconsign.dblite" % common_flags, - stdout = expect_r) +test.run_sconsign( + arguments="%s -r -f dblite my_sconsign.dblite" % common_flags, stdout=expect_r +) test.pass_test() -- cgit v0.12 From 53e33adc2a7cbd7d68020350060f73ea5977f002 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 31 May 2022 09:11:42 -0600 Subject: Fix sider-detected typo in test change Signed-off-by: Mats Wichmann --- test/Copy-Action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Copy-Action.py b/test/Copy-Action.py index 2179744..e135a9f 100644 --- a/test/Copy-Action.py +++ b/test/Copy-Action.py @@ -166,7 +166,7 @@ def must_be_same(f1, f2): for value in ['ST_MODE', 'ST_MTIME']: v = getattr(stat, value) if s1[v] != s2[v]: - msg = f"{f1!r}[{value}] {s1[v1]} != {f2!r}[{value}] {s2[v]}\n" + msg = f"{f1!r}[{value}] {s1[v]} != {f2!r}[{value}] {s2[v]}\n" sys.stderr.write(msg) errors += 1 -- cgit v0.12 From c9c678683a3944b0500a8ce68c80d7c5d48c9230 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 1 Jun 2022 14:28:03 -0700 Subject: Updated CHANGES/RELEASE to be a bit simpler. Fixed typo in docs for SHELL_ENV_GENERATORS. --- CHANGES.txt | 9 ++++++--- RELEASE.txt | 9 ++++++--- SCons/Action.py | 10 ++++++++-- SCons/Action.xml | 8 ++++---- test/Actions/subst_shell_env.py | 2 +- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d5aba9f..84ec06b 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -99,9 +99,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected format. Compiler tools will also configure the variable automatically. - - Added SHELL_ENV_GENERATORS construction variable. This variable - is an iterable which will contain functions in which each are called and each can allow - the user a method to customize the execution environment. + - Added SHELL_ENV_GENERATORS construction variable. This variable should be set to a list + (or an iterable) which contains functions to be called in order + when constructing the execution environment (Generally this is the shell environment + variables). This allows the user to customize how (for example) PATH is constructed. + Note that these are called for every build command run by SCons. It could have considerable + performance impact if not used carefully. - Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. - Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry to connect to the server during start up. diff --git a/RELEASE.txt b/RELEASE.txt index 195e325..e7fa31c 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -16,9 +16,12 @@ NEW FUNCTIONALITY - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. - Added Configure.CheckMember() checker to check if struct/class has the specified member. -- Added SHELL_ENV_GENERATORS construction variable. This variable - is an iterable which will contain functions in which each are called and each can allow - the user a method to customize the execution environment. +- Added SHELL_ENV_GENERATORS construction variable. This variable should be set to a list + (or an iterable) which contains functions to be called in order + when constructing the execution environment (Generally this is the shell environment + variables). This allows the user to customize how (for example) PATH is constructed. + Note that these are called for every build command run by SCons. It could have considerable + performance impact if not used carefully. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. diff --git a/SCons/Action.py b/SCons/Action.py index e29c4e9..6e67c7f 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -756,9 +756,15 @@ def get_default_ENV(env): def _resolve_shell_env(env, target, source): + """ + First get default environment. + Then if SHELL_ENV_GENERATORS is set and is iterable, + call each callable in that list to allow it to alter + the created execution environment. + """ ENV = get_default_ENV(env) - shell_gen = env.get('SHELL_ENV_GENERATORS') - if shell_gen is not None: + shell_gen = env.get('SHELL_ENV_GENERATORS') + if shell_gen: try: shell_gens = iter(shell_gen) except TypeError: diff --git a/SCons/Action.xml b/SCons/Action.xml index 8994adb..1346db2 100644 --- a/SCons/Action.xml +++ b/SCons/Action.xml @@ -203,7 +203,7 @@ in which the command should be executed. -Must be an iterable containing functions where each function generates or +Must be a list (or an iterable) containing functions where each function generates or alters the environment dictionary which will be used when executing the &cv-link-SPAWN; function. The functions will initially be passed a reference of the current execution environment (e.g. env['ENV']), @@ -214,14 +214,14 @@ values. This primary purpose of this construction variable is to give the user the ability to substitute execution environment variables based on env, targets, and sources. -If desired, the user can completly customize the execution environment for particular +If desired, the user can completely customize the execution environment for particular targets. def custom_shell_env(env, target, source, shell_env): - # customize shell_env if desired - if str(target[0]) == 'special_target'is: + """customize shell_env if desired""" + if str(target[0]) == 'special_target': shell_env['SPECIAL_VAR'] = env.subst('SOME_VAR', target=target, source=source) return shell_env diff --git a/test/Actions/subst_shell_env.py b/test/Actions/subst_shell_env.py index d26f0ae..02ba640 100644 --- a/test/Actions/subst_shell_env.py +++ b/test/Actions/subst_shell_env.py @@ -25,7 +25,7 @@ """ Verify that shell environment variables can be expanded per target/source -when exectuting actions on the command line. +when executing actions on the command line. """ import os -- cgit v0.12 From 00dd82bf9a1f0b3332449a0add3a5f3d5c78c8c2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 2 Jun 2022 10:25:48 -0700 Subject: Move logic which waits for a process to die into SCons.Util wait_for_process_to_die(pid) --- SCons/Tool/ninja/NinjaState.py | 29 ++--------------------------- SCons/Util.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 5f8d5c7..770da07 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -37,7 +37,7 @@ import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Util import is_List +from SCons.Util import is_List, wait_for_process_to_die from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS @@ -644,32 +644,7 @@ class NinjaState: pass # wait for the server process to fully killed - try: - import psutil - while True: - if pid not in [proc.pid for proc in psutil.process_iter()]: - break - else: - time.sleep(0.1) - except ImportError: - # if psutil is not installed we can do this the hard way - while True: - if sys.platform == 'win32': - import ctypes - PROCESS_QUERY_INFORMATION = 0x1000 - processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) - if processHandle == 0: - break - else: - ctypes.windll.kernel32.CloseHandle(processHandle) - time.sleep(0.1) - else: - try: - os.kill(pid, 0) - except OSError: - break - else: - time.sleep(0.1) + wait_for_process_to_die(pid) if os.path.exists(scons_daemon_dirty): os.unlink(scons_daemon_dirty) diff --git a/SCons/Util.py b/SCons/Util.py index 093ca41..0fe42c8 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -29,6 +29,7 @@ import os import pprint import re import sys +import time from collections import UserDict, UserList, UserString, OrderedDict from collections.abc import MappingView from contextlib import suppress @@ -2123,6 +2124,40 @@ def print_time(): from SCons.Script.Main import print_time return print_time + +def wait_for_process_to_die(pid): + """ + Wait for specified process to die, or alternatively kill it + NOTE: This function operates best with psutil pypi package + """ + # wait for the process to fully killed + try: + import psutil + while True: + if pid not in [proc.pid for proc in psutil.process_iter()]: + break + else: + time.sleep(0.1) + except ImportError: + # if psutil is not installed we can do this the hard way + while True: + if sys.platform == 'win32': + import ctypes + PROCESS_QUERY_INFORMATION = 0x1000 + processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) + if processHandle == 0: + break + else: + ctypes.windll.kernel32.CloseHandle(processHandle) + time.sleep(0.1) + else: + try: + os.kill(pid, 0) + except OSError: + break + else: + time.sleep(0.1) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From a2dffb888472f55fad92e539d24b929b292fcb07 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 2 Jun 2022 11:27:25 -0700 Subject: Updated CHANGES/RELEASE contents --- CHANGES.txt | 18 +++++++++--------- RELEASE.txt | 13 ++++++++----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d21aa60..08fdd34 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -81,8 +81,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER only used for ninja) - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - - Added ninja mingw support and improved ninja CommandGeneratorAction support. - - Update ninja file generation to only create response files for build commands + - Ninja: Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Ninja: Update ninja file generation to only create response files for build commands which exceed MAXLINELENGTH - Ninja: Added NINJA_GENERATED_SOURCE_ALIAS_NAME which allows user to specify an Alias() which the ninja tool can use to determine which files are generated sources. @@ -93,16 +93,16 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER required by other build targets. Code contributed by MongoDB The downstream commit is here: https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f - - Added special case for ninja scons daemon to work in win32 python3.6 environments. + - Ninja: Added special case for ninja scons daemon to work in win32 python3.6 environments. This particular environment does a bad job managing popen standard file handles, so some special workarounds are needed. - - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + - Ninja:Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected format. Compiler tools will also configure the variable automatically. - - Made ninja tool force the ninja file as the only target. Also improved the default - targets setup and made sure there is always a default target for + - Ninja: Made ninja tool force the ninja file as the only target. + - Ninja: Improved the default targets setup and made sure there is always a default target for the ninja file, which excludes targets that start and stop the daemon. - - Update ninja tool so targets passed to SCons are propgated to ninja when scons + - Ninja: Update ninja tool so targets passed to SCons are propagated to ninja when scons automatically executes ninja. - Small refactor of scons daemons using a shared StateInfo class for communication between the scons interactive thread and the http server thread. Added error handling @@ -110,8 +110,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define a function which will be called to generate or alter the execution environment which will be used in the shell command of some Action. - - Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. - - Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry + - Ninja: Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. + - Ninja: Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry to connect to the server during start up. From Mats Wichmann: diff --git a/RELEASE.txt b/RELEASE.txt index ce2b8e5..832698d 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -58,17 +58,17 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY ".scons_msvc_cache" so there should be no transition problem if using the default; if using a custom cache file name, the cache should still be manually removed if there are problems to transition to the new style. -- Update ninja file generation to only create response files for build commands +- Ninja: Update ninja file generation to only create response files for build commands which exceed MAXLINELENGTH - Update the debug output written to stdout for MSVC initialization which is enabled by setting SCONS_MSCOMMON_DEBUG=- to use the logging module. Also changed the debug output format written to stdout to include more information about the source for each message of MSVC initialization debugging output. A single space was added before the message for all debugging output records written to stdout and to files. -- Made ninja tool force the ninja file as the only target. Also improved the default +- Ninja: Made ninja tool force the ninja file as the only target. Also improved the default targets setup and made sure there is always a default target for the ninja file, which excludes targets that start and stop the daemon. -- Update ninja tool so targets passed to SCons are propgated to ninja when scons +- Ninja: Update ninja tool so targets passed to SCons are propgated to ninja when scons automatically executes ninja. - Add JavaScanner to include JAVACLASSPATH as a dependency when using the Java tool. - The build argument (i.e., x86) is no longer passed to the MSVC 6.0 to 7.1 batch @@ -138,7 +138,7 @@ IMPROVEMENTS - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception immediately when the user specified msvc script does not exist. - Add cache-debug messages for push failures. -- Added ninja mingw support and improved ninja CommandGeneratorAction support. +- Ninja: Added ninja mingw support and improved ninja CommandGeneratorAction support. - Command-line help is now sensitive to the size of the terminal window: the width of the help text will scale for terminals other than 80 chars wide. - Refactor the msvc code so that the same data structures are used during initial msvc detection @@ -148,7 +148,10 @@ IMPROVEMENTS - Small refactor of scons daemons using a shared StateInfo class for communication between the scons interactive thread and the http server thread. Added error handling for scons interactive failing to startup. -- Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. +- Ninja: Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. +- Ninja: Ensure that all targets set as default via Default() in SConstruct/SConscripts are + default targets in the generated ninja.build file. + PACKAGING --------- -- cgit v0.12 From b5ebe82c9867ad9da154e57c70d1f63f2d98d3a1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 2 Jun 2022 12:11:10 -0700 Subject: Updated ninja docs in users guilde. --- SCons/Util.py | 2 +- doc/user/external.xml | 54 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/SCons/Util.py b/SCons/Util.py index 0fe42c8..0eeef2a 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2138,7 +2138,7 @@ def wait_for_process_to_die(pid): break else: time.sleep(0.1) - except ImportError: + except ImportError: # if psutil is not installed we can do this the hard way while True: if sys.platform == 'win32': diff --git a/doc/user/external.xml b/doc/user/external.xml index b483196..e15f998 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -303,25 +303,45 @@
Ninja Build Generator - - - + + This is an experimental new feature. It is subject to change and/or removal without a depreciation cycle. - + + + + + Loading the &t-link-ninja; tool into SCons will make significant changes + in SCons' normal functioning. + + + - To enable this feature you'll need to use one of the following: + SCons will no longer execute any commands directly and will only create the build.ninja and + run ninja. - - -# On the command line ---experimental=ninja + + + + Any targets specified on the command line will be passed along to &ninja; + + + + + + To enable this feature you'll need to use one of the following: + + + +# On the command line --experimental=ninja # Or in your SConstruct SetOption('experimental', 'ninja') - - + + + + Ninja is a small build system that tries to be fast @@ -366,8 +386,10 @@ conda install -c conda-forge ninja It is not expected that the &b-link-Ninja; builder will work for all builds at this point. It is still under active - development. If you find that your build doesn't work with &ninja; please bring this to the users mailing list - or #scons-help channel on our Discord server. + development. If you find that your build doesn't work with &ninja; please bring this to the users mailing list + or + #scons-help + channel on our Discord server. @@ -377,6 +399,12 @@ conda install -c conda-forge ninja implement those actions via shell commands in the &ninja; build file. + + When ninja runs the generated &ninja; build file, ninja will launch &scons; as a daemon and feed commands + to that &scons; process which &ninja; is unable to build directly. This daemon will stay alive until + explicitly killed or it times out. The timeout is set by &cv-link-NINJA_SCONS_DAEMON_KEEP_ALIVE; . + + See: -- cgit v0.12 From a3d7d83d64a711b8b1738091c7eb6eebe4666716 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 2 Jun 2022 13:42:36 -0700 Subject: Fix sider complaints --- SCons/Tool/ninja/NinjaState.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index 770da07..8f7e04b 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -29,7 +29,6 @@ import signal import tempfile import shutil import sys -import time from os.path import splitext from tempfile import NamedTemporaryFile import ninja @@ -37,7 +36,7 @@ import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS -from SCons.Util import is_List, wait_for_process_to_die +from SCons.Util import wait_for_process_to_die from SCons.Errors import InternalError from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS -- cgit v0.12 From b511d618966ff82b2200824123793301039cf579 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 3 Jun 2022 10:30:20 -0700 Subject: [ci skip] Add note to handle processes not dying properly by raising exception and handling in calling function --- SCons/Tool/ninja/NinjaState.py | 2 ++ SCons/Util.py | 1 + 2 files changed, 3 insertions(+) diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index c9cacce..c168c7f 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -643,6 +643,8 @@ class NinjaState: pass # wait for the server process to fully killed + # TODO: update wait_for_process_to_die() to handle timeout and then catch exception + # here and do something smart. wait_for_process_to_die(pid) if os.path.exists(scons_daemon_dirty): diff --git a/SCons/Util.py b/SCons/Util.py index 0eeef2a..49a3a0f 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2129,6 +2129,7 @@ def wait_for_process_to_die(pid): """ Wait for specified process to die, or alternatively kill it NOTE: This function operates best with psutil pypi package + TODO: Add timeout which raises exception """ # wait for the process to fully killed try: -- cgit v0.12 From ae46e4569da500e0a9270fda61dd983384f68a3a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 3 Jun 2022 10:53:50 -0700 Subject: [ci skip] Added blurb about ninja restarting the scons daemon if it detects need to regenerate the build.ninja --- doc/user/external.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/user/external.xml b/doc/user/external.xml index e15f998..0eab32e 100644 --- a/doc/user/external.xml +++ b/doc/user/external.xml @@ -400,9 +400,14 @@ conda install -c conda-forge ninja - When ninja runs the generated &ninja; build file, ninja will launch &scons; as a daemon and feed commands + When &ninja; runs the generated &ninja; build file, &ninja; will launch &scons; as a daemon and feed commands to that &scons; process which &ninja; is unable to build directly. This daemon will stay alive until - explicitly killed or it times out. The timeout is set by &cv-link-NINJA_SCONS_DAEMON_KEEP_ALIVE; . + explicitly killed, or it times out. The timeout is set by &cv-link-NINJA_SCONS_DAEMON_KEEP_ALIVE; . + + + + The daemon will be restarted if any &SConscript; file(s) change or the build changes in a way that &ninja; determines + it needs to regenerate the build.ninja file See: -- cgit v0.12 From 48d31b80ce66afba4b495c1dcad44d9d38d2c242 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 3 Jun 2022 14:22:21 -0700 Subject: If no newline at end of message supplied to skip_test(), then we write one to stdout after the original message --- testing/framework/TestCommon.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index 1999f74..e8045ca 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -782,6 +782,8 @@ class TestCommon(TestCmd): """ if message: sys.stdout.write(message) + if not message.endswith('\n'): + sys.stdout.write('\n') sys.stdout.flush() pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS') if pass_skips in [None, 0, '0']: -- cgit v0.12 From 23a35cec7896e567ff81a8afaab99ab87ed92b8b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 4 Jun 2022 07:24:05 -0600 Subject: Update ActionTests for Python 3.11 Required all new bytecode strings. Some reformatting done (this is not a full formatting run). Partial fix for #4162, do not close only for this PR. Signed-off-by: Mats Wichmann --- SCons/ActionTests.py | 344 +++++++++++++++++++++++++++------------------------ 1 file changed, 185 insertions(+), 159 deletions(-) diff --git a/SCons/ActionTests.py b/SCons/ActionTests.py index 29ed4a7..0a7f25b 100644 --- a/SCons/ActionTests.py +++ b/SCons/ActionTests.py @@ -71,16 +71,16 @@ with open(sys.argv[1], 'w') as f: if 'ACTPY_PIPE' in os.environ: if 'PIPE_STDOUT_FILE' in os.environ: - with open(os.environ['PIPE_STDOUT_FILE'], 'r') as f: - stdout_msg = f.read() + with open(os.environ['PIPE_STDOUT_FILE'], 'r') as f: + stdout_msg = f.read() else: - stdout_msg = "act.py: stdout: executed act.py %s\\n" % ' '.join(sys.argv[1:]) - sys.stdout.write( stdout_msg ) + stdout_msg = "act.py: stdout: executed act.py %s\\n" % ' '.join(sys.argv[1:]) + sys.stdout.write(stdout_msg) if 'PIPE_STDERR_FILE' in os.environ: - with open(os.environ['PIPE_STDERR_FILE'], 'r') as f: - stderr_msg = f.read() + with open(os.environ['PIPE_STDERR_FILE'], 'r') as f: + stderr_msg = f.read() else: - stderr_msg = "act.py: stderr: executed act.py %s\\n" % ' '.join(sys.argv[1:]) + stderr_msg = "act.py: stderr: executed act.py %s\\n" % ' '.join(sys.argv[1:]) sys.stderr.write(stderr_msg) sys.exit(0) """) @@ -113,8 +113,9 @@ class CmdStringHolder: return self.literal def escape(self, escape_func): - """Escape the string with the supplied function. The - function is expected to take an arbitrary string, then + """Escape the string with the supplied function. + + The function is expected to take an arbitrary string, then return it with all special characters escaped and ready for passing to the command interpreter. @@ -251,8 +252,8 @@ def test_varlist(pos_call, str_call, cmd, cmdstrfunc, **kw): def test_positional_args(pos_callback, cmd, **kw): - """Test that Action() returns the expected type and that positional args work. - """ + """Test that Action returns the expected type and positional args work.""" + act = SCons.Action.Action(cmd, **kw) pos_callback(act) assert act.varlist == (), act.varlist @@ -293,7 +294,7 @@ def test_positional_args(pos_callback, cmd, **kw): test_varlist(pos_callback, none, cmd, None, **kw) - """Test handling of bad cmdstrfunc arguments """ + # Test handling of bad cmdstrfunc arguments try: a = SCons.Action.Action(cmd, [], **kw) except SCons.Errors.UserError as e: @@ -310,8 +311,7 @@ class ActionTestCase(unittest.TestCase): """Test the Action() factory function""" def test_FunctionAction(self): - """Test the Action() factory's creation of FunctionAction objects - """ + """Test the Action() factory's creation of FunctionAction objects.""" def foo(): pass @@ -325,8 +325,7 @@ class ActionTestCase(unittest.TestCase): test_positional_args(func_action, [foo]) def test_CommandAction(self): - """Test the Action() factory's creation of CommandAction objects - """ + """Test the Action() factory's creation of CommandAction objects.""" def cmd_action(a): assert isinstance(a, SCons.Action.CommandAction), a @@ -343,8 +342,8 @@ class ActionTestCase(unittest.TestCase): test_positional_args(line_action, [["explicit", "command", "line"]]) def test_ListAction(self): - """Test the Action() factory's creation of ListAction objects - """ + """Test the Action() factory's creation of ListAction objects.""" + a1 = SCons.Action.Action(["x", "y", "z", ["a", "b", "c"]]) assert isinstance(a1, SCons.Action.ListAction), a1 assert a1.varlist == (), a1.varlist @@ -401,8 +400,7 @@ class ActionTestCase(unittest.TestCase): assert a5.list[1].strfunction == foo, a5.list[1].strfunction def test_CommandGeneratorAction(self): - """Test the Action() factory's creation of CommandGeneratorAction objects - """ + """Test the Action factory's creation of CommandGeneratorAction objects.""" def foo(): pass @@ -413,8 +411,7 @@ class ActionTestCase(unittest.TestCase): test_positional_args(gen_action, foo, generator=1) def test_LazyCmdGeneratorAction(self): - """Test the Action() factory's creation of lazy CommandGeneratorAction objects - """ + """Test the Action factory's creation of lazy CommandGeneratorAction objects.""" def lazy_action(a): assert isinstance(a, SCons.Action.LazyAction), a @@ -425,8 +422,8 @@ class ActionTestCase(unittest.TestCase): test_positional_args(lazy_action, "${FOO}") def test_no_action(self): - """Test when the Action() factory can't create an action object - """ + """Test when the Action() factory can't create an action object.""" + try: a5 = SCons.Action.Action(1) except TypeError: @@ -435,8 +432,8 @@ class ActionTestCase(unittest.TestCase): assert 0, "Should have thrown a TypeError creating Action from an int." def test_reentrance(self): - """Test the Action() factory when the action is already an Action object - """ + """Test the Action factory when the action is already an Action object.""" + a1 = SCons.Action.Action("foo") a2 = SCons.Action.Action(a1) assert a2 is a1, a2 @@ -445,8 +442,7 @@ class ActionTestCase(unittest.TestCase): class _ActionActionTestCase(unittest.TestCase): def test__init__(self): - """Test creation of _ActionAction objects - """ + """Test creation of _ActionAction objects.""" def func1(): pass @@ -519,8 +515,7 @@ class _ActionActionTestCase(unittest.TestCase): assert a.varlist is t, a.varlist def test_dup_keywords(self): - """Test handling of both cmdstr and strfunction arguments - """ + """Test handling of both cmdstr and strfunction arguments.""" def func(): pass @@ -535,8 +530,8 @@ class _ActionActionTestCase(unittest.TestCase): raise Exception("did not catch expected UserError") def test___cmp__(self): - """Test Action comparison - """ + """Test Action comparison.""" + a1 = SCons.Action.Action("x") a2 = SCons.Action.Action("x") assert a1 == a2 @@ -545,8 +540,8 @@ class _ActionActionTestCase(unittest.TestCase): assert a2 != a3 def test_print_cmd_lines(self): - """Test the print_cmd_lines() method - """ + """Test the print_cmd_lines() method.""" + save_stdout = sys.stdout try: @@ -565,8 +560,8 @@ class _ActionActionTestCase(unittest.TestCase): sys.stdout = save_stdout def test___call__(self): - """Test calling an Action - """ + """Test calling an Action.""" + save_stdout = sys.stdout save_print_actions = SCons.Action.print_actions @@ -753,8 +748,8 @@ class _ActionActionTestCase(unittest.TestCase): SCons.Action.execute_actions = save_execute_actions def test_presub_lines(self): - """Test the presub_lines() method - """ + """Test the presub_lines() method.""" + env = Environment() a = SCons.Action.Action("x") s = a.presub_lines(env) @@ -875,8 +870,8 @@ class _ActionActionTestCase(unittest.TestCase): class CommandActionTestCase(unittest.TestCase): def test___init__(self): - """Test creation of a command Action - """ + """Test creation of a command Action.""" + a = SCons.Action.CommandAction(["xyzzy"]) assert a.cmd_list == ["xyzzy"], a.cmd_list assert a.cmdstr is _null, a.cmdstr @@ -886,8 +881,8 @@ class CommandActionTestCase(unittest.TestCase): assert a.cmdstr == "cadabra", a.cmdstr def test___str__(self): - """Test fetching the pre-substitution string for command Actions - """ + """Test fetching the pre-substitution string for command Actions.""" + env = Environment() act = SCons.Action.CommandAction('xyzzy $TARGET $SOURCE') s = str(act) @@ -900,8 +895,7 @@ class CommandActionTestCase(unittest.TestCase): assert s == "xyzzy $TARGET $SOURCE $TARGETS $SOURCES", s def test_genstring(self): - """Test the genstring() method for command Actions - """ + """Test the genstring() method for command Actions.""" env = Environment() t1 = DummyNode('t1') @@ -938,8 +932,7 @@ class CommandActionTestCase(unittest.TestCase): assert s == expect, s def test_strfunction(self): - """Test fetching the string representation of command Actions - """ + """Test fetching the string representation of command Actions.""" env = Environment() t1 = DummyNode('t1') @@ -1090,9 +1083,8 @@ class CommandActionTestCase(unittest.TestCase): assert s == "foo bar", s def test_execute(self): - """Test execution of command Actions + """Test execution of command Actions.""" - """ try: env = self.env except AttributeError: @@ -1247,8 +1239,7 @@ class CommandActionTestCase(unittest.TestCase): assert r == 0, r def test_set_handler(self): - """Test setting the command handler... - """ + """Test setting the command handler...""" class Test: def __init__(self): @@ -1299,8 +1290,7 @@ class CommandActionTestCase(unittest.TestCase): assert t.executed == ['**xyzzy**'], t.executed def test_get_contents(self): - """Test fetching the contents of a command Action - """ + """Test fetching the contents of a command Action.""" def CmdGen(target, source, env, for_signature): assert for_signature @@ -1373,6 +1363,7 @@ class CommandActionTestCase(unittest.TestCase): def test_get_implicit_deps(self): """Test getting the implicit dependencies of a command Action.""" + class SpecialEnvironment(Environment): def WhereIs(self, prog): return prog @@ -1413,11 +1404,11 @@ class CommandGeneratorActionTestCase(unittest.TestCase): def factory(self, act, **kw): """Pass any keywords as a dict""" + return SCons.Action.CommandGeneratorAction(act, kw) def test___init__(self): - """Test creation of a command generator Action - """ + """Test creation of a command generator Action.""" def f(target, source, env): pass @@ -1426,8 +1417,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert a.generator == f def test___str__(self): - """Test the pre-substitution strings for command generator Actions - """ + """Test the pre-substitution strings for command generator Actions.""" def f(target, source, env, for_signature, self=self): # See if "env" is really a construction environment (or @@ -1444,8 +1434,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert s == 'FOO', s def test_genstring(self): - """Test the command generator Action genstring() method - """ + """Test the command generator Action genstring() method.""" def f(target, source, env, for_signature, self=self): dummy = env['dummy'] @@ -1459,8 +1448,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert s == "$FOO $TARGET $SOURCE $TARGETS $SOURCES", s def test_execute(self): - """Test executing a command generator Action - """ + """Test executing a command generator Action.""" def f(target, source, env, for_signature, self=self): dummy = env['dummy'] @@ -1519,8 +1507,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert self.rfile_called def test_get_contents(self): - """Test fetching the contents of a command generator Action - """ + """Test fetching the contents of a command generator Action.""" def f(target, source, env, for_signature): foo = env['foo'] @@ -1540,8 +1527,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): assert c == b"guux FFF BBB test", c def test_get_contents_of_function_action(self): - """Test contents of a CommandGeneratorAction-generated FunctionAction - """ + """Test contents of a CommandGeneratorAction-generated FunctionAction.""" def LocalFunc(): pass @@ -1554,6 +1540,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): (3, 8): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } meth_matches = [ @@ -1571,12 +1558,12 @@ class CommandGeneratorActionTestCase(unittest.TestCase): a = self.factory(f_global) c = a.get_contents(target=[], source=[], env=env) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + repr( + assert c == func_matches[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( func_matches[sys.version_info[:2]]) a = self.factory(f_local) c = a.get_contents(target=[], source=[], env=env) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + repr( + assert c == func_matches[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( func_matches[sys.version_info[:2]]) def f_global2(target, source, env, for_signature): @@ -1599,8 +1586,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): class FunctionActionTestCase(unittest.TestCase): def test___init__(self): - """Test creation of a function Action - """ + """Test creation of a function Action.""" def func1(): pass @@ -1623,8 +1609,7 @@ class FunctionActionTestCase(unittest.TestCase): assert a.strfunction == func3, a.strfunction def test___str__(self): - """Test the __str__() method for function Actions - """ + """Test the __str__() method for function Actions.""" def func1(): pass @@ -1642,8 +1627,8 @@ class FunctionActionTestCase(unittest.TestCase): assert s == "class1(target, source, env)", s def test_execute(self): - """Test executing a function Action - """ + """Test executing a function Action.""" + self.inc = 0 def f(target, source, env): @@ -1721,8 +1706,7 @@ class FunctionActionTestCase(unittest.TestCase): assert self.string_it def test_get_contents(self): - """Test fetching the contents of a function Action - """ + """Test fetching the contents of a function Action.""" def LocalFunc(): pass @@ -1734,6 +1718,7 @@ class FunctionActionTestCase(unittest.TestCase): (3, 8): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } @@ -1744,6 +1729,7 @@ class FunctionActionTestCase(unittest.TestCase): (3, 8): bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 9): bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), + (3, 11): bytearray(b'1, 1, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } def factory(act, **kw): @@ -1751,20 +1737,23 @@ class FunctionActionTestCase(unittest.TestCase): a = factory(GlobalFunc) c = a.get_contents(target=[], source=[], env=Environment()) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) a = factory(LocalFunc) c = a.get_contents(target=[], source=[], env=Environment()) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) matches_foo = func_matches[sys.version_info[:2]] + b'foo' a = factory(GlobalFunc, varlist=['XYZ']) c = a.get_contents(target=[], source=[], env=Environment()) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) # assert c in func_matches, repr(c) c = a.get_contents(target=[], source=[], env=Environment(XYZ='foo')) @@ -1775,8 +1764,9 @@ class FunctionActionTestCase(unittest.TestCase): a = factory(GlobalFunc, varlist='XYZ') c = a.get_contents(target=[], source=[], env=Environment()) # assert c in func_matches, repr(c) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) c = a.get_contents(target=[], source=[], env=Environment(XYZ='foo')) assert c in matches_foo, repr(c) @@ -1796,12 +1786,12 @@ class FunctionActionTestCase(unittest.TestCase): lc = LocalClass() a = factory(lc.LocalMethod) c = a.get_contents(target=[], source=[], env=Environment()) - assert c == meth_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - meth_matches[sys.version_info[:2]]) + assert ( + c == meth_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(meth_matches[sys.version_info[:2]]) def test_strfunction(self): - """Test the FunctionAction.strfunction() method - """ + """Test the FunctionAction.strfunction() method.""" def func(): pass @@ -1825,8 +1815,7 @@ class FunctionActionTestCase(unittest.TestCase): class ListActionTestCase(unittest.TestCase): def test___init__(self): - """Test creation of a list of subsidiary Actions - """ + """Test creation of a list of subsidiary Actions.""" def func(): pass @@ -1838,8 +1827,7 @@ class ListActionTestCase(unittest.TestCase): assert a.list[2].list[0].cmd_list == 'y' def test___str__(self): - """Test the __str__() method for a list of subsidiary Actions - """ + """Test the __str__() method for a list of subsidiary Actions.""" def f(target, source, env): pass @@ -1852,8 +1840,7 @@ class ListActionTestCase(unittest.TestCase): assert s == "f(target, source, env)\ng(target, source, env)\nXXX\nf(target, source, env)", s def test_genstring(self): - """Test the genstring() method for a list of subsidiary Actions - """ + """Test the genstring() method for a list of subsidiary Actions.""" def f(target, source, env): pass @@ -1867,8 +1854,7 @@ class ListActionTestCase(unittest.TestCase): assert s == "f(target, source, env)\ngenerated foo.x bar.y\nXXX\nf(target, source, env)", s def test_execute(self): - """Test executing a list of subsidiary Actions - """ + """Test executing a list of subsidiary Actions.""" self.inc = 0 def f(target, source, env): @@ -1904,8 +1890,8 @@ class ListActionTestCase(unittest.TestCase): assert c == "act.py: 'syzygy'\nfunction2\nclass2a\nclass2b\n", c def test_get_contents(self): - """Test fetching the contents of a list of subsidiary Actions - """ + """Test fetching the contents of a list of subsidiary Actions.""" + self.foo = 0 def gen(target, source, env, for_signature): @@ -1923,8 +1909,8 @@ class ListActionTestCase(unittest.TestCase): class LazyActionTestCase(unittest.TestCase): def test___init__(self): - """Test creation of a lazy-evaluation Action - """ + """Test creation of a lazy-evaluation Action.""" + # Environment variable references should create a special type # of LazyAction that lazily evaluates the variable for whether # it's a string or something else before doing anything. @@ -1937,8 +1923,7 @@ class LazyActionTestCase(unittest.TestCase): assert a10.var == 'FOO', a10.var def test_genstring(self): - """Test the lazy-evaluation Action genstring() method - """ + """Test the lazy-evaluation Action genstring() method.""" def f(target, source, env): pass @@ -1952,8 +1937,7 @@ class LazyActionTestCase(unittest.TestCase): assert s == 'xxx', s def test_execute(self): - """Test executing a lazy-evaluation Action - """ + """Test executing a lazy-evaluation Action.""" def f(target, source, env): s = env['s'] @@ -1969,16 +1953,15 @@ class LazyActionTestCase(unittest.TestCase): assert c == "act.py: 'lazy'\n", c def test_get_contents(self): - """Test fetching the contents of a lazy-evaluation Action - """ + """Test fetching the contents of a lazy-evaluation Action.""" + a = SCons.Action.Action("${FOO}") env = Environment(FOO=[["This", "is", "a", "test"]]) c = a.get_contents(target=[], source=[], env=env) assert c == b"This is a test", c def test_get_contents_of_function_action(self): - """Test fetching the contents of a lazy-evaluation FunctionAction - """ + """Test fetching the contents of a lazy-evaluation FunctionAction.""" def LocalFunc(): pass @@ -1990,6 +1973,7 @@ class LazyActionTestCase(unittest.TestCase): (3, 8): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), + (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } meth_matches = [ @@ -2005,21 +1989,24 @@ class LazyActionTestCase(unittest.TestCase): env = Environment(FOO=factory(GlobalFunc)) c = a.get_contents(target=[], source=[], env=env) # assert c in func_matches, "Got\n"+repr(c)+"\nExpected one of \n"+"\n".join([repr(f) for f in func_matches]) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) env = Environment(FOO=factory(LocalFunc)) c = a.get_contents(target=[], source=[], env=env) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) # matches_foo = [x + b"foo" for x in func_matches] matches_foo = func_matches[sys.version_info[:2]] + b'foo' env = Environment(FOO=factory(GlobalFunc, varlist=['XYZ'])) c = a.get_contents(target=[], source=[], env=env) - assert c == func_matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - func_matches[sys.version_info[:2]]) + assert ( + c == func_matches[sys.version_info[:2]] + ), f"Got\n{c!r}\nExpected one of \n" + repr(func_matches[sys.version_info[:2]]) env['XYZ'] = 'foo' c = a.get_contents(target=[], source=[], env=env) @@ -2029,6 +2016,7 @@ class LazyActionTestCase(unittest.TestCase): class ActionCallerTestCase(unittest.TestCase): def test___init__(self): """Test creation of an ActionCaller""" + ac = SCons.Action.ActionCaller(1, [2, 3], {'FOO': 4, 'BAR': 5}) assert ac.parent == 1, ac.parent assert ac.args == [2, 3], ac.args @@ -2050,20 +2038,24 @@ class ActionCallerTestCase(unittest.TestCase): (3, 8): b'd\x00S\x00', (3, 9): b'd\x00S\x00', (3, 10): b'd\x00S\x00', - + (3, 11): b'\x97\x00d\x00S\x00', } af = SCons.Action.ActionFactory(GlobalFunc, strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c == matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - matches[sys.version_info[:2]]) + assert c == matches[sys.version_info[:2]], ( + f"Got\n{c!r}\nExpected one of \n" + + repr(matches[sys.version_info[:2]]) + ) af = SCons.Action.ActionFactory(LocalFunc, strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c == matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - matches[sys.version_info[:2]]) + assert c == matches[sys.version_info[:2]], ( + f"Got\n{c!r}\nExpected one of \n" + + repr(matches[sys.version_info[:2]]) + ) class LocalActFunc: def __call__(self): @@ -2072,14 +2064,18 @@ class ActionCallerTestCase(unittest.TestCase): af = SCons.Action.ActionFactory(GlobalActFunc(), strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c == matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - matches[sys.version_info[:2]]) + assert c == matches[sys.version_info[:2]], ( + f"Got\n{c!r}\nExpected one of \n" + + repr(matches[sys.version_info[:2]]) + ) af = SCons.Action.ActionFactory(LocalActFunc(), strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c == matches[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected one of \n" + repr( - matches[sys.version_info[:2]]) + assert c == matches[sys.version_info[:2]], ( + f"Got\n{c!r}\nExpected one of \n" + + repr(matches[sys.version_info[:2]]) + ) matches = [ b"", @@ -2089,13 +2085,12 @@ class ActionCallerTestCase(unittest.TestCase): af = SCons.Action.ActionFactory(str, strfunc) ac = SCons.Action.ActionCaller(af, [], {}) c = ac.get_contents([], [], Environment()) - assert c == "" or \ - c == "" or \ - c == "", repr(c) + assert c in ("", "", ""), repr(c) # ^^ class str for python3 def test___call__(self): """Test calling an ActionCaller""" + actfunc_args = [] def actfunc(a1, a2, a3, args=actfunc_args): @@ -2123,6 +2118,7 @@ class ActionCallerTestCase(unittest.TestCase): def test_strfunction(self): """Test calling the ActionCaller strfunction() method""" + strfunc_args = [] def actfunc(a1, a2, a3, a4): @@ -2159,6 +2155,7 @@ class ActionFactoryTestCase(unittest.TestCase): def test___call__(self): """Test calling whatever's returned from an ActionFactory""" + actfunc_args = [] strfunc_args = [] @@ -2175,12 +2172,12 @@ class ActionFactoryTestCase(unittest.TestCase): class ActionCompareTestCase(unittest.TestCase): - def test_1_solo_name(self): """Test Lazy Cmd Generator Action get_name alone. Basically ensures we can locate the builder, comparing it to itself along the way.""" + bar = SCons.Builder.Builder(action={}) env = Environment(BUILDERS={'BAR': bar}) name = bar.get_name(env) @@ -2191,12 +2188,12 @@ class ActionCompareTestCase(unittest.TestCase): Ensure that we can compare builders (and thereby actions) to each other safely.""" + foo = SCons.Builder.Builder(action='$FOO', suffix='.foo') bar = SCons.Builder.Builder(action={}) assert foo != bar assert foo.action != bar.action - env = Environment(BUILDERS={'FOO': foo, - 'BAR': bar}) + env = Environment(BUILDERS={'FOO': foo, 'BAR': bar}) name = foo.get_name(env) assert name == 'FOO', name name = bar.get_name(env) @@ -2214,10 +2211,7 @@ class ActionCompareTestCase(unittest.TestCase): bar = SCons.Builder.Builder(action={}, suffix={None: '.bar'}) bar.add_action('.cow', "$MOO") dog = SCons.Builder.Builder(suffix='.bar') - - env = Environment(BUILDERS={'FOO': foo, - 'BAR': bar, - 'DOG': dog}) + env = Environment(BUILDERS={'FOO': foo, 'BAR': bar, 'DOG': dog}) assert foo.get_name(env) == 'FOO', foo.get_name(env) assert bar.get_name(env) == 'BAR', bar.get_name(env) @@ -2236,7 +2230,6 @@ class TestClass: class ObjectContentsTestCase(unittest.TestCase): - def test_function_contents(self): """Test that Action._function_contents works""" @@ -2244,20 +2237,25 @@ class ObjectContentsTestCase(unittest.TestCase): """A test function""" return a - # Since the python bytecode has per version differences, we need different expected results per version + # Since the python bytecode has per version differences, + # we need different expected results per version expected = { (3, 5): (bytearray(b'3, 3, 0, 0,(),(),(|\x00\x00S),(),()')), (3, 6): (bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()')), (3, 7): (bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()')), (3, 8): (bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()')), (3, 9): (bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()')), - (3, 10): (bytearray(b'3, 3, 0, 0,(N.),(),(|\x00S\x00),(),()'), - bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()')) # 3.10.1, 3.10.2 + (3, 10): ( + bytearray(b'3, 3, 0, 0,(N.),(),(|\x00S\x00),(),()'), + bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()'), + ), # 3.10.1, 3.10.2 + (3, 11): bytearray(b'3, 3, 0, 0,(),(),(\x97\x00|\x00S\x00),(),()'), } c = SCons.Action._function_contents(func1) - assert c in expected[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + "\n" + repr( - expected[sys.version_info[:2]]) + assert c in expected[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( + expected[sys.version_info[:2]] + ) def test_object_contents(self): """Test that Action._object_contents works""" @@ -2268,24 +2266,34 @@ class ObjectContentsTestCase(unittest.TestCase): # c = SCons.Action._object_instance_content(o) - # Since the python bytecode has per version differences, we need different expected results per version + # Since the python bytecode has per version differences, + # we need different expected results per version expected = { (3, 5): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01\x00|\x00\x00_\x00\x00d\x02\x00|\x00\x00_\x01\x00d\x00\x00S),(),(),2, 2, 0, 0,(),(),(d\x00\x00S),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01\x00|\x00\x00_\x00\x00d\x02\x00|\x00\x00_\x01\x00d\x00\x00S),(),(),2, 2, 0, 0,(),(),(d\x00\x00S),(),()}}{{{a=a,b=b}}}" + ), (3, 6): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), (3, 7): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), (3, 8): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), (3, 9): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), (3, 10): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), + (3, 11): bytearray( + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(\x97\x00d\x01|\x00_\x00\x00\x00\x00\x00\x00\x00\x00\x00d\x02|\x00_\x01\x00\x00\x00\x00\x00\x00\x00\x00d\x00S\x00),(),(),2, 2, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()}}{{{a=a,b=b}}}"), } - assert c == expected[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + "\n" + repr( - expected[sys.version_info[:2]]) + assert c == expected[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( + expected[sys.version_info[:2]] + ) def test_code_contents(self): """Test that Action._code_contents works""" @@ -2295,25 +2303,43 @@ class ObjectContentsTestCase(unittest.TestCase): # Since the python bytecode has per version differences, we need different expected results per version expected = { - (3, 5): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00\x00d\x00\x00\x83\x01\x00\x01d\x01\x00S)'), - (3, 6): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), - (3, 7): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), - (3, 8): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), - (3, 9): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), - (3, 10): bytearray(b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)'), + (3, 5): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00\x00d\x00\x00\x83\x01\x00\x01d\x01\x00S)' + ), + (3, 6): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' + ), + (3, 7): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' + ), + (3, 8): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' + ), + (3, 9): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' + ), + (3, 10): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' + ), + (3, 11): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(\x97\x00\x02\x00e\x00d\x00\xa6\x01\x00\x00\xab\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00d\x01S\x00)'), } - assert c == expected[sys.version_info[:2]], "Got\n" + repr(c) + "\nExpected \n" + "\n" + repr(expected[ - sys.version_info[:2]]) + assert c == expected[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( + expected[sys.version_info[:2]] + ) def test_uncaught_exception_bubbles(self): """Test that _subproc bubbles uncaught exceptions""" + try: - pobj = SCons.Action._subproc(Environment(), - None, - stdin='devnull', - stderr='devnull', - stdout=subprocess.PIPE) + pobj = SCons.Action._subproc( + Environment(), + None, + stdin='devnull', + stderr='devnull', + stdout=subprocess.PIPE, + ) pobj.wait() except EnvironmentError: pass -- cgit v0.12 From eed3b5845a6a1258cd979d39b3f47b7133053552 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 5 Jun 2022 18:23:01 -0700 Subject: Added test for invalid env['MSVC_NOTFOUND_POLICY'] value --- test/MSVC/msvc_badversion.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/MSVC/msvc_badversion.py b/test/MSVC/msvc_badversion.py index a36bd2b..65dc789 100644 --- a/test/MSVC/msvc_badversion.py +++ b/test/MSVC/msvc_badversion.py @@ -44,25 +44,35 @@ installed_msvc_versions = msvc.get_installed_vcs() # skip_if_not_msvc() function would have skipped the test test.write('SConstruct', """\ +DefaultEnvironment(tools=[]) env = Environment(MSVC_VERSION='12.9') """) test.run(arguments='-Q -s', stdout='') test.write('SConstruct', """\ +DefaultEnvironment(tools=[]) env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='ignore') """) test.run(arguments='-Q -s', stdout='') test.write('SConstruct', """\ +DefaultEnvironment(tools=[]) env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='warning') """) test.run(arguments='-Q -s', stdout='') test.write('SConstruct', """\ +DefaultEnvironment(tools=[]) env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='error') """) test.run(arguments='-Q -s', status=2, stderr=r"^.*MSVCVersionNotFound.+", match=TestSCons.match_re_dotall) +test.write('SConstruct', """\ +env = Environment(MSVC_VERSION='12.9', MSVC_NOTFOUND_POLICY='bad_value') +""") +test.run(arguments='-Q -s', status=2, stderr=r"^.* Value specified for MSVC_NOTFOUND_POLICY.+", match=TestSCons.match_re_dotall) + + test.pass_test() # Local Variables: -- cgit v0.12 From de86183d3307c32b7b917e8b2dc989921b632ec7 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 5 Jun 2022 18:29:46 -0700 Subject: Add Blurb for MSVC_NOTFOUND_POLICY --- SCons/Tool/msvc.xml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 4a45d26..59a285e 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -74,6 +74,7 @@ Sets construction variables for the Microsoft Visual C/C++ compiler. PCH PCHSTOP PDB +MSVC_NOTFOUND_POLICY @@ -578,5 +579,28 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools:
+ + + +Specify how to handle missing MSVC. + + + + The MSVC_NOTFOUND_POLICY specifies how &scons; should handle when there is no installed + MSVC or if a specific version is requested, that version is not available. + + + +Valid values for MSVC_NOTFOUND_POLICY are listed here. Note that each group below is equivalent. + + +'error', 'Error', 'ERROR', 'exception', 'Exception', 'EXCEPTION' +'warning', 'Warning', 'WARNING', 'warn', 'Warn', 'WARN' +'suppress', 'Suppress', 'SUPPRESS' + + + + + -- cgit v0.12 From 294629587a8cadfa1b6e409d2a4751a7be4880bf Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 6 Jun 2022 09:55:50 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 59a285e..84c0e90 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -582,22 +582,48 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools: -Specify how to handle missing MSVC. +Specify the &scons; action to take when an msvc installation is not found. - The MSVC_NOTFOUND_POLICY specifies how &scons; should handle when there is no installed - MSVC or if a specific version is requested, that version is not available. + The MSVC_NOTFOUND_POLICY specifies the action &scons; should take when there are no + msvc versions installed or when the requested msvc version is not installed. -Valid values for MSVC_NOTFOUND_POLICY are listed here. Note that each group below is equivalent. +The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; action taken are: - -'error', 'Error', 'ERROR', 'exception', 'Exception', 'EXCEPTION' -'warning', 'Warning', 'WARNING', 'warn', 'Warn', 'WARN' -'suppress', 'Suppress', 'SUPPRESS' - + + + + +'error', 'Error', 'ERROR', 'exception', 'Exception', 'EXCEPTION' + + +Raise an exception when there are no msvc versions installed or when the requested msvc version is not installed. + + + + + +'warning', 'Warning', 'WARNING', 'warn', 'Warn', 'WARN' + + +Issue a warning when there are no msvc versions installed or when the requested msvc version is not installed. + + + + + +'ignore', 'Ignore', 'IGNORE', 'suppress', 'Suppress', 'SUPPRESS' + + +Take no action and continue when there are no msvc versions installed or when the requested msvc version is not installed. Depending on usage, this could result in build failure(s). + + + + + -- cgit v0.12 From 69d6b9acceeda73780592d1b8e09b3aa4f2c3813 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 6 Jun 2022 10:11:51 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 84c0e90..bb484c5 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -582,11 +582,11 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools: -Specify the &scons; action to take when an msvc installation is not found. +Specify the action taken when the Microsoft Visual C/C++ compiler is not found. - The MSVC_NOTFOUND_POLICY specifies the action &scons; should take when there are no + The MSVC_NOTFOUND_POLICY specifies the &scons; action taken when there are no msvc versions installed or when the requested msvc version is not installed. -- cgit v0.12 From e7c4d5bb062d979da2a27560db7a658cdef9c262 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 6 Jun 2022 11:56:37 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index bb484c5..187f3b5 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -597,7 +597,7 @@ The valid values for MSVC_NOTFOUND_POLICY and the corresponding & -'error', 'Error', 'ERROR', 'exception', 'Exception', 'EXCEPTION' +'Error' or 'Exception' Raise an exception when there are no msvc versions installed or when the requested msvc version is not installed. @@ -606,25 +606,36 @@ Raise an exception when there are no msvc versions installed or when the request -'warning', 'Warning', 'WARNING', 'warn', 'Warn', 'WARN' +'Warning' or 'Warn' -Issue a warning when there are no msvc versions installed or when the requested msvc version is not installed. +Issue a warning and continue when there are no msvc versions installed or when the requested msvc version is not installed. +Depending on usage, this could result in build failure(s). -'ignore', 'Ignore', 'IGNORE', 'suppress', 'Suppress', 'SUPPRESS' +'Ignore' or 'Suppress' -Take no action and continue when there are no msvc versions installed or when the requested msvc version is not installed. Depending on usage, this could result in build failure(s). +Take no action and continue when there are no msvc versions installed or when the requested msvc version is not installed. +Depending on usage, this could result in build failure(s). + +Note: in addition to the camel case values shown above, lower case and upper case values are accepted as well. + + + +The default scons action taken when there are no msvc versions installed or when the requested msvc version is +not installed is to issue a warning and continue. This may change in the future. + + -- cgit v0.12 From 1d3e0e3b0edd4b723d868434c49997215da45fc1 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 6 Jun 2022 13:00:20 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 187f3b5..59b8944 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -582,16 +582,16 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools: -Specify the action taken when the Microsoft Visual C/C++ compiler is not found. +Specify the scons behavior when the Microsoft Visual C/C++ compiler is not detected. - The MSVC_NOTFOUND_POLICY specifies the &scons; action taken when there are no - msvc versions installed or when the requested msvc version is not installed. + The MSVC_NOTFOUND_POLICY specifies the &scons; behavior when no msvc versions are detected or + when the requested msvc version is not detected. -The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; action taken are: +The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; behavior are: @@ -600,7 +600,7 @@ The valid values for MSVC_NOTFOUND_POLICY and the corresponding & 'Error' or 'Exception' -Raise an exception when there are no msvc versions installed or when the requested msvc version is not installed. +Raise an exception when no msvc versions are detected or when the requested msvc version is not detected. @@ -609,7 +609,7 @@ Raise an exception when there are no msvc versions installed or when the request 'Warning' or 'Warn' -Issue a warning and continue when there are no msvc versions installed or when the requested msvc version is not installed. +Issue a warning and continue when no msvc versions are detected or when the requested msvc version is not detected. Depending on usage, this could result in build failure(s). @@ -619,7 +619,7 @@ Depending on usage, this could result in build failure(s). 'Ignore' or 'Suppress' -Take no action and continue when there are no msvc versions installed or when the requested msvc version is not installed. +Take no action and continue when no msvc versions are detected or when the requested msvc version is not detected. Depending on usage, this could result in build failure(s). @@ -632,8 +632,8 @@ Note: in addition to the camel case values shown above, lower case and upper cas -The default scons action taken when there are no msvc versions installed or when the requested msvc version is -not installed is to issue a warning and continue. This may change in the future. +The default scons behavior when no msvc versions are detected or when the requested msvc version is not detected +is to issue a warning and continue. This may change in the future. -- cgit v0.12 From bd3cc3b1f0218643a6c8b610bc1328e169f45c6c Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Mon, 6 Jun 2022 16:11:28 -0500 Subject: Updated lex emitter to respect escaped spaces when climbing out of a the SCosncript dir --- CHANGES.txt | 1 + RELEASE.txt | 1 + SCons/Tool/lex.py | 2 +- test/LEX/lex_headerfile.py | 44 ++++++++++++++++++++++ test/LEX/lex_headerfile/spaced path/SConstruct | 2 + test/LEX/lex_headerfile/spaced path/src/SConscript | 10 +++++ test/LEX/lex_headerfile/spaced path/src/lexer.l | 1 + test/LEX/lex_headerfile/spaced path/src/lexer2.l | 1 + 8 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 test/LEX/lex_headerfile.py create mode 100644 test/LEX/lex_headerfile/spaced path/SConstruct create mode 100644 test/LEX/lex_headerfile/spaced path/src/SConscript create mode 100644 test/LEX/lex_headerfile/spaced path/src/lexer.l create mode 100644 test/LEX/lex_headerfile/spaced path/src/lexer2.l diff --git a/CHANGES.txt b/CHANGES.txt index 03692d6..efea669 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -116,6 +116,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. + - Updated lex emitter to respect escaped spaces when climbing out of a the SCosncript dir From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index cfa8afc..a393a36 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -134,6 +134,7 @@ FIXES - The system environment variable names imported for MSVC 7.0 and 6.0 were updated to be consistent with the variables names defined by their respective installers. This fixes an error caused when bypassing MSVC detection by specifying the MSVC 7.0 batch file directly. +- Updated lex emitter to respect escaped spaces when climbing out of a the SCosncript dir IMPROVEMENTS ------------ diff --git a/SCons/Tool/lex.py b/SCons/Tool/lex.py index d8d8de4..c33d0fa 100644 --- a/SCons/Tool/lex.py +++ b/SCons/Tool/lex.py @@ -58,7 +58,7 @@ def lexEmitter(target, source, env): # Different options that are used to trigger the creation of extra files. fileGenOptions = ["--header-file=", "--tables-file="] - lexflags = env.subst("$LEXFLAGS", target=target, source=source) + lexflags = env.subst_list("$LEXFLAGS", target=target, source=source) for option in SCons.Util.CLVar(lexflags): for fileGenOption in fileGenOptions: l = len(fileGenOption) diff --git a/test/LEX/lex_headerfile.py b/test/LEX/lex_headerfile.py new file mode 100644 index 0000000..c2e2e8b --- /dev/null +++ b/test/LEX/lex_headerfile.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Test the headerfile option for lex tool. +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.dir_fixture('lex_headerfile') + +test.run(chdir='spaced path', arguments='.') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/LEX/lex_headerfile/spaced path/SConstruct b/test/LEX/lex_headerfile/spaced path/SConstruct new file mode 100644 index 0000000..aa4aca0 --- /dev/null +++ b/test/LEX/lex_headerfile/spaced path/SConstruct @@ -0,0 +1,2 @@ +DefaultEnvironment(tools=[]) +SConscript("src/SConscript") \ No newline at end of file diff --git a/test/LEX/lex_headerfile/spaced path/src/SConscript b/test/LEX/lex_headerfile/spaced path/src/SConscript new file mode 100644 index 0000000..a3f4bfd --- /dev/null +++ b/test/LEX/lex_headerfile/spaced path/src/SConscript @@ -0,0 +1,10 @@ +env = Environment(tools=['lex']) + +def make_header_path(env, target, source, for_signature): + return target[1] + +env.Replace(LEX_HEADER_FILE_GEN=make_header_path) +env.Append(LEXFLAGS=['--header-file=$LEX_HEADER_FILE_GEN']) + +env.CFile(target=['#gen_src/lexer.c', '#gen_src/lexer.l.h'], source='lexer.l') +env.CFile(target=['#gen_src/lexer2.c', '#gen_src/lexer2.l.h'], source='lexer2.l') \ No newline at end of file diff --git a/test/LEX/lex_headerfile/spaced path/src/lexer.l b/test/LEX/lex_headerfile/spaced path/src/lexer.l new file mode 100644 index 0000000..66b82a4 --- /dev/null +++ b/test/LEX/lex_headerfile/spaced path/src/lexer.l @@ -0,0 +1 @@ +%% \ No newline at end of file diff --git a/test/LEX/lex_headerfile/spaced path/src/lexer2.l b/test/LEX/lex_headerfile/spaced path/src/lexer2.l new file mode 100644 index 0000000..66b82a4 --- /dev/null +++ b/test/LEX/lex_headerfile/spaced path/src/lexer2.l @@ -0,0 +1 @@ +%% \ No newline at end of file -- cgit v0.12 From 0ed5eea9af9358a0bfbeefa42507775947fe2d5f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 12:30:50 -0500 Subject: Added option to allow scons to determine if it should skip ninja regeneration. --- CHANGES.txt | 2 + RELEASE.txt | 2 + SCons/Script/SConsOptions.py | 3 +- SCons/Tool/ninja/NinjaState.py | 119 ++++++++++++++------- SCons/Tool/ninja/Utils.py | 25 ++++- SCons/Tool/ninja/__init__.py | 11 +- test/ninja/ninja_file_deterministic.py | 105 ++++++++++++++++++ .../sconstruct_ninja_determinism | 20 ++++ 8 files changed, 241 insertions(+), 46 deletions(-) create mode 100644 test/ninja/ninja_file_deterministic.py create mode 100644 test/ninja/ninja_test_sconscripts/sconstruct_ninja_determinism diff --git a/CHANGES.txt b/CHANGES.txt index 03692d6..4541168 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -116,6 +116,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. + - Ninja: added option to skip ninja regeneration if scons can determine the ninja file does + not need to be regenerated. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index cfa8afc..0dae549 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -88,6 +88,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY require delayed expansion to be enabled which is currently not supported and is typically not enabled by default on the host system. The batch files may also require environment variables that are not included by default in the msvc environment. +- Ninja: added option to skip ninja regeneration if scons can determine the ninja file does + not need to be regenerated. FIXES ----- diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index e2631fb..0210b79 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -151,7 +151,8 @@ class SConsValues(optparse.Values): # Requested setable flag in : https://github.com/SCons/scons/issues/3983 # From experimental ninja 'disable_execute_ninja', - 'disable_ninja' + 'disable_ninja', + 'skip_ninja_regen' ] def set_option(self, name, value): diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index c168c7f..c75c966 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -29,6 +29,8 @@ import signal import tempfile import shutil import sys +import random +import filecmp from os.path import splitext from tempfile import NamedTemporaryFile import ninja @@ -42,7 +44,7 @@ from .Globals import COMMAND_TYPES, NINJA_RULES, NINJA_POOLS, \ NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS from .Rules import _install_action_function, _mkdir_action_function, _lib_symlink_action_function, _copy_action_function from .Utils import get_path, alias_to_ninja_build, generate_depfile, ninja_noop, get_order_only, \ - get_outputs, get_inputs, get_dependencies, get_rule, get_command_env, to_escaped_list + get_outputs, get_inputs, get_dependencies, get_rule, get_command_env, to_escaped_list, ninja_sorted_build from .Methods import get_command @@ -82,7 +84,26 @@ class NinjaState: # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) - scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT',-1)) + + # The daemon port should be the same across runs, unless explicitly set + # or if the portfile is deleted. This ensures the ninja file is deterministic + # across regen's if nothings changed. The construction var should take preference, + # then portfile is next, and then otherwise create a new random port to persist in + # use. + scons_daemon_port = None + os.makedirs(get_path(self.env.get("NINJA_DIR")), exist_ok=True) + scons_daemon_port_file = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_portfile") + + if env.get('NINJA_SCONS_DAEMON_PORT') is not None: + scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT')) + elif os.path.exists(scons_daemon_port_file): + with open(scons_daemon_port_file) as f: + scons_daemon_port = int(f.read()) + else: + scons_daemon_port = random.randint(10000, 60000) + + with open(scons_daemon_port_file, 'w') as f: + f.write(str(scons_daemon_port)) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script @@ -381,13 +402,13 @@ class NinjaState: ninja.variable("builddir", get_path(self.env.Dir(self.env['NINJA_DIR']).path)) - for pool_name, size in self.pools.items(): + for pool_name, size in sorted(self.pools.items()): ninja.pool(pool_name, min(self.env.get('NINJA_MAX_JOBS', size), size)) - for var, val in self.variables.items(): + for var, val in sorted(self.variables.items()): ninja.variable(var, val) - for rule, kwargs in self.rules.items(): + for rule, kwargs in sorted(self.rules.items()): if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs: kwargs['pool'] = 'local_pool' ninja.rule(rule, **kwargs) @@ -529,8 +550,9 @@ class NinjaState: ) if remaining_outputs: - ninja.build( - outputs=sorted(remaining_outputs), rule="phony", implicit=first_output, + ninja_sorted_build( + ninja, + outputs=remaining_outputs, rule="phony", implicit=first_output, ) build["outputs"] = first_output @@ -548,12 +570,18 @@ class NinjaState: if "inputs" in build: build["inputs"].sort() - ninja.build(**build) + ninja_sorted_build( + ninja, + **build + ) scons_daemon_dirty = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_dirty") for template_builder in template_builders: template_builder["implicit"] += [scons_daemon_dirty] - ninja.build(**template_builder) + ninja_sorted_build( + ninja, + **template_builder + ) # We have to glob the SCons files here to teach the ninja file # how to regenerate itself. We'll never see ourselves in the @@ -563,17 +591,19 @@ class NinjaState: ninja_file_path = self.env.File(self.ninja_file).path regenerate_deps = to_escaped_list(self.env, self.env['NINJA_REGENERATE_DEPS']) - ninja.build( - ninja_file_path, + ninja_sorted_build( + ninja, + outputs=ninja_file_path, rule="REGENERATE", implicit=regenerate_deps, variables={ - "self": ninja_file_path, + "self": ninja_file_path } ) - ninja.build( - regenerate_deps, + ninja_sorted_build( + ninja, + outputs=regenerate_deps, rule="phony", variables={ "self": ninja_file_path, @@ -584,8 +614,9 @@ class NinjaState: # If we ever change the name/s of the rules that include # compile commands (i.e. something like CC) we will need to # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", + ninja_sorted_build( + ninja, + outputs="compile_commands.json", rule="CMD", pool="console", implicit=[str(self.ninja_file)], @@ -601,12 +632,14 @@ class NinjaState: }, ) - ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], + ninja_sorted_build( + ninja, + outputs="compiledb", rule="phony", implicit=["compile_commands.json"], ) - ninja.build( - ["run_ninja_scons_daemon_phony", scons_daemon_dirty], + ninja_sorted_build( + ninja, + outputs=["run_ninja_scons_daemon_phony", scons_daemon_dirty], rule="SCONS_DAEMON", ) @@ -620,39 +653,45 @@ class NinjaState: if len(all_targets) == 0: all_targets = ["phony_default"] - ninja.build( + ninja_sorted_build( + ninja, outputs=all_targets, rule="phony", ) ninja.default([self.ninja_syntax.escape_path(path) for path in sorted(all_targets)]) - daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) - pidfile = None - if os.path.exists(scons_daemon_dirty): - pidfile = scons_daemon_dirty - elif os.path.exists(daemon_dir / 'pidfile'): - pidfile = daemon_dir / 'pidfile' - - if pidfile: - with open(pidfile) as f: - pid = int(f.readline()) - try: - os.kill(pid, signal.SIGINT) - except OSError: - pass + with NamedTemporaryFile(delete=False, mode='w') as temp_ninja_file: + temp_ninja_file.write(content.getvalue()) + + if self.env.GetOption('skip_ninja_regen') and os.path.exists(ninja_file_path) and filecmp.cmp(temp_ninja_file.name, ninja_file_path): + os.unlink(temp_ninja_file.name) + else: + + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + pidfile = None + if os.path.exists(scons_daemon_dirty): + pidfile = scons_daemon_dirty + elif os.path.exists(daemon_dir / 'pidfile'): + pidfile = daemon_dir / 'pidfile' + + if pidfile: + with open(pidfile) as f: + pid = int(f.readline()) + try: + os.kill(pid, signal.SIGINT) + except OSError: + pass # wait for the server process to fully killed # TODO: update wait_for_process_to_die() to handle timeout and then catch exception # here and do something smart. wait_for_process_to_die(pid) - if os.path.exists(scons_daemon_dirty): - os.unlink(scons_daemon_dirty) + if os.path.exists(scons_daemon_dirty): + os.unlink(scons_daemon_dirty) - with NamedTemporaryFile(delete=False, mode='w') as temp_ninja_file: - temp_ninja_file.write(content.getvalue()) - shutil.move(temp_ninja_file.name, ninja_file_path) + shutil.move(temp_ninja_file.name, ninja_file_path) self.__generated = True diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index b2c3cd3..dec6d2c 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -23,6 +23,7 @@ import os import shutil from os.path import join as joinpath +from collections import OrderedDict import SCons from SCons.Action import get_default_ENV, _string_from_cmd_list @@ -52,6 +53,15 @@ def ninja_add_command_line_options(): help='Disable ninja generation and build with scons even if tool is loaded. '+ 'Also used by ninja to build targets which only scons can build.') + AddOption('--skip-ninja-regen', + dest='skip_ninja_regen', + metavar='BOOL', + action="store_true", + default=False, + help='Allow scons to skip regeneration of the ninja file and restarting of the daemon. ' + + 'Care should be taken in cases where Glob is in use or generated files are used in ' + + 'command lines.') + def is_valid_dependent_node(node): """ @@ -126,7 +136,7 @@ def get_dependencies(node, skip_sources=False): get_path(src_file(child)) for child in filter_ninja_nodes(node.children()) if child not in node.sources - ] + ] return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] @@ -261,6 +271,19 @@ def ninja_noop(*_args, **_kwargs): """ return None +def ninja_recursive_sorted_dict(build): + sorted_dict = OrderedDict() + for key, val in sorted(build.items()): + if isinstance(val, dict): + sorted_dict[key] = ninja_recursive_sorted_dict(val) + else: + sorted_dict[key] = val + return sorted_dict + + +def ninja_sorted_build(ninja, **build): + sorted_dict = ninja_recursive_sorted_dict(build) + ninja.build(**sorted_dict) def get_command_env(env, target, source): """ diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 04a7abb..ad75978 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,7 +26,7 @@ import importlib import os -import random +import traceback import subprocess import sys @@ -66,8 +66,12 @@ def ninja_builder(env, target, source): print("Generating:", str(target[0])) generated_build_ninja = target[0].get_abspath() - NINJA_STATE.generate() - + try: + NINJA_STATE.generate() + except Exception as e: + raise SCons.Errors.BuildError( + errstr=f"ERROR: an excetion occurred while generating the ninja file:\n{traceback.format_exc()}", + node=target) if env["PLATFORM"] == "win32": # TODO: Is this necessary as you set env variable in the ninja build file per target? # this is not great, its doesn't consider specific @@ -194,7 +198,6 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) - env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000, 60000)) if GetOption("disable_ninja"): env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) diff --git a/test/ninja/ninja_file_deterministic.py b/test/ninja/ninja_file_deterministic.py new file mode 100644 index 0000000..2ac5e1a --- /dev/null +++ b/test/ninja/ninja_file_deterministic.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import shutil +import filecmp + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_ninja_determinism', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) +shutil.copyfile(test.workpath('build.ninja'), test.workpath('build.ninja.orig')) + +# generate same build again +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja', 'ninja: no work to do.']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) + +# make sure the ninja file was deterministic +if not filecmp.cmp(test.workpath('build.ninja'), test.workpath('build.ninja.orig')): + test.fail_test() + +# clean build and ninja files +os.unlink(test.workpath('build.ninja.orig')) +test.run(arguments='-c', stdout=None) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.must_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) +shutil.copyfile(test.workpath('build.ninja'), test.workpath('build.ninja.orig')) + +# only generate the ninja file again +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) + +# run ninja independently again +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.must_contain_all_lines(test.stdout(), ['ninja: no work to do.']) +test.must_exist([test.workpath('out1.txt'), test.workpath('out2.txt')]) + +# make sure the ninja file was deterministic +if not filecmp.cmp(test.workpath('build.ninja'), test.workpath('build.ninja.orig')): + test.fail_test() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_ninja_determinism b/test/ninja/ninja_test_sconscripts/sconstruct_ninja_determinism new file mode 100644 index 0000000..6bd7c26 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_ninja_determinism @@ -0,0 +1,20 @@ +import SCons +import random +SetOption('experimental','ninja') +SetOption('skip_ninja_regen', True) +DefaultEnvironment(tools=[]) + +env = Environment(tools=[]) + +env.Tool('ninja') + +# make the dependency list vary in order. Ninja tool should sort them to be deterministic. +for i in range(1, 10): + node = env.Command(f'out{i}.txt', 'foo.c', 'echo test > $TARGET') + deps = list(range(1, i)) + random.shuffle(deps) + for j in deps: + if j == i: + continue + env.Depends(node, f'out{j}.txt') + -- cgit v0.12 From 60b6724b28a909f63fa99f970bc39111f039024d Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 12:35:08 -0500 Subject: improve help text slightly --- SCons/Tool/ninja/Utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index dec6d2c..583301e 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -59,7 +59,7 @@ def ninja_add_command_line_options(): action="store_true", default=False, help='Allow scons to skip regeneration of the ninja file and restarting of the daemon. ' + - 'Care should be taken in cases where Glob is in use or generated files are used in ' + + 'Care should be taken in cases where Glob is in use or SCons generated files are used in ' + 'command lines.') -- cgit v0.12 From 97a265e6da13a6a3827c599c23271b60aadef176 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 14:23:10 -0500 Subject: Added new alias 'shutdown-ninja-scons-daemon' to allow ninja to shutdown the daemon --- CHANGES.txt | 2 + RELEASE.txt | 2 + SCons/Tool/ninja/NinjaState.py | 11 +++++ SCons/Tool/ninja/__init__.py | 1 + SCons/Tool/ninja/ninja_daemon_build.py | 14 +++++- SCons/Tool/ninja/ninja_scons_daemon.py | 5 ++- test/ninja/shutdown_scons_daemon.py | 79 ++++++++++++++++++++++++++++++++++ testing/framework/TestCmd.py | 32 ++++++++++++++ 8 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 test/ninja/shutdown_scons_daemon.py diff --git a/CHANGES.txt b/CHANGES.txt index 03692d6..9db19a8 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -116,6 +116,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. + -Ninja: Added new alias "shutdown-ninja-scons-daemon" to allow ninja to shutdown the daemon. + Also added cleanup to test framework to kill ninja scons daemons and clean ip daemon logs. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index cfa8afc..811d957 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -24,6 +24,8 @@ NEW FUNCTIONALITY performance impact if not used carefully. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. +- Ninja: Added new alias "shutdown-ninja-scons-daemon" to allow ninja to shutdown the daemon. + Also added cleanup to test framework to kill ninja scons daemons and clean ip daemon logs. DEPRECATED FUNCTIONALITY diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index c168c7f..553d8cc 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -214,6 +214,12 @@ class NinjaState: "pool": "local_pool", "restat": 1 }, + "EXIT_SCONS_DAEMON": { + "command": "$PYTHON_BIN $NINJA_TOOL_DIR/ninja_daemon_build.py $PORT $NINJA_DIR_PATH --exit", + "description": "Shutting down ninja scons daemon server", + "pool": "local_pool", + "restat": 1 + }, "SCONS": { "command": "$SCONS_INVOCATION $out", "description": "$SCONS_INVOCATION $out", @@ -610,6 +616,11 @@ class NinjaState: rule="SCONS_DAEMON", ) + ninja.build( + "shutdown_ninja_scons_daemon_phony", + rule="EXIT_SCONS_DAEMON", + ) + if all_targets is None: # Look in SCons's list of DEFAULT_TARGETS, find the ones that diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 04a7abb..6f474ca 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -495,3 +495,4 @@ def generate(env): env.Alias('run-ninja-scons-daemon', 'run_ninja_scons_daemon_phony') + env.Alias('shutdown-ninja-scons-daemon', 'shutdown_ninja_scons_daemon_phony') \ No newline at end of file diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 48156f5..63eb136 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -60,11 +60,22 @@ def log_error(msg): while True: try: + if not os.path.exists(daemon_dir / "pidfile"): + if sys.argv[3] != '--exit': + logging.debug(f"ERROR: Server pid not found {daemon_dir / 'pidfile'} for request {sys.argv[3]}") + exit(1) + else: + logging.debug(f"WARNING: Unecessary request to shutfown server, its already shutdown.") + exit(0) + logging.debug(f"Sending request: {sys.argv[3]}") conn = http.client.HTTPConnection( "127.0.0.1", port=int(sys.argv[1]), timeout=60 ) - conn.request("GET", "/?build=" + sys.argv[3]) + if sys.argv[3] == '--exit': + conn.request("GET", "/?exit=1") + else: + conn.request("GET", "/?build=" + sys.argv[3]) response = None while not response: @@ -81,6 +92,7 @@ while True: if status != 200: log_error(msg.decode("utf-8")) exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") exit(0) diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index c4a1d11..35a7789 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -168,6 +168,7 @@ def daemon_thread_func(): te.start() daemon_ready = False + building_node = None startup_complete = False @@ -218,12 +219,14 @@ def daemon_thread_func(): except queue.Empty: break if "exit" in building_node: + daemon_log("input: " + "exit") p.stdin.write("exit\n".encode("utf-8")) p.stdin.flush() with building_cv: shared_state.finished_building += [building_node] daemon_ready = False - raise + daemon_needs_to_shutdown = True + break else: input_command = "build " + building_node + "\n" diff --git a/test/ninja/shutdown_scons_daemon.py b/test/ninja/shutdown_scons_daemon.py new file mode 100644 index 0000000..fffbadb --- /dev/null +++ b/test/ninja/shutdown_scons_daemon.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import psutil +from timeit import default_timer as timer + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath( + os.path.join(ninja.__file__, os.pardir, "data", "bin", "ninja" + _exe) +) + +test.dir_fixture("ninja-fixture") + +test.file_fixture( + "ninja_test_sconscripts/sconstruct_force_scons_callback", "SConstruct" +) + +test.run(stdout=None) +pid = None +test.must_exist(test.workpath('.ninja/scons_daemon_dirty')) +with open(test.workpath('.ninja/scons_daemon_dirty')) as f: + pid = int(f.read()) + if pid not in [proc.pid for proc in psutil.process_iter()]: + test.fail_test(message="daemon not running!") + +program = test.workpath("run_ninja_env.bat") if IS_WINDOWS else ninja_bin +test.run(program=program, arguments='shutdown-ninja-scons-daemon', stdout=None) + +wait_time = 10 +start_time = timer() +while True: + if wait_time > (timer() - start_time): + if pid not in [proc.pid for proc in psutil.process_iter()]: + break + else: + test.fail_test(message=f"daemon still not shutdown after {wait_time} seconds.") + +test.must_not_exist(test.workpath('.ninja/scons_daemon_dirty')) +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 5759121..34acb4d 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -299,9 +299,12 @@ __version__ = "1.3" import atexit import difflib import errno +import hashlib import os import re +import psutil import shutil +import signal import stat import subprocess import sys @@ -310,6 +313,7 @@ import threading import time import traceback from collections import UserList, UserString +from pathlib import Path from subprocess import PIPE, STDOUT from typing import Optional @@ -382,6 +386,31 @@ def _caller(tblist, skip): atfrom = "\tfrom" return string +def clean_up_ninja_daemon(self, result_type): + if self: + for path in Path(self.workdir).rglob('.ninja'): + daemon_dir = Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(path.resolve()).encode()).hexdigest()) + ) + pidfiles = [daemon_dir / 'pidfile', path / 'scons_daemon_dirty'] + for pidfile in pidfiles: + if pidfile.exists(): + with open(daemon_dir / 'pidfile') as f: + try: + pid = int(f.read()) + os.kill(pid, signal.SIGINT) + except OSError: + pass + + while True: + if pid not in [proc.pid for proc in psutil.process_iter()]: + break + else: + time.sleep(0.1) + + if not self._preserve[result_type]: + if daemon_dir.exists(): + shutil.rmtree(daemon_dir) def fail_test(self=None, condition=True, function=None, skip=0, message=None): """Causes a test to exit with a fail. @@ -402,6 +431,7 @@ def fail_test(self=None, condition=True, function=None, skip=0, message=None): return if function is not None: function() + clean_up_ninja_daemon(self, 'fail_test') of = "" desc = "" sep = " " @@ -447,6 +477,7 @@ def no_result(self=None, condition=True, function=None, skip=0): return if function is not None: function() + clean_up_ninja_daemon(self, 'no_result') of = "" desc = "" sep = " " @@ -483,6 +514,7 @@ def pass_test(self=None, condition=True, function=None): return if function is not None: function() + clean_up_ninja_daemon(self, 'pass_test') sys.stderr.write("PASSED\n") sys.exit(0) -- cgit v0.12 From b1ede4ce01b6163086c748f509c51616b9dec051 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 14:28:15 -0500 Subject: fixed sider complaint --- SCons/Tool/ninja/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index ad75978..7c32676 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -68,7 +68,7 @@ def ninja_builder(env, target, source): generated_build_ninja = target[0].get_abspath() try: NINJA_STATE.generate() - except Exception as e: + except Exception: raise SCons.Errors.BuildError( errstr=f"ERROR: an excetion occurred while generating the ninja file:\n{traceback.format_exc()}", node=target) -- cgit v0.12 From e3b2dd31e326a0d5b948fa35eae0cb1eefbbebf4 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 14:34:28 -0500 Subject: install psutil for testing and fix sider complaints --- .github/workflows/experimental_tests.yml | 2 +- .github/workflows/runtest.yml | 2 +- SCons/Tool/ninja/ninja_daemon_build.py | 2 +- SCons/Tool/ninja/ninja_scons_daemon.py | 2 +- testing/framework/TestCmd.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/experimental_tests.yml b/.github/workflows/experimental_tests.yml index 3672144..92fc134 100644 --- a/.github/workflows/experimental_tests.yml +++ b/.github/workflows/experimental_tests.yml @@ -44,7 +44,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel python -m pip install ninja - # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements.txt ]; then python -m pip install -r requirements.txt; fi # sudo apt-get update - name: Test experimental packages ${{ matrix.os }} run: | diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index cfc585e..f37111b 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -38,7 +38,7 @@ jobs: run: | python -m pip install --upgrade pip setuptools wheel python -m pip install ninja - # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements.txt ]; then python -m pip install -r requirements.txt; fi # sudo apt-get update - name: runtest ${{ matrix.os }} run: | diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index 63eb136..c21ea0b 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -65,7 +65,7 @@ while True: logging.debug(f"ERROR: Server pid not found {daemon_dir / 'pidfile'} for request {sys.argv[3]}") exit(1) else: - logging.debug(f"WARNING: Unecessary request to shutfown server, its already shutdown.") + logging.debug("WARNING: Unnecessary request to shutfown server, its already shutdown.") exit(0) logging.debug(f"Sending request: {sys.argv[3]}") diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 35a7789..6802af2 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -225,7 +225,7 @@ def daemon_thread_func(): with building_cv: shared_state.finished_building += [building_node] daemon_ready = False - daemon_needs_to_shutdown = True + shared_state.daemon_needs_to_shutdown = True break else: diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 34acb4d..2d3428c 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -399,7 +399,7 @@ def clean_up_ninja_daemon(self, result_type): try: pid = int(f.read()) os.kill(pid, signal.SIGINT) - except OSError: + except OSError: pass while True: -- cgit v0.12 From a20c08c1309486f0f51ecd8cf34ba521da0718f0 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 14:36:39 -0500 Subject: fix formatting in changes.txt --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 9db19a8..a4ef7b2 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -116,7 +116,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. - -Ninja: Added new alias "shutdown-ninja-scons-daemon" to allow ninja to shutdown the daemon. + - Ninja: Added new alias "shutdown-ninja-scons-daemon" to allow ninja to shutdown the daemon. Also added cleanup to test framework to kill ninja scons daemons and clean ip daemon logs. From Mats Wichmann: -- cgit v0.12 From 719ad105165e942ee39f42749ee48d06e7567c90 Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 15:05:42 -0500 Subject: Added command line variable to pass ninja args through scons. --- CHANGES.txt | 2 ++ RELEASE.txt | 1 + SCons/Tool/ninja/__init__.py | 12 +++++++++++- SCons/Tool/ninja/ninja.xml | 11 ++++++++++- test/ninja/generate_and_build.py | 3 ++- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 03692d6..90e2d4e 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -116,6 +116,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. + - Ninja: Added command line variable NINJA_CMD_ARGS that allows to pass through ninja command line args. + From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to diff --git a/RELEASE.txt b/RELEASE.txt index cfa8afc..699e51e 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -24,6 +24,7 @@ NEW FUNCTIONALITY performance impact if not used carefully. - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. +- Ninja: Added command line variable NINJA_CMD_ARGS that allows to pass through ninja command line args. DEPRECATED FUNCTIONALITY diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 04a7abb..27a2957 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -33,6 +33,7 @@ import sys import SCons import SCons.Script import SCons.Tool.ninja.Globals +from SCons.Script import Variables from SCons.Script import GetOption from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS, NINJA_CMDLINE_TARGETS @@ -87,7 +88,7 @@ def ninja_builder(env, target, source): if str(env.get("NINJA_DISABLE_AUTO_RUN")).lower() not in ['1', 'true']: num_jobs = env.get('NINJA_MAX_JOBS', env.GetOption("num_jobs")) - cmd += ['-j' + str(num_jobs)] + NINJA_CMDLINE_TARGETS + cmd += ['-j' + str(num_jobs)] + env.get('NINJA_CMD_ARGS', '').split() + NINJA_CMDLINE_TARGETS print(f"ninja will be run with command line targets: {' '.join(NINJA_CMDLINE_TARGETS)}") print("Executing:", str(' '.join(cmd))) @@ -185,6 +186,15 @@ def generate(env): env["NINJA_DISABLE_AUTO_RUN"] = env.get("NINJA_DISABLE_AUTO_RUN", GetOption('disable_execute_ninja')) env["NINJA_FILE_NAME"] = env.get("NINJA_FILE_NAME", "build.ninja") + if env.get("NINJA_CMD_ARGS") is not None: + env["NINJA_CMD_ARGS"] = env.get("NINJA_CMD_ARGS") + else: + vars = Variables() + vars.Add("NINJA_CMD_ARGS") + var_env = env.Clone() + vars.Update(var_env) + env["NINJA_CMD_ARGS"] = var_env.get("NINJA_CMD_ARGS", '') + # Add the Ninja builder. always_exec_ninja_action = AlwaysExecAction(ninja_builder, {}) ninja_builder_obj = SCons.Builder.Builder(action=always_exec_ninja_action, diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 6b247d0..0929684 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -77,7 +77,7 @@ See its __doc__ string for a discussion of the format. IMPLICIT_COMMAND_DEPENDENCIES NINJA_SCONS_DAEMON_KEEP_ALIVE NINJA_SCONS_DAEMON_PORT - + NINJA_CMD_ARGS @@ -395,5 +395,14 @@ python -m pip install ninja + + + + A string which will pass arguments through SCons to the ninja command when scons executes ninja. + Has no effect if &cv-NINJA_DISABLE_AUTO_RUN; is set. + + + + diff --git a/test/ninja/generate_and_build.py b/test/ninja/generate_and_build.py index 91be108..83b7387 100644 --- a/test/ninja/generate_and_build.py +++ b/test/ninja/generate_and_build.py @@ -49,10 +49,11 @@ test.dir_fixture('ninja-fixture') test.file_fixture('ninja_test_sconscripts/sconstruct_generate_and_build', 'SConstruct') # generate simple build -test.run(stdout=None) +test.run(stdout=None, arguments='NINJA_CMD_ARGS=-v') test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) test.must_contain_all(test.stdout(), 'Executing:') test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_contain_all(test.stdout(), ' -j1 -v') test.run(program=test.workpath('foo' + _exe), stdout="foo.c") # clean build and ninja files -- cgit v0.12 From fc6a0e35e7bc1c25aa0359a2391e31ce4ab2dd1f Mon Sep 17 00:00:00 2001 From: Daniel Moody Date: Tue, 7 Jun 2022 15:10:39 -0500 Subject: fix psutil install --- .github/workflows/experimental_tests.yml | 3 +-- .github/workflows/runtest.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/experimental_tests.yml b/.github/workflows/experimental_tests.yml index 92fc134..52078d5 100644 --- a/.github/workflows/experimental_tests.yml +++ b/.github/workflows/experimental_tests.yml @@ -43,8 +43,7 @@ jobs: - name: Install dependencies including ninja ${{ matrix.os }} run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ninja - if [ -f requirements.txt ]; then python -m pip install -r requirements.txt; fi + python -m pip install ninja psutil # sudo apt-get update - name: Test experimental packages ${{ matrix.os }} run: | diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index f37111b..dd18013 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -37,8 +37,7 @@ jobs: - name: Install dependencies including ninja ${{ matrix.os }} run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install ninja - if [ -f requirements.txt ]; then python -m pip install -r requirements.txt; fi + python -m pip install ninja psutil # sudo apt-get update - name: runtest ${{ matrix.os }} run: | -- cgit v0.12 From e30a6efe2cbf887a501b2a18be854ea0346e299d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 8 Jun 2022 12:29:04 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 59b8944..d99bcef 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -632,10 +632,28 @@ Note: in addition to the camel case values shown above, lower case and upper cas -The default scons behavior when no msvc versions are detected or when the requested msvc version is not detected -is to issue a warning and continue. This may change in the future. +The MSVC_NOTFOUND_POLICY is enforced if any of the following conditions are satisfied: + +&cv-MSVC_VERSION; is specified, the default tools list is implicitly defined (i.e., the tools list is not specified), and the default tools list contains one or more of the msvc tools. +&cv-MSVC_VERSION; is specified, the default tools list is explicitly specified (e.g., tools=['default']), and the default tools list contains one or more of the msvc tools. +A non-default tools list is specified that contains one or more of the msvc tools (e.g., tools=['msvc', 'mslink']). + + +The MSVC_NOTFOUND_POLICY is ignored if any of the following conditions are satisfied: + +&cv-MSVC_VERSION; is not specified and the default tools list is implicitly defined (i.e., the tools list is not specified). +&cv-MSVC_VERSION; is not specified and the default tools list is explicitly specified (e.g., tools=['default']). +A non-default tool list is specified that does not contain any of the msvc tools (e.g., tools=['mingw']). + + + + +When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue subject to the enforcement conditions listed above. The default &scons; behavior may change in the future. + + + -- cgit v0.12 From be45d8d5f77d15bbca6cee1108f2d7dbfe5ef79b Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 8 Jun 2022 12:44:27 -0400 Subject: Add preliminary docstrings for set_msvc_notfound_policy and get_msvc_notfound_policy --- SCons/Tool/MSCommon/vc.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 32d0e61..f0c286a 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1070,6 +1070,19 @@ def _msvc_notfound_policy_lookup(symbol): return notfound_policy_def def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): + """ Set the default policy when MSVC is not found. + + Args: + MSVC_NOTFOUND_POLICY: + string representing the policy behavior + when MSVC is not found or None + + Returns: + The previous policy is returned when the MSVC_NOTFOUND_POLICY argument + is not None. The active policy is returned when the MSVC_NOTFOUND_POLICY + argument is None. + + """ global _MSVC_NOTFOUND_POLICY_DEF prev_policy = _MSVC_NOTFOUND_POLICY_DEF.symbol @@ -1087,6 +1100,7 @@ def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): return prev_policy def get_msvc_notfound_policy(): + """Return the active policy when MSVC is not found.""" debug( 'policy.symbol=%s, policy.value=%s', repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) -- cgit v0.12 From 609b79f538dc025a2b2d4dfd4d8814f96481949e Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 8 Jun 2022 14:12:59 -0400 Subject: Update MSVC_NOTFOUND_POLICY documentation --- SCons/Tool/msvc.xml | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index d99bcef..e8df128 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -632,25 +632,40 @@ Note: in addition to the camel case values shown above, lower case and upper cas -The MSVC_NOTFOUND_POLICY is enforced if any of the following conditions are satisfied: +The MSVC_NOTFOUND_POLICY is applied when any of the following conditions are satisfied: -&cv-MSVC_VERSION; is specified, the default tools list is implicitly defined (i.e., the tools list is not specified), and the default tools list contains one or more of the msvc tools. -&cv-MSVC_VERSION; is specified, the default tools list is explicitly specified (e.g., tools=['default']), and the default tools list contains one or more of the msvc tools. -A non-default tools list is specified that contains one or more of the msvc tools (e.g., tools=['msvc', 'mslink']). + +&cv-MSVC_VERSION; is specified, the default tools list is implicitly defined (i.e., the tools list is not specified), +and the default tools list contains one or more of the msvc tools. + + +&cv-MSVC_VERSION; is specified, the default tools list is explicitly specified (e.g., tools=['default']), +and the default tools list contains one or more of the msvc tools. + + +A non-default tools list is specified that contains one or more of the msvc tools (e.g., tools=['msvc', 'mslink']). + -The MSVC_NOTFOUND_POLICY is ignored if any of the following conditions are satisfied: +The MSVC_NOTFOUND_POLICY is ignored when any of the following conditions are satisfied: -&cv-MSVC_VERSION; is not specified and the default tools list is implicitly defined (i.e., the tools list is not specified). -&cv-MSVC_VERSION; is not specified and the default tools list is explicitly specified (e.g., tools=['default']). -A non-default tool list is specified that does not contain any of the msvc tools (e.g., tools=['mingw']). + +&cv-MSVC_VERSION; is not specified and the default tools list is implicitly defined (i.e., the tools list is not specified). + + +&cv-MSVC_VERSION; is not specified and the default tools list is explicitly specified (e.g., tools=['default']). + + +A non-default tool list is specified that does not contain any of the msvc tools (e.g., tools=['mingw']). + -When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue subject to the enforcement conditions listed above. The default &scons; behavior may change in the future. +When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue +subject to the conditions listed above. The default &scons; behavior may change in the future. -- cgit v0.12 From 3d9345e2d086f3b39a419452c9b74f763698dd04 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 9 Jun 2022 09:03:58 -0600 Subject: doc: FORTRANCOM doesn't include cpp vars [skip appveyor] FORTRANCOM and SHFORTRANCOM don't add the C preprocessor variables by default, the docs suggest they do. The PP variants, which run through the preprocessor, do add these. Adjust docs. Fixes #2128 Signed-off-by: Mats Wichmann --- SCons/Tool/fortran.xml | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/SCons/Tool/fortran.xml b/SCons/Tool/fortran.xml index a4f0cec..4f0517b 100644 --- a/SCons/Tool/fortran.xml +++ b/SCons/Tool/fortran.xml @@ -1,6 +1,6 @@ + - # On the command line - --experimental=ninja +# On the command line +--experimental=ninja - # Or in your SConstruct - SetOption('experimental', 'ninja') +# Or in your SConstruct +SetOption('experimental', 'ninja') This functionality is subject to change and/or removal without deprecation cycle. - - To use this tool you must install pypi's ninja - package. - This can be done via - pip install ninja + To use this tool you need to install the &Python; &ninja; package, + as the tool by default depends on being able to do an + import of the package + + This can be done via: + +python -m pip install ninja + diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index c982b96..a2d9acd 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -63,7 +63,7 @@ Added methods propagate through &f-env-Clone; calls. -Examples: +More examples: @@ -472,7 +472,7 @@ and/or suffix, so the contents are treated as a list of strings, that is, adding a string will result in a separate string entry, not a combined string. For &cv-CPPDEFINES; as well as -for &cv-link-LIBS;, and the various *PATH +for &cv-link-LIBS;, and the various *PATH; variables, &SCons; will supply the compiler-specific syntax (e.g. adding a -D or /D prefix for &cv-CPPDEFINES;), so this syntax should be omitted when @@ -550,7 +550,7 @@ do not make sense and a &Python; exception will be raised. When using &f-env-Append; to modify &consvars; which are path specifications (conventionally, -the names of such end in PATH), +the names of such end in PATH), it is recommended to add the values as a list of strings, even if there is only a single string to add. The same goes for adding library names to &cv-LIBS;. @@ -570,26 +570,26 @@ See also &f-link-env-AppendUnique;, env.AppendENVPath(name, newpath, [envname, sep, delete_existing=False]) -Append new path elements to the given path in the -specified external environment (&cv-link-ENV; by default). -This will only add -any particular path once (leaving the last one it encounters and -ignoring the rest, to preserve path order), -and to help assure this, -will normalize all paths (using -os.path.normpath +Append path elements specified by newpath +to the given search path string or list name +in mapping envname in the &consenv;. +Supplying envname is optional: +the default is the execution environment &cv-link-ENV;. +Optional sep is used as the search path separator, +the default is the platform's separator (os.pathsep). +A path element will only appear once. +Any duplicates in newpath are dropped, +keeping the last appearing (to preserve path order). +If delete_existing +is False (the default) +any addition duplicating an existing path element is ignored; +if delete_existing +is True the existing value will +be dropped and the path element will be added at the end. +To help maintain uniqueness all paths are normalized (using +os.path.normpath and -os.path.normcase). -This can also handle the -case where the given old path variable is a list instead of a -string, in which case a list will be returned instead of a string. - - - -If -delete_existing -is False, then adding a path that already exists -will not move it to the end; it will stay where it is in the list. +os.path.normcase). @@ -608,6 +608,11 @@ print('after:', env['ENV']['INCLUDE']) before: /foo:/biz after: /biz:/foo/bar:/foo + + +See also &f-link-env-PrependENVPath;. + + @@ -718,7 +723,7 @@ is being used and &scons; finds a derived file that needs to be rebuilt, it will first look in the cache to see if a -file with matching build signature exists +file with matching &buildsig; exists (indicating the input file(s) and build action(s) were identical to those for the current target), and if so, will retrieve the file from the cache. @@ -730,7 +735,7 @@ If the derived file is not present in the cache, &scons; will build it and then place a copy of the built file in the cache, -identified by its build signature, for future use. +identified by its &buildsig;, for future use. @@ -787,6 +792,13 @@ method can be used to disable caching of specific files. This can be useful if inputs and/or outputs of some tool are impossible to predict or prohibitively large. + + +Note that (at this time) &SCons; provides no facilities +for managing the derived-file cache. It is up to the developer +to arrange for cache pruning, expiry, etc. if needed. + + @@ -1194,7 +1206,7 @@ was built. This can be consulted to match various file characteristics such as the timestamp, -size, or content signature. +size, or &contentsig;. @@ -1390,10 +1402,11 @@ Find an executable from one or more choices: progs may be a string or a list of strings. Returns the first value from progs that was found, or None. -Executable is searched by checking the paths specified -by env['ENV']['PATH']. +Executable is searched by checking the paths in the execution environment +(env['ENV']['PATH']). On Windows systems, additionally applies the filename suffixes found in -env['ENV']['PATHEXT'] +the execution environment +(env['ENV']['PATHEXT']) but will not include any such extension in the return value. &f-env-Detect; is a wrapper around &f-link-env-WhereIs;. @@ -2640,12 +2653,8 @@ not as separate arguments to -By default, -duplicate values are eliminated; -you can, however, specify -unique=False -to allow duplicate -values to be added. +If unique is true (the default), +duplicate values are not stored. When eliminating duplicate values, any &consvars; that end with the string @@ -2653,6 +2662,8 @@ the string keep the left-most unique value. All other &consvars; keep the right-most unique value. +If unique is false, +values are added even if they are duplicates. @@ -2669,9 +2680,13 @@ env.MergeFlags(['!pkg-config gtk+-2.0 --cflags', '-O3']) # Combine an optimization flag with the flags returned from running pkg-config # twice and merge the result into the construction variables. -env.MergeFlags(['-O3', - '!pkg-config gtk+-2.0 --cflags --libs', - '!pkg-config libpng12 --cflags --libs']) +env.MergeFlags( + [ + '-O3', + '!pkg-config gtk+-2.0 --cflags --libs', + '!pkg-config libpng12 --cflags --libs', + ] +) @@ -2773,15 +2788,13 @@ NoClean(env.Program('hello', 'hello.c')) env.ParseConfig(command, [function, unique]) Updates the current &consenv; with the values extracted -from the output from running external command, -by calling a helper function function -which understands -the output of command. +from the output of running external command, +by passing it to a helper function. command may be a string or a list of strings representing the command and its arguments. If function -is not given, +is omitted or None, &f-link-env-MergeFlags; is used. By default, duplicate values are not @@ -2792,33 +2805,32 @@ to allow duplicate values to be added. -If &f-env-MergeFlags; is used, -it expects a response in the style of a -*-config -command typical of the POSIX programming environment -(for example, -gtk-config) -and adds the options -to the appropriate construction variables. -Interpreted options -and the construction variables they affect -are as specified for the -&f-link-env-ParseFlags; -method (which -&f-env-MergeFlags; calls). -See that method's description -for a table of options and corresponding construction variables. +command is executed using the +SCons execution environment (that is, the &consvar; +&cv-link-ENV; in the current &consenv;). +If command needs additional information +to operate properly, that needs to be set in the execution environment. +For example, pkg-config +may need a custom value set in the PKG_CONFIG_PATH +environment variable. -If &f-env-MergeFlags; cannot interpret the results of +&f-env-MergeFlags; needs to understand +the output produced by command +in order to distribute it to appropriate &consvars;. +&f-env-MergeFlags; uses a separate function to +do that processing - +see &f-link-env-ParseFlags; for the details, including a +a table of options and corresponding construction variables. +To provide alternative processing of the output of command, you can suppply a custom -function to do so. -function -must accept three arguments: -the &consenv; to modify, the string returned -by running command, +function, +which must accept three arguments: +the &consenv; to modify, +a string argument containing the output from running +command, and the optional unique flag. @@ -2828,8 +2840,7 @@ and the optional ParseDepends(filename, [must_exist, only_one]) env.ParseDepends(filename, [must_exist, only_one]) -Parses the contents of the specified -filename +Parses the contents of filename as a list of dependencies in the style of &Make; or @@ -2840,27 +2851,21 @@ and explicitly establishes all of the listed dependencies. By default, it is not an error -if the specified -filename +if filename does not exist. The optional must_exist -argument may be set to a non-zero -value to have -scons -throw an exception and -generate an error if the file does not exist, +argument may be set to True +to have &SCons; +raise an exception if the file does not exist, or is otherwise inaccessible. The optional only_one -argument may be set to a non-zero -value to have -scons -thrown an exception and -generate an error +argument may be set to True +to have &SCons; raise an exception if the file contains dependency information for more than one target. This can provide a small sanity check @@ -2876,7 +2881,6 @@ file. -The filename and all of the files listed therein will be interpreted relative to @@ -2892,10 +2896,10 @@ function. env.ParseFlags(flags, ...) Parses one or more strings containing -typical command-line flags for GCC tool chains +typical command-line flags for GCC-style tool chains and returns a dictionary with the flag values separated into the appropriate SCons construction variables. -This is intended as a companion to the +Intended as a companion to the &f-link-env-MergeFlags; method, but allows for the values in the returned dictionary to be modified, if necessary, @@ -2910,11 +2914,20 @@ directly unless you want to manipulate the values.) If the first character in any string is -an exclamation mark (!), +an exclamation mark (!), the rest of the string is executed as a command, and the output from the command is parsed as GCC tool chain command-line flags and added to the resulting dictionary. +This can be used to call a *-config +command typical of the POSIX programming environment +(for example, +pkg-config). +Note that such a comamnd is executed using the +SCons execution environment; +if the command needs additional information, +that information needs to be explcitly provided. +See &f-link-ParseConfig; for more details. @@ -3051,30 +3064,28 @@ and &f-link-env-PrependUnique;. - env.PrependENVPath(name, newpath, [envname, sep, delete_existing]) + env.PrependENVPath(name, newpath, [envname, sep, delete_existing=True]) -Prepend new path elements to the given path in the -specified external environment (&cv-link-ENV; by default). -This will only add -any particular path once (leaving the first one it encounters and -ignoring the rest, to preserve path order), -and to help assure this, -will normalize all paths (using +Prepend path elements specified by newpath +to the given search path string or list name +in mapping envname in the &consenv;. +Supplying envname is optional: +the default is the execution environment &cv-link-ENV;. +Optional sep is used as the search path separator, +the default is the platform's separator (os.pathsep). +A path element will only appear once. +Any duplicates in newpath are dropped, +keeping the first appearing (to preserve path order). +If delete_existing +is False +any addition duplicating an existing path element is ignored; +if delete_existing +is True (the default) the existing value will +be dropped and the path element will be inserted at the beginning. +To help maintain uniqueness all paths are normalized (using os.path.normpath and os.path.normcase). -This can also handle the -case where the given old path variable is a list instead of a -string, in which case a list will be returned instead of a string. - - - -If -delete_existing -is False, -then adding a path that already exists -will not move it to the beginning; -it will stay where it is in the list. @@ -3094,6 +3105,11 @@ print('after:', env['ENV']['INCLUDE']) before: /biz:/foo after: /foo/bar:/foo:/biz + + +See also &f-link-env-AppendENVPath;. + + @@ -3103,7 +3119,7 @@ Prepend values to &consvars; in the current &consenv;, maintaining uniqueness. Works like &f-link-env-Append; (see for details), except that values are added to the front, -rather than the end, of any existing value of the the &consvar;, +rather than the end, of any existing value of the &consvar;, and values already present in the &consvar; will not be added again. If delete_existing @@ -3461,40 +3477,36 @@ for a complete explanation of the arguments and behavior. SConscript(scripts, [exports, variant_dir, duplicate, must_exist]) env.SConscript(scripts, [exports, variant_dir, duplicate, must_exist]) - SConscript(dirs=subdirs, [name=script, exports, variant_dir, duplicate, must_exist]) - env.SConscript(dirs=subdirs, [name=script, exports, variant_dir, duplicate, must_exist]) + SConscript(dirs=subdirs, [name=scriptname, exports, variant_dir, duplicate, must_exist]) + env.SConscript(dirs=subdirs, [name=scriptname, exports, variant_dir, duplicate, must_exist]) -Execute one or more subsidiary SConscript (configuration) files. +Executes one or more subsidiary SConscript (configuration) files. There are two ways to call the &f-SConscript; function. -The first calling style -is to explicitly specify one or more -scripts -as the first argument. +The first calling style is to supply +one or more SConscript file names +as the first (positional) argument. A single script may be specified as a string; -multiple scripts must be specified as a list +multiple scripts must be specified as a list of strings (either explicitly or as created by a function like &f-link-Split;). Examples: -SConscript('SConscript') # run SConscript in the current directory +SConscript('SConscript') # run SConscript in the current directory SConscript('src/SConscript') # run SConscript in the src directory SConscript(['src/SConscript', 'doc/SConscript']) config = SConscript('MyConfig.py') -The second way to call -&f-SConscript; -is to specify a list of (sub)directory names -as a -dirs=subdirs -keyword argument. +The other calling style is to omit the positional argument naming +scripts and instead specify a list of directory names using the +dirs keyword argument. In this case, &scons; will @@ -3504,14 +3516,14 @@ in each of the specified directories. You may specify a name other than &SConscript; by supplying an optional -name=script +name=scriptname keyword argument. The first three examples below have the same effect as the first three examples above: -SConscript(dirs='.') # run SConscript in the current directory -SConscript(dirs='src') # run SConscript in the src directory +SConscript(dirs='.') # run SConscript in the current directory +SConscript(dirs='src') # run SConscript in the src directory SConscript(dirs=['src', 'doc']) SConscript(dirs=['sub1', 'sub2'], name='MySConscript') @@ -3519,8 +3531,12 @@ SConscript(dirs=['sub1', 'sub2'], name='MySConscript') The optional exports -argument provides a string or list of strings representing +keyword argument provides a string or list of strings representing variable names, or a dictionary of named values, to export. +For the first calling style only, a second positional argument +will be interpreted as exports; the +second calling style must use the keyword argument form +for exports. These variables are locally exported only to the called SConscript file(s) and do not affect the global pool of variables managed by the @@ -3544,38 +3560,24 @@ SConscript(dirs=['one', 'two', 'three'], exports='shared_info') If the optional variant_dir argument is present, it causes an effect equivalent to the -&f-link-VariantDir; function. +&f-link-VariantDir; function, +but in effect only within the scope of the &f-SConscript; call. The variant_dir -argument is interpreted relative to the directory of the calling -SConscript file. -The optional -duplicate argument is -interpreted as for &f-link-VariantDir;. -If variant_dir -is omitted, the duplicate argument is ignored. -See the description of -&f-link-VariantDir; -below for additional details and restrictions. - - - -If -variant_dir -is present, -the source directory is the directory in which the -SConscript -file resides and the -SConscript +argument is interpreted relative to the directory of the +calling SConscript file. +The source directory is the directory in which the +called SConscript +file resides and the SConscript file is evaluated as if it were in the variant_dir -directory: +directory. Thus: SConscript('src/SConscript', variant_dir='build') -is equivalent to +is equivalent to: @@ -3584,9 +3586,8 @@ SConscript('build/SConscript') -This later paradigm is often used when the sources are -in the same directory as the -&SConstruct;: +If the sources are in the same directory as the +&SConstruct;, @@ -3594,7 +3595,7 @@ SConscript('SConscript', variant_dir='build') -is equivalent to +is equivalent to: @@ -3603,6 +3604,17 @@ SConscript('build/SConscript') +The optional +duplicate argument is +interpreted as for &f-link-VariantDir;. +If the variant_dir argument +is omitted, the duplicate argument is ignored. +See the description of +&f-link-VariantDir; +for additional details and restrictions. + + + $__LDMODULEVERSIONFLAGS"> -$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -38,6 +37,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $CC"> $CCCOM"> $CCCOMSTR"> +$CCDEPFLAGS"> $CCFLAGS"> $CCPCHFLAGS"> $CCPDBFLAGS"> @@ -320,7 +320,10 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MSSDK_DIR"> $MSSDK_VERSION"> $MSVC_BATCH"> +$MSVC_NOTFOUND_POLICY"> $MSVC_USE_SCRIPT"> +$MSVC_USE_SCRIPT_ARGS"> +$MSVC_USE_SETTINGS"> $MSVC_UWP_APP"> $MSVC_VERSION"> $MSVS"> @@ -351,17 +354,22 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSIONS"> $NAME"> $NINJA_ALIAS_NAME"> +$NINJA_CMD_ARGS"> $NINJA_COMPDB_EXPAND"> +$NINJA_DEPFILE_PARSE_FORMAT"> $NINJA_DIR"> $NINJA_DISABLE_AUTO_RUN"> $NINJA_ENV_VAR_CACHE"> $NINJA_FILE_NAME"> $NINJA_FORCE_SCONS_BUILD"> +$NINJA_GENERATED_SOURCE_ALIAS_NAME"> $NINJA_GENERATED_SOURCE_SUFFIXES"> $NINJA_MSVC_DEPS_PREFIX"> $NINJA_POOL"> $NINJA_REGENERATE_DEPS"> $_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SCONS_DAEMON_KEEP_ALIVE"> +$NINJA_SCONS_DAEMON_PORT"> $NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> @@ -480,6 +488,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $SHDLINKCOM"> $SHDLINKFLAGS"> $SHELL"> +$SHELL_ENV_GENERATORS"> $SHF03"> $SHF03COM"> $SHF03COMSTR"> @@ -664,7 +673,6 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. --> $__LDMODULEVERSIONFLAGS"> -$__NINJA_NO"> $__SHLIBVERSIONFLAGS"> $APPLELINK_COMPATIBILITY_VERSION"> $_APPLELINK_COMPATIBILITY_VERSION"> @@ -693,6 +701,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $CC"> $CCCOM"> $CCCOMSTR"> +$CCDEPFLAGS"> $CCFLAGS"> $CCPCHFLAGS"> $CCPDBFLAGS"> @@ -975,7 +984,10 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MSSDK_DIR"> $MSSDK_VERSION"> $MSVC_BATCH"> +$MSVC_NOTFOUND_POLICY"> $MSVC_USE_SCRIPT"> +$MSVC_USE_SCRIPT_ARGS"> +$MSVC_USE_SETTINGS"> $MSVC_UWP_APP"> $MSVC_VERSION"> $MSVS"> @@ -1006,17 +1018,22 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MWCW_VERSIONS"> $NAME"> $NINJA_ALIAS_NAME"> +$NINJA_CMD_ARGS"> $NINJA_COMPDB_EXPAND"> +$NINJA_DEPFILE_PARSE_FORMAT"> $NINJA_DIR"> $NINJA_DISABLE_AUTO_RUN"> $NINJA_ENV_VAR_CACHE"> $NINJA_FILE_NAME"> $NINJA_FORCE_SCONS_BUILD"> +$NINJA_GENERATED_SOURCE_ALIAS_NAME"> $NINJA_GENERATED_SOURCE_SUFFIXES"> $NINJA_MSVC_DEPS_PREFIX"> $NINJA_POOL"> $NINJA_REGENERATE_DEPS"> $_NINJA_REGENERATE_DEPS_FUNC"> +$NINJA_SCONS_DAEMON_KEEP_ALIVE"> +$NINJA_SCONS_DAEMON_PORT"> $NINJA_SYNTAX"> $no_import_lib"> $OBJPREFIX"> @@ -1135,6 +1152,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $SHDLINKCOM"> $SHDLINKFLAGS"> $SHELL"> +$SHELL_ENV_GENERATORS"> $SHF03"> $SHF03COM"> $SHF03COMSTR"> diff --git a/doc/man/scons.xml b/doc/man/scons.xml index a98f4e8..4578d1c 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -5207,7 +5207,7 @@ the dependency graph related to their sources. An alias is checked for up to date by checking if its sources are up to date. An alias is built by making sure its sources have been built, -and if any building took place, +and if any building took place, applying any Actions that are defined as part of the alias. @@ -7470,7 +7470,7 @@ There are, however, a few portability issues waiting to trap the unwary. -.C file suffix +.C File Suffix &scons; handles the upper-case .C @@ -7490,50 +7490,125 @@ suffix as a C source file. -Fortran file suffixes +Fortran File Suffixes -&scons; handles upper-case -Fortran file suffixes differently -depending on the capabilities of -the underlying system. -On a case-sensitive system -such as Linux or UNIX, -&scons; treats a file with a -.F -as a Fortran source file -that is to be first run through -the standard C preprocessor, -while the lower-case version is not. -This matches the convention of gfortran, -which may also be followed by other Fortran compilers. -This also applies to other naming variants, + +There are several ways source file suffixes impact the +behavior of &SCons; when working with Fortran language code +(not all are system-specific, but they are included here +for completeness). + + + +As the Fortran language has evolved through multiple +standards editions, projects might have a need to handle +files from different language generations differently. +To this end, &SCons; dispatches to a different compiler +dialect setup (expressed as a set of &consvars;) +depending on the file suffix. +By default, all of these setups start out the same, +but individual &consvars; can be modified as needed to tune a given dialect. +Each of these dialacts has a tool specification module +whose documentation describes the &consvars; associated +with that dialect: .f +(as well as .for and .ftn) +in &t-link-fortran;; (&consvars; start with FORTRAN) +.f77 in &t-link-f77;; +(&consvars; start with F77) +.f90 in &t-link-f90;; +(&consvars; start with F90) +.f95 in &t-link-f95;; +(&consvars; start with F95) +.f03 in &t-link-f03;; +(&consvars; start with F03) +.f08 in &t-link-f08; +(&consvars; start with F08). + + + +While &SCons; recognizes multiple internal dialects +based on filename suffixes, +the convention of various available Fortran compilers is +to assign an actual meaning to only two of these suffixes: +.f +(as well as .for and .ftn) +refers to the fixed-format source +code that was the only available option in FORTRAN 77 and earlier, +and .f90 refers to free-format source code +which became available as of the Fortran 90 standard. +Some compilers recognize suffixes which correspond to Fortran +specifications later then F90 as equivalent to +.f90 for this purpose, +while some do not - check the documentation for your compiler. +An occasionally suggested policy suggestion is to use only +.f and .f90 +as Fortran filename suffixes. +The fixed/free form determination can usually be controlled +explicitly with compiler flags +(e.g. for gfortran), +overriding any assumption that may be made based on the source file suffix. + + + +The source file suffix does not imply conformance +with the similarly-named Fortran standard - a suffix of +.f08 does not mean you are compiling +specifically for Fortran 2008. Normally, compilers +provide command-line options for making this selection +(e.g. for gfortran). + + + +For dialects from F90 on (including the generic FORTRAN dialect), +a suffix of .mod is recognized for Fortran modules. +These files are a side effect of compiling a Fortran +source file containing module declarations, +and must be available when other code which declares +that it uses the module is processed. +&SCons; does not currently have integrated support for submodules, +introduced in the Fortran 2008 standard - +the invoked compiler will produce results, +but &SCons; will not recognize +.smod files as tracked objects. + + + +On a case-sensitive system such as Linux or UNIX, +a file with a an upper-cased suffix from the set +.F, .FOR, .FTN, .F90, .F95, .F03 and -.F08; -files suffixed with +.F08 +is treated as a Fortran source file +which shall first be run through +the standard C preprocessor. +The lower-cased versions of these suffixes do not +trigger this behavior. +On systems which do not distinguish between uppper +and lower case in filenames, +this behavior is not available, +but files suffixed with either .FPP -and .fpp -are both run through the preprocessor, -as indicated by the pp -part of the name. -On a case-insensitive system -such as Windows, -&scons; treats a file with a -.F -suffix as a Fortran source file that should -not -be run through the C preprocessor. +or .fpp +are always passed to the preprocessor first. +This matches the convention of gfortran +from the GNU Compiler Collection, +and also followed by certain other Fortran compilers. +For these two suffixes, +the generic FORTRAN dialect will be selected. + + -Run through the C preprocessor -here means that a different set of &consvars; will -be applied in constructed commands, for example +&SCons; itself does not invoke the preprocessor, +that is handled by the compiler, +but it adds &consvars; which are applicable to the preprocessor run. +You can see this difference by examining &cv-link-FORTRANPPCOM; and &cv-link-FORTRANPPCOMSTR; -instead of -&cv-link-FORTRANCOM; and &cv-link-FORTRANCOMSTR;. -See the Fortran-related &consvars; for more details. +which are used instead of +&cv-link-FORTRANCOM; and &cv-link-FORTRANCOMSTR; for that dialect. @@ -7542,10 +7617,10 @@ See the Fortran-related &consvars; for more details. Cygwin supplies a set of tools and utilities that let users work on a -Windows system using a more POSIX-like environment. -The Cygwin tools, including Cygwin Python, +Windows system using a POSIX-like environment. +The Cygwin tools, including Cygwin &Python;, do this, in part, -by sharing an ability to interpret UNIX-like path names. +by sharing an ability to interpret POSIX-style path names. For example, the Cygwin tools will internally translate a Cygwin path name like /cygdrive/c/mydir @@ -7553,11 +7628,11 @@ to an equivalent Windows pathname of C:/mydir (equivalent to C:\mydir). -Versions of Python +Versions of &Python; that are built for native Windows execution, such as the python.org and ActiveState versions, -do not have the Cygwin path name semantics. -This means that using a native Windows version of Python +do not understand the Cygwin path name semantics. +This means that using a native Windows version of &Python; to build compiled programs using Cygwin tools (such as &gcc;, &bison; and &flex;) may yield unpredictable results. @@ -7567,14 +7642,22 @@ but it requires careful attention to the use of path names in your SConscript files. In practice, users can sidestep -the issue by adopting the following rules: +the issue by adopting the following guidelines: When using Cygwin's &gcc; for compiling, -use the Cygwin-supplied Python interpreter +use the Cygwin-supplied &Python; interpreter to run &scons;; when using Microsoft Visual C/C++ -(or some other Windows compiler) -use the python.org or Microsoft Store or ActiveState version of Python -to run &scons;. +(or some other "native" Windows compiler) +use the python.org, Microsoft Store, ActiveState or other +native version of &Python; to run &scons;. + + + +This discussion largely applies to the msys2 environment +as well (with the use of the mingw compiler toolchain), +in particular the recommendation to use the msys2 version of +&Python; if running &scons; from inside an msys2 shell. + @@ -7584,7 +7667,7 @@ to run &scons;. scons.bat batch file, there are (at least) two ramifications. Note this is no longer the default - &scons; installed -via Python's pip installer +via &Python;''s pip installer will have an scons.exe which does not have these limitations: @@ -7624,7 +7707,7 @@ directory must be in your PATH environment variable or the ['ENV']['PATH'] &consvar; for &scons; to detect and use the MinGW tools. When running under the native Windows -Python interpreter, &scons; will prefer the MinGW tools over the Cygwin +Python; interpreter, &scons; will prefer the MinGW tools over the Cygwin tools, if they are both installed, regardless of the order of the bin directories in the PATH variable. If you have both MSVC and MinGW -- cgit v0.12 From 5187917d8a99b86e965ddeb676b0f8fe6c670318 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 16 Jun 2022 11:41:17 -0400 Subject: Add SDK version support and validate all arguments. --- SCons/Tool/MSCommon/vc.py | 1637 ++++++++++++++++++++++++++++----------------- 1 file changed, 1010 insertions(+), 627 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 0c9fcee..c44c698 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -101,6 +101,17 @@ class BatchFileExecutionWarning(SCons.Warnings.WarningOnByDefault): class _Const: + BOOLEAN_KEYS = {} + BOOLEAN_SYMBOLS = {} + + for bool, symbol_list in [ + (False, (0, '0', False, 'False', 'FALSE', 'false', 'No', 'NO', 'no', None, '')), + (True, (1, '1', True, 'True', 'TRUE', 'true', 'Yes', 'YES', 'yes', )), + ]: + BOOLEAN_KEYS[bool] = symbol_list + for symbol in symbol_list: + BOOLEAN_SYMBOLS[symbol] = bool + MSVC_RUNTIME_DEFINITION = namedtuple('MSVCRuntime', [ 'vc_runtime', 'vc_runtime_numeric', @@ -1197,7 +1208,8 @@ def reset_installed_vcs(): global __INSTALLED_VCS_RUN __INSTALLED_VCS_RUN = None _MSVCSetupEnvDefault.reset() - _MSVCScriptArguments.reset() + _WindowsSDK.reset() + _ScriptArguments.reset() # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" @@ -1622,287 +1634,543 @@ def msvc_setup_env_once(env, tool=None): " Requested tool(s) are: {}".format(req_tools) _msvc_notfound_policy_handler(env, msg) -class _MSVCScriptArguments: - - # Force -vcvars_ver argument for default toolset - MSVC_TOOLSET_DEFAULT_VCVARSVER = False +def msvc_find_valid_batch_script(env, version): + """Find and execute appropriate batch script to set up build env. - # MSVC batch file arguments: - # - # VS2022: UWP, SDK, TOOLSET, SPECTRE - # VS2019: UWP, SDK, TOOLSET, SPECTRE - # VS2017: UWP, SDK, TOOLSET, SPECTRE - # VS2015: UWP, SDK - # - # MSVC_SCRIPT_ARGS: VS2015+ - # - # MSVC_UWP_APP: VS2015+ - # MSVC_SDK_VERSION: VS2015+ - # MSVC_TOOLSET_VERSION: VS2017+ - # MSVC_SPECTRE_LIBS: VS2017+ + The MSVC build environment depends heavily on having the shell + environment set. SCons does not inherit that, and does not count + on that being set up correctly anyway, so it tries to find the right + MSVC batch script, or the right arguments to the generic batch script + vcvarsall.bat, and run that, so we have a valid environment to build in. + There are dragons here: the batch scripts don't fail (see comments + elsewhere), they just leave you with a bad setup, so try hard to + get it right. + """ - @enum.unique - class SortOrder(enum.IntEnum): - ARCH = 0 # arch - UWP = 1 # MSVC_UWP_APP - SDK = 2 # MSVC_SDK_VERSION - TOOLSET = 3 # MSVC_TOOLSET_VERSION - SPECTRE = 4 # MSVC_SPECTRE_LIBS - USER = 5 # MSVC_SCRIPT_ARGS + # Find the host, target, and all candidate (host, target) platform combinations: + platforms = get_host_target(env, version) + debug("host_platform %s, target_platform %s host_target_list %s", *platforms) + host_platform, target_platform, host_target_list = platforms - VS2019 = _Const.MSVS_VERSION_INTERNAL['2019'] - VS2017 = _Const.MSVS_VERSION_INTERNAL['2017'] - VS2015 = _Const.MSVS_VERSION_INTERNAL['2015'] + d = None + version_installed = False + for host_arch, target_arch, in host_target_list: + # Set to current arch. + env['TARGET_ARCH'] = target_arch + arg = '' - MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ - 'version', # fully qualified msvc version (e.g., '14.1Exp') - 'vs_def', - ]) + # Try to locate a batch file for this host/target platform combo + try: + (vc_script, arg, vc_dir, sdk_script) = find_batch_file(env, version, host_arch, target_arch) + debug('vc_script:%s vc_script_arg:%s sdk_script:%s', vc_script, arg, sdk_script) + version_installed = True + except VisualCException as e: + msg = str(e) + debug('Caught exception while looking for batch file (%s)', msg) + version_installed = False + continue - @classmethod - def _msvc_version(cls, version): + # Try to use the located batch file for this host/target platform combo + debug('use_script 2 %s, args:%s', repr(vc_script), arg) + found = None + if vc_script: + arg = _ScriptArguments.msvc_script_arguments(env, version, vc_dir, arg) + try: + d = script_env(vc_script, args=arg) + found = vc_script + except BatchFileExecutionError as e: + debug('use_script 3: failed running VC script %s: %s: Error:%s', repr(vc_script), arg, e) + vc_script=None + continue + if not vc_script and sdk_script: + debug('use_script 4: trying sdk script: %s', sdk_script) + try: + d = script_env(sdk_script) + found = sdk_script + except BatchFileExecutionError as e: + debug('use_script 5: failed running SDK script %s: Error:%s', repr(sdk_script), e) + continue + elif not vc_script and not sdk_script: + debug('use_script 6: Neither VC script nor SDK script found') + continue - verstr = get_msvc_version_numeric(version) - vs_def = _Const.MSVC_VERSION_INTERNAL[verstr] + debug("Found a working script/target: %s/%s", repr(found), arg) + break # We've found a working target_platform, so stop looking - version_args = cls.MSVC_VERSION_ARGS_DEFINITION( - version = version, - vs_def = vs_def, - ) + # If we cannot find a viable installed compiler, reset the TARGET_ARCH + # To it's initial value + if not d: + env['TARGET_ARCH'] = target_platform - return version_args + if version_installed: + msg = "MSVC version '{}' working host/target script was not found.\n" \ + " Host = '{}', Target = '{}'\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format( + version, host_platform, target_platform + ) + else: + installed_vcs = get_installed_vcs(env) + if installed_vcs: + msg = "MSVC version '{}' was not found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly.\n" \ + " Installed versions are: {}".format(version, installed_vcs) + else: + msg = "MSVC version '{}' was not found.\n" \ + " No versions of the MSVC compiler were found.\n" \ + " Visual Studio C/C++ compilers may not be set correctly".format(version) - @classmethod - def _msvc_script_argument_uwp(cls, env, msvc, arglist): + _msvc_notfound_policy_handler(env, msg) - uwp_app = env['MSVC_UWP_APP'] - debug('MSVC_VERSION=%s, MSVC_UWP_APP=%s', repr(msvc.version), repr(uwp_app)) + return d - if not uwp_app: - return None +_undefined = None - if uwp_app not in (True, '1'): - return None +def get_use_script_use_settings(env): + global _undefined - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_UWP_APP ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(uwp_app), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - raise MSVCArgumentError(err_msg) + if _undefined is None: + _undefined = object() - # VS2017+ rewrites uwp => store for 14.0 toolset - uwp_arg = msvc.vs_def.vc_uwp + # use_script use_settings return values action + # value ignored (value, None) use script or bypass detection + # undefined value not None (False, value) use dictionary + # undefined undefined/None (True, None) msvc detection - # uwp may not be installed - argpair = (cls.SortOrder.UWP, uwp_arg) - arglist.append(argpair) + # None (documentation) or evaluates False (code): bypass detection + # need to distinguish between undefined and None + use_script = env.get('MSVC_USE_SCRIPT', _undefined) - return uwp_arg + if use_script != _undefined: + # use_script defined, use_settings ignored (not type checked) + return (use_script, None) - # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? - re_sdk_version_10 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') + # undefined or None: use_settings ignored + use_settings = env.get('MSVC_USE_SETTINGS', None) - @classmethod - def _msvc_script_argument_sdk_constraints(cls, msvc, sdk_version): + if use_settings is not None: + # use script undefined, use_settings defined and not None (type checked) + return (False, use_settings) - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc_version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_SDK_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(sdk_version), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - return err_msg + # use script undefined, use_settings undefined or None + return (True, None) - # TODO: check sdk against vs_def/vc_buildtools_def +def msvc_setup_env(env): + debug('called') + version = get_default_version(env) + if version is None: + if not msvc_setup_env_user(env): + _MSVCSetupEnvDefault.set_nodefault() + return None - if sdk_version == '8.1': - debug('valid: sdk_version=%s', repr(sdk_version)) - return None + # XXX: we set-up both MSVS version for backward + # compatibility with the msvs tool + env['MSVC_VERSION'] = version + env['MSVS_VERSION'] = version + env['MSVS'] = {} - if cls.re_sdk_version_10.match(sdk_version): - debug('valid: sdk_version=%s', repr(sdk_version)) - return None + use_script, use_settings = get_use_script_use_settings(env) + if SCons.Util.is_String(use_script): + use_script = use_script.strip() + if not os.path.exists(use_script): + raise MSVCScriptNotFound('Script specified by MSVC_USE_SCRIPT not found: "{}"'.format(use_script)) + args = env.subst('$MSVC_USE_SCRIPT_ARGS') + debug('use_script 1 %s %s', repr(use_script), repr(args)) + d = script_env(use_script, args) + elif use_script: + d = msvc_find_valid_batch_script(env,version) + debug('use_script 2 %s', d) + if not d: + return d + elif use_settings is not None: + if not SCons.Util.is_Dict(use_settings): + error_msg = 'MSVC_USE_SETTINGS type error: expected a dictionary, found {}'.format(type(use_settings).__name__) + raise MSVCUseSettingsError(error_msg) + d = use_settings + debug('use_settings %s', d) + else: + debug('MSVC_USE_SCRIPT set to False') + warn_msg = "MSVC_USE_SCRIPT set to False, assuming environment " \ + "set correctly." + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + return None - debug('invalid: method exit: sdk_version=%s', repr(sdk_version)) - err_msg = "MSVC_SDK_VERSION ({}) is not supported".format(repr(sdk_version)) - return err_msg + for k, v in d.items(): + env.PrependENVPath(k, v, delete_existing=True) + debug("env['ENV']['%s'] = %s", k, env['ENV'][k]) - @classmethod - def _msvc_script_argument_sdk(cls, env, msvc, is_uwp, arglist): + # final check to issue a warning if the compiler is not present + if not find_program_path(env, 'cl'): + debug("did not find %s", _CL_EXE_NAME) + if CONFIG_CACHE: + propose = "SCONS_CACHE_MSVC_CONFIG caching enabled, remove cache file {} if out of date.".format(CONFIG_CACHE) + else: + propose = "It may need to be installed separately with Visual Studio." + warn_msg = "Could not find MSVC compiler 'cl'. {}".format(propose) + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) - sdk_version = env['MSVC_SDK_VERSION'] - debug( - 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, uwp=%s', - repr(msvc.version), repr(sdk_version), repr(is_uwp) - ) +def msvc_exists(env=None, version=None): + debug('version=%s', repr(version)) + vcs = get_installed_vcs(env) + if version is None: + rval = len(vcs) > 0 + else: + rval = version in vcs + debug('version=%s, return=%s', repr(version), rval) + return rval - if not sdk_version: - return None +def msvc_setup_env_user(env=None): + rval = False + if env: - err_msg = cls._msvc_script_argument_sdk_constraints(msvc, sdk_version) - if err_msg: - raise MSVCArgumentError(err_msg) + # Intent is to use msvc tools: + # MSVC_VERSION or MSVS_VERSION: defined and is True + # MSVC_USE_SCRIPT: defined and (is string or is False) + # MSVC_USE_SETTINGS: defined and is not None - # sdk folder may not exist - argpair = (cls.SortOrder.SDK, sdk_version) - arglist.append(argpair) + # defined and is True + for key in ['MSVC_VERSION', 'MSVS_VERSION']: + if key in env and env[key]: + rval = True + debug('key=%s, return=%s', repr(key), rval) + return rval - return sdk_version + # defined and (is string or is False) + for key in ['MSVC_USE_SCRIPT']: + if key in env and (SCons.Util.is_String(env[key]) or not env[key]): + rval = True + debug('key=%s, return=%s', repr(key), rval) + return rval - @classmethod - def _msvc_read_toolset_file(cls, msvc, filename): - toolset_version = None + # defined and is not None + for key in ['MSVC_USE_SETTINGS']: + if key in env and env[key] is not None: + rval = True + debug('key=%s, return=%s', repr(key), rval) + return rval + + debug('return=%s', rval) + return rval + +def msvc_setup_env_tool(env=None, version=None, tool=None): + debug('tool=%s, version=%s', repr(tool), repr(version)) + _MSVCSetupEnvDefault.register_tool(env, tool) + rval = False + if not rval and msvc_exists(env, version): + rval = True + if not rval and msvc_setup_env_user(env): + rval = True + debug('tool=%s, version=%s, return=%s', repr(tool), repr(version), rval) + return rval + +class _Util: + + @staticmethod + def listdir_dirs(p): + dirs = [] + for dir_name in os.listdir(p): + dir_path = os.path.join(p, dir_name) + if os.path.isdir(dir_path): + dirs.append((dir_name, dir_path)) + return dirs + + @staticmethod + def process_path(p): + if p: + p = os.path.normpath(p) + p = os.path.realpath(p) + p = os.path.normcase(p) + return p + +class _Registry: + + def read_value(hkey, subkey_valname): try: - with open(filename) as f: - toolset_version = f.readlines()[0].strip() - debug( - 'msvc_version=%s, filename=%s, toolset_version=%s', - repr(msvc.version), repr(filename), repr(toolset_version) - ) + rval = common.read_reg(subkey_valname, hkroot=hkey) except OSError: - debug('OSError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) + debug('OSError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) + return None except IndexError: - debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) - return toolset_version + debug('IndexError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) + return None + debug('hkey=%s, subkey=%s, rval=%s', repr(hkey), repr(subkey_valname), repr(rval)) + return rval @classmethod - def _msvc_read_toolset_folders(cls, msvc, vc_dir): + def registry_query_path(cls, key, val, suffix): + extval = val + '\\' + suffix if suffix else val + qpath = cls.read_value(key, extval) + if qpath and os.path.exists(qpath): + qpath = _Util.process_path(qpath) + else: + qpath = None + return (qpath, key, val, extval) + + REG_SOFTWARE_MICROSOFT = [ + (SCons.Util.HKEY_LOCAL_MACHINE, r'Software\Wow6432Node\Microsoft'), + (SCons.Util.HKEY_CURRENT_USER, r'Software\Wow6432Node\Microsoft'), # SDK queries + (SCons.Util.HKEY_LOCAL_MACHINE, r'Software\Microsoft'), + (SCons.Util.HKEY_CURRENT_USER, r'Software\Microsoft'), + ] - toolsets_sxs = {} - toolsets_full = [] + @classmethod + def microsoft_query_paths(cls, suffix, usrval=None): + paths = [] + records = [] + for key, val in cls.REG_SOFTWARE_MICROSOFT: + extval = val + '\\' + suffix if suffix else val + qpath = cls.read_value(key, extval) + if qpath and os.path.exists(qpath): + qpath = _Util.process_path(qpath) + if qpath not in paths: + paths.append(qpath) + records.append((qpath, key, val, extval, usrval)) + return records - build_dir = os.path.join(vc_dir, "Auxiliary", "Build") - sxs_toolsets = [f.name for f in os.scandir(build_dir) if f.is_dir()] - for sxs_toolset in sxs_toolsets: - filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) - filepath = os.path.join(build_dir, sxs_toolset, filename) - debug('sxs toolset: check file=%s', repr(filepath)) - if os.path.exists(filepath): - toolset_version = cls._msvc_read_toolset_file(msvc, filepath) - if not toolset_version: - continue - toolsets_sxs[sxs_toolset] = toolset_version - debug( - 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', - repr(msvc.version), repr(sxs_toolset), toolset_version - ) + @classmethod + def microsoft_query_keys(cls, suffix, usrval=None): + records = [] + for key, val in cls.REG_SOFTWARE_MICROSOFT: + extval = val + '\\' + suffix if suffix else val + rval = cls.read_value(key, extval) + if rval: + records.append((key, val, extval, usrval)) + return records - toolset_dir = os.path.join(vc_dir, "Tools", "MSVC") - toolsets = [f.name for f in os.scandir(toolset_dir) if f.is_dir()] - for toolset_version in toolsets: - binpath = os.path.join(toolset_dir, toolset_version, "bin") - debug('toolset: check binpath=%s', repr(binpath)) - if os.path.exists(binpath): - toolsets_full.append(toolset_version) - debug( - 'toolset: msvc_version=%s, toolset_version=%s', - repr(msvc.version), repr(toolset_version) - ) + @classmethod + def microsoft_sdks(cls, version): + return '\\'.join([r'Microsoft SDKs\Windows', 'v' + version, r'InstallationFolder']) - toolsets_full.sort(reverse=True) - debug('msvc_version=%s, toolsets=%s', repr(msvc.version), repr(toolsets_full)) + @classmethod + def sdk_query_paths(cls, version): + q = cls.microsoft_sdks(version) + return cls.microsoft_query_paths(q) - return toolsets_sxs, toolsets_full + @classmethod + def windows_kits(cls, version): + return r'Windows Kits\Installed Roots\KitsRoot' + version @classmethod - def _msvc_read_toolset_default(cls, msvc, vc_dir): + def windows_kit_query_paths(cls, version): + q = cls.windows_kits(version) + return cls.microsoft_query_paths(q) - build_dir = os.path.join(vc_dir, "Auxiliary", "Build") +class _WindowsSDK: - # VS2019+ - filename = "Microsoft.VCToolsVersion.{}.default.txt".format(msvc.vs_def.vc_buildtools_def.vc_buildtools) - filepath = os.path.join(build_dir, filename) + sdk_map_cache = {} + sdk_cache = {} - debug('default toolset: check file=%s', repr(filepath)) - toolset_buildtools = None - if os.path.exists(filepath): - toolset_buildtools = cls._msvc_read_toolset_file(msvc, filepath) - if toolset_buildtools: - return toolset_buildtools + @classmethod + def reset(cls): + cls.sdk_map_cache = {} + cls.sdk_cache = {} - # VS2017+ - filename = "Microsoft.VCToolsVersion.default.txt" - filepath = os.path.join(build_dir, filename) + @classmethod + def _new_sdk_map(cls): + sdk_map = { + 'desktop': [], + 'uwp': [], + } + return sdk_map - debug('default toolset: check file=%s', repr(filepath)) - toolset_default = None - if os.path.exists(filepath): - toolset_default = cls._msvc_read_toolset_file(msvc, filepath) - if toolset_default: - return toolset_default + @classmethod + def _sdk_10_layout(cls, version): - return None + folder_prefix = version + '.' + + sdk_map = cls._new_sdk_map() + + sdk_roots = _Registry.sdk_query_paths(version) + + sdk_version_platform_seen = set() + sdk_roots_seen = set() + + for sdk_t in sdk_roots: + + sdk_root = sdk_t[0] + if sdk_root in sdk_roots_seen: + continue + sdk_roots_seen.add(sdk_root) + + if not os.path.exists(sdk_root): + continue + + sdk_include_path = os.path.join(sdk_root, 'include') + if not os.path.exists(sdk_include_path): + continue + + for version_nbr, version_nbr_path in _Util.listdir_dirs(sdk_include_path): + + if not version_nbr.startswith(folder_prefix): + continue + + sdk_inc_path = _Util.process_path(os.path.join(version_nbr_path, 'um')) + if not os.path.exists(sdk_inc_path): + continue + + for platform, sdk_inc_file in [ + ('desktop', 'winsdkver.h'), + ('uwp', 'windows.h'), + ]: + + if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): + continue + + key = (version_nbr, platform) + if key in sdk_version_platform_seen: + continue + sdk_version_platform_seen.add(key) + + sdk_map[platform].append(version_nbr) + + for key, val in sdk_map.items(): + val.sort(reverse=True) + + return sdk_map @classmethod - def _reset_toolsets(cls): - debug('reset: toolset cache') - cls._toolset_version_cache = {} - cls._toolset_default_cache = {} + def _sdk_81_layout(cls, version): - _toolset_version_cache = {} - _toolset_default_cache = {} + version_nbr = version + + sdk_map = cls._new_sdk_map() + + sdk_roots = _Registry.sdk_query_paths(version) + + sdk_version_platform_seen = set() + sdk_roots_seen = set() + + sdk_targets = [] + + for sdk_t in sdk_roots: + + sdk_root = sdk_t[0] + if sdk_root in sdk_roots_seen: + continue + sdk_roots_seen.add(sdk_root) + + # msvc does not check for existence of root or other files + + sdk_inc_path = _Util.process_path(os.path.join(sdk_root, r'include\um')) + if not os.path.exists(sdk_inc_path): + continue + + for platform, sdk_inc_file in [ + ('desktop', 'winsdkver.h'), + ('uwp', 'windows.h'), + ]: + + if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): + continue + + key = (version_nbr, platform) + if key in sdk_version_platform_seen: + continue + sdk_version_platform_seen.add(key) + + sdk_map[platform].append(version_nbr) + + for key, val in sdk_map.items(): + val.sort(reverse=True) + + return sdk_map @classmethod - def _msvc_version_toolsets(cls, msvc, vc_dir): + def _sdk_10(cls, key, reg_version): + if key in cls.sdk_map_cache: + sdk_map = cls.sdk_map_cache[key] + else: + sdk_map = cls._sdk_10_layout(reg_version) + cls.sdk_map_cache[key] = sdk_map + return sdk_map - if msvc.version in cls._toolset_version_cache: - toolsets_sxs, toolsets_full = cls._toolset_version_cache[msvc.version] + @classmethod + def _sdk_81(cls, key, reg_version): + if key in cls.sdk_map_cache: + sdk_map = cls.sdk_map_cache[key] else: - toolsets_sxs, toolsets_full = cls._msvc_read_toolset_folders(msvc, vc_dir) - cls._toolset_version_cache[msvc.version] = toolsets_sxs, toolsets_full + sdk_map = cls._sdk_81_layout(reg_version) + cls.sdk_map_cache[key] = sdk_map + return sdk_map - return toolsets_sxs, toolsets_full + @classmethod + def _combine_sdk_map_list(cls, sdk_map_list): + combined_sdk_map = cls._new_sdk_map() + for sdk_map in sdk_map_list: + for key, val in sdk_map.items(): + combined_sdk_map[key].extend(val) + return combined_sdk_map + + sdk_dispatch_map = None @classmethod - def _msvc_default_toolset(cls, msvc, vc_dir): + def _version_list_sdk_map(cls, version_list): - if msvc.version in cls._toolset_default_cache: - toolset_default = cls._toolset_default_cache[msvc.version] - else: - toolset_default = cls._msvc_read_toolset_default(msvc, vc_dir) - cls._toolset_default_cache[msvc.version] = toolset_default + if not cls.sdk_dispatch_map: + cls.sdk_dispatch_map = { + '10.0': (cls._sdk_10, '10.0'), + '8.1': (cls._sdk_81, '8.1'), + } - return toolset_default + sdk_map_list = [] + for version in version_list: + func, reg_version = cls.sdk_dispatch_map[version] + sdk_map = func(version, reg_version) + sdk_map_list.append(sdk_map) + + combined_sdk_map = cls._combine_sdk_map_list(sdk_map_list) + return combined_sdk_map @classmethod - def _msvc_version_toolset_vcvars(cls, msvc, vc_dir, toolset_version): + def _sdk_map(cls, version_list): + key = tuple(version_list) + if key in cls.sdk_cache: + sdk_map = cls.sdk_cache[key] + else: + version_numlist = [float(v) for v in version_list] + version_numlist.sort(reverse=True) + key = tuple([str(v) for v in version_numlist]) + sdk_map = cls._version_list_sdk_map(key) + cls.sdk_cache[key] = sdk_map + return sdk_map - if toolset_version == '14.0': - return toolset_version + @classmethod + def _get_sdk_version_list(cls, version_list, platform): + sdk_map = cls._sdk_map(version_list) + sdk_list = sdk_map.get(platform, []) + return sdk_list - toolsets_sxs, toolsets_full = cls._msvc_version_toolsets(msvc, vc_dir) +def get_sdk_versions(MSVC_VERSION=None, MSVC_UWP_APP=False): - if msvc.vs_def.vc_buildtools_def.vc_version_numeric == cls.VS2019.vc_buildtools_def.vc_version_numeric: - # necessary to detect toolset not found - if toolset_version == '14.28.16.8': - new_toolset_version = '14.28' - # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 - # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files - # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. - debug( - 'rewrite toolset_version=%s => toolset_version=%s', - repr(toolset_version), repr(new_toolset_version) - ) - toolset_version = new_toolset_version + sdk_versions = [] - if toolset_version in toolsets_sxs: - toolset_vcvars = toolsets_sxs[toolset_version] - return toolset_vcvars + if not MSVC_VERSION: + vcs = get_installed_vcs() + if not vcs: + return sdk_versions + MSVC_VERSION = vcs[0] - for toolset_full in toolsets_full: - if toolset_full.startswith(toolset_version): - toolset_vcvars = toolset_full - return toolset_vcvars + verstr = get_msvc_version_numeric(MSVC_VERSION) + vs_def = _Const.MSVC_VERSION_EXTERNAL.get(verstr, None) + if not vs_def: + return sdk_versions - return None + is_uwp = True if MSVC_UWP_APP in _Const.BOOLEAN_KEYS[True] else False + platform = 'uwp' if is_uwp else 'desktop' + sdk_list = _WindowsSDK._get_sdk_version_list(vs_def.vc_sdk_versions, platform) + + sdk_versions.extend(sdk_list) + return sdk_versions + +class _ScriptArguments: + + # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? + re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') + re_sdk_version_81 = re.compile(r'^8[.]1$') + + re_sdk_dispatch_map = { + '10.0': re_sdk_version_100, + '8.1': re_sdk_version_81, + } # capture msvc version re_toolset_version = re.compile(r'^(?P[1-9][0-9]?[.][0-9])[0-9.]*$', re.IGNORECASE) @@ -1920,223 +2188,210 @@ class _MSVCScriptArguments: # valid SxS formats will be matched with re_toolset_full: match 3 '.' format re_toolset_sxs = re.compile(r'^[1-9][0-9][.][0-9]{2}[.][0-9]{2}[.][0-9]{1,2}$') - @classmethod - def _msvc_script_argument_toolset_constraints(cls, msvc, toolset_version): + # MSVC_SCRIPT_ARGS + re_vcvars_uwp = re.compile(r'(?:(?(?:uwp|store))(?:(?!\S)|$)',re.IGNORECASE) + re_vcvars_sdk = re.compile(r'(?:(?(?:[1-9][0-9]*[.]\S*))(?:(?!\S)|$)',re.IGNORECASE) + re_vcvars_toolset = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_ver[=](?P\S*))(?:(?!\S)|$)', re.IGNORECASE) + re_vcvars_spectre = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_spectre_libs[=](?P\S*))(?:(?!\S)|$)',re.IGNORECASE) - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2017', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( - repr(toolset_version), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) - ) - return err_msg + # Force default sdk argument + MSVC_FORCE_DEFAULT_SDK = False - m = cls.re_toolset_version.match(toolset_version) - if not m: - debug('invalid: re_toolset_version: toolset_version=%s', repr(toolset_version)) - err_msg = 'MSVC_TOOLSET_VERSION {} format is not supported'.format( - repr(toolset_version) - ) - return err_msg + # Force default toolset argument + MSVC_FORCE_DEFAULT_TOOLSET = False - toolset_ver = m.group('version') - toolset_vernum = float(toolset_ver) + # MSVC batch file arguments: + # + # VS2022: UWP, SDK, TOOLSET, SPECTRE + # VS2019: UWP, SDK, TOOLSET, SPECTRE + # VS2017: UWP, SDK, TOOLSET, SPECTRE + # VS2015: UWP, SDK + # + # MSVC_SCRIPT_ARGS: VS2015+ + # + # MSVC_UWP_APP: VS2015+ + # MSVC_SDK_VERSION: VS2015+ + # MSVC_TOOLSET_VERSION: VS2017+ + # MSVC_SPECTRE_LIBS: VS2017+ - if toolset_vernum < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: toolset version constraint: %s < %s VS2015', - repr(toolset_vernum), repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} < {} VS2015".format( - repr(toolset_version), repr(toolset_ver), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - return err_msg + @enum.unique + class SortOrder(enum.IntEnum): + ARCH = 0 # arch + UWP = 1 # MSVC_UWP_APP + SDK = 2 # MSVC_SDK_VERSION + TOOLSET = 3 # MSVC_TOOLSET_VERSION + SPECTRE = 4 # MSVC_SPECTRE_LIBS + USER = 5 # MSVC_SCRIPT_ARGS - if toolset_vernum > msvc.vs_def.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: toolset version constraint: toolset %s > %s msvc', - repr(toolset_vernum), repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} > {} MSVC_VERSION".format( - repr(toolset_version), repr(toolset_ver), repr(msvc.version) - ) - return err_msg + VS2019 = _Const.MSVS_VERSION_INTERNAL['2019'] + VS2017 = _Const.MSVS_VERSION_INTERNAL['2017'] + VS2015 = _Const.MSVS_VERSION_INTERNAL['2015'] - if toolset_vernum == cls.VS2015.vc_buildtools_def.vc_version_numeric: - # tooset = 14.0 - if cls.re_toolset_full.match(toolset_version): - if not cls.re_toolset_140.match(toolset_version): - debug( - 'invalid: toolset version 14.0 constraint: %s != 14.0', - repr(toolset_version) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} != '14.0'".format( - repr(toolset_version), repr(toolset_version) - ) - return err_msg - return None + MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ + 'version', # fully qualified msvc version (e.g., '14.1Exp') + 'vs_def', + ]) - if cls.re_toolset_full.match(toolset_version): - debug('valid: re_toolset_full: toolset_version=%s', repr(toolset_version)) - return None + @classmethod + def _msvc_version(cls, version): - if cls.re_toolset_sxs.match(toolset_version): - debug('valid: re_toolset_sxs: toolset_version=%s', repr(toolset_version)) - return None + verstr = get_msvc_version_numeric(version) + vs_def = _Const.MSVC_VERSION_INTERNAL[verstr] - debug('invalid: method exit: toolset_version=%s', repr(toolset_version)) - err_msg = "MSVC_TOOLSET_VERSION ({}) format is not supported".format(repr(toolset_version)) - return err_msg + version_args = cls.MSVC_VERSION_ARGS_DEFINITION( + version = version, + vs_def = vs_def, + ) + + return version_args @classmethod - def _msvc_script_argument_toolset(cls, env, msvc, vc_dir, arglist): + def _msvc_script_argument_uwp(cls, env, msvc, arglist): - toolset_version = env['MSVC_TOOLSET_VERSION'] - debug('MSVC_VERSION=%s, MSVC_TOOLSET_VERSION=%s', repr(msvc.version), repr(toolset_version)) + uwp_app = env['MSVC_UWP_APP'] + debug('MSVC_VERSION=%s, MSVC_UWP_APP=%s', repr(msvc.version), repr(uwp_app)) - if not toolset_version: + if not uwp_app: return None - err_msg = cls._msvc_script_argument_toolset_constraints(msvc, toolset_version) - if err_msg: - raise MSVCArgumentError(err_msg) + if uwp_app not in _Const.BOOLEAN_KEYS[True]: + return None - if toolset_version.startswith('14.0') and len(toolset_version) > len('14.0'): - new_toolset_version = '14.0' + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: debug( - 'rewrite toolset_version=%s => toolset_version=%s', - repr(toolset_version), repr(new_toolset_version) + 'invalid: msvc version constraint: %s < %s VS2015', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) ) - toolset_version = new_toolset_version - - toolset_vcvars = cls._msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version) - debug( - 'toolset: toolset_version=%s, toolset_vcvars=%s', - repr(toolset_version), repr(toolset_vcvars) - ) - - if not toolset_vcvars: - err_msg = "MSVC_TOOLSET_VERSION {} not found for MSVC_VERSION {}".format( - repr(toolset_version), repr(msvc.version) + err_msg = "MSVC_UWP_APP ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(uwp_app), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) ) raise MSVCArgumentError(err_msg) - argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_vcvars)) + # VS2017+ rewrites uwp => store for 14.0 toolset + uwp_arg = msvc.vs_def.vc_uwp + + # uwp may not be installed + argpair = (cls.SortOrder.UWP, uwp_arg) arglist.append(argpair) - return toolset_vcvars + return uwp_arg @classmethod - def _msvc_script_default_toolset(cls, env, msvc, vc_dir, arglist): + def _user_script_argument_uwp(cls, env, uwp, user_argstr): - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: + matches = [m for m in cls.re_vcvars_uwp.finditer(user_argstr)] + if not matches: return None - toolset_default = cls._msvc_default_toolset(msvc, vc_dir) - if not toolset_default: + if len(matches) > 1: + debug('multiple uwp declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple uwp declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not uwp: return None - debug('MSVC_VERSION=%s, toolset_default=%s', repr(msvc.version), repr(toolset_default)) + env_argstr = env.get('MSVC_UWP_APP','') + debug('multiple uwp declarations: MSVC_UWP_APP=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_default)) - arglist.append(argpair) + err_msg = "multiple uwp declarations: MSVC_UWP_APP={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) - return toolset_default + raise MSVCArgumentError(err_msg) @classmethod - def _msvc_script_argument_spectre(cls, env, msvc, arglist): - - spectre_libs = env['MSVC_SPECTRE_LIBS'] - debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) - - if not spectre_libs: - return None - - if spectre_libs not in (True, '1'): - return None + def _msvc_script_argument_sdk_constraints(cls, msvc, sdk_version): - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: debug( - 'invalid: msvc version constraint: %s < %s VS2017', + 'invalid: msvc_version constraint: %s < %s VS2015', repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) + repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) ) - err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( - repr(spectre_libs), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) + err_msg = "MSVC_SDK_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(sdk_version), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) ) - raise MSVCArgumentError(err_msg) - - spectre_arg = 'spectre' + return err_msg - # spectre libs may not be installed - argpair = (cls.SortOrder.SPECTRE, '-vcvars_spectre_libs={}'.format(spectre_arg)) - arglist.append(argpair) + for msvc_sdk_version in msvc.vs_def.vc_sdk_versions: + re_sdk_version = cls.re_sdk_dispatch_map[msvc_sdk_version] + if re_sdk_version.match(sdk_version): + debug('valid: sdk_version=%s', repr(sdk_version)) + return None - return spectre_arg + debug('invalid: method exit: sdk_version=%s', repr(sdk_version)) + err_msg = "MSVC_SDK_VERSION ({}) is not supported".format(repr(sdk_version)) + return err_msg @classmethod - def _msvc_script_argument_user(cls, env, msvc, arglist): + def _msvc_script_argument_sdk(cls, env, msvc, platform, arglist): - # subst None -> empty string - script_args = env.subst('$MSVC_SCRIPT_ARGS') - debug('MSVC_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(msvc.version), repr(script_args)) + sdk_version = env['MSVC_SDK_VERSION'] + debug( + 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform=%s', + repr(msvc.version), repr(sdk_version), repr(platform) + ) - if not script_args: + if not sdk_version: return None - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_SCRIPT_ARGS ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(script_args), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) + err_msg = cls._msvc_script_argument_sdk_constraints(msvc, sdk_version) + if err_msg: + raise MSVCArgumentError(err_msg) + + sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform) + + if sdk_version not in sdk_list: + err_msg = "MSVC_SDK_VERSION {} not found for platform {}".format( + repr(sdk_version), repr(platform) ) raise MSVCArgumentError(err_msg) - # user arguments are not validated - argpair = (cls.SortOrder.USER, script_args) + # sdk folder may not exist + argpair = (cls.SortOrder.SDK, sdk_version) arglist.append(argpair) - return script_args - - re_vcvars_uwp = re.compile(r'(?:\s|^)(?P(?:uwp|store))(?:\s|$)',re.IGNORECASE) - re_vcvars_sdk = re.compile(r'(?:\s|^)(?P(?:[1-9][0-9]*[.]\S*))(?:\s|$)',re.IGNORECASE) - re_vcvars_spectre = re.compile(r'(?:\s|^)(?P(?:[-]{1,2}|[/])vcvars_spectre_libs[=](?P\S*))(?:\s|$)',re.IGNORECASE) - re_vcvars_toolset = re.compile(r'(?:\s|^)(?P(?:[-]{1,2}|[/])vcvars_ver[=](?P\S*))(?:\s|$)', re.IGNORECASE) + return sdk_version @classmethod - def _user_script_argument_uwp(cls, env, uwp, user_argstr): + def _msvc_script_default_sdk(cls, env, msvc, platform, arglist): - if not uwp: + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: return None - m = cls.re_vcvars_uwp.search(user_argstr) - if not m: + sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform) + if not len(sdk_list): return None - env_argstr = env.get('MSVC_UWP_APP','') - debug('multiple uwp declarations: MSVC_UWP_APP=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + sdk_default = sdk_list[0] - err_msg = "multiple uwp declarations: MSVC_UWP_APP={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) + debug( + 'MSVC_VERSION=%s, sdk_default=%s, platform=%s', + repr(msvc.version), repr(sdk_default), repr(platform) ) - raise MSVCArgumentError(err_msg) + argpair = (cls.SortOrder.SDK, sdk_default) + arglist.append(argpair) + + return sdk_default @classmethod def _user_script_argument_sdk(cls, env, sdk_version, user_argstr): - if not sdk_version: + matches = [m for m in cls.re_vcvars_sdk.finditer(user_argstr)] + if not matches: return None - m = cls.re_vcvars_sdk.search(user_argstr) - if not m: - return None + if len(matches) > 1: + debug('multiple sdk version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple sdk version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not sdk_version: + user_sdk = matches[0].group('sdk') + return user_sdk env_argstr = env.get('MSVC_SDK_VERSION','') debug('multiple sdk version declarations: MSVC_SDK_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) @@ -2148,33 +2403,374 @@ class _MSVCScriptArguments: raise MSVCArgumentError(err_msg) @classmethod - def _user_script_argument_toolset(cls, env, toolset_version, user_argstr): + def _msvc_read_toolset_file(cls, msvc, filename): + toolset_version = None + try: + with open(filename) as f: + toolset_version = f.readlines()[0].strip() + debug( + 'msvc_version=%s, filename=%s, toolset_version=%s', + repr(msvc.version), repr(filename), repr(toolset_version) + ) + except OSError: + debug('OSError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) + except IndexError: + debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) + return toolset_version - m = cls.re_vcvars_toolset.search(user_argstr) - if not m: - return None + @classmethod + def _msvc_read_toolset_folders(cls, msvc, vc_dir): - if not toolset_version: - user_toolset = m.group('toolset') - return user_toolset + toolsets_sxs = {} + toolsets_full = [] - env_argstr = env.get('MSVC_TOOLSET_VERSION','') - debug('multiple toolset version declarations: MSVC_TOOLSET_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + build_dir = os.path.join(vc_dir, "Auxiliary", "Build") + sxs_toolsets = [f.name for f in os.scandir(build_dir) if f.is_dir()] + for sxs_toolset in sxs_toolsets: + filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) + filepath = os.path.join(build_dir, sxs_toolset, filename) + debug('sxs toolset: check file=%s', repr(filepath)) + if os.path.exists(filepath): + toolset_version = cls._msvc_read_toolset_file(msvc, filepath) + if not toolset_version: + continue + toolsets_sxs[sxs_toolset] = toolset_version + debug( + 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', + repr(msvc.version), repr(sxs_toolset), toolset_version + ) - err_msg = "multiple toolset version declarations: MSVC_TOOLSET_VERSION={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) + toolset_dir = os.path.join(vc_dir, "Tools", "MSVC") + toolsets = [f.name for f in os.scandir(toolset_dir) if f.is_dir()] + for toolset_version in toolsets: + binpath = os.path.join(toolset_dir, toolset_version, "bin") + debug('toolset: check binpath=%s', repr(binpath)) + if os.path.exists(binpath): + toolsets_full.append(toolset_version) + debug( + 'toolset: msvc_version=%s, toolset_version=%s', + repr(msvc.version), repr(toolset_version) + ) - raise MSVCArgumentError(err_msg) + toolsets_full.sort(reverse=True) + debug('msvc_version=%s, toolsets=%s', repr(msvc.version), repr(toolsets_full)) + + return toolsets_sxs, toolsets_full + + @classmethod + def _msvc_read_toolset_default(cls, msvc, vc_dir): + + build_dir = os.path.join(vc_dir, "Auxiliary", "Build") + + # VS2019+ + filename = "Microsoft.VCToolsVersion.{}.default.txt".format(msvc.vs_def.vc_buildtools_def.vc_buildtools) + filepath = os.path.join(build_dir, filename) + + debug('default toolset: check file=%s', repr(filepath)) + toolset_buildtools = None + if os.path.exists(filepath): + toolset_buildtools = cls._msvc_read_toolset_file(msvc, filepath) + if toolset_buildtools: + return toolset_buildtools + + # VS2017+ + filename = "Microsoft.VCToolsVersion.default.txt" + filepath = os.path.join(build_dir, filename) + + debug('default toolset: check file=%s', repr(filepath)) + toolset_default = None + if os.path.exists(filepath): + toolset_default = cls._msvc_read_toolset_file(msvc, filepath) + if toolset_default: + return toolset_default + + return None + + @classmethod + def _reset_toolsets(cls): + debug('reset: toolset cache') + cls._toolset_version_cache = {} + cls._toolset_default_cache = {} + + _toolset_version_cache = {} + _toolset_default_cache = {} + + @classmethod + def _msvc_version_toolsets(cls, msvc, vc_dir): + + if msvc.version in cls._toolset_version_cache: + toolsets_sxs, toolsets_full = cls._toolset_version_cache[msvc.version] + else: + toolsets_sxs, toolsets_full = cls._msvc_read_toolset_folders(msvc, vc_dir) + cls._toolset_version_cache[msvc.version] = toolsets_sxs, toolsets_full + + return toolsets_sxs, toolsets_full + + @classmethod + def _msvc_default_toolset(cls, msvc, vc_dir): + + if msvc.version in cls._toolset_default_cache: + toolset_default = cls._toolset_default_cache[msvc.version] + else: + toolset_default = cls._msvc_read_toolset_default(msvc, vc_dir) + cls._toolset_default_cache[msvc.version] = toolset_default + + return toolset_default + + @classmethod + def _msvc_version_toolset_vcvars(cls, msvc, vc_dir, toolset_version): + + if toolset_version == '14.0': + return toolset_version + + toolsets_sxs, toolsets_full = cls._msvc_version_toolsets(msvc, vc_dir) + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric == cls.VS2019.vc_buildtools_def.vc_version_numeric: + # necessary to detect toolset not found + if toolset_version == '14.28.16.8': + new_toolset_version = '14.28' + # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 + # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files + # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. + debug( + 'rewrite toolset_version=%s => toolset_version=%s', + repr(toolset_version), repr(new_toolset_version) + ) + toolset_version = new_toolset_version + + if toolset_version in toolsets_sxs: + toolset_vcvars = toolsets_sxs[toolset_version] + return toolset_vcvars + + for toolset_full in toolsets_full: + if toolset_full.startswith(toolset_version): + toolset_vcvars = toolset_full + return toolset_vcvars + + return None + + @classmethod + def _msvc_script_argument_toolset_constraints(cls, msvc, toolset_version): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2017', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( + repr(toolset_version), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) + ) + return err_msg + + m = cls.re_toolset_version.match(toolset_version) + if not m: + debug('invalid: re_toolset_version: toolset_version=%s', repr(toolset_version)) + err_msg = 'MSVC_TOOLSET_VERSION {} format is not supported'.format( + repr(toolset_version) + ) + return err_msg + + toolset_ver = m.group('version') + toolset_vernum = float(toolset_ver) + + if toolset_vernum < cls.VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: toolset version constraint: %s < %s VS2015', + repr(toolset_vernum), repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} < {} VS2015".format( + repr(toolset_version), repr(toolset_ver), repr(cls.VS2015.vc_buildtools_def.vc_version) + ) + return err_msg + + if toolset_vernum > msvc.vs_def.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: toolset version constraint: toolset %s > %s msvc', + repr(toolset_vernum), repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} > {} MSVC_VERSION".format( + repr(toolset_version), repr(toolset_ver), repr(msvc.version) + ) + return err_msg + + if toolset_vernum == cls.VS2015.vc_buildtools_def.vc_version_numeric: + # tooset = 14.0 + if cls.re_toolset_full.match(toolset_version): + if not cls.re_toolset_140.match(toolset_version): + debug( + 'invalid: toolset version 14.0 constraint: %s != 14.0', + repr(toolset_version) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} != '14.0'".format( + repr(toolset_version), repr(toolset_version) + ) + return err_msg + return None + + if cls.re_toolset_full.match(toolset_version): + debug('valid: re_toolset_full: toolset_version=%s', repr(toolset_version)) + return None + + if cls.re_toolset_sxs.match(toolset_version): + debug('valid: re_toolset_sxs: toolset_version=%s', repr(toolset_version)) + return None + + debug('invalid: method exit: toolset_version=%s', repr(toolset_version)) + err_msg = "MSVC_TOOLSET_VERSION ({}) format is not supported".format(repr(toolset_version)) + return err_msg + + @classmethod + def _msvc_script_argument_toolset(cls, env, msvc, vc_dir, arglist): + + toolset_version = env['MSVC_TOOLSET_VERSION'] + debug('MSVC_VERSION=%s, MSVC_TOOLSET_VERSION=%s', repr(msvc.version), repr(toolset_version)) + + if not toolset_version: + return None + + err_msg = cls._msvc_script_argument_toolset_constraints(msvc, toolset_version) + if err_msg: + raise MSVCArgumentError(err_msg) + + if toolset_version.startswith('14.0') and len(toolset_version) > len('14.0'): + new_toolset_version = '14.0' + debug( + 'rewrite toolset_version=%s => toolset_version=%s', + repr(toolset_version), repr(new_toolset_version) + ) + toolset_version = new_toolset_version + + toolset_vcvars = cls._msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version) + debug( + 'toolset: toolset_version=%s, toolset_vcvars=%s', + repr(toolset_version), repr(toolset_vcvars) + ) + + if not toolset_vcvars: + err_msg = "MSVC_TOOLSET_VERSION {} not found for MSVC_VERSION {}".format( + repr(toolset_version), repr(msvc.version) + ) + raise MSVCArgumentError(err_msg) + + argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_vcvars)) + arglist.append(argpair) + + return toolset_vcvars + + @classmethod + def _msvc_script_default_toolset(cls, env, msvc, vc_dir, arglist): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: + return None + + toolset_default = cls._msvc_default_toolset(msvc, vc_dir) + if not toolset_default: + return None + + debug('MSVC_VERSION=%s, toolset_default=%s', repr(msvc.version), repr(toolset_default)) + + argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_default)) + arglist.append(argpair) + + return toolset_default + + @classmethod + def _user_script_argument_toolset(cls, env, toolset_version, user_argstr): + + matches = [m for m in cls.re_vcvars_toolset.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple toolset version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple toolset version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not toolset_version: + user_toolset = matches[0].group('toolset') + return user_toolset + + env_argstr = env.get('MSVC_TOOLSET_VERSION','') + debug('multiple toolset version declarations: MSVC_TOOLSET_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple toolset version declarations: MSVC_TOOLSET_VERSION={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + + @classmethod + def _msvc_script_argument_spectre(cls, env, msvc, arglist): + + spectre_libs = env['MSVC_SPECTRE_LIBS'] + debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) + + if not spectre_libs: + return None + + if spectre_libs not in (True, '1'): + return None + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2017', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( + repr(spectre_libs), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) + ) + raise MSVCArgumentError(err_msg) + + spectre_arg = 'spectre' + + # spectre libs may not be installed + argpair = (cls.SortOrder.SPECTRE, '-vcvars_spectre_libs={}'.format(spectre_arg)) + arglist.append(argpair) + + return spectre_arg + + @classmethod + def _msvc_script_argument_user(cls, env, msvc, arglist): + + # subst None -> empty string + script_args = env.subst('$MSVC_SCRIPT_ARGS') + debug('MSVC_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(msvc.version), repr(script_args)) + + if not script_args: + return None + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2015', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SCRIPT_ARGS ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(script_args), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) + ) + raise MSVCArgumentError(err_msg) + + # user arguments are not validated + argpair = (cls.SortOrder.USER, script_args) + arglist.append(argpair) + + return script_args @classmethod def _user_script_argument_spectre(cls, env, spectre, user_argstr): - if not spectre: + matches = [m for m in cls.re_vcvars_spectre.finditer(user_argstr)] + if not matches: return None - m = cls.re_vcvars_spectre.search(user_argstr) - if not m: + if len(matches) > 1: + debug('multiple spectre declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple spectre declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not spectre: return None env_argstr = env.get('MSVC_SPECTRE_LIBS','') @@ -2189,55 +2785,70 @@ class _MSVCScriptArguments: @classmethod def msvc_script_arguments(cls, env, version, vc_dir, arg): - msvc = cls._msvc_version(version) - - argstr = '' arglist = [] + msvc = cls._msvc_version(version) + if arg: argpair = (cls.SortOrder.ARCH, arg) arglist.append(argpair) - user_argstr = None - user_toolset = None - - uwp = None - sdk_version = None - toolset_version = None - spectre = None - if 'MSVC_SCRIPT_ARGS' in env: user_argstr = cls._msvc_script_argument_user(env, msvc, arglist) + else: + user_argstr = None if 'MSVC_UWP_APP' in env: uwp = cls._msvc_script_argument_uwp(env, msvc, arglist) - if uwp and user_argstr: - cls._user_script_argument_uwp(env, uwp, user_argstr) + else: + uwp = None + + if user_argstr: + cls._user_script_argument_uwp(env, uwp, user_argstr) + + platform = 'uwp' if uwp else 'desktop' if 'MSVC_SDK_VERSION' in env: - is_uwp = True if uwp else False - sdk_version = cls._msvc_script_argument_sdk(env, msvc, is_uwp, arglist) - if sdk_version and user_argstr: - cls._user_script_argument_sdk(env, sdk_version, user_argstr) + sdk_version = cls._msvc_script_argument_sdk(env, msvc, platform, arglist) + else: + sdk_version = None + + if user_argstr: + user_sdk = cls._user_script_argument_sdk(env, sdk_version, user_argstr) + else: + user_sdk = None + + if cls.MSVC_FORCE_DEFAULT_SDK: + if not sdk_version and not user_sdk: + sdk_version = cls._msvc_script_default_sdk(env, msvc, platform, arglist) if 'MSVC_TOOLSET_VERSION' in env: toolset_version = cls._msvc_script_argument_toolset(env, msvc, vc_dir, arglist) + else: + toolset_version = None if user_argstr: user_toolset = cls._user_script_argument_toolset(env, toolset_version, user_argstr) + else: + user_toolset = None - if cls.MSVC_TOOLSET_DEFAULT_VCVARSVER: + if cls.MSVC_FORCE_DEFAULT_TOOLSET: if not toolset_version and not user_toolset: toolset_version = cls._msvc_script_default_toolset(env, msvc, vc_dir, arglist) if 'MSVC_SPECTRE_LIBS' in env: spectre = cls._msvc_script_argument_spectre(env, msvc, arglist) - if spectre and user_argstr: - cls._user_script_argument_spectre(env, spectre, user_argstr) + else: + spectre = None + + if user_argstr: + cls._user_script_argument_spectre(env, spectre, user_argstr) if arglist: arglist.sort() argstr = ' '.join([argpair[-1] for argpair in arglist]).strip() + else: + argstr = '' debug('arguments: %s', repr(argstr)) return argstr @@ -2247,231 +2858,3 @@ class _MSVCScriptArguments: debug('reset') cls._reset_toolsets() -def msvc_find_valid_batch_script(env, version): - """Find and execute appropriate batch script to set up build env. - - The MSVC build environment depends heavily on having the shell - environment set. SCons does not inherit that, and does not count - on that being set up correctly anyway, so it tries to find the right - MSVC batch script, or the right arguments to the generic batch script - vcvarsall.bat, and run that, so we have a valid environment to build in. - There are dragons here: the batch scripts don't fail (see comments - elsewhere), they just leave you with a bad setup, so try hard to - get it right. - """ - - # Find the host, target, and all candidate (host, target) platform combinations: - platforms = get_host_target(env, version) - debug("host_platform %s, target_platform %s host_target_list %s", *platforms) - host_platform, target_platform, host_target_list = platforms - - d = None - version_installed = False - for host_arch, target_arch, in host_target_list: - # Set to current arch. - env['TARGET_ARCH'] = target_arch - arg = '' - - # Try to locate a batch file for this host/target platform combo - try: - (vc_script, arg, vc_dir, sdk_script) = find_batch_file(env, version, host_arch, target_arch) - debug('vc_script:%s vc_script_arg:%s sdk_script:%s', vc_script, arg, sdk_script) - version_installed = True - except VisualCException as e: - msg = str(e) - debug('Caught exception while looking for batch file (%s)', msg) - version_installed = False - continue - - # Try to use the located batch file for this host/target platform combo - debug('use_script 2 %s, args:%s', repr(vc_script), arg) - found = None - if vc_script: - arg = _MSVCScriptArguments.msvc_script_arguments(env, version, vc_dir, arg) - try: - d = script_env(vc_script, args=arg) - found = vc_script - except BatchFileExecutionError as e: - debug('use_script 3: failed running VC script %s: %s: Error:%s', repr(vc_script), arg, e) - vc_script=None - continue - if not vc_script and sdk_script: - debug('use_script 4: trying sdk script: %s', sdk_script) - try: - d = script_env(sdk_script) - found = sdk_script - except BatchFileExecutionError as e: - debug('use_script 5: failed running SDK script %s: Error:%s', repr(sdk_script), e) - continue - elif not vc_script and not sdk_script: - debug('use_script 6: Neither VC script nor SDK script found') - continue - - debug("Found a working script/target: %s/%s", repr(found), arg) - break # We've found a working target_platform, so stop looking - - # If we cannot find a viable installed compiler, reset the TARGET_ARCH - # To it's initial value - if not d: - env['TARGET_ARCH'] = target_platform - - if version_installed: - msg = "MSVC version '{}' working host/target script was not found.\n" \ - " Host = '{}', Target = '{}'\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format( - version, host_platform, target_platform - ) - else: - installed_vcs = get_installed_vcs(env) - if installed_vcs: - msg = "MSVC version '{}' was not found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly.\n" \ - " Installed versions are: {}".format(version, installed_vcs) - else: - msg = "MSVC version '{}' was not found.\n" \ - " No versions of the MSVC compiler were found.\n" \ - " Visual Studio C/C++ compilers may not be set correctly".format(version) - - _msvc_notfound_policy_handler(env, msg) - - return d - -_undefined = None - -def get_use_script_use_settings(env): - global _undefined - - if _undefined is None: - _undefined = object() - - # use_script use_settings return values action - # value ignored (value, None) use script or bypass detection - # undefined value not None (False, value) use dictionary - # undefined undefined/None (True, None) msvc detection - - # None (documentation) or evaluates False (code): bypass detection - # need to distinguish between undefined and None - use_script = env.get('MSVC_USE_SCRIPT', _undefined) - - if use_script != _undefined: - # use_script defined, use_settings ignored (not type checked) - return (use_script, None) - - # undefined or None: use_settings ignored - use_settings = env.get('MSVC_USE_SETTINGS', None) - - if use_settings is not None: - # use script undefined, use_settings defined and not None (type checked) - return (False, use_settings) - - # use script undefined, use_settings undefined or None - return (True, None) - -def msvc_setup_env(env): - debug('called') - version = get_default_version(env) - if version is None: - if not msvc_setup_env_user(env): - _MSVCSetupEnvDefault.set_nodefault() - return None - - # XXX: we set-up both MSVS version for backward - # compatibility with the msvs tool - env['MSVC_VERSION'] = version - env['MSVS_VERSION'] = version - env['MSVS'] = {} - - use_script, use_settings = get_use_script_use_settings(env) - if SCons.Util.is_String(use_script): - use_script = use_script.strip() - if not os.path.exists(use_script): - raise MSVCScriptNotFound('Script specified by MSVC_USE_SCRIPT not found: "{}"'.format(use_script)) - args = env.subst('$MSVC_USE_SCRIPT_ARGS') - debug('use_script 1 %s %s', repr(use_script), repr(args)) - d = script_env(use_script, args) - elif use_script: - d = msvc_find_valid_batch_script(env,version) - debug('use_script 2 %s', d) - if not d: - return d - elif use_settings is not None: - if not SCons.Util.is_Dict(use_settings): - error_msg = 'MSVC_USE_SETTINGS type error: expected a dictionary, found {}'.format(type(use_settings).__name__) - raise MSVCUseSettingsError(error_msg) - d = use_settings - debug('use_settings %s', d) - else: - debug('MSVC_USE_SCRIPT set to False') - warn_msg = "MSVC_USE_SCRIPT set to False, assuming environment " \ - "set correctly." - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) - return None - - for k, v in d.items(): - env.PrependENVPath(k, v, delete_existing=True) - debug("env['ENV']['%s'] = %s", k, env['ENV'][k]) - - # final check to issue a warning if the compiler is not present - if not find_program_path(env, 'cl'): - debug("did not find %s", _CL_EXE_NAME) - if CONFIG_CACHE: - propose = "SCONS_CACHE_MSVC_CONFIG caching enabled, remove cache file {} if out of date.".format(CONFIG_CACHE) - else: - propose = "It may need to be installed separately with Visual Studio." - warn_msg = "Could not find MSVC compiler 'cl'. {}".format(propose) - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) - -def msvc_exists(env=None, version=None): - debug('version=%s', repr(version)) - vcs = get_installed_vcs(env) - if version is None: - rval = len(vcs) > 0 - else: - rval = version in vcs - debug('version=%s, return=%s', repr(version), rval) - return rval - -def msvc_setup_env_user(env=None): - rval = False - if env: - - # Intent is to use msvc tools: - # MSVC_VERSION or MSVS_VERSION: defined and is True - # MSVC_USE_SCRIPT: defined and (is string or is False) - # MSVC_USE_SETTINGS: defined and is not None - - # defined and is True - for key in ['MSVC_VERSION', 'MSVS_VERSION']: - if key in env and env[key]: - rval = True - debug('key=%s, return=%s', repr(key), rval) - return rval - - # defined and (is string or is False) - for key in ['MSVC_USE_SCRIPT']: - if key in env and (SCons.Util.is_String(env[key]) or not env[key]): - rval = True - debug('key=%s, return=%s', repr(key), rval) - return rval - - # defined and is not None - for key in ['MSVC_USE_SETTINGS']: - if key in env and env[key] is not None: - rval = True - debug('key=%s, return=%s', repr(key), rval) - return rval - - debug('return=%s', rval) - return rval - -def msvc_setup_env_tool(env=None, version=None, tool=None): - debug('tool=%s, version=%s', repr(tool), repr(version)) - _MSVCSetupEnvDefault.register_tool(env, tool) - rval = False - if not rval and msvc_exists(env, version): - rval = True - if not rval and msvc_setup_env_user(env): - rval = True - debug('tool=%s, version=%s, return=%s', repr(tool), repr(version), rval) - return rval - -- cgit v0.12 From 33bef70b9ccaac0106d5e7f0c37211d224b6a585 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 16 Jun 2022 12:43:22 -0400 Subject: Rename msvc batch platform (uwp/store/desktop) to platform_type. --- SCons/Tool/MSCommon/vc.py | 50 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index c44c698..acffc7c 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -106,7 +106,7 @@ class _Const: for bool, symbol_list in [ (False, (0, '0', False, 'False', 'FALSE', 'false', 'No', 'NO', 'no', None, '')), - (True, (1, '1', True, 'True', 'TRUE', 'true', 'Yes', 'YES', 'yes', )), + (True, (1, '1', True, 'True', 'TRUE', 'true', 'Yes', 'YES', 'yes')), ]: BOOLEAN_KEYS[bool] = symbol_list for symbol in symbol_list: @@ -2007,7 +2007,7 @@ class _WindowsSDK: if not os.path.exists(sdk_inc_path): continue - for platform, sdk_inc_file in [ + for platform_type, sdk_inc_file in [ ('desktop', 'winsdkver.h'), ('uwp', 'windows.h'), ]: @@ -2015,12 +2015,12 @@ class _WindowsSDK: if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): continue - key = (version_nbr, platform) + key = (version_nbr, platform_type) if key in sdk_version_platform_seen: continue sdk_version_platform_seen.add(key) - sdk_map[platform].append(version_nbr) + sdk_map[platform_type].append(version_nbr) for key, val in sdk_map.items(): val.sort(reverse=True) @@ -2039,8 +2039,6 @@ class _WindowsSDK: sdk_version_platform_seen = set() sdk_roots_seen = set() - sdk_targets = [] - for sdk_t in sdk_roots: sdk_root = sdk_t[0] @@ -2054,7 +2052,7 @@ class _WindowsSDK: if not os.path.exists(sdk_inc_path): continue - for platform, sdk_inc_file in [ + for platform_type, sdk_inc_file in [ ('desktop', 'winsdkver.h'), ('uwp', 'windows.h'), ]: @@ -2062,12 +2060,12 @@ class _WindowsSDK: if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): continue - key = (version_nbr, platform) + key = (version_nbr, platform_type) if key in sdk_version_platform_seen: continue sdk_version_platform_seen.add(key) - sdk_map[platform].append(version_nbr) + sdk_map[platform_type].append(version_nbr) for key, val in sdk_map.items(): val.sort(reverse=True) @@ -2134,9 +2132,9 @@ class _WindowsSDK: return sdk_map @classmethod - def _get_sdk_version_list(cls, version_list, platform): + def _get_sdk_version_list(cls, version_list, platform_type): sdk_map = cls._sdk_map(version_list) - sdk_list = sdk_map.get(platform, []) + sdk_list = sdk_map.get(platform_type, []) return sdk_list def get_sdk_versions(MSVC_VERSION=None, MSVC_UWP_APP=False): @@ -2155,8 +2153,8 @@ def get_sdk_versions(MSVC_VERSION=None, MSVC_UWP_APP=False): return sdk_versions is_uwp = True if MSVC_UWP_APP in _Const.BOOLEAN_KEYS[True] else False - platform = 'uwp' if is_uwp else 'desktop' - sdk_list = _WindowsSDK._get_sdk_version_list(vs_def.vc_sdk_versions, platform) + platform_type = 'uwp' if is_uwp else 'desktop' + sdk_list = _WindowsSDK._get_sdk_version_list(vs_def.vc_sdk_versions, platform_type) sdk_versions.extend(sdk_list) return sdk_versions @@ -2326,12 +2324,12 @@ class _ScriptArguments: return err_msg @classmethod - def _msvc_script_argument_sdk(cls, env, msvc, platform, arglist): + def _msvc_script_argument_sdk(cls, env, msvc, platform_type, arglist): sdk_version = env['MSVC_SDK_VERSION'] debug( - 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform=%s', - repr(msvc.version), repr(sdk_version), repr(platform) + 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform_type=%s', + repr(msvc.version), repr(sdk_version), repr(platform_type) ) if not sdk_version: @@ -2341,11 +2339,11 @@ class _ScriptArguments: if err_msg: raise MSVCArgumentError(err_msg) - sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform) + sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) if sdk_version not in sdk_list: - err_msg = "MSVC_SDK_VERSION {} not found for platform {}".format( - repr(sdk_version), repr(platform) + err_msg = "MSVC_SDK_VERSION {} not found for platform type {}".format( + repr(sdk_version), repr(platform_type) ) raise MSVCArgumentError(err_msg) @@ -2356,20 +2354,20 @@ class _ScriptArguments: return sdk_version @classmethod - def _msvc_script_default_sdk(cls, env, msvc, platform, arglist): + def _msvc_script_default_sdk(cls, env, msvc, platform_type, arglist): if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: return None - sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform) + sdk_list = _WindowsSDK._get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) if not len(sdk_list): return None sdk_default = sdk_list[0] debug( - 'MSVC_VERSION=%s, sdk_default=%s, platform=%s', - repr(msvc.version), repr(sdk_default), repr(platform) + 'MSVC_VERSION=%s, sdk_default=%s, platform_type=%s', + repr(msvc.version), repr(sdk_default), repr(platform_type) ) argpair = (cls.SortOrder.SDK, sdk_default) @@ -2806,10 +2804,10 @@ class _ScriptArguments: if user_argstr: cls._user_script_argument_uwp(env, uwp, user_argstr) - platform = 'uwp' if uwp else 'desktop' + platform_type = 'uwp' if uwp else 'desktop' if 'MSVC_SDK_VERSION' in env: - sdk_version = cls._msvc_script_argument_sdk(env, msvc, platform, arglist) + sdk_version = cls._msvc_script_argument_sdk(env, msvc, platform_type, arglist) else: sdk_version = None @@ -2820,7 +2818,7 @@ class _ScriptArguments: if cls.MSVC_FORCE_DEFAULT_SDK: if not sdk_version and not user_sdk: - sdk_version = cls._msvc_script_default_sdk(env, msvc, platform, arglist) + sdk_version = cls._msvc_script_default_sdk(env, msvc, platform_type, arglist) if 'MSVC_TOOLSET_VERSION' in env: toolset_version = cls._msvc_script_argument_toolset(env, msvc, vc_dir, arglist) -- cgit v0.12 From 716f93a5207d8e7c941f3ff38ef2b3b0c923c479 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 16 Jun 2022 13:04:52 -0400 Subject: Update comments --- SCons/Tool/MSCommon/vc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index acffc7c..e29fba4 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -2269,7 +2269,7 @@ class _ScriptArguments: # VS2017+ rewrites uwp => store for 14.0 toolset uwp_arg = msvc.vs_def.vc_uwp - # uwp may not be installed + # store/uwp may not be fully installed argpair = (cls.SortOrder.UWP, uwp_arg) arglist.append(argpair) @@ -2347,7 +2347,6 @@ class _ScriptArguments: ) raise MSVCArgumentError(err_msg) - # sdk folder may not exist argpair = (cls.SortOrder.SDK, sdk_version) arglist.append(argpair) -- cgit v0.12 From e2033e5f52002d404383c97e3a6a010efef174c8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 9 Jun 2022 11:30:47 -0600 Subject: Fortran vairants now include FORTRANCOMMONFLAGS The documentation suggested $FORTRANFLAGS was included in the build lines for all variants, but it was not. Turns out the test suite quite explicitly checks that FORTRANFLAGS doesn't leak through to other variants, so instead define a new FORTRANCOMMONFLAGS to serve the purpose of a flag that applies to all variants. Assorted cleanup. And f-strings. Fixes #2257 Signed-off-by: Mats Wichmann --- CHANGES.txt | 6 ++ RELEASE.txt | 2 + SCons/Tool/FortranCommon.py | 178 ++++++++++++++++---------------- SCons/Tool/f03.xml | 5 +- SCons/Tool/f08.xml | 5 +- SCons/Tool/f77.xml | 7 +- SCons/Tool/f90.xml | 5 +- SCons/Tool/f95.xml | 5 +- SCons/Tool/fortran.xml | 13 ++- SCons/Tool/g77.py | 2 - SCons/Tool/g77.xml | 5 +- SCons/Tool/gfortran.py | 18 ++-- SCons/Tool/gfortran.xml | 19 +--- SCons/Tool/ifl.py | 21 ++-- SCons/Tool/ifort.py | 13 +-- test/Fortran/F77PATH.py | 78 ++++++-------- test/Fortran/F90PATH.py | 66 +++++------- test/Fortran/FORTRANCOMMONFLAGS.py | 132 +++++++++++++++++++++++ test/Fortran/FORTRANFILESUFFIXES2.py | 43 ++++---- test/Fortran/FORTRANPATH.py | 54 +++++----- test/Fortran/fixture/myfortran.py | 17 +-- test/Fortran/fixture/myfortran_flags.py | 17 +-- 22 files changed, 406 insertions(+), 305 deletions(-) create mode 100644 test/Fortran/FORTRANCOMMONFLAGS.py diff --git a/CHANGES.txt b/CHANGES.txt index 28785e1..d221603 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -194,6 +194,12 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER to add a path to the execution environment if it was discovered in default_paths. Previously, the routine, called by many tool modules, never altered the execution environment, leaving it to the tools. + - A new construction variable FORTRANCOMMONFLAGS is added which is + applied to all Fortran dialects, in case someone needs to set some + flags globally. FORTRANFLAGS looked like it was intended for that, + but was not applied to other dialects, and e2e tests explicitly checked + that FORTRANFLAGS did not propagate outside the FORTRAN dialect, + so the conclusion is that behavior is intentional (issue #2257) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/RELEASE.txt b/RELEASE.txt index abecf39..4f42e8c 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,6 +43,8 @@ NEW FUNCTIONALITY is taken and the constructed environment is likely incomplete. As implemented, the default global policy is "warning". The ability to set the global policy via an SCons command-line option may be added in a future enhancement. +- Fortran: a new construction variable FORTRANCOMMONFLAGS is added which is + applied to all Fortran dialects, to enable global (all-dialect) settings. DEPRECATED FUNCTIONALITY diff --git a/SCons/Tool/FortranCommon.py b/SCons/Tool/FortranCommon.py index cad16b6..f889383 100644 --- a/SCons/Tool/FortranCommon.py +++ b/SCons/Tool/FortranCommon.py @@ -21,43 +21,48 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -""" - -Stuff for processing Fortran, common to all fortran dialects. - -""" +"""Routines for setting up Fortran, common to all dialects.""" import re import os.path +from typing import Tuple -import SCons.Action import SCons.Scanner.Fortran import SCons.Tool import SCons.Util +from SCons.Action import Action + +def isfortran(env, source) -> bool: + """Returns True if source has any fortran files in it. -def isfortran(env, source): - """Return 1 if any of code in source has fortran files in it, 0 - otherwise.""" + Only checks based on filename suffixes, does not examine code. + """ try: fsuffixes = env['FORTRANSUFFIXES'] except KeyError: # If no FORTRANSUFFIXES, no fortran tool, so there is no need to look # for fortran sources. - return 0 + return False if not source: # Source might be None for unusual cases like SConf. - return 0 + return False for s in source: if s.sources: ext = os.path.splitext(str(s.sources[0]))[1] if ext in fsuffixes: - return 1 - return 0 + return True + return False -def _fortranEmitter(target, source, env): +def _fortranEmitter(target, source, env) -> Tuple: + """Common code for Fortran emitter. + + Called by both the static and shared object emitters, + mainly to account for generated module files. + """ + node = source[0].rfile() if not node.exists() and not node.is_derived(): print("Could not locate " + str(node.name)) @@ -78,21 +83,29 @@ def _fortranEmitter(target, source, env): return (target, source) -def FortranEmitter(target, source, env): +def FortranEmitter(target, source, env) -> Tuple: import SCons.Defaults target, source = _fortranEmitter(target, source, env) return SCons.Defaults.StaticObjectEmitter(target, source, env) -def ShFortranEmitter(target, source, env): +def ShFortranEmitter(target, source, env) -> Tuple: import SCons.Defaults target, source = _fortranEmitter(target, source, env) return SCons.Defaults.SharedObjectEmitter(target, source, env) -def ComputeFortranSuffixes(suffixes, ppsuffixes): - """suffixes are fortran source files, and ppsuffixes the ones to be - pre-processed. Both should be sequences, not strings.""" +def ComputeFortranSuffixes(suffixes, ppsuffixes) -> None: + """Update the suffix lists to reflect the platform requirements. + + If upper-cased suffixes can be distinguished from lower, those are + added to *ppsuffixes*. If not, they are added to *suffixes*. + + Args: + suffixes (list): indicate regular Fortran source files + ppsuffixes (list): indicate Fortran source files that should be + be run through the pre-processor + """ assert len(suffixes) > 0 s = suffixes[0] sup = s.upper() @@ -102,31 +115,34 @@ def ComputeFortranSuffixes(suffixes, ppsuffixes): else: suffixes.extend(upper_suffixes) - -def CreateDialectActions(dialect): +def CreateDialectActions(dialect) -> Tuple[Action, Action, Action, Action]: """Create dialect specific actions.""" - CompAction = SCons.Action.Action('$%sCOM ' % dialect, '$%sCOMSTR' % dialect) - CompPPAction = SCons.Action.Action('$%sPPCOM ' % dialect, '$%sPPCOMSTR' % dialect) - ShCompAction = SCons.Action.Action('$SH%sCOM ' % dialect, '$SH%sCOMSTR' % dialect) - ShCompPPAction = SCons.Action.Action('$SH%sPPCOM ' % dialect, '$SH%sPPCOMSTR' % dialect) - + CompAction = Action(f'${dialect}COM ', cmdstr=f'${dialect}COMSTR') + CompPPAction = Action(f'${dialect}PPCOM ', cmdstr=f'${dialect}PPCOMSTR') + ShCompAction = Action(f'$SH{dialect}COM ', cmdstr=f'$SH{dialect}COMSTR') + ShCompPPAction = Action(f'$SH{dialect}PPCOM ', cmdstr=f'$SH{dialect}PPCOMSTR') return CompAction, CompPPAction, ShCompAction, ShCompPPAction -def DialectAddToEnv(env, dialect, suffixes, ppsuffixes, support_module=False): - """Add dialect specific construction variables.""" - ComputeFortranSuffixes(suffixes, ppsuffixes) +def DialectAddToEnv(env, dialect, suffixes, ppsuffixes, support_mods=False) -> None: + """Add dialect specific construction variables. - fscan = SCons.Scanner.Fortran.FortranScan("%sPATH" % dialect) + Args: + dialect (str): dialect name + suffixes (list): suffixes associated with this dialect + ppsuffixes (list): suffixes using cpp associated with this dialect + support_mods (bool): whether this dialect supports modules + """ + ComputeFortranSuffixes(suffixes, ppsuffixes) + fscan = SCons.Scanner.Fortran.FortranScan(f"{dialect}PATH") for suffix in suffixes + ppsuffixes: SCons.Tool.SourceFileScanner.add_scanner(suffix, fscan) - env.AppendUnique(FORTRANSUFFIXES = suffixes + ppsuffixes) + env.AppendUnique(FORTRANSUFFIXES=suffixes + ppsuffixes) compaction, compppaction, shcompaction, shcompppaction = \ CreateDialectActions(dialect) - static_obj, shared_obj = SCons.Tool.createObjBuilders(env) for suffix in suffixes: @@ -141,64 +157,60 @@ def DialectAddToEnv(env, dialect, suffixes, ppsuffixes, support_module=False): static_obj.add_emitter(suffix, FortranEmitter) shared_obj.add_emitter(suffix, ShFortranEmitter) - if '%sFLAGS' % dialect not in env: - env['%sFLAGS' % dialect] = SCons.Util.CLVar('') - - if 'SH%sFLAGS' % dialect not in env: - env['SH%sFLAGS' % dialect] = SCons.Util.CLVar('$%sFLAGS' % dialect) + if f'{dialect}FLAGS' not in env: + env[f'{dialect}FLAGS'] = SCons.Util.CLVar('') + if f'SH{dialect}FLAGS' not in env: + env[f'SH{dialect}FLAGS'] = SCons.Util.CLVar(f'${dialect}FLAGS') # If a tool does not define fortran prefix/suffix for include path, use C ones - if 'INC%sPREFIX' % dialect not in env: - env['INC%sPREFIX' % dialect] = '$INCPREFIX' - - if 'INC%sSUFFIX' % dialect not in env: - env['INC%sSUFFIX' % dialect] = '$INCSUFFIX' - - env['_%sINCFLAGS' % dialect] = '${_concat(INC%sPREFIX, %sPATH, INC%sSUFFIX, __env__, RDirs, TARGET, SOURCE, affect_signature=False)}' % (dialect, dialect, dialect) - - if support_module: - env['%sCOM' % dialect] = '$%s -o $TARGET -c $%sFLAGS $_%sINCFLAGS $_FORTRANMODFLAG $SOURCES' % (dialect, dialect, dialect) - env['%sPPCOM' % dialect] = '$%s -o $TARGET -c $%sFLAGS $CPPFLAGS $_CPPDEFFLAGS $_%sINCFLAGS $_FORTRANMODFLAG $SOURCES' % (dialect, dialect, dialect) - env['SH%sCOM' % dialect] = '$SH%s -o $TARGET -c $SH%sFLAGS $_%sINCFLAGS $_FORTRANMODFLAG $SOURCES' % (dialect, dialect, dialect) - env['SH%sPPCOM' % dialect] = '$SH%s -o $TARGET -c $SH%sFLAGS $CPPFLAGS $_CPPDEFFLAGS $_%sINCFLAGS $_FORTRANMODFLAG $SOURCES' % (dialect, dialect, dialect) + if f'INC{dialect}PREFIX' not in env: + env[f'INC{dialect}PREFIX'] = '$INCPREFIX' + if f'INC{dialect}SUFFIX' not in env: + env[f'INC{dialect}SUFFIX'] = '$INCSUFFIX' + + env[f'_{dialect}INCFLAGS'] = f'${{_concat(INC{dialect}PREFIX, {dialect}PATH, INC{dialect}SUFFIX, __env__, RDirs, TARGET, SOURCE, affect_signature=False)}}' + + if support_mods: + env[f'{dialect}COM'] = f'${dialect} -o $TARGET -c $FORTRANCOMMONFLAGS ${dialect}FLAGS $_{dialect}INCFLAGS $_FORTRANMODFLAG $SOURCES #XXX{dialect}' + env[f'{dialect}PPCOM'] = f'${dialect} -o $TARGET -c $FORTRANCOMMONFLAGS ${dialect}FLAGS $CPPFLAGS $_CPPDEFFLAGS $_{dialect}INCFLAGS $_FORTRANMODFLAG $SOURCES #XXXPP{dialect}' + env[f'SH{dialect}COM'] = f'$SH{dialect} -o $TARGET -c $FORTRANCOMMONFLAGS $SH{dialect}FLAGS $_{dialect}INCFLAGS $_FORTRANMODFLAG $SOURCES #XXX{dialect}' + env[f'SH{dialect}PPCOM'] = f'$SH{dialect} -o $TARGET -c $FORTRANCOMMONFLAGS $SH{dialect}FLAGS $CPPFLAGS $_CPPDEFFLAGS $_{dialect}INCFLAGS $_FORTRANMODFLAG $SOURCES #XXXPP{dialect}' else: - env['%sCOM' % dialect] = '$%s -o $TARGET -c $%sFLAGS $_%sINCFLAGS $SOURCES' % (dialect, dialect, dialect) - env['%sPPCOM' % dialect] = '$%s -o $TARGET -c $%sFLAGS $CPPFLAGS $_CPPDEFFLAGS $_%sINCFLAGS $SOURCES' % (dialect, dialect, dialect) - env['SH%sCOM' % dialect] = '$SH%s -o $TARGET -c $SH%sFLAGS $_%sINCFLAGS $SOURCES' % (dialect, dialect, dialect) - env['SH%sPPCOM' % dialect] = '$SH%s -o $TARGET -c $SH%sFLAGS $CPPFLAGS $_CPPDEFFLAGS $_%sINCFLAGS $SOURCES' % (dialect, dialect, dialect) + env[f'{dialect}COM'] = f'${dialect} -o $TARGET -c $FORTRANCOMMONFLAGS ${dialect}FLAGS $_{dialect}INCFLAGS $SOURCES #XXX{dialect}' + env[f'{dialect}PPCOM'] = f'${dialect} -o $TARGET -c $FORTRANCOMMONFLAGS ${dialect}FLAGS $CPPFLAGS $_CPPDEFFLAGS $_{dialect}INCFLAGS $SOURCES #XXXPP{dialect}' + env[f'SH{dialect}COM'] = f'$SH{dialect} -o $TARGET -c $FORTRANCOMMONFLAGS $SH{dialect}FLAGS $_{dialect}INCFLAGS $SOURCES #XXX{dialect}' + env[f'SH{dialect}PPCOM'] = f'$SH{dialect} -o $TARGET -c $FORTRANCOMMONFLAGS $SH{dialect}FLAGS $CPPFLAGS $_CPPDEFFLAGS $_{dialect}INCFLAGS $SOURCES #XXXPP{dialect}' + -def add_fortran_to_env(env): - """Add Builders and construction variables for Fortran to an Environment.""" +def add_fortran_to_env(env) -> None: + """Add Builders and construction variables for Fortran/generic.""" try: FortranSuffixes = env['FORTRANFILESUFFIXES'] except KeyError: FortranSuffixes = ['.f', '.for', '.ftn'] - #print("Adding %s to fortran suffixes" % FortranSuffixes) try: FortranPPSuffixes = env['FORTRANPPFILESUFFIXES'] except KeyError: FortranPPSuffixes = ['.fpp', '.FPP'] - DialectAddToEnv(env, "FORTRAN", FortranSuffixes, - FortranPPSuffixes, support_module=True) + DialectAddToEnv(env, "FORTRAN", FortranSuffixes, FortranPPSuffixes, support_mods=True) + # Module support env['FORTRANMODPREFIX'] = '' # like $LIBPREFIX env['FORTRANMODSUFFIX'] = '.mod' # like $LIBSUFFIX - env['FORTRANMODDIR'] = '' # where the compiler should place .mod files env['FORTRANMODDIRPREFIX'] = '' # some prefix to $FORTRANMODDIR - similar to $INCPREFIX env['FORTRANMODDIRSUFFIX'] = '' # some suffix to $FORTRANMODDIR - similar to $INCSUFFIX env['_FORTRANMODFLAG'] = '$( ${_concat(FORTRANMODDIRPREFIX, FORTRANMODDIR, FORTRANMODDIRSUFFIX, __env__, RDirs, TARGET, SOURCE)} $)' -def add_f77_to_env(env): - """Add Builders and construction variables for f77 to an Environment.""" +def add_f77_to_env(env) -> None: + """Add Builders and construction variables for f77 dialect.""" try: F77Suffixes = env['F77FILESUFFIXES'] except KeyError: F77Suffixes = ['.f77'] - #print("Adding %s to f77 suffixes" % F77Suffixes) try: F77PPSuffixes = env['F77PPFILESUFFIXES'] except KeyError: @@ -206,59 +218,50 @@ def add_f77_to_env(env): DialectAddToEnv(env, "F77", F77Suffixes, F77PPSuffixes) -def add_f90_to_env(env): - """Add Builders and construction variables for f90 to an Environment.""" +def add_f90_to_env(env) -> None: + """Add Builders and construction variables for f90 dialect.""" try: F90Suffixes = env['F90FILESUFFIXES'] except KeyError: F90Suffixes = ['.f90'] - #print("Adding %s to f90 suffixes" % F90Suffixes) try: F90PPSuffixes = env['F90PPFILESUFFIXES'] except KeyError: F90PPSuffixes = [] - DialectAddToEnv(env, "F90", F90Suffixes, F90PPSuffixes, - support_module=True) - + DialectAddToEnv(env, "F90", F90Suffixes, F90PPSuffixes, support_mods=True) -def add_f95_to_env(env): - """Add Builders and construction variables for f95 to an Environment.""" +def add_f95_to_env(env) -> None: + """Add Builders and construction variables for f95 dialect.""" try: F95Suffixes = env['F95FILESUFFIXES'] except KeyError: F95Suffixes = ['.f95'] - #print("Adding %s to f95 suffixes" % F95Suffixes) try: F95PPSuffixes = env['F95PPFILESUFFIXES'] except KeyError: F95PPSuffixes = [] - DialectAddToEnv(env, "F95", F95Suffixes, F95PPSuffixes, - support_module=True) - + DialectAddToEnv(env, "F95", F95Suffixes, F95PPSuffixes, support_mods=True) -def add_f03_to_env(env): - """Add Builders and construction variables for f03 to an Environment.""" +def add_f03_to_env(env) -> None: + """Add Builders and construction variables for f03 dialect.""" try: F03Suffixes = env['F03FILESUFFIXES'] except KeyError: F03Suffixes = ['.f03'] - #print("Adding %s to f95 suffixes" % F95Suffixes) try: F03PPSuffixes = env['F03PPFILESUFFIXES'] except KeyError: F03PPSuffixes = [] - DialectAddToEnv(env, "F03", F03Suffixes, F03PPSuffixes, - support_module=True) + DialectAddToEnv(env, "F03", F03Suffixes, F03PPSuffixes, support_mods=True) - -def add_f08_to_env(env): - """Add Builders and construction variables for f08 to an Environment.""" +def add_f08_to_env(env) -> None: + """Add Builders and construction variables for f08 dialect.""" try: F08Suffixes = env['F08FILESUFFIXES'] except KeyError: @@ -269,13 +272,10 @@ def add_f08_to_env(env): except KeyError: F08PPSuffixes = [] - DialectAddToEnv(env, "F08", F08Suffixes, F08PPSuffixes, - support_module=True) - + DialectAddToEnv(env, "F08", F08Suffixes, F08PPSuffixes, support_mods=True) -def add_all_to_env(env): - """Add builders and construction variables for all supported fortran - dialects.""" +def add_all_to_env(env) -> None: + """Add builders and construction variables for all supported dialects.""" add_fortran_to_env(env) add_f77_to_env(env) add_f90_to_env(env) diff --git a/SCons/Tool/f03.xml b/SCons/Tool/f03.xml index fc14ace..c989385 100644 --- a/SCons/Tool/f03.xml +++ b/SCons/Tool/f03.xml @@ -1,6 +1,6 @@ @@ -5585,38 +5605,37 @@ and the list of sources for this builder. def e(target, source, env): - return (target + ['foo.foo'], source + ['foo.src']) + return target + ['foo.foo'], source + ['foo.src'] # Simple association of an emitter function with a Builder. -b = Builder("my_build < $TARGET > $SOURCE", - emitter = e) +b = Builder("my_build < $TARGET > $SOURCE", emitter=e) def e2(target, source, env): - return (target + ['bar.foo'], source + ['bar.src']) + return target + ['bar.foo'], source + ['bar.src'] # Simple association of a list of emitter functions with a Builder. -b = Builder("my_build < $TARGET > $SOURCE", - emitter = [e, e2]) +b = Builder("my_build < $TARGET > $SOURCE", emitter=[e, e2]) -# Calling an emitter function through a &consvar;. +# Calling an emitter function through a construction variable. env = Environment(MY_EMITTER=e) -b = Builder("my_build < $TARGET > $SOURCE", - emitter='$MY_EMITTER') +b = Builder("my_build < $TARGET > $SOURCE", emitter='$MY_EMITTER') -# Calling a list of emitter functions through a &consvar;. +# Calling a list of emitter functions through a construction variable. env = Environment(EMITTER_LIST=[e, e2]) -b = Builder("my_build < $TARGET > $SOURCE", - emitter='$EMITTER_LIST') +b = Builder("my_build < $TARGET > $SOURCE", emitter='$EMITTER_LIST') # Associating multiple emitters with different file # suffixes using a dictionary. def e_suf1(target, source, env): - return (target + ['another_target_file'], source) + return target + ['another_target_file'], source + def e_suf2(target, source, env): - return (target, source + ['another_source_file']) -b = Builder("my_build < $TARGET > $SOURCE", - emitter={'.suf1' : e_suf1, - '.suf2' : e_suf2}) + return target, source + ['another_source_file'] + +b = Builder( + action="my_build < $TARGET > $SOURCE", + emitter={'.suf1': e_suf1, '.suf2': e_suf2} +) -- cgit v0.12 From 869c75679492039a5dcb1f66fd9a78d59a4340da Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 20 Jun 2022 08:58:35 -0600 Subject: A few more tweaks in Builder Methods intro [skip appveyor] Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 8e64290..2e7784c 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2675,7 +2675,7 @@ and a same-named environment method that splits a single string into a list, using strings of white-space characters as the delimiter -(similar to the Python string split +(similar to the &Python; string split method, but succeeds even if the input isn't a string). @@ -2697,7 +2697,7 @@ env.Program('bar', source='bar.c foo.c'.split()) Sources and targets can be specified as a a scalar or as a list, either of strings or nodes (more on nodes below). When specifying path strings, -Python follows the POSIX pathname convention: +&Python; follows the POSIX pathname convention: if a string begins with the operating system pathname separator (on Windows both the slash and backslash separator are accepted, and any leading drive specifier is ignored for @@ -2806,7 +2806,7 @@ env.Program('build/prog', ['f1.c', 'f2.c'], srcdir='src') The optional parse_flags -keyword argument causes behavior similarly to the +keyword argument causes behavior similarl to the &f-link-env-MergeFlags; method, where the argument value is broken into individual settings and merged into the appropriate &consvars;. @@ -2861,15 +2861,13 @@ env.Command('sub/dir/foo.out', 'sub/dir/foo.in', "cp foo.in foo.out", chdir=True not automatically modify its expansion of -&consvars; like -$TARGET -and -$SOURCE +&consvars; like &cv-link-TARGET; +and &cv-link-SOURCE; when using the chdir keyword argument--that is, the expanded file names will still be relative to -the top-level directory where &SConstruct; was found, +the top-level directory where the &SConstruct; was found, and consequently incorrect relative to the chdir directory. If you use the chdir keyword argument, @@ -2880,7 +2878,7 @@ expansions like and ${SOURCE.file} to use just the filename portion of the -targets and source. +target and source. Keyword arguments that are not specifically recognized are treated as &consvar; @@ -2911,7 +2909,7 @@ env.SharedLibrary( and &cv-link-LIBSUFFIXES; &consvars; must be set if you want &scons; to search automatically for dependencies on the non-standard library names; -see the descriptions below of these variables for more information. +see the descriptions of these variables for more information. Although the builder methods defined by &scons; @@ -2930,11 +2928,11 @@ SharedLibrary('word', 'word.cpp') has determined are appropriate for the local system. Builder methods that can be called without an explicit -environment (indicated in the listing of builders without -a leading env.) -may be called from custom Python modules that you +environment (indicated in the listing of builders below +without a leading env.) +may be called from custom &Python; modules that you import into an SConscript file by adding the following -to the Python module: +to the &Python; module: from SCons.Script import * @@ -2942,17 +2940,17 @@ from SCons.Script import * A builder may add additional targets -(and, in fact, sources) beyond those requested, +beyond those requested if an attached Emitter chooses to do so -(see and, as an example, -&cv-link-PROGEMITTER;, for more information). +(see for more information. +&cv-link-PROGEMITTER; is an example). For example, the GNU linker takes a command-line argument , which causes it to produce a linker map file in addition to the executable file actually being linked. If the &b-link-Program; builder's emitter is configured to add this mapfile if the option is set, -two targets will be returned when you only asked for one. +then two targets will be returned when you only asked for one. @@ -3000,7 +2998,7 @@ contain elements which are themselves lists, such as bar_obj_list returned by the &b-link-StaticObject; call. If you need to manipulate a list of lists returned by builders -directly in Python code, +directly in &Python; code, you can either build a new list by hand: @@ -3025,19 +3023,19 @@ for obj in objects: Since builder calls return -a list-like object, not an actual Python list, -it is not appropriate to use the Python add +a list-like object, not an actual &Python; list, +it is not appropriate to use the &Python; add operator (+ or +=) -to append builder results to a Python list. +to append builder results to a &Python; list. Because the list and the object are different types, -Python will not update the original list in place, +&Python; will not update the original list in place, but will instead create a new NodeList object containing the concatenation of the list elements and the builder results. -This will cause problems for any other Python variables +This will cause problems for any other &Python; variables in your SCons configuration that still hold on to a reference to the original list. -Instead, use the Python list +Instead, use the &Python; list extend method to make sure the list is updated in-place. Example: @@ -3055,7 +3053,7 @@ object_files.extend(Object('bar.c')) The path name for a Node's file may be used -by passing the Node to Python's builtin +by passing the Node to &Python;'s builtin str function: -- cgit v0.12 From 94b23f77472aabaabc372b0facb26c4f384413c5 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:11:24 -0400 Subject: Rename _Const to _Config. Rework msvc sdk version function. Minor cleanup. --- SCons/Tool/MSCommon/__init__.py | 1 + SCons/Tool/MSCommon/vc.py | 68 ++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/SCons/Tool/MSCommon/__init__.py b/SCons/Tool/MSCommon/__init__.py index 9d8a8ff..de78f84 100644 --- a/SCons/Tool/MSCommon/__init__.py +++ b/SCons/Tool/MSCommon/__init__.py @@ -40,6 +40,7 @@ from SCons.Tool.MSCommon.vc import ( msvc_find_vswhere, set_msvc_notfound_policy, get_msvc_notfound_policy, + get_msvc_sdk_versions, ) from SCons.Tool.MSCommon.vs import ( diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index e193d91..3abd606 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -132,7 +132,7 @@ class _Dispatcher: func = getattr(classref, method) func() -class _Const: +class _Config: BOOLEAN_SYMBOLS = {} BOOLEAN_EXTERNAL = {} @@ -1255,6 +1255,12 @@ def reset_installed_vcs(): __INSTALLED_VCS_RUN = None _Dispatcher.reset() +def get_default_installed_msvc(env=None): + vcs = get_installed_vcs(env) + msvc_version = vcs[0] if vcs else None + debug('msvc_version=%s', repr(msvc_version)) + return msvc_version + # Running these batch files isn't cheap: most of the time spent in # msvs.generate() is due to vcvars*.bat. In a build that uses "tools='msvs'" # in multiple environments, for example: @@ -1648,13 +1654,11 @@ def get_default_version(env): return msvs_version if not msvc_version: - installed_vcs = get_installed_vcs(env) - debug('installed_vcs:%s', installed_vcs) - if not installed_vcs: + msvc_version = get_default_installed_msvc(env) + if not msvc_version: #SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) debug('No installed VCs') return None - msvc_version = installed_vcs[0] debug('using default installed MSVC version %s', repr(msvc_version)) else: debug('using specified MSVC version %s', repr(msvc_version)) @@ -1769,13 +1773,9 @@ def msvc_find_valid_batch_script(env, version): return d -_undefined = None +_UNDEFINED = object() def get_use_script_use_settings(env): - global _undefined - - if _undefined is None: - _undefined = object() # use_script use_settings return values action # value ignored (value, None) use script or bypass detection @@ -1784,9 +1784,9 @@ def get_use_script_use_settings(env): # None (documentation) or evaluates False (code): bypass detection # need to distinguish between undefined and None - use_script = env.get('MSVC_USE_SCRIPT', _undefined) + use_script = env.get('MSVC_USE_SCRIPT', _UNDEFINED) - if use_script != _undefined: + if use_script != _UNDEFINED: # use_script defined, use_settings ignored (not type checked) return (use_script, None) @@ -1908,6 +1908,21 @@ def msvc_setup_env_tool(env=None, version=None, tool=None): debug('tool=%s, version=%s, return=%s', repr(tool), repr(version), rval) return rval +def get_msvc_sdk_versions(msvc_version=None, msvc_uwp_app=False): + debug('msvc_version=%s, msvc_uwp_app=%s', repr(msvc_version), repr(msvc_uwp_app)) + + rval = [] + + if not msvc_version: + msvc_version = get_default_installed_msvc() + + if not msvc_version: + debug('no msvc versions detected') + return rval + + rval = MSVC.WinSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) + return rval + class _Util: @staticmethod @@ -2158,7 +2173,7 @@ class _WindowsSDK: def _verify_sdk_dispatch_map(cls): debug('%s verify sdk_dispatch_map', cls.__name__) cls._init_sdk_dispatch_map() - for sdk_version in _Const.MSVC_SDK_VERSIONS: + for sdk_version in _Config.MSVC_SDK_VERSIONS: if sdk_version in cls.sdk_dispatch_map: continue err_msg = 'sdk version {} not in {}.sdk_dispatch_map'.format(sdk_version, cls.__name__) @@ -2205,20 +2220,13 @@ class _WindowsSDK: sdk_versions = [] - if not msvc_version: - vcs = get_installed_vcs() - if not vcs: - debug('no msvc versions detected') - return sdk_versions - msvc_version = vcs[0] - verstr = get_msvc_version_numeric(msvc_version) - vs_def = _Const.MSVC_VERSION_EXTERNAL.get(verstr, None) + vs_def = _Config.MSVC_VERSION_EXTERNAL.get(verstr, None) if not vs_def: debug('vs_def is not defined') return sdk_versions - is_uwp = True if msvc_uwp_app in _Const.BOOLEAN_SYMBOLS[True] else False + is_uwp = True if msvc_uwp_app in _Config.BOOLEAN_SYMBOLS[True] else False platform_type = 'uwp' if is_uwp else 'desktop' sdk_list = _WindowsSDK.get_sdk_version_list(vs_def.vc_sdk_versions, platform_type) @@ -2239,10 +2247,6 @@ class _WindowsSDK: _Dispatcher.register(_WindowsSDK) -def get_sdk_versions(MSVC_VERSION=None, MSVC_UWP_APP=False): - debug('MSVC_VERSION=%s, MSVC_UWP_APP=%s', repr(MSVC_VERSION), repr(MSVC_UWP_APP)) - return _WindowsSDK.get_msvc_sdk_version_list(msvc_version=MSVC_VERSION, msvc_uwp_app=MSVC_UWP_APP) - class _ScriptArguments: # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? @@ -2257,7 +2261,7 @@ class _ScriptArguments: @classmethod def _verify_re_sdk_dispatch_map(cls): debug('%s verify re_sdk_dispatch_map', cls.__name__) - for sdk_version in _Const.MSVC_SDK_VERSIONS: + for sdk_version in _Config.MSVC_SDK_VERSIONS: if sdk_version in cls.re_sdk_dispatch_map: continue err_msg = 'sdk version {} not in {}.re_sdk_dispatch_map'.format(sdk_version, cls.__name__) @@ -2315,9 +2319,9 @@ class _ScriptArguments: SPECTRE = 4 # MSVC_SPECTRE_LIBS USER = 5 # MSVC_SCRIPT_ARGS - VS2019 = _Const.MSVS_VERSION_INTERNAL['2019'] - VS2017 = _Const.MSVS_VERSION_INTERNAL['2017'] - VS2015 = _Const.MSVS_VERSION_INTERNAL['2015'] + VS2019 = _Config.MSVS_VERSION_INTERNAL['2019'] + VS2017 = _Config.MSVS_VERSION_INTERNAL['2017'] + VS2015 = _Config.MSVS_VERSION_INTERNAL['2015'] MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ 'version', # fully qualified msvc version (e.g., '14.1Exp') @@ -2328,7 +2332,7 @@ class _ScriptArguments: def _msvc_version(cls, version): verstr = get_msvc_version_numeric(version) - vs_def = _Const.MSVC_VERSION_INTERNAL[verstr] + vs_def = _Config.MSVC_VERSION_INTERNAL[verstr] version_args = cls.MSVC_VERSION_ARGS_DEFINITION( version = version, @@ -2346,7 +2350,7 @@ class _ScriptArguments: if not uwp_app: return None - if uwp_app not in _Const.BOOLEAN_SYMBOLS[True]: + if uwp_app not in _Config.BOOLEAN_SYMBOLS[True]: return None if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: -- cgit v0.12 From dbd301878f8f7bb004658b25a62d57b48ee4c8a3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 12:19:39 -0400 Subject: Fix windows SDK reference --- SCons/Tool/MSCommon/vc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 3abd606..ab05323 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1920,7 +1920,7 @@ def get_msvc_sdk_versions(msvc_version=None, msvc_uwp_app=False): debug('no msvc versions detected') return rval - rval = MSVC.WinSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) + rval = _WindowsSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) return rval class _Util: -- cgit v0.12 From 010c2ad46c32827b7ed8082ec11b83404a039faa Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 16:53:31 -0400 Subject: Refactor recent portions of vc.py into MSVC module --- SCons/Tool/MSCommon/MSVC/Config.py | 284 +++++ SCons/Tool/MSCommon/MSVC/Dispatcher.py | 62 + SCons/Tool/MSCommon/MSVC/Exceptions.py | 39 + SCons/Tool/MSCommon/MSVC/NotFound.py | 132 +++ SCons/Tool/MSCommon/MSVC/Registry.py | 110 ++ SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 733 ++++++++++++ SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py | 235 ++++ SCons/Tool/MSCommon/MSVC/Util.py | 55 + SCons/Tool/MSCommon/MSVC/WinSDK.py | 250 ++++ SCons/Tool/MSCommon/MSVC/__init__.py | 50 + SCons/Tool/MSCommon/__init__.py | 7 +- SCons/Tool/MSCommon/vc.py | 1638 +-------------------------- 12 files changed, 1968 insertions(+), 1627 deletions(-) create mode 100644 SCons/Tool/MSCommon/MSVC/Config.py create mode 100644 SCons/Tool/MSCommon/MSVC/Dispatcher.py create mode 100644 SCons/Tool/MSCommon/MSVC/Exceptions.py create mode 100644 SCons/Tool/MSCommon/MSVC/NotFound.py create mode 100644 SCons/Tool/MSCommon/MSVC/Registry.py create mode 100644 SCons/Tool/MSCommon/MSVC/ScriptArguments.py create mode 100644 SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py create mode 100644 SCons/Tool/MSCommon/MSVC/Util.py create mode 100644 SCons/Tool/MSCommon/MSVC/WinSDK.py create mode 100644 SCons/Tool/MSCommon/MSVC/__init__.py diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py new file mode 100644 index 0000000..476dcb3 --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -0,0 +1,284 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Constants and initialized data structures for Microsoft Visual C/C++. +""" + +from collections import ( + namedtuple, +) + +from . import Dispatcher + +Dispatcher.register_modulename(__name__) + +UNDEFINED = object() + +BOOLEAN_SYMBOLS = {} +BOOLEAN_EXTERNAL = {} + +for bool, symbol_list, symbol_case_list in [ + (False, (False, 0, '0', None, ''), ('False', 'No', 'F', 'N')), + (True, (True, 1, '1'), ('True', 'Yes', 'T', 'Y')), +]: + BOOLEAN_SYMBOLS[bool] = list(symbol_list) + for symbol in symbol_case_list: + BOOLEAN_SYMBOLS[bool].extend([symbol, symbol.lower(), symbol.upper()]) + + for symbol in BOOLEAN_SYMBOLS[bool]: + BOOLEAN_EXTERNAL[symbol] = bool + +MSVC_RUNTIME_DEFINITION = namedtuple('MSVCRuntime', [ + 'vc_runtime', + 'vc_runtime_numeric', + 'vc_runtime_alias_list', + 'vc_runtime_vsdef_list', +]) + +MSVC_RUNTIME_DEFINITION_LIST = [] + +MSVC_RUNTIME_INTERNAL = {} +MSVC_RUNTIME_EXTERNAL = {} + +for vc_runtime, vc_runtime_numeric, vc_runtime_alias_list in [ + ('140', 140, ['ucrt']), + ('120', 120, ['msvcr120']), + ('110', 110, ['msvcr110']), + ('100', 100, ['msvcr100']), + ('90', 90, ['msvcr90']), + ('80', 80, ['msvcr80']), + ('71', 71, ['msvcr71']), + ('70', 70, ['msvcr70']), + ('60', 60, ['msvcrt']), +]: + vc_runtime_def = MSVC_RUNTIME_DEFINITION( + vc_runtime = vc_runtime, + vc_runtime_numeric = vc_runtime_numeric, + vc_runtime_alias_list = vc_runtime_alias_list, + vc_runtime_vsdef_list = [], + ) + + MSVC_RUNTIME_DEFINITION_LIST.append(vc_runtime_def) + + MSVC_RUNTIME_INTERNAL[vc_runtime] = vc_runtime_def + MSVC_RUNTIME_EXTERNAL[vc_runtime] = vc_runtime_def + + for vc_runtime_alias in vc_runtime_alias_list: + MSVC_RUNTIME_EXTERNAL[vc_runtime_alias] = vc_runtime_def + +MSVC_BUILDTOOLS_DEFINITION = namedtuple('MSVCBuildtools', [ + 'vc_buildtools', + 'vc_buildtools_numeric', + 'vc_version', + 'vc_version_numeric', + 'cl_version', + 'cl_version_numeric', + 'vc_runtime_def', +]) + +MSVC_BUILDTOOLS_DEFINITION_LIST = [] + +MSVC_BUILDTOOLS_INTERNAL = {} +MSVC_BUILDTOOLS_EXTERNAL = {} + +VC_VERSION_MAP = {} + +for vc_buildtools, vc_version, cl_version, vc_runtime in [ + ('v143', '14.3', '19.3', '140'), + ('v142', '14.2', '19.2', '140'), + ('v141', '14.1', '19.1', '140'), + ('v140', '14.0', '19.0', '140'), + ('v120', '12.0', '18.0', '120'), + ('v110', '11.0', '17.0', '110'), + ('v100', '10.0', '16.0', '100'), + ('v90', '9.0', '15.0', '90'), + ('v80', '8.0', '14.0', '80'), + ('v71', '7.1', '13.1', '71'), + ('v70', '7.0', '13.0', '70'), + ('v60', '6.0', '12.0', '60'), +]: + + vc_runtime_def = MSVC_RUNTIME_INTERNAL[vc_runtime] + + vc_buildtools_def = MSVC_BUILDTOOLS_DEFINITION( + vc_buildtools = vc_buildtools, + vc_buildtools_numeric = int(vc_buildtools[1:]), + vc_version = vc_version, + vc_version_numeric = float(vc_version), + cl_version = cl_version, + cl_version_numeric = float(cl_version), + vc_runtime_def = vc_runtime_def, + ) + + MSVC_BUILDTOOLS_DEFINITION_LIST.append(vc_buildtools_def) + + MSVC_BUILDTOOLS_INTERNAL[vc_buildtools] = vc_buildtools_def + MSVC_BUILDTOOLS_EXTERNAL[vc_buildtools] = vc_buildtools_def + MSVC_BUILDTOOLS_EXTERNAL[vc_version] = vc_buildtools_def + + VC_VERSION_MAP[vc_version] = vc_buildtools_def + +MSVS_VERSION_INTERNAL = {} +MSVS_VERSION_EXTERNAL = {} + +MSVC_VERSION_INTERNAL = {} +MSVC_VERSION_EXTERNAL = {} + +MSVS_VERSION_MAJOR_MAP = {} + +CL_VERSION_MAP = {} + +MSVC_SDK_VERSIONS = set() + +VISUALSTUDIO_DEFINITION = namedtuple('VisualStudioDefinition', [ + 'vs_product', + 'vs_product_alias_list', + 'vs_version', + 'vs_version_major', + 'vs_envvar', + 'vs_express', + 'vs_lookup', + 'vc_sdk_versions', + 'vc_ucrt_versions', + 'vc_uwp', + 'vc_buildtools_def', + 'vc_buildtools_all', +]) + +VISUALSTUDIO_DEFINITION_LIST = [] + +VS_PRODUCT_ALIAS = { + '1998': ['6'] +} + +# vs_envvar: VisualStudioVersion defined in environment for MSVS 2012 and later +# MSVS 2010 and earlier cl_version -> vs_def is a 1:1 mapping +# SDK attached to product or buildtools? +for vs_product, vs_version, vs_envvar, vs_express, vs_lookup, vc_sdk, vc_ucrt, vc_uwp, vc_buildtools_all in [ + ('2022', '17.0', True, False, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v143', 'v142', 'v141', 'v140']), + ('2019', '16.0', True, False, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v142', 'v141', 'v140']), + ('2017', '15.0', True, True, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v141', 'v140']), + ('2015', '14.0', True, True, 'registry', ['10.0', '8.1'], ['10'], 'store', ['v140']), + ('2013', '12.0', True, True, 'registry', None, None, None, ['v120']), + ('2012', '11.0', True, True, 'registry', None, None, None, ['v110']), + ('2010', '10.0', False, True, 'registry', None, None, None, ['v100']), + ('2008', '9.0', False, True, 'registry', None, None, None, ['v90']), + ('2005', '8.0', False, True, 'registry', None, None, None, ['v80']), + ('2003', '7.1', False, False, 'registry', None, None, None, ['v71']), + ('2002', '7.0', False, False, 'registry', None, None, None, ['v70']), + ('1998', '6.0', False, False, 'registry', None, None, None, ['v60']), +]: + + vs_version_major = vs_version.split('.')[0] + + vc_buildtools_def = MSVC_BUILDTOOLS_INTERNAL[vc_buildtools_all[0]] + + vs_def = VISUALSTUDIO_DEFINITION( + vs_product = vs_product, + vs_product_alias_list = [], + vs_version = vs_version, + vs_version_major = vs_version_major, + vs_envvar = vs_envvar, + vs_express = vs_express, + vs_lookup = vs_lookup, + vc_sdk_versions = vc_sdk, + vc_ucrt_versions = vc_ucrt, + vc_uwp = vc_uwp, + vc_buildtools_def = vc_buildtools_def, + vc_buildtools_all = vc_buildtools_all, + ) + + VISUALSTUDIO_DEFINITION_LIST.append(vs_def) + + vc_buildtools_def.vc_runtime_def.vc_runtime_vsdef_list.append(vs_def) + + MSVS_VERSION_INTERNAL[vs_product] = vs_def + MSVS_VERSION_EXTERNAL[vs_product] = vs_def + MSVS_VERSION_EXTERNAL[vs_version] = vs_def + + MSVC_VERSION_INTERNAL[vc_buildtools_def.vc_version] = vs_def + MSVC_VERSION_EXTERNAL[vs_product] = vs_def + MSVC_VERSION_EXTERNAL[vc_buildtools_def.vc_version] = vs_def + MSVC_VERSION_EXTERNAL[vc_buildtools_def.vc_buildtools] = vs_def + + if vs_product in VS_PRODUCT_ALIAS: + for vs_product_alias in VS_PRODUCT_ALIAS[vs_product]: + vs_def.vs_product_alias_list.append(vs_product_alias) + MSVS_VERSION_EXTERNAL[vs_product_alias] = vs_def + MSVC_VERSION_EXTERNAL[vs_product_alias] = vs_def + + MSVS_VERSION_MAJOR_MAP[vs_version_major] = vs_def + + CL_VERSION_MAP[vc_buildtools_def.cl_version] = vs_def + + if not vc_sdk: + continue + + MSVC_SDK_VERSIONS.update(vc_sdk) + +# convert string version set to string version list ranked in descending order +MSVC_SDK_VERSIONS = [str(f) for f in sorted([float(s) for s in MSVC_SDK_VERSIONS], reverse=True)] + +MSVS_VERSION_LEGACY = {} +MSVC_VERSION_LEGACY = {} + +for vdict in (MSVS_VERSION_EXTERNAL, MSVC_VERSION_INTERNAL): + for key, vs_def in vdict.items(): + if key not in MSVS_VERSION_LEGACY: + MSVS_VERSION_LEGACY[key] = vs_def + MSVC_VERSION_LEGACY[key] = vs_def + +# MSVC_NOTFOUND_POLICY definition: +# error: raise exception +# warning: issue warning and continue +# ignore: continue + +MSVC_NOTFOUND_POLICY_DEFINITION = namedtuple('MSVCNotFoundPolicyDefinition', [ + 'value', + 'symbol', +]) + +MSVC_NOTFOUND_DEFINITION_LIST = [] + +MSVC_NOTFOUND_POLICY_INTERNAL = {} +MSVC_NOTFOUND_POLICY_EXTERNAL = {} + +for policy_value, policy_symbol_list in [ + (True, ['Error', 'Exception']), + (False, ['Warning', 'Warn']), + (None, ['Ignore', 'Suppress']), +]: + + policy_symbol = policy_symbol_list[0].lower() + policy_def = MSVC_NOTFOUND_POLICY_DEFINITION(policy_value, policy_symbol) + + MSVC_NOTFOUND_DEFINITION_LIST.append(vs_def) + + MSVC_NOTFOUND_POLICY_INTERNAL[policy_symbol] = policy_def + + for policy_symbol in policy_symbol_list: + MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.lower()] = policy_def + MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol] = policy_def + MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.upper()] = policy_def + diff --git a/SCons/Tool/MSCommon/MSVC/Dispatcher.py b/SCons/Tool/MSCommon/MSVC/Dispatcher.py new file mode 100644 index 0000000..ebcd704 --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/Dispatcher.py @@ -0,0 +1,62 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Internal method dispatcher for Microsoft Visual C/C++. +""" + +import sys + +from ..common import ( + debug, +) + +_refs = [] + +def register_class(ref): + _refs.append(ref) + +def register_modulename(modname): + module = sys.modules[modname] + _refs.append(module) + +def reset(): + debug('') + for ref in _refs: + for method in ['reset', '_reset']: + if not hasattr(ref, method) or not callable(getattr(ref, method, None)): + continue + debug('call %s.%s()', ref.__name__, method) + func = getattr(ref, method) + func() + +def verify(): + debug('') + for ref in _refs: + for method in ['verify', '_verify']: + if not hasattr(ref, method) or not callable(getattr(ref, method, None)): + continue + debug('call %s.%s()', ref.__name__, method) + func = getattr(ref, method) + func() + diff --git a/SCons/Tool/MSCommon/MSVC/Exceptions.py b/SCons/Tool/MSCommon/MSVC/Exceptions.py new file mode 100644 index 0000000..7a61ec5 --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/Exceptions.py @@ -0,0 +1,39 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Exceptions for Microsoft Visual C/C++. +""" + +class VisualCException(Exception): + pass + +class MSVCVersionNotFound(VisualCException): + pass + +class MSVCInternalError(VisualCException): + pass + +class MSVCArgumentError(VisualCException): + pass + diff --git a/SCons/Tool/MSCommon/MSVC/NotFound.py b/SCons/Tool/MSCommon/MSVC/NotFound.py new file mode 100644 index 0000000..7abe5ad --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/NotFound.py @@ -0,0 +1,132 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Microsoft Visual C/C++ not found policy. + +Notes: + * As implemented, the default is that a warning is issued. This can + be changed globally via the function set_msvc_notfound_policy and/or + through the environment via the MSVC_NOTFOUND_POLICY variable. +""" + +import SCons.Warnings + +from ..common import ( + debug, +) + +from . import Dispatcher +from . import Config + +from .Exceptions import ( + MSVCVersionNotFound, +) + +Dispatcher.register_modulename(__name__) + +_MSVC_NOTFOUND_POLICY_DEF = Config.MSVC_NOTFOUND_POLICY_INTERNAL['warning'] + +def _msvc_notfound_policy_lookup(symbol): + + try: + notfound_policy_def = Config.MSVC_NOTFOUND_POLICY_EXTERNAL[symbol] + except KeyError: + err_msg = "Value specified for MSVC_NOTFOUND_POLICY is not supported: {}.\n" \ + " Valid values are: {}".format( + repr(symbol), + ', '.join([repr(s) for s in Config.MSVC_NOTFOUND_POLICY_EXTERNAL.keys()]) + ) + raise ValueError(err_msg) + + return notfound_policy_def + +def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): + """ Set the default policy when MSVC is not found. + + Args: + MSVC_NOTFOUND_POLICY: + string representing the policy behavior + when MSVC is not found or None + + Returns: + The previous policy is returned when the MSVC_NOTFOUND_POLICY argument + is not None. The active policy is returned when the MSVC_NOTFOUND_POLICY + argument is None. + + """ + global _MSVC_NOTFOUND_POLICY_DEF + + prev_policy = _MSVC_NOTFOUND_POLICY_DEF.symbol + + policy = MSVC_NOTFOUND_POLICY + if policy is not None: + _MSVC_NOTFOUND_POLICY_DEF = _msvc_notfound_policy_lookup(policy) + + debug( + 'prev_policy=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', + repr(prev_policy), repr(policy), + repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) + ) + + return prev_policy + +def get_msvc_notfound_policy(): + """Return the active policy when MSVC is not found.""" + debug( + 'policy.symbol=%s, policy.value=%s', + repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) + ) + return _MSVC_NOTFOUND_POLICY_DEF.symbol + +def policy_handler(env, msg): + + if env and 'MSVC_NOTFOUND_POLICY' in env: + # environment setting + notfound_policy_src = 'environment' + policy = env['MSVC_NOTFOUND_POLICY'] + if policy is not None: + # user policy request + notfound_policy_def = _msvc_notfound_policy_lookup(policy) + else: + # active global setting + notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF + else: + # active global setting + notfound_policy_src = 'default' + policy = None + notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF + + debug( + 'source=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', + notfound_policy_src, repr(policy), repr(notfound_policy_def.symbol), repr(notfound_policy_def.value) + ) + + if notfound_policy_def.value is None: + # ignore + pass + elif notfound_policy_def.value: + raise MSVCVersionNotFound(msg) + else: + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) + diff --git a/SCons/Tool/MSCommon/MSVC/Registry.py b/SCons/Tool/MSCommon/MSVC/Registry.py new file mode 100644 index 0000000..848b125 --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/Registry.py @@ -0,0 +1,110 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Windows registry functions for Microsoft Visual C/C++. +""" + +import os + +from SCons.Util import ( + HKEY_LOCAL_MACHINE, + HKEY_CURRENT_USER, + HKEY_LOCAL_MACHINE, + HKEY_CURRENT_USER, +) + +from .. common import ( + debug, + read_reg, +) + +from . import Dispatcher +from . import Util + +Dispatcher.register_modulename(__name__) + +def read_value(hkey, subkey_valname): + try: + rval = read_reg(subkey_valname, hkroot=hkey) + except OSError: + debug('OSError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) + return None + except IndexError: + debug('IndexError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) + return None + debug('hkey=%s, subkey=%s, rval=%s', repr(hkey), repr(subkey_valname), repr(rval)) + return rval + +def registry_query_path(key, val, suffix): + extval = val + '\\' + suffix if suffix else val + qpath = read_value(key, extval) + if qpath and os.path.exists(qpath): + qpath = Util.process_path(qpath) + else: + qpath = None + return (qpath, key, val, extval) + +REG_SOFTWARE_MICROSOFT = [ + (HKEY_LOCAL_MACHINE, r'Software\Wow6432Node\Microsoft'), + (HKEY_CURRENT_USER, r'Software\Wow6432Node\Microsoft'), # SDK queries + (HKEY_LOCAL_MACHINE, r'Software\Microsoft'), + (HKEY_CURRENT_USER, r'Software\Microsoft'), +] + +def microsoft_query_paths(suffix, usrval=None): + paths = [] + records = [] + for key, val in REG_SOFTWARE_MICROSOFT: + extval = val + '\\' + suffix if suffix else val + qpath = read_value(key, extval) + if qpath and os.path.exists(qpath): + qpath = Util.process_path(qpath) + if qpath not in paths: + paths.append(qpath) + records.append((qpath, key, val, extval, usrval)) + return records + +def microsoft_query_keys(suffix, usrval=None): + records = [] + for key, val in REG_SOFTWARE_MICROSOFT: + extval = val + '\\' + suffix if suffix else val + rval = read_value(key, extval) + if rval: + records.append((key, val, extval, usrval)) + return records + +def microsoft_sdks(version): + return '\\'.join([r'Microsoft SDKs\Windows', 'v' + version, r'InstallationFolder']) + +def sdk_query_paths(version): + q = microsoft_sdks(version) + return microsoft_query_paths(q) + +def windows_kits(version): + return r'Windows Kits\Installed Roots\KitsRoot' + version + +def windows_kit_query_paths(version): + q = windows_kits(version) + return microsoft_query_paths(q) + diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py new file mode 100644 index 0000000..324f8be --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -0,0 +1,733 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Batch file argument functions for Microsoft Visual C/C++. +""" + +import os +import re +import enum + +from collections import ( + namedtuple, +) + +from ..common import ( + debug, +) + +from . import Dispatcher +from . import Util +from . import Config +from . import WinSDK + +from .Exceptions import ( + MSVCInternalError, + MSVCArgumentError, +) + +Dispatcher.register_modulename(__name__) + +# TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? +re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') +re_sdk_version_81 = re.compile(r'^8[.]1$') + +re_sdk_dispatch_map = { + '10.0': re_sdk_version_100, + '8.1': re_sdk_version_81, +} + +def _verify_re_sdk_dispatch_map(): + debug('') + for sdk_version in Config.MSVC_SDK_VERSIONS: + if sdk_version in re_sdk_dispatch_map: + continue + err_msg = 'sdk version {} not in re_sdk_dispatch_map'.format(sdk_version) + raise MSVCInternalError(err_msg) + return None + +# capture msvc version +re_toolset_version = re.compile(r'^(?P[1-9][0-9]?[.][0-9])[0-9.]*$', re.IGNORECASE) + +re_toolset_full = re.compile(r'''^(?: + (?:[1-9][0-9][.][0-9]{1,2})| # XX.Y - XX.YY + (?:[1-9][0-9][.][0-9]{2}[.][0-9]{1,5}) # XX.YY.Z - XX.YY.ZZZZZ +)$''', re.VERBOSE) + +re_toolset_140 = re.compile(r'''^(?: + (?:14[.]0{1,2})| # 14.0 - 14.00 + (?:14[.]0{2}[.]0{1,5}) # 14.00.0 - 14.00.00000 +)$''', re.VERBOSE) + +# valid SxS formats will be matched with re_toolset_full: match 3 '.' format +re_toolset_sxs = re.compile(r'^[1-9][0-9][.][0-9]{2}[.][0-9]{2}[.][0-9]{1,2}$') + +# MSVC_SCRIPT_ARGS +re_vcvars_uwp = re.compile(r'(?:(?(?:uwp|store))(?:(?!\S)|$)',re.IGNORECASE) +re_vcvars_sdk = re.compile(r'(?:(?(?:[1-9][0-9]*[.]\S*))(?:(?!\S)|$)',re.IGNORECASE) +re_vcvars_toolset = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_ver[=](?P\S*))(?:(?!\S)|$)', re.IGNORECASE) +re_vcvars_spectre = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_spectre_libs[=](?P\S*))(?:(?!\S)|$)',re.IGNORECASE) + +# Force default sdk argument +MSVC_FORCE_DEFAULT_SDK = False + +# Force default toolset argument +MSVC_FORCE_DEFAULT_TOOLSET = False + +# MSVC batch file arguments: +# +# VS2022: UWP, SDK, TOOLSET, SPECTRE +# VS2019: UWP, SDK, TOOLSET, SPECTRE +# VS2017: UWP, SDK, TOOLSET, SPECTRE +# VS2015: UWP, SDK +# +# MSVC_SCRIPT_ARGS: VS2015+ +# +# MSVC_UWP_APP: VS2015+ +# MSVC_SDK_VERSION: VS2015+ +# MSVC_TOOLSET_VERSION: VS2017+ +# MSVC_SPECTRE_LIBS: VS2017+ + +@enum.unique +class SortOrder(enum.IntEnum): + ARCH = 0 # arch + UWP = 1 # MSVC_UWP_APP + SDK = 2 # MSVC_SDK_VERSION + TOOLSET = 3 # MSVC_TOOLSET_VERSION + SPECTRE = 4 # MSVC_SPECTRE_LIBS + USER = 5 # MSVC_SCRIPT_ARGS + +VS2019 = Config.MSVS_VERSION_INTERNAL['2019'] +VS2017 = Config.MSVS_VERSION_INTERNAL['2017'] +VS2015 = Config.MSVS_VERSION_INTERNAL['2015'] + +MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ + 'version', # fully qualified msvc version (e.g., '14.1Exp') + 'vs_def', +]) + +def _msvc_version(version): + + verstr = Util.get_version_prefix(version) + vs_def = Config.MSVC_VERSION_INTERNAL[verstr] + + version_args = MSVC_VERSION_ARGS_DEFINITION( + version = version, + vs_def = vs_def, + ) + + return version_args + +def _msvc_script_argument_uwp(env, msvc, arglist): + + uwp_app = env['MSVC_UWP_APP'] + debug('MSVC_VERSION=%s, MSVC_UWP_APP=%s', repr(msvc.version), repr(uwp_app)) + + if not uwp_app: + return None + + if uwp_app not in Config.BOOLEAN_SYMBOLS[True]: + return None + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2015', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_UWP_APP ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(uwp_app), repr(msvc.version), repr(VS2015.vc_buildtools_def.vc_version) + ) + raise MSVCArgumentError(err_msg) + + # VS2017+ rewrites uwp => store for 14.0 toolset + uwp_arg = msvc.vs_def.vc_uwp + + # store/uwp may not be fully installed + argpair = (SortOrder.UWP, uwp_arg) + arglist.append(argpair) + + return uwp_arg + +def _user_script_argument_uwp(env, uwp, user_argstr): + + matches = [m for m in re_vcvars_uwp.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple uwp declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple uwp declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not uwp: + return None + + env_argstr = env.get('MSVC_UWP_APP','') + debug('multiple uwp declarations: MSVC_UWP_APP=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple uwp declarations: MSVC_UWP_APP={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + +def _msvc_script_argument_sdk_constraints(msvc, sdk_version): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc_version constraint: %s < %s VS2015', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SDK_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(sdk_version), repr(msvc.version), repr(VS2015.vc_buildtools_def.vc_version) + ) + return err_msg + + for msvc_sdk_version in msvc.vs_def.vc_sdk_versions: + re_sdk_version = re_sdk_dispatch_map[msvc_sdk_version] + if re_sdk_version.match(sdk_version): + debug('valid: sdk_version=%s', repr(sdk_version)) + return None + + debug('invalid: method exit: sdk_version=%s', repr(sdk_version)) + err_msg = "MSVC_SDK_VERSION ({}) is not supported".format(repr(sdk_version)) + return err_msg + +def _msvc_script_argument_sdk(env, msvc, platform_type, arglist): + + sdk_version = env['MSVC_SDK_VERSION'] + debug( + 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform_type=%s', + repr(msvc.version), repr(sdk_version), repr(platform_type) + ) + + if not sdk_version: + return None + + err_msg = _msvc_script_argument_sdk_constraints(msvc, sdk_version) + if err_msg: + raise MSVCArgumentError(err_msg) + + sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) + + if sdk_version not in sdk_list: + err_msg = "MSVC_SDK_VERSION {} not found for platform type {}".format( + repr(sdk_version), repr(platform_type) + ) + raise MSVCArgumentError(err_msg) + + argpair = (SortOrder.SDK, sdk_version) + arglist.append(argpair) + + return sdk_version + +def _msvc_script_default_sdk(env, msvc, platform_type, arglist): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: + return None + + sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) + if not len(sdk_list): + return None + + sdk_default = sdk_list[0] + + debug( + 'MSVC_VERSION=%s, sdk_default=%s, platform_type=%s', + repr(msvc.version), repr(sdk_default), repr(platform_type) + ) + + argpair = (SortOrder.SDK, sdk_default) + arglist.append(argpair) + + return sdk_default + +def _user_script_argument_sdk(env, sdk_version, user_argstr): + + matches = [m for m in re_vcvars_sdk.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple sdk version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple sdk version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not sdk_version: + user_sdk = matches[0].group('sdk') + return user_sdk + + env_argstr = env.get('MSVC_SDK_VERSION','') + debug('multiple sdk version declarations: MSVC_SDK_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple sdk version declarations: MSVC_SDK_VERSION={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + +def _msvc_read_toolset_file(msvc, filename): + toolset_version = None + try: + with open(filename) as f: + toolset_version = f.readlines()[0].strip() + debug( + 'msvc_version=%s, filename=%s, toolset_version=%s', + repr(msvc.version), repr(filename), repr(toolset_version) + ) + except OSError: + debug('OSError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) + except IndexError: + debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) + return toolset_version + +def _msvc_read_toolset_folders(msvc, vc_dir): + + toolsets_sxs = {} + toolsets_full = [] + + build_dir = os.path.join(vc_dir, "Auxiliary", "Build") + sxs_toolsets = [f.name for f in os.scandir(build_dir) if f.is_dir()] + for sxs_toolset in sxs_toolsets: + filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) + filepath = os.path.join(build_dir, sxs_toolset, filename) + debug('sxs toolset: check file=%s', repr(filepath)) + if os.path.exists(filepath): + toolset_version = _msvc_read_toolset_file(msvc, filepath) + if not toolset_version: + continue + toolsets_sxs[sxs_toolset] = toolset_version + debug( + 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', + repr(msvc.version), repr(sxs_toolset), toolset_version + ) + + toolset_dir = os.path.join(vc_dir, "Tools", "MSVC") + toolsets = [f.name for f in os.scandir(toolset_dir) if f.is_dir()] + for toolset_version in toolsets: + binpath = os.path.join(toolset_dir, toolset_version, "bin") + debug('toolset: check binpath=%s', repr(binpath)) + if os.path.exists(binpath): + toolsets_full.append(toolset_version) + debug( + 'toolset: msvc_version=%s, toolset_version=%s', + repr(msvc.version), repr(toolset_version) + ) + + toolsets_full.sort(reverse=True) + debug('msvc_version=%s, toolsets=%s', repr(msvc.version), repr(toolsets_full)) + + return toolsets_sxs, toolsets_full + +def _msvc_read_toolset_default(msvc, vc_dir): + + build_dir = os.path.join(vc_dir, "Auxiliary", "Build") + + # VS2019+ + filename = "Microsoft.VCToolsVersion.{}.default.txt".format(msvc.vs_def.vc_buildtools_def.vc_buildtools) + filepath = os.path.join(build_dir, filename) + + debug('default toolset: check file=%s', repr(filepath)) + if os.path.exists(filepath): + toolset_buildtools = _msvc_read_toolset_file(msvc, filepath) + if toolset_buildtools: + return toolset_buildtools + + # VS2017+ + filename = "Microsoft.VCToolsVersion.default.txt" + filepath = os.path.join(build_dir, filename) + + debug('default toolset: check file=%s', repr(filepath)) + if os.path.exists(filepath): + toolset_default = _msvc_read_toolset_file(msvc, filepath) + if toolset_default: + return toolset_default + + return None + +_toolset_version_cache = {} +_toolset_default_cache = {} + +def _reset_toolset_cache(): + debug('reset: toolset cache') + _toolset_version_cache = {} + _toolset_default_cache = {} + +def _msvc_version_toolsets(msvc, vc_dir): + + if msvc.version in _toolset_version_cache: + toolsets_sxs, toolsets_full = _toolset_version_cache[msvc.version] + else: + toolsets_sxs, toolsets_full = _msvc_read_toolset_folders(msvc, vc_dir) + _toolset_version_cache[msvc.version] = toolsets_sxs, toolsets_full + + return toolsets_sxs, toolsets_full + +def _msvc_default_toolset(msvc, vc_dir): + + if msvc.version in _toolset_default_cache: + toolset_default = _toolset_default_cache[msvc.version] + else: + toolset_default = _msvc_read_toolset_default(msvc, vc_dir) + _toolset_default_cache[msvc.version] = toolset_default + + return toolset_default + +def _msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version): + + if toolset_version == '14.0': + return toolset_version + + toolsets_sxs, toolsets_full = _msvc_version_toolsets(msvc, vc_dir) + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric == VS2019.vc_buildtools_def.vc_version_numeric: + # necessary to detect toolset not found + if toolset_version == '14.28.16.8': + new_toolset_version = '14.28' + # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 + # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files + # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. + debug( + 'rewrite toolset_version=%s => toolset_version=%s', + repr(toolset_version), repr(new_toolset_version) + ) + toolset_version = new_toolset_version + + if toolset_version in toolsets_sxs: + toolset_vcvars = toolsets_sxs[toolset_version] + return toolset_vcvars + + for toolset_full in toolsets_full: + if toolset_full.startswith(toolset_version): + toolset_vcvars = toolset_full + return toolset_vcvars + + return None + +def _msvc_script_argument_toolset_constraints(msvc, toolset_version): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2017', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2017.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( + repr(toolset_version), repr(msvc.version), repr(VS2017.vc_buildtools_def.vc_version) + ) + return err_msg + + m = re_toolset_version.match(toolset_version) + if not m: + debug('invalid: re_toolset_version: toolset_version=%s', repr(toolset_version)) + err_msg = 'MSVC_TOOLSET_VERSION {} format is not supported'.format( + repr(toolset_version) + ) + return err_msg + + toolset_ver = m.group('version') + toolset_vernum = float(toolset_ver) + + if toolset_vernum < VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: toolset version constraint: %s < %s VS2015', + repr(toolset_vernum), repr(VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} < {} VS2015".format( + repr(toolset_version), repr(toolset_ver), repr(VS2015.vc_buildtools_def.vc_version) + ) + return err_msg + + if toolset_vernum > msvc.vs_def.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: toolset version constraint: toolset %s > %s msvc', + repr(toolset_vernum), repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} > {} MSVC_VERSION".format( + repr(toolset_version), repr(toolset_ver), repr(msvc.version) + ) + return err_msg + + if toolset_vernum == VS2015.vc_buildtools_def.vc_version_numeric: + # tooset = 14.0 + if re_toolset_full.match(toolset_version): + if not re_toolset_140.match(toolset_version): + debug( + 'invalid: toolset version 14.0 constraint: %s != 14.0', + repr(toolset_version) + ) + err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} != '14.0'".format( + repr(toolset_version), repr(toolset_version) + ) + return err_msg + return None + + if re_toolset_full.match(toolset_version): + debug('valid: re_toolset_full: toolset_version=%s', repr(toolset_version)) + return None + + if re_toolset_sxs.match(toolset_version): + debug('valid: re_toolset_sxs: toolset_version=%s', repr(toolset_version)) + return None + + debug('invalid: method exit: toolset_version=%s', repr(toolset_version)) + err_msg = "MSVC_TOOLSET_VERSION ({}) format is not supported".format(repr(toolset_version)) + return err_msg + +def _msvc_script_argument_toolset(env, msvc, vc_dir, arglist): + + toolset_version = env['MSVC_TOOLSET_VERSION'] + debug('MSVC_VERSION=%s, MSVC_TOOLSET_VERSION=%s', repr(msvc.version), repr(toolset_version)) + + if not toolset_version: + return None + + err_msg = _msvc_script_argument_toolset_constraints(msvc, toolset_version) + if err_msg: + raise MSVCArgumentError(err_msg) + + if toolset_version.startswith('14.0') and len(toolset_version) > len('14.0'): + new_toolset_version = '14.0' + debug( + 'rewrite toolset_version=%s => toolset_version=%s', + repr(toolset_version), repr(new_toolset_version) + ) + toolset_version = new_toolset_version + + toolset_vcvars = _msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version) + debug( + 'toolset: toolset_version=%s, toolset_vcvars=%s', + repr(toolset_version), repr(toolset_vcvars) + ) + + if not toolset_vcvars: + err_msg = "MSVC_TOOLSET_VERSION {} not found for MSVC_VERSION {}".format( + repr(toolset_version), repr(msvc.version) + ) + raise MSVCArgumentError(err_msg) + + argpair = (SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_vcvars)) + arglist.append(argpair) + + return toolset_vcvars + +def _msvc_script_default_toolset(env, msvc, vc_dir, arglist): + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: + return None + + toolset_default = _msvc_default_toolset(msvc, vc_dir) + if not toolset_default: + return None + + debug('MSVC_VERSION=%s, toolset_default=%s', repr(msvc.version), repr(toolset_default)) + + argpair = (SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_default)) + arglist.append(argpair) + + return toolset_default + +def _user_script_argument_toolset(env, toolset_version, user_argstr): + + matches = [m for m in re_vcvars_toolset.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple toolset version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple toolset version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not toolset_version: + user_toolset = matches[0].group('toolset') + return user_toolset + + env_argstr = env.get('MSVC_TOOLSET_VERSION','') + debug('multiple toolset version declarations: MSVC_TOOLSET_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple toolset version declarations: MSVC_TOOLSET_VERSION={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + +def _msvc_script_argument_spectre(env, msvc, arglist): + + spectre_libs = env['MSVC_SPECTRE_LIBS'] + debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) + + if not spectre_libs: + return None + + if spectre_libs not in (True, '1'): + return None + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2017', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2017.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( + repr(spectre_libs), repr(msvc.version), repr(VS2017.vc_buildtools_def.vc_version) + ) + raise MSVCArgumentError(err_msg) + + spectre_arg = 'spectre' + + # spectre libs may not be installed + argpair = (SortOrder.SPECTRE, '-vcvars_spectre_libs={}'.format(spectre_arg)) + arglist.append(argpair) + + return spectre_arg + +def _msvc_script_argument_user(env, msvc, arglist): + + # subst None -> empty string + script_args = env.subst('$MSVC_SCRIPT_ARGS') + debug('MSVC_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(msvc.version), repr(script_args)) + + if not script_args: + return None + + if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: msvc version constraint: %s < %s VS2015', + repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2015.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SCRIPT_ARGS ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( + repr(script_args), repr(msvc.version), repr(VS2015.vc_buildtools_def.vc_version) + ) + raise MSVCArgumentError(err_msg) + + # user arguments are not validated + argpair = (SortOrder.USER, script_args) + arglist.append(argpair) + + return script_args + +def _user_script_argument_spectre(env, spectre, user_argstr): + + matches = [m for m in re_vcvars_spectre.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple spectre declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple spectre declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not spectre: + return None + + env_argstr = env.get('MSVC_SPECTRE_LIBS','') + debug('multiple spectre declarations: MSVC_SPECTRE_LIBS=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple spectre declarations: MSVC_SPECTRE_LIBS={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + +def msvc_script_arguments(env, version, vc_dir, arg): + + arglist = [] + + msvc = _msvc_version(version) + + if arg: + argpair = (SortOrder.ARCH, arg) + arglist.append(argpair) + + if 'MSVC_SCRIPT_ARGS' in env: + user_argstr = _msvc_script_argument_user(env, msvc, arglist) + else: + user_argstr = None + + if 'MSVC_UWP_APP' in env: + uwp = _msvc_script_argument_uwp(env, msvc, arglist) + else: + uwp = None + + if user_argstr: + _user_script_argument_uwp(env, uwp, user_argstr) + + platform_type = 'uwp' if uwp else 'desktop' + + if 'MSVC_SDK_VERSION' in env: + sdk_version = _msvc_script_argument_sdk(env, msvc, platform_type, arglist) + else: + sdk_version = None + + if user_argstr: + user_sdk = _user_script_argument_sdk(env, sdk_version, user_argstr) + else: + user_sdk = None + + if MSVC_FORCE_DEFAULT_SDK: + if not sdk_version and not user_sdk: + sdk_version = _msvc_script_default_sdk(env, msvc, platform_type, arglist) + + if 'MSVC_TOOLSET_VERSION' in env: + toolset_version = _msvc_script_argument_toolset(env, msvc, vc_dir, arglist) + else: + toolset_version = None + + if user_argstr: + user_toolset = _user_script_argument_toolset(env, toolset_version, user_argstr) + else: + user_toolset = None + + if MSVC_FORCE_DEFAULT_TOOLSET: + if not toolset_version and not user_toolset: + toolset_version = _msvc_script_default_toolset(env, msvc, vc_dir, arglist) + + if 'MSVC_SPECTRE_LIBS' in env: + spectre = _msvc_script_argument_spectre(env, msvc, arglist) + else: + spectre = None + + if user_argstr: + _user_script_argument_spectre(env, spectre, user_argstr) + + if arglist: + arglist.sort() + argstr = ' '.join([argpair[-1] for argpair in arglist]).strip() + else: + argstr = '' + + debug('arguments: %s', repr(argstr)) + return argstr + +def reset(): + debug('') + _reset_toolset_cache() + +def verify(): + debug('') + _verify_re_sdk_dispatch_map() + diff --git a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py new file mode 100644 index 0000000..8a79007 --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py @@ -0,0 +1,235 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Determine if and/or when an error/warning should be issued when there +are no versions of msvc installed. If there is at least one version of +msvc installed, these routines do (almost) nothing. + +Notes: + * When msvc is the default compiler because there are no compilers + installed, a build may fail due to the cl.exe command not being + recognized. Currently, there is no easy way to detect during + msvc initialization if the default environment will be used later + to build a program and/or library. There is no error/warning + as there are legitimate SCons uses that do not require a c compiler. + * An error is indicated by returning a non-empty tool list from the + function register_iserror. +""" + +import re + +from .. common import ( + debug, +) + +from . import Dispatcher + +Dispatcher.register_modulename(__name__) + +class _Data: + + separator = r';' + + need_init = True + + @classmethod + def reset(cls): + debug('msvc default:init') + cls.n_setup = 0 # number of calls to msvc_setup_env_once + cls.default_ismsvc = False # is msvc the default compiler + cls.default_tools_re_list = [] # list of default tools regular expressions + cls.msvc_tools_init = set() # tools registered via msvc_exists + cls.msvc_tools = None # tools registered via msvc_setup_env_once + cls.msvc_installed = False # is msvc installed (vcs_installed > 0) + cls.msvc_nodefault = False # is there a default version of msvc + cls.need_init = True # reset initialization indicator + +def _initialize(env, msvc_exists_func): + if _Data.need_init: + _Data.reset() + _Data.need_init = False + _Data.msvc_installed = msvc_exists_func(env) + debug('msvc default:msvc_installed=%s', _Data.msvc_installed) + +def register_tool(env, tool, msvc_exists_func): + debug('msvc default:tool=%s', tool) + if _Data.need_init: + _initialize(env, msvc_exists_func) + if _Data.msvc_installed: + return None + if not tool: + return None + if _Data.n_setup == 0: + if tool not in _Data.msvc_tools_init: + _Data.msvc_tools_init.add(tool) + debug('msvc default:tool=%s, msvc_tools_init=%s', tool, _Data.msvc_tools_init) + return None + if tool not in _Data.msvc_tools: + _Data.msvc_tools.add(tool) + debug('msvc default:tool=%s, msvc_tools=%s', tool, _Data.msvc_tools) + +def register_setup(env, msvc_exists_func): + debug('msvc default') + if _Data.need_init: + _initialize(env, msvc_exists_func) + _Data.n_setup += 1 + if not _Data.msvc_installed: + _Data.msvc_tools = set(_Data.msvc_tools_init) + if _Data.n_setup == 1: + tool_list = env.get('TOOLS', None) + if tool_list and tool_list[0] == 'default': + if len(tool_list) > 1 and tool_list[1] in _Data.msvc_tools: + # msvc tools are the default compiler + _Data.default_ismsvc = True + _Data.msvc_nodefault = False + debug( + 'msvc default:n_setup=%d, msvc_installed=%s, default_ismsvc=%s', + _Data.n_setup, _Data.msvc_installed, _Data.default_ismsvc + ) + +def set_nodefault(): + # default msvc version, msvc not installed + _Data.msvc_nodefault = True + debug('msvc default:msvc_nodefault=%s', _Data.msvc_nodefault) + +def register_iserror(env, tool, msvc_exists_func): + + register_tool(env, tool, msvc_exists_func) + + if _Data.msvc_installed: + # msvc installed + return None + + if not _Data.msvc_nodefault: + # msvc version specified + return None + + tool_list = env.get('TOOLS', None) + if not tool_list: + # tool list is empty + return None + + debug( + 'msvc default:n_setup=%s, default_ismsvc=%s, msvc_tools=%s, tool_list=%s', + _Data.n_setup, _Data.default_ismsvc, _Data.msvc_tools, tool_list + ) + + if not _Data.default_ismsvc: + + # Summary: + # * msvc is not installed + # * msvc version not specified (default) + # * msvc is not the default compiler + + # construct tools set + tools_set = set(tool_list) + + else: + + if _Data.n_setup == 1: + # first setup and msvc is default compiler: + # build default tools regex for current tool state + tools = _Data.separator.join(tool_list) + tools_nchar = len(tools) + debug('msvc default:add regex:nchar=%d, tools=%s', tools_nchar, tools) + re_default_tools = re.compile(re.escape(tools)) + _Data.default_tools_re_list.insert(0, (tools_nchar, re_default_tools)) + # early exit: no error for default environment when msvc is not installed + return None + + # Summary: + # * msvc is not installed + # * msvc version not specified (default) + # * environment tools list is not empty + # * default tools regex list constructed + # * msvc tools set constructed + # + # Algorithm using tools string and sets: + # * convert environment tools list to a string + # * iteratively remove default tools sequences via regex + # substition list built from longest sequence (first) + # to shortest sequence (last) + # * build environment tools set with remaining tools + # * compute intersection of environment tools and msvc tools sets + # * if the intersection is: + # empty - no error: default tools and/or no additional msvc tools + # not empty - error: user specified one or more msvc tool(s) + # + # This will not produce an error or warning when there are no + # msvc installed instances nor any other recognized compilers + # and the default environment is needed for a build. The msvc + # compiler is forcibly added to the environment tools list when + # there are no compilers installed on win32. In this case, cl.exe + # will not be found on the path resulting in a failed build. + + # construct tools string + tools = _Data.separator.join(tool_list) + tools_nchar = len(tools) + + debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) + + # iteratively remove default tool sequences (longest to shortest) + re_nchar_min, re_tools_min = _Data.default_tools_re_list[-1] + if tools_nchar >= re_nchar_min and re_tools_min.search(tools): + # minimum characters satisfied and minimum pattern exists + for re_nchar, re_default_tool in _Data.default_tools_re_list: + if tools_nchar < re_nchar: + # not enough characters for pattern + continue + tools = re_default_tool.sub('', tools).strip(_Data.separator) + tools_nchar = len(tools) + debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) + if tools_nchar < re_nchar_min or not re_tools_min.search(tools): + # less than minimum characters or minimum pattern does not exist + break + + # construct non-default list(s) tools set + tools_set = {msvc_tool for msvc_tool in tools.split(_Data.separator) if msvc_tool} + + debug('msvc default:tools=%s', tools_set) + if not tools_set: + return None + + # compute intersection of remaining tools set and msvc tools set + tools_found = _Data.msvc_tools.intersection(tools_set) + debug('msvc default:tools_exist=%s', tools_found) + if not tools_found: + return None + + # construct in same order as tools list + tools_found_list = [] + seen_tool = set() + for tool in tool_list: + if tool not in seen_tool: + seen_tool.add(tool) + if tool in tools_found: + tools_found_list.append(tool) + + # return tool list in order presented + return tools_found_list + +def reset(): + debug('') + _Data.reset() + diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py new file mode 100644 index 0000000..15abdcd --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -0,0 +1,55 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Helper functions for Microsoft Visual C/C++. +""" + +import os +import re + +def listdir_dirs(p): + dirs = [] + for dir_name in os.listdir(p): + dir_path = os.path.join(p, dir_name) + if os.path.isdir(dir_path): + dirs.append((dir_name, dir_path)) + return dirs + +def process_path(p): + if p: + p = os.path.normpath(p) + p = os.path.realpath(p) + p = os.path.normcase(p) + return p + +re_version_prefix = re.compile(r'^(?P[0-9.]+).*') + +def get_version_prefix(version): + m = re_version_prefix.match(version) + if m: + rval = m.group('version') + else: + rval = '' + return rval + diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py new file mode 100644 index 0000000..e95c72e --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -0,0 +1,250 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Windows SDK functions for Microsoft Visual C/C++. +""" + +import os + +from ..common import ( + debug, +) + +from . import Dispatcher +from . import Util +from . import Config +from . import Registry + +from .Exceptions import ( + MSVCInternalError, +) + +Dispatcher.register_modulename(__name__) + +def _new_sdk_map(): + sdk_map = { + 'desktop': [], + 'uwp': [], + } + return sdk_map + +def _sdk_10_layout(version): + + folder_prefix = version + '.' + + sdk_map = _new_sdk_map() + + sdk_roots = Registry.sdk_query_paths(version) + + sdk_version_platform_seen = set() + sdk_roots_seen = set() + + for sdk_t in sdk_roots: + + sdk_root = sdk_t[0] + if sdk_root in sdk_roots_seen: + continue + sdk_roots_seen.add(sdk_root) + + if not os.path.exists(sdk_root): + continue + + sdk_include_path = os.path.join(sdk_root, 'include') + if not os.path.exists(sdk_include_path): + continue + + for version_nbr, version_nbr_path in Util.listdir_dirs(sdk_include_path): + + if not version_nbr.startswith(folder_prefix): + continue + + sdk_inc_path = Util.process_path(os.path.join(version_nbr_path, 'um')) + if not os.path.exists(sdk_inc_path): + continue + + for platform_type, sdk_inc_file in [ + ('desktop', 'winsdkver.h'), + ('uwp', 'windows.h'), + ]: + + if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): + continue + + key = (version_nbr, platform_type) + if key in sdk_version_platform_seen: + continue + sdk_version_platform_seen.add(key) + + sdk_map[platform_type].append(version_nbr) + + for key, val in sdk_map.items(): + val.sort(reverse=True) + + return sdk_map + +def _sdk_81_layout(version): + + version_nbr = version + + sdk_map = _new_sdk_map() + + sdk_roots = Registry.sdk_query_paths(version) + + sdk_version_platform_seen = set() + sdk_roots_seen = set() + + for sdk_t in sdk_roots: + + sdk_root = sdk_t[0] + if sdk_root in sdk_roots_seen: + continue + sdk_roots_seen.add(sdk_root) + + # msvc does not check for existence of root or other files + + sdk_inc_path = Util.process_path(os.path.join(sdk_root, r'include\um')) + if not os.path.exists(sdk_inc_path): + continue + + for platform_type, sdk_inc_file in [ + ('desktop', 'winsdkver.h'), + ('uwp', 'windows.h'), + ]: + + if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): + continue + + key = (version_nbr, platform_type) + if key in sdk_version_platform_seen: + continue + sdk_version_platform_seen.add(key) + + sdk_map[platform_type].append(version_nbr) + + for key, val in sdk_map.items(): + val.sort(reverse=True) + + return sdk_map + +_sdk_map_cache = {} +_sdk_cache = {} + +def _reset_sdk_cache(): + debug('') + _sdk_map_cache = {} + _sdk_cache = {} + +def _sdk_10(key, reg_version): + if key in _sdk_map_cache: + sdk_map = _sdk_map_cache[key] + else: + sdk_map = _sdk_10_layout(reg_version) + _sdk_map_cache[key] = sdk_map + return sdk_map + +def _sdk_81(key, reg_version): + if key in _sdk_map_cache: + sdk_map = _sdk_map_cache[key] + else: + sdk_map = _sdk_81_layout(reg_version) + _sdk_map_cache[key] = sdk_map + return sdk_map + +def _combine_sdk_map_list(sdk_map_list): + combined_sdk_map = _new_sdk_map() + for sdk_map in sdk_map_list: + for key, val in sdk_map.items(): + combined_sdk_map[key].extend(val) + return combined_sdk_map + +_sdk_dispatch_map = { + '10.0': (_sdk_10, '10.0'), + '8.1': (_sdk_81, '8.1'), +} + +def _verify_sdk_dispatch_map(): + debug('') + for sdk_version in Config.MSVC_SDK_VERSIONS: + if sdk_version in _sdk_dispatch_map: + continue + err_msg = 'sdk version {} not in sdk_dispatch_map'.format(sdk_version) + raise MSVCInternalError(err_msg) + return None + +def _version_list_sdk_map(version_list): + + sdk_map_list = [] + for version in version_list: + func, reg_version = _sdk_dispatch_map[version] + sdk_map = func(version, reg_version) + sdk_map_list.append(sdk_map) + + combined_sdk_map = _combine_sdk_map_list(sdk_map_list) + return combined_sdk_map + +def _sdk_map(version_list): + key = tuple(version_list) + if key in _sdk_cache: + sdk_map = _sdk_cache[key] + else: + version_numlist = [float(v) for v in version_list] + version_numlist.sort(reverse=True) + key = tuple([str(v) for v in version_numlist]) + sdk_map = _version_list_sdk_map(key) + _sdk_cache[key] = sdk_map + return sdk_map + +def get_sdk_version_list(version_list, platform_type): + sdk_map = _sdk_map(version_list) + sdk_list = sdk_map.get(platform_type, []) + return sdk_list + +def get_msvc_sdk_version_list(msvc_version, msvc_uwp_app=False): + debug('msvc_version=%s, msvc_uwp_app=%s', repr(msvc_version), repr(msvc_uwp_app)) + + sdk_versions = [] + + verstr = Util.get_version_prefix(msvc_version) + vs_def = Config.MSVC_VERSION_EXTERNAL.get(verstr, None) + if not vs_def: + debug('vs_def is not defined') + return sdk_versions + + is_uwp = True if msvc_uwp_app in Config.BOOLEAN_SYMBOLS[True] else False + platform_type = 'uwp' if is_uwp else 'desktop' + sdk_list = get_sdk_version_list(vs_def.vc_sdk_versions, platform_type) + + sdk_versions.extend(sdk_list) + debug('sdk_versions=%s', repr(sdk_versions)) + + return sdk_versions + +def reset(): + debug('') + _reset_sdk_cache() + +def verify(): + debug('') + _verify_sdk_dispatch_map() + diff --git a/SCons/Tool/MSCommon/MSVC/__init__.py b/SCons/Tool/MSCommon/MSVC/__init__.py new file mode 100644 index 0000000..afd993f --- /dev/null +++ b/SCons/Tool/MSCommon/MSVC/__init__.py @@ -0,0 +1,50 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Functions for Microsoft Visual C/C++. +""" + +from . import Exceptions +from . import Util + +from . import Dispatcher as _Dispatcher + +from . import Config +from . import Registry +from . import SetupEnvDefault +from . import NotFound +from . import WinSDK +from . import ScriptArguments + +from .NotFound import ( + set_msvc_notfound_policy, + get_msvc_notfound_policy, +) + +def reset(): + _Dispatcher.reset() + +#reset() # testing +_Dispatcher.verify() + diff --git a/SCons/Tool/MSCommon/__init__.py b/SCons/Tool/MSCommon/__init__.py index de78f84..9f35e94 100644 --- a/SCons/Tool/MSCommon/__init__.py +++ b/SCons/Tool/MSCommon/__init__.py @@ -32,14 +32,17 @@ import SCons.Util from SCons.Tool.MSCommon.sdk import mssdk_exists, mssdk_setup_env +from SCons.Tool.MSCommon.MSVC import ( + set_msvc_notfound_policy, + get_msvc_notfound_policy, +) + from SCons.Tool.MSCommon.vc import ( msvc_exists, msvc_setup_env_tool, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere, - set_msvc_notfound_policy, - get_msvc_notfound_policy, get_msvc_sdk_versions, ) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index ab05323..5a27f44 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -51,7 +51,6 @@ from collections import ( namedtuple, OrderedDict, ) -import enum import SCons.Util import SCons.Warnings @@ -61,9 +60,10 @@ from . import common from .common import CONFIG_CACHE, debug from .sdk import get_installed_sdks - -class VisualCException(Exception): - pass +from . import MSVC +from .MSVC.Exceptions import ( + VisualCException +) class UnsupportedVersion(VisualCException): pass @@ -89,296 +89,10 @@ class MSVCScriptNotFound(VisualCException): class MSVCUseSettingsError(VisualCException): pass -class MSVCVersionNotFound(VisualCException): - pass - -class MSVCArgumentError(VisualCException): - pass - -class MSVCInternalError(VisualCException): - pass - class BatchFileExecutionWarning(SCons.Warnings.WarningOnByDefault): pass -class _Dispatcher: - - classrefs = [] - - @classmethod - def register(cls, classref): - cls.classrefs.append(classref) - - @classmethod - def reset(cls): - debug('reset %s', cls.__name__) - for classref in cls.classrefs: - for method in ['reset', '_reset']: - if not hasattr(classref, method) or not callable(getattr(classref, method, None)): - continue - debug('call %s.%s()', classref.__name__, method) - func = getattr(classref, method) - func() - - @classmethod - def verify(cls): - debug('verify %s', cls.__name__) - for classref in cls.classrefs: - for method in ['verify', '_verify']: - if not hasattr(classref, method) or not callable(getattr(classref, method, None)): - continue - debug('call %s.%s()', classref.__name__, method) - func = getattr(classref, method) - func() - -class _Config: - - BOOLEAN_SYMBOLS = {} - BOOLEAN_EXTERNAL = {} - - for bool, symbol_list, symbol_case_list in [ - (False, (False, 0, '0', None, ''), ('False', 'No', 'F', 'N')), - (True, (True, 1, '1'), ('True', 'Yes', 'T', 'Y')), - ]: - BOOLEAN_SYMBOLS[bool] = list(symbol_list) - for symbol in symbol_case_list: - BOOLEAN_SYMBOLS[bool].extend([symbol, symbol.lower(), symbol.upper()]) - - for symbol in BOOLEAN_SYMBOLS[bool]: - BOOLEAN_EXTERNAL[symbol] = bool - - MSVC_RUNTIME_DEFINITION = namedtuple('MSVCRuntime', [ - 'vc_runtime', - 'vc_runtime_numeric', - 'vc_runtime_alias_list', - 'vc_runtime_vsdef_list', - ]) - - MSVC_RUNTIME_DEFINITION_LIST = [] - - MSVC_RUNTIME_INTERNAL = {} - MSVC_RUNTIME_EXTERNAL = {} - - for vc_runtime, vc_runtime_numeric, vc_runtime_alias_list in [ - ('140', 140, ['ucrt']), - ('120', 120, ['msvcr120']), - ('110', 110, ['msvcr110']), - ('100', 100, ['msvcr100']), - ('90', 90, ['msvcr90']), - ('80', 80, ['msvcr80']), - ('71', 71, ['msvcr71']), - ('70', 70, ['msvcr70']), - ('60', 60, ['msvcrt']), - ]: - vc_runtime_def = MSVC_RUNTIME_DEFINITION( - vc_runtime = vc_runtime, - vc_runtime_numeric = vc_runtime_numeric, - vc_runtime_alias_list = vc_runtime_alias_list, - vc_runtime_vsdef_list = [], - ) - - MSVC_RUNTIME_DEFINITION_LIST.append(vc_runtime_def) - - MSVC_RUNTIME_INTERNAL[vc_runtime] = vc_runtime_def - MSVC_RUNTIME_EXTERNAL[vc_runtime] = vc_runtime_def - - for vc_runtime_alias in vc_runtime_alias_list: - MSVC_RUNTIME_EXTERNAL[vc_runtime_alias] = vc_runtime_def - - MSVC_BUILDTOOLS_DEFINITION = namedtuple('MSVCBuildtools', [ - 'vc_buildtools', - 'vc_buildtools_numeric', - 'vc_version', - 'vc_version_numeric', - 'cl_version', - 'cl_version_numeric', - 'vc_runtime_def', - ]) - - MSVC_BUILDTOOLS_DEFINITION_LIST = [] - - MSVC_BUILDTOOLS_INTERNAL = {} - MSVC_BUILDTOOLS_EXTERNAL = {} - - VC_VERSION_MAP = {} - - for vc_buildtools, vc_version, cl_version, vc_runtime in [ - ('v143', '14.3', '19.3', '140'), - ('v142', '14.2', '19.2', '140'), - ('v141', '14.1', '19.1', '140'), - ('v140', '14.0', '19.0', '140'), - ('v120', '12.0', '18.0', '120'), - ('v110', '11.0', '17.0', '110'), - ('v100', '10.0', '16.0', '100'), - ('v90', '9.0', '15.0', '90'), - ('v80', '8.0', '14.0', '80'), - ('v71', '7.1', '13.1', '71'), - ('v70', '7.0', '13.0', '70'), - ('v60', '6.0', '12.0', '60'), - ]: - - vc_runtime_def = MSVC_RUNTIME_INTERNAL[vc_runtime] - - vc_buildtools_def = MSVC_BUILDTOOLS_DEFINITION( - vc_buildtools = vc_buildtools, - vc_buildtools_numeric = int(vc_buildtools[1:]), - vc_version = vc_version, - vc_version_numeric = float(vc_version), - cl_version = cl_version, - cl_version_numeric = float(cl_version), - vc_runtime_def = vc_runtime_def, - ) - - MSVC_BUILDTOOLS_DEFINITION_LIST.append(vc_buildtools_def) - - MSVC_BUILDTOOLS_INTERNAL[vc_buildtools] = vc_buildtools_def - MSVC_BUILDTOOLS_EXTERNAL[vc_buildtools] = vc_buildtools_def - MSVC_BUILDTOOLS_EXTERNAL[vc_version] = vc_buildtools_def - - VC_VERSION_MAP[vc_version] = vc_buildtools_def - - MSVS_VERSION_INTERNAL = {} - MSVS_VERSION_EXTERNAL = {} - - MSVC_VERSION_INTERNAL = {} - MSVC_VERSION_EXTERNAL = {} - - MSVS_VERSION_MAJOR_MAP = {} - - CL_VERSION_MAP = {} - - MSVC_SDK_VERSIONS = set() - - VISUALSTUDIO_DEFINITION = namedtuple('VisualStudioDefinition', [ - 'vs_product', - 'vs_product_alias_list', - 'vs_version', - 'vs_version_major', - 'vs_envvar', - 'vs_express', - 'vs_lookup', - 'vc_sdk_versions', - 'vc_ucrt_versions', - 'vc_uwp', - 'vc_buildtools_def', - 'vc_buildtools_all', - ]) - - VISUALSTUDIO_DEFINITION_LIST = [] - - VS_PRODUCT_ALIAS = { - '1998': ['6'] - } - - # vs_envvar: VisualStudioVersion defined in environment for MSVS 2012 and later - # MSVS 2010 and earlier cl_version -> vs_def is a 1:1 mapping - # SDK attached to product or buildtools? - for vs_product, vs_version, vs_envvar, vs_express, vs_lookup, vc_sdk, vc_ucrt, vc_uwp, vc_buildtools_all in [ - ('2022', '17.0', True, False, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v143', 'v142', 'v141', 'v140']), - ('2019', '16.0', True, False, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v142', 'v141', 'v140']), - ('2017', '15.0', True, True, 'vswhere' , ['10.0', '8.1'], ['10'], 'uwp', ['v141', 'v140']), - ('2015', '14.0', True, True, 'registry', ['10.0', '8.1'], ['10'], 'store', ['v140']), - ('2013', '12.0', True, True, 'registry', None, None, None, ['v120']), - ('2012', '11.0', True, True, 'registry', None, None, None, ['v110']), - ('2010', '10.0', False, True, 'registry', None, None, None, ['v100']), - ('2008', '9.0', False, True, 'registry', None, None, None, ['v90']), - ('2005', '8.0', False, True, 'registry', None, None, None, ['v80']), - ('2003', '7.1', False, False, 'registry', None, None, None, ['v71']), - ('2002', '7.0', False, False, 'registry', None, None, None, ['v70']), - ('1998', '6.0', False, False, 'registry', None, None, None, ['v60']), - ]: - - vs_version_major = vs_version.split('.')[0] - - vc_buildtools_def = MSVC_BUILDTOOLS_INTERNAL[vc_buildtools_all[0]] - - vs_def = VISUALSTUDIO_DEFINITION( - vs_product = vs_product, - vs_product_alias_list = [], - vs_version = vs_version, - vs_version_major = vs_version_major, - vs_envvar = vs_envvar, - vs_express = vs_express, - vs_lookup = vs_lookup, - vc_sdk_versions = vc_sdk, - vc_ucrt_versions = vc_ucrt, - vc_uwp = vc_uwp, - vc_buildtools_def = vc_buildtools_def, - vc_buildtools_all = vc_buildtools_all, - ) - - VISUALSTUDIO_DEFINITION_LIST.append(vs_def) - - vc_buildtools_def.vc_runtime_def.vc_runtime_vsdef_list.append(vs_def) - - MSVS_VERSION_INTERNAL[vs_product] = vs_def - MSVS_VERSION_EXTERNAL[vs_product] = vs_def - MSVS_VERSION_EXTERNAL[vs_version] = vs_def - - MSVC_VERSION_INTERNAL[vc_buildtools_def.vc_version] = vs_def - MSVC_VERSION_EXTERNAL[vs_product] = vs_def - MSVC_VERSION_EXTERNAL[vc_buildtools_def.vc_version] = vs_def - MSVC_VERSION_EXTERNAL[vc_buildtools_def.vc_buildtools] = vs_def - - if vs_product in VS_PRODUCT_ALIAS: - for vs_product_alias in VS_PRODUCT_ALIAS[vs_product]: - vs_def.vs_product_alias_list.append(vs_product_alias) - MSVS_VERSION_EXTERNAL[vs_product_alias] = vs_def - MSVC_VERSION_EXTERNAL[vs_product_alias] = vs_def - - MSVS_VERSION_MAJOR_MAP[vs_version_major] = vs_def - - CL_VERSION_MAP[vc_buildtools_def.cl_version] = vs_def - - if not vc_sdk: - continue - - MSVC_SDK_VERSIONS.update(vc_sdk) - - # convert string version set to string version list ranked in descending order - MSVC_SDK_VERSIONS = [str(f) for f in sorted([float(s) for s in MSVC_SDK_VERSIONS], reverse=True)] - - MSVS_VERSION_LEGACY = {} - MSVC_VERSION_LEGACY = {} - - for vdict in (MSVS_VERSION_EXTERNAL, MSVC_VERSION_INTERNAL): - for key, vs_def in vdict.items(): - if key not in MSVS_VERSION_LEGACY: - MSVS_VERSION_LEGACY[key] = vs_def - MSVC_VERSION_LEGACY[key] = vs_def - -# MSVC_NOTFOUND_POLICY definition: -# error: raise exception -# warning: issue warning and continue -# ignore: continue - -_MSVC_NOTFOUND_POLICY_DEFINITION = namedtuple('MSVCNotFoundPolicyDefinition', [ - 'value', - 'symbol', -]) - -_MSVC_NOTFOUND_POLICY_INTERNAL = {} -_MSVC_NOTFOUND_POLICY_EXTERNAL = {} - -for policy_value, policy_symbol_list in [ - (True, ['Error', 'Exception']), - (False, ['Warning', 'Warn']), - (None, ['Ignore', 'Suppress']), -]: - - policy_symbol = policy_symbol_list[0].lower() - policy_def = _MSVC_NOTFOUND_POLICY_DEFINITION(policy_value, policy_symbol) - - _MSVC_NOTFOUND_POLICY_INTERNAL[policy_symbol] = policy_def - - for policy_symbol in policy_symbol_list: - _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.lower()] = policy_def - _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol] = policy_def - _MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.upper()] = policy_def - -_MSVC_NOTFOUND_POLICY_DEF = _MSVC_NOTFOUND_POLICY_INTERNAL['warning'] - # Dict to 'canonalize' the arch _ARCH_TO_CANONICAL = { "amd64" : "amd64", @@ -1253,7 +967,7 @@ def reset_installed_vcs(): """Make it try again to find VC. This is just for the tests.""" global __INSTALLED_VCS_RUN __INSTALLED_VCS_RUN = None - _Dispatcher.reset() + MSVC.reset() def get_default_installed_msvc(env=None): vcs = get_installed_vcs(env) @@ -1344,295 +1058,6 @@ def script_env(script, args=None): return cache_data -def _msvc_notfound_policy_lookup(symbol): - - try: - notfound_policy_def = _MSVC_NOTFOUND_POLICY_EXTERNAL[symbol] - except KeyError: - err_msg = "Value specified for MSVC_NOTFOUND_POLICY is not supported: {}.\n" \ - " Valid values are: {}".format( - repr(symbol), - ', '.join([repr(s) for s in _MSVC_NOTFOUND_POLICY_EXTERNAL.keys()]) - ) - raise ValueError(err_msg) - - return notfound_policy_def - -def set_msvc_notfound_policy(MSVC_NOTFOUND_POLICY=None): - """ Set the default policy when MSVC is not found. - - Args: - MSVC_NOTFOUND_POLICY: - string representing the policy behavior - when MSVC is not found or None - - Returns: - The previous policy is returned when the MSVC_NOTFOUND_POLICY argument - is not None. The active policy is returned when the MSVC_NOTFOUND_POLICY - argument is None. - - """ - global _MSVC_NOTFOUND_POLICY_DEF - - prev_policy = _MSVC_NOTFOUND_POLICY_DEF.symbol - - policy = MSVC_NOTFOUND_POLICY - if policy is not None: - _MSVC_NOTFOUND_POLICY_DEF = _msvc_notfound_policy_lookup(policy) - - debug( - 'prev_policy=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', - repr(prev_policy), repr(policy), - repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) - ) - - return prev_policy - -def get_msvc_notfound_policy(): - """Return the active policy when MSVC is not found.""" - debug( - 'policy.symbol=%s, policy.value=%s', - repr(_MSVC_NOTFOUND_POLICY_DEF.symbol), repr(_MSVC_NOTFOUND_POLICY_DEF.value) - ) - return _MSVC_NOTFOUND_POLICY_DEF.symbol - -def _msvc_notfound_policy_handler(env, msg): - - if env and 'MSVC_NOTFOUND_POLICY' in env: - # environment setting - notfound_policy_src = 'environment' - policy = env['MSVC_NOTFOUND_POLICY'] - if policy is not None: - # user policy request - notfound_policy_def = _msvc_notfound_policy_lookup(policy) - else: - # active global setting - notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF - else: - # active global setting - notfound_policy_src = 'default' - policy = None - notfound_policy_def = _MSVC_NOTFOUND_POLICY_DEF - - debug( - 'source=%s, set_policy=%s, policy.symbol=%s, policy.value=%s', - notfound_policy_src, repr(policy), repr(notfound_policy_def.symbol), repr(notfound_policy_def.value) - ) - - if notfound_policy_def.value is None: - # ignore - pass - elif notfound_policy_def.value: - raise MSVCVersionNotFound(msg) - else: - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, msg) - -class _MSVCSetupEnvDefault: - """ - Determine if and/or when an error/warning should be issued when there - are no versions of msvc installed. If there is at least one version of - msvc installed, these routines do (almost) nothing. - - Notes: - * When msvc is the default compiler because there are no compilers - installed, a build may fail due to the cl.exe command not being - recognized. Currently, there is no easy way to detect during - msvc initialization if the default environment will be used later - to build a program and/or library. There is no error/warning - as there are legitimate SCons uses that do not require a c compiler. - * As implemented, the default is that a warning is issued. This can - be changed globally via the function set_msvc_notfound_policy and/or - through the environment via the MSVC_NOTFOUND_POLICY variable. - """ - - separator = r';' - - need_init = True - - @classmethod - def reset(cls): - debug('msvc default:init') - cls.n_setup = 0 # number of calls to msvc_setup_env_once - cls.default_ismsvc = False # is msvc the default compiler - cls.default_tools_re_list = [] # list of default tools regular expressions - cls.msvc_tools_init = set() # tools registered via msvc_exists - cls.msvc_tools = None # tools registered via msvc_setup_env_once - cls.msvc_installed = False # is msvc installed (vcs_installed > 0) - cls.msvc_nodefault = False # is there a default version of msvc - cls.need_init = True # reset initialization indicator - - @classmethod - def _initialize(cls, env): - if cls.need_init: - cls.reset() - cls.need_init = False - vcs = get_installed_vcs(env) - cls.msvc_installed = len(vcs) > 0 - debug('msvc default:msvc_installed=%s', cls.msvc_installed) - - @classmethod - def register_tool(cls, env, tool): - debug('msvc default:tool=%s', tool) - if cls.need_init: - cls._initialize(env) - if cls.msvc_installed: - return None - if not tool: - return None - if cls.n_setup == 0: - if tool not in cls.msvc_tools_init: - cls.msvc_tools_init.add(tool) - debug('msvc default:tool=%s, msvc_tools_init=%s', tool, cls.msvc_tools_init) - return None - if tool not in cls.msvc_tools: - cls.msvc_tools.add(tool) - debug('msvc default:tool=%s, msvc_tools=%s', tool, cls.msvc_tools) - - @classmethod - def register_setup(cls, env): - debug('msvc default') - if cls.need_init: - cls._initialize(env) - cls.n_setup += 1 - if not cls.msvc_installed: - cls.msvc_tools = set(cls.msvc_tools_init) - if cls.n_setup == 1: - tool_list = env.get('TOOLS', None) - if tool_list and tool_list[0] == 'default': - if len(tool_list) > 1 and tool_list[1] in cls.msvc_tools: - # msvc tools are the default compiler - cls.default_ismsvc = True - cls.msvc_nodefault = False - debug( - 'msvc default:n_setup=%d, msvc_installed=%s, default_ismsvc=%s', - cls.n_setup, cls.msvc_installed, cls.default_ismsvc - ) - - @classmethod - def set_nodefault(cls): - # default msvc version, msvc not installed - cls.msvc_nodefault = True - debug('msvc default:msvc_nodefault=%s', cls.msvc_nodefault) - - @classmethod - def register_iserror(cls, env, tool): - - cls.register_tool(env, tool) - - if cls.msvc_installed: - # msvc installed - return None - - if not cls.msvc_nodefault: - # msvc version specified - return None - - tool_list = env.get('TOOLS', None) - if not tool_list: - # tool list is empty - return None - - debug( - 'msvc default:n_setup=%s, default_ismsvc=%s, msvc_tools=%s, tool_list=%s', - cls.n_setup, cls.default_ismsvc, cls.msvc_tools, tool_list - ) - - if not cls.default_ismsvc: - - # Summary: - # * msvc is not installed - # * msvc version not specified (default) - # * msvc is not the default compiler - - # construct tools set - tools_set = set(tool_list) - - else: - - if cls.n_setup == 1: - # first setup and msvc is default compiler: - # build default tools regex for current tool state - tools = cls.separator.join(tool_list) - tools_nchar = len(tools) - debug('msvc default:add regex:nchar=%d, tools=%s', tools_nchar, tools) - re_default_tools = re.compile(re.escape(tools)) - cls.default_tools_re_list.insert(0, (tools_nchar, re_default_tools)) - # early exit: no error for default environment when msvc is not installed - return None - - # Summary: - # * msvc is not installed - # * msvc version not specified (default) - # * environment tools list is not empty - # * default tools regex list constructed - # * msvc tools set constructed - # - # Algorithm using tools string and sets: - # * convert environment tools list to a string - # * iteratively remove default tools sequences via regex - # substition list built from longest sequence (first) - # to shortest sequence (last) - # * build environment tools set with remaining tools - # * compute intersection of environment tools and msvc tools sets - # * if the intersection is: - # empty - no error: default tools and/or no additional msvc tools - # not empty - error: user specified one or more msvc tool(s) - # - # This will not produce an error or warning when there are no - # msvc installed instances nor any other recognized compilers - # and the default environment is needed for a build. The msvc - # compiler is forcibly added to the environment tools list when - # there are no compilers installed on win32. In this case, cl.exe - # will not be found on the path resulting in a failed build. - - # construct tools string - tools = cls.separator.join(tool_list) - tools_nchar = len(tools) - - debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) - - # iteratively remove default tool sequences (longest to shortest) - re_nchar_min, re_tools_min = cls.default_tools_re_list[-1] - if tools_nchar >= re_nchar_min and re_tools_min.search(tools): - # minimum characters satisfied and minimum pattern exists - for re_nchar, re_default_tool in cls.default_tools_re_list: - if tools_nchar < re_nchar: - # not enough characters for pattern - continue - tools = re_default_tool.sub('', tools).strip(cls.separator) - tools_nchar = len(tools) - debug('msvc default:check tools:nchar=%d, tools=%s', tools_nchar, tools) - if tools_nchar < re_nchar_min or not re_tools_min.search(tools): - # less than minimum characters or minimum pattern does not exist - break - - # construct non-default list(s) tools set - tools_set = {msvc_tool for msvc_tool in tools.split(cls.separator) if msvc_tool} - - debug('msvc default:tools=%s', tools_set) - if not tools_set: - return None - - # compute intersection of remaining tools set and msvc tools set - tools_found = cls.msvc_tools.intersection(tools_set) - debug('msvc default:tools_exist=%s', tools_found) - if not tools_found: - return None - - # construct in same order as tools list - tools_found_list = [] - seen_tool = set() - for tool in tool_list: - if tool not in seen_tool: - seen_tool.add(tool) - if tool in tools_found: - tools_found_list.append(tool) - - # return tool list in order presented - return tools_found_list - -_Dispatcher.register(_MSVCSetupEnvDefault) - def get_default_version(env): msvc_version = env.get('MSVC_VERSION') msvs_version = env.get('MSVS_VERSION') @@ -1673,16 +1098,16 @@ def msvc_setup_env_once(env, tool=None): if not has_run: debug('tool=%s', repr(tool)) - _MSVCSetupEnvDefault.register_setup(env) + MSVC.SetupEnvDefault.register_setup(env, msvc_exists) msvc_setup_env(env) env["MSVC_SETUP_RUN"] = True - req_tools = _MSVCSetupEnvDefault.register_iserror(env, tool) + req_tools = MSVC.SetupEnvDefault.register_iserror(env, tool, msvc_exists) if req_tools: msg = "No versions of the MSVC compiler were found.\n" \ " Visual Studio C/C++ compilers may not be set correctly.\n" \ " Requested tool(s) are: {}".format(req_tools) - _msvc_notfound_policy_handler(env, msg) + MSVC.Notfound.policy_handler(env, msg) def msvc_find_valid_batch_script(env, version): """Find and execute appropriate batch script to set up build env. @@ -1724,7 +1149,7 @@ def msvc_find_valid_batch_script(env, version): debug('use_script 2 %s, args:%s', repr(vc_script), arg) found = None if vc_script: - arg = _ScriptArguments.msvc_script_arguments(env, version, vc_dir, arg) + arg = MSVC.ScriptArguments.msvc_script_arguments(env, version, vc_dir, arg) try: d = script_env(vc_script, args=arg) found = vc_script @@ -1769,7 +1194,7 @@ def msvc_find_valid_batch_script(env, version): " No versions of the MSVC compiler were found.\n" \ " Visual Studio C/C++ compilers may not be set correctly".format(version) - _msvc_notfound_policy_handler(env, msg) + MSVC.NotFound.policy_handler(env, msg) return d @@ -1805,7 +1230,7 @@ def msvc_setup_env(env): version = get_default_version(env) if version is None: if not msvc_setup_env_user(env): - _MSVCSetupEnvDefault.set_nodefault() + MSVC.SetupEnvDefault.set_nodefault() return None # XXX: we set-up both MSVS version for backward @@ -1899,7 +1324,7 @@ def msvc_setup_env_user(env=None): def msvc_setup_env_tool(env=None, version=None, tool=None): debug('tool=%s, version=%s', repr(tool), repr(version)) - _MSVCSetupEnvDefault.register_tool(env, tool) + MSVC.SetupEnvDefault.register_tool(env, tool, msvc_exists) rval = False if not rval and msvc_exists(env, version): rval = True @@ -1920,1043 +1345,6 @@ def get_msvc_sdk_versions(msvc_version=None, msvc_uwp_app=False): debug('no msvc versions detected') return rval - rval = _WindowsSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) + rval = MSVC.WinSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) return rval -class _Util: - - @staticmethod - def listdir_dirs(p): - dirs = [] - for dir_name in os.listdir(p): - dir_path = os.path.join(p, dir_name) - if os.path.isdir(dir_path): - dirs.append((dir_name, dir_path)) - return dirs - - @staticmethod - def process_path(p): - if p: - p = os.path.normpath(p) - p = os.path.realpath(p) - p = os.path.normcase(p) - return p - -class _Registry: - - def read_value(hkey, subkey_valname): - try: - rval = common.read_reg(subkey_valname, hkroot=hkey) - except OSError: - debug('OSError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) - return None - except IndexError: - debug('IndexError: hkey=%s, subkey=%s', repr(hkey), repr(subkey_valname)) - return None - debug('hkey=%s, subkey=%s, rval=%s', repr(hkey), repr(subkey_valname), repr(rval)) - return rval - - @classmethod - def registry_query_path(cls, key, val, suffix): - extval = val + '\\' + suffix if suffix else val - qpath = cls.read_value(key, extval) - if qpath and os.path.exists(qpath): - qpath = _Util.process_path(qpath) - else: - qpath = None - return (qpath, key, val, extval) - - REG_SOFTWARE_MICROSOFT = [ - (SCons.Util.HKEY_LOCAL_MACHINE, r'Software\Wow6432Node\Microsoft'), - (SCons.Util.HKEY_CURRENT_USER, r'Software\Wow6432Node\Microsoft'), # SDK queries - (SCons.Util.HKEY_LOCAL_MACHINE, r'Software\Microsoft'), - (SCons.Util.HKEY_CURRENT_USER, r'Software\Microsoft'), - ] - - @classmethod - def microsoft_query_paths(cls, suffix, usrval=None): - paths = [] - records = [] - for key, val in cls.REG_SOFTWARE_MICROSOFT: - extval = val + '\\' + suffix if suffix else val - qpath = cls.read_value(key, extval) - if qpath and os.path.exists(qpath): - qpath = _Util.process_path(qpath) - if qpath not in paths: - paths.append(qpath) - records.append((qpath, key, val, extval, usrval)) - return records - - @classmethod - def microsoft_query_keys(cls, suffix, usrval=None): - records = [] - for key, val in cls.REG_SOFTWARE_MICROSOFT: - extval = val + '\\' + suffix if suffix else val - rval = cls.read_value(key, extval) - if rval: - records.append((key, val, extval, usrval)) - return records - - @classmethod - def microsoft_sdks(cls, version): - return '\\'.join([r'Microsoft SDKs\Windows', 'v' + version, r'InstallationFolder']) - - @classmethod - def sdk_query_paths(cls, version): - q = cls.microsoft_sdks(version) - return cls.microsoft_query_paths(q) - - @classmethod - def windows_kits(cls, version): - return r'Windows Kits\Installed Roots\KitsRoot' + version - - @classmethod - def windows_kit_query_paths(cls, version): - q = cls.windows_kits(version) - return cls.microsoft_query_paths(q) - -class _WindowsSDK: - - @classmethod - def _new_sdk_map(cls): - sdk_map = { - 'desktop': [], - 'uwp': [], - } - return sdk_map - - @classmethod - def _sdk_10_layout(cls, version): - - folder_prefix = version + '.' - - sdk_map = cls._new_sdk_map() - - sdk_roots = _Registry.sdk_query_paths(version) - - sdk_version_platform_seen = set() - sdk_roots_seen = set() - - for sdk_t in sdk_roots: - - sdk_root = sdk_t[0] - if sdk_root in sdk_roots_seen: - continue - sdk_roots_seen.add(sdk_root) - - if not os.path.exists(sdk_root): - continue - - sdk_include_path = os.path.join(sdk_root, 'include') - if not os.path.exists(sdk_include_path): - continue - - for version_nbr, version_nbr_path in _Util.listdir_dirs(sdk_include_path): - - if not version_nbr.startswith(folder_prefix): - continue - - sdk_inc_path = _Util.process_path(os.path.join(version_nbr_path, 'um')) - if not os.path.exists(sdk_inc_path): - continue - - for platform_type, sdk_inc_file in [ - ('desktop', 'winsdkver.h'), - ('uwp', 'windows.h'), - ]: - - if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): - continue - - key = (version_nbr, platform_type) - if key in sdk_version_platform_seen: - continue - sdk_version_platform_seen.add(key) - - sdk_map[platform_type].append(version_nbr) - - for key, val in sdk_map.items(): - val.sort(reverse=True) - - return sdk_map - - @classmethod - def _sdk_81_layout(cls, version): - - version_nbr = version - - sdk_map = cls._new_sdk_map() - - sdk_roots = _Registry.sdk_query_paths(version) - - sdk_version_platform_seen = set() - sdk_roots_seen = set() - - for sdk_t in sdk_roots: - - sdk_root = sdk_t[0] - if sdk_root in sdk_roots_seen: - continue - sdk_roots_seen.add(sdk_root) - - # msvc does not check for existence of root or other files - - sdk_inc_path = _Util.process_path(os.path.join(sdk_root, r'include\um')) - if not os.path.exists(sdk_inc_path): - continue - - for platform_type, sdk_inc_file in [ - ('desktop', 'winsdkver.h'), - ('uwp', 'windows.h'), - ]: - - if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): - continue - - key = (version_nbr, platform_type) - if key in sdk_version_platform_seen: - continue - sdk_version_platform_seen.add(key) - - sdk_map[platform_type].append(version_nbr) - - for key, val in sdk_map.items(): - val.sort(reverse=True) - - return sdk_map - - sdk_map_cache = {} - sdk_cache = {} - - @classmethod - def _reset_sdk_cache(cls): - debug('reset %s: sdk cache', cls.__name__) - cls._sdk_map_cache = {} - cls._sdk_cache = {} - - @classmethod - def _sdk_10(cls, key, reg_version): - if key in cls.sdk_map_cache: - sdk_map = cls.sdk_map_cache[key] - else: - sdk_map = cls._sdk_10_layout(reg_version) - cls.sdk_map_cache[key] = sdk_map - return sdk_map - - @classmethod - def _sdk_81(cls, key, reg_version): - if key in cls.sdk_map_cache: - sdk_map = cls.sdk_map_cache[key] - else: - sdk_map = cls._sdk_81_layout(reg_version) - cls.sdk_map_cache[key] = sdk_map - return sdk_map - - @classmethod - def _combine_sdk_map_list(cls, sdk_map_list): - combined_sdk_map = cls._new_sdk_map() - for sdk_map in sdk_map_list: - for key, val in sdk_map.items(): - combined_sdk_map[key].extend(val) - return combined_sdk_map - - sdk_dispatch_map = None - - @classmethod - def _init_sdk_dispatch_map(cls): - cls.sdk_dispatch_map = { - '10.0': (cls._sdk_10, '10.0'), - '8.1': (cls._sdk_81, '8.1'), - } - - @classmethod - def _verify_sdk_dispatch_map(cls): - debug('%s verify sdk_dispatch_map', cls.__name__) - cls._init_sdk_dispatch_map() - for sdk_version in _Config.MSVC_SDK_VERSIONS: - if sdk_version in cls.sdk_dispatch_map: - continue - err_msg = 'sdk version {} not in {}.sdk_dispatch_map'.format(sdk_version, cls.__name__) - raise MSVCInternalError(err_msg) - return None - - @classmethod - def _version_list_sdk_map(cls, version_list): - - if not cls.sdk_dispatch_map: - cls._init_sdk_dispatch_map() - - sdk_map_list = [] - for version in version_list: - func, reg_version = cls.sdk_dispatch_map[version] - sdk_map = func(version, reg_version) - sdk_map_list.append(sdk_map) - - combined_sdk_map = cls._combine_sdk_map_list(sdk_map_list) - return combined_sdk_map - - @classmethod - def _sdk_map(cls, version_list): - key = tuple(version_list) - if key in cls.sdk_cache: - sdk_map = cls.sdk_cache[key] - else: - version_numlist = [float(v) for v in version_list] - version_numlist.sort(reverse=True) - key = tuple([str(v) for v in version_numlist]) - sdk_map = cls._version_list_sdk_map(key) - cls.sdk_cache[key] = sdk_map - return sdk_map - - @classmethod - def get_sdk_version_list(cls, version_list, platform_type): - sdk_map = cls._sdk_map(version_list) - sdk_list = sdk_map.get(platform_type, []) - return sdk_list - - @classmethod - def get_msvc_sdk_version_list(cls, msvc_version=None, msvc_uwp_app=False): - debug('msvc_version=%s, msvc_uwp_app=%s', repr(msvc_version), repr(msvc_uwp_app)) - - sdk_versions = [] - - verstr = get_msvc_version_numeric(msvc_version) - vs_def = _Config.MSVC_VERSION_EXTERNAL.get(verstr, None) - if not vs_def: - debug('vs_def is not defined') - return sdk_versions - - is_uwp = True if msvc_uwp_app in _Config.BOOLEAN_SYMBOLS[True] else False - platform_type = 'uwp' if is_uwp else 'desktop' - sdk_list = _WindowsSDK.get_sdk_version_list(vs_def.vc_sdk_versions, platform_type) - - sdk_versions.extend(sdk_list) - debug('sdk_versions=%s', repr(sdk_versions)) - - return sdk_versions - - @classmethod - def reset(cls): - debug('reset %s', cls.__name__) - cls._reset_sdk_cache() - - @classmethod - def verify(cls): - debug('verify %s', cls.__name__) - cls._verify_sdk_dispatch_map() - -_Dispatcher.register(_WindowsSDK) - -class _ScriptArguments: - - # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? - re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') - re_sdk_version_81 = re.compile(r'^8[.]1$') - - re_sdk_dispatch_map = { - '10.0': re_sdk_version_100, - '8.1': re_sdk_version_81, - } - - @classmethod - def _verify_re_sdk_dispatch_map(cls): - debug('%s verify re_sdk_dispatch_map', cls.__name__) - for sdk_version in _Config.MSVC_SDK_VERSIONS: - if sdk_version in cls.re_sdk_dispatch_map: - continue - err_msg = 'sdk version {} not in {}.re_sdk_dispatch_map'.format(sdk_version, cls.__name__) - raise MSVCInternalError(err_msg) - return None - - # capture msvc version - re_toolset_version = re.compile(r'^(?P[1-9][0-9]?[.][0-9])[0-9.]*$', re.IGNORECASE) - - re_toolset_full = re.compile(r'''^(?: - (?:[1-9][0-9][.][0-9]{1,2})| # XX.Y - XX.YY - (?:[1-9][0-9][.][0-9]{2}[.][0-9]{1,5}) # XX.YY.Z - XX.YY.ZZZZZ - )$''', re.VERBOSE) - - re_toolset_140 = re.compile(r'''^(?: - (?:14[.]0{1,2})| # 14.0 - 14.00 - (?:14[.]0{2}[.]0{1,5}) # 14.00.0 - 14.00.00000 - )$''', re.VERBOSE) - - # valid SxS formats will be matched with re_toolset_full: match 3 '.' format - re_toolset_sxs = re.compile(r'^[1-9][0-9][.][0-9]{2}[.][0-9]{2}[.][0-9]{1,2}$') - - # MSVC_SCRIPT_ARGS - re_vcvars_uwp = re.compile(r'(?:(?(?:uwp|store))(?:(?!\S)|$)',re.IGNORECASE) - re_vcvars_sdk = re.compile(r'(?:(?(?:[1-9][0-9]*[.]\S*))(?:(?!\S)|$)',re.IGNORECASE) - re_vcvars_toolset = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_ver[=](?P\S*))(?:(?!\S)|$)', re.IGNORECASE) - re_vcvars_spectre = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_spectre_libs[=](?P\S*))(?:(?!\S)|$)',re.IGNORECASE) - - # Force default sdk argument - MSVC_FORCE_DEFAULT_SDK = False - - # Force default toolset argument - MSVC_FORCE_DEFAULT_TOOLSET = False - - # MSVC batch file arguments: - # - # VS2022: UWP, SDK, TOOLSET, SPECTRE - # VS2019: UWP, SDK, TOOLSET, SPECTRE - # VS2017: UWP, SDK, TOOLSET, SPECTRE - # VS2015: UWP, SDK - # - # MSVC_SCRIPT_ARGS: VS2015+ - # - # MSVC_UWP_APP: VS2015+ - # MSVC_SDK_VERSION: VS2015+ - # MSVC_TOOLSET_VERSION: VS2017+ - # MSVC_SPECTRE_LIBS: VS2017+ - - @enum.unique - class SortOrder(enum.IntEnum): - ARCH = 0 # arch - UWP = 1 # MSVC_UWP_APP - SDK = 2 # MSVC_SDK_VERSION - TOOLSET = 3 # MSVC_TOOLSET_VERSION - SPECTRE = 4 # MSVC_SPECTRE_LIBS - USER = 5 # MSVC_SCRIPT_ARGS - - VS2019 = _Config.MSVS_VERSION_INTERNAL['2019'] - VS2017 = _Config.MSVS_VERSION_INTERNAL['2017'] - VS2015 = _Config.MSVS_VERSION_INTERNAL['2015'] - - MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ - 'version', # fully qualified msvc version (e.g., '14.1Exp') - 'vs_def', - ]) - - @classmethod - def _msvc_version(cls, version): - - verstr = get_msvc_version_numeric(version) - vs_def = _Config.MSVC_VERSION_INTERNAL[verstr] - - version_args = cls.MSVC_VERSION_ARGS_DEFINITION( - version = version, - vs_def = vs_def, - ) - - return version_args - - @classmethod - def _msvc_script_argument_uwp(cls, env, msvc, arglist): - - uwp_app = env['MSVC_UWP_APP'] - debug('MSVC_VERSION=%s, MSVC_UWP_APP=%s', repr(msvc.version), repr(uwp_app)) - - if not uwp_app: - return None - - if uwp_app not in _Config.BOOLEAN_SYMBOLS[True]: - return None - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_UWP_APP ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(uwp_app), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - raise MSVCArgumentError(err_msg) - - # VS2017+ rewrites uwp => store for 14.0 toolset - uwp_arg = msvc.vs_def.vc_uwp - - # store/uwp may not be fully installed - argpair = (cls.SortOrder.UWP, uwp_arg) - arglist.append(argpair) - - return uwp_arg - - @classmethod - def _user_script_argument_uwp(cls, env, uwp, user_argstr): - - matches = [m for m in cls.re_vcvars_uwp.finditer(user_argstr)] - if not matches: - return None - - if len(matches) > 1: - debug('multiple uwp declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) - err_msg = "multiple uwp declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) - raise MSVCArgumentError(err_msg) - - if not uwp: - return None - - env_argstr = env.get('MSVC_UWP_APP','') - debug('multiple uwp declarations: MSVC_UWP_APP=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - - err_msg = "multiple uwp declarations: MSVC_UWP_APP={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) - - raise MSVCArgumentError(err_msg) - - @classmethod - def _msvc_script_argument_sdk_constraints(cls, msvc, sdk_version): - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc_version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_SDK_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(sdk_version), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - return err_msg - - for msvc_sdk_version in msvc.vs_def.vc_sdk_versions: - re_sdk_version = cls.re_sdk_dispatch_map[msvc_sdk_version] - if re_sdk_version.match(sdk_version): - debug('valid: sdk_version=%s', repr(sdk_version)) - return None - - debug('invalid: method exit: sdk_version=%s', repr(sdk_version)) - err_msg = "MSVC_SDK_VERSION ({}) is not supported".format(repr(sdk_version)) - return err_msg - - @classmethod - def _msvc_script_argument_sdk(cls, env, msvc, platform_type, arglist): - - sdk_version = env['MSVC_SDK_VERSION'] - debug( - 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform_type=%s', - repr(msvc.version), repr(sdk_version), repr(platform_type) - ) - - if not sdk_version: - return None - - err_msg = cls._msvc_script_argument_sdk_constraints(msvc, sdk_version) - if err_msg: - raise MSVCArgumentError(err_msg) - - sdk_list = _WindowsSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) - - if sdk_version not in sdk_list: - err_msg = "MSVC_SDK_VERSION {} not found for platform type {}".format( - repr(sdk_version), repr(platform_type) - ) - raise MSVCArgumentError(err_msg) - - argpair = (cls.SortOrder.SDK, sdk_version) - arglist.append(argpair) - - return sdk_version - - @classmethod - def _msvc_script_default_sdk(cls, env, msvc, platform_type, arglist): - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - return None - - sdk_list = _WindowsSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) - if not len(sdk_list): - return None - - sdk_default = sdk_list[0] - - debug( - 'MSVC_VERSION=%s, sdk_default=%s, platform_type=%s', - repr(msvc.version), repr(sdk_default), repr(platform_type) - ) - - argpair = (cls.SortOrder.SDK, sdk_default) - arglist.append(argpair) - - return sdk_default - - @classmethod - def _user_script_argument_sdk(cls, env, sdk_version, user_argstr): - - matches = [m for m in cls.re_vcvars_sdk.finditer(user_argstr)] - if not matches: - return None - - if len(matches) > 1: - debug('multiple sdk version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) - err_msg = "multiple sdk version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) - raise MSVCArgumentError(err_msg) - - if not sdk_version: - user_sdk = matches[0].group('sdk') - return user_sdk - - env_argstr = env.get('MSVC_SDK_VERSION','') - debug('multiple sdk version declarations: MSVC_SDK_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - - err_msg = "multiple sdk version declarations: MSVC_SDK_VERSION={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) - - raise MSVCArgumentError(err_msg) - - @classmethod - def _msvc_read_toolset_file(cls, msvc, filename): - toolset_version = None - try: - with open(filename) as f: - toolset_version = f.readlines()[0].strip() - debug( - 'msvc_version=%s, filename=%s, toolset_version=%s', - repr(msvc.version), repr(filename), repr(toolset_version) - ) - except OSError: - debug('OSError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) - except IndexError: - debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) - return toolset_version - - @classmethod - def _msvc_read_toolset_folders(cls, msvc, vc_dir): - - toolsets_sxs = {} - toolsets_full = [] - - build_dir = os.path.join(vc_dir, "Auxiliary", "Build") - sxs_toolsets = [f.name for f in os.scandir(build_dir) if f.is_dir()] - for sxs_toolset in sxs_toolsets: - filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) - filepath = os.path.join(build_dir, sxs_toolset, filename) - debug('sxs toolset: check file=%s', repr(filepath)) - if os.path.exists(filepath): - toolset_version = cls._msvc_read_toolset_file(msvc, filepath) - if not toolset_version: - continue - toolsets_sxs[sxs_toolset] = toolset_version - debug( - 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', - repr(msvc.version), repr(sxs_toolset), toolset_version - ) - - toolset_dir = os.path.join(vc_dir, "Tools", "MSVC") - toolsets = [f.name for f in os.scandir(toolset_dir) if f.is_dir()] - for toolset_version in toolsets: - binpath = os.path.join(toolset_dir, toolset_version, "bin") - debug('toolset: check binpath=%s', repr(binpath)) - if os.path.exists(binpath): - toolsets_full.append(toolset_version) - debug( - 'toolset: msvc_version=%s, toolset_version=%s', - repr(msvc.version), repr(toolset_version) - ) - - toolsets_full.sort(reverse=True) - debug('msvc_version=%s, toolsets=%s', repr(msvc.version), repr(toolsets_full)) - - return toolsets_sxs, toolsets_full - - @classmethod - def _msvc_read_toolset_default(cls, msvc, vc_dir): - - build_dir = os.path.join(vc_dir, "Auxiliary", "Build") - - # VS2019+ - filename = "Microsoft.VCToolsVersion.{}.default.txt".format(msvc.vs_def.vc_buildtools_def.vc_buildtools) - filepath = os.path.join(build_dir, filename) - - debug('default toolset: check file=%s', repr(filepath)) - if os.path.exists(filepath): - toolset_buildtools = cls._msvc_read_toolset_file(msvc, filepath) - if toolset_buildtools: - return toolset_buildtools - - # VS2017+ - filename = "Microsoft.VCToolsVersion.default.txt" - filepath = os.path.join(build_dir, filename) - - debug('default toolset: check file=%s', repr(filepath)) - if os.path.exists(filepath): - toolset_default = cls._msvc_read_toolset_file(msvc, filepath) - if toolset_default: - return toolset_default - - return None - - _toolset_version_cache = {} - _toolset_default_cache = {} - - @classmethod - def _reset_toolset_cache(cls): - debug('reset %s: toolset cache', cls.__name__) - cls._toolset_version_cache = {} - cls._toolset_default_cache = {} - - @classmethod - def _msvc_version_toolsets(cls, msvc, vc_dir): - - if msvc.version in cls._toolset_version_cache: - toolsets_sxs, toolsets_full = cls._toolset_version_cache[msvc.version] - else: - toolsets_sxs, toolsets_full = cls._msvc_read_toolset_folders(msvc, vc_dir) - cls._toolset_version_cache[msvc.version] = toolsets_sxs, toolsets_full - - return toolsets_sxs, toolsets_full - - @classmethod - def _msvc_default_toolset(cls, msvc, vc_dir): - - if msvc.version in cls._toolset_default_cache: - toolset_default = cls._toolset_default_cache[msvc.version] - else: - toolset_default = cls._msvc_read_toolset_default(msvc, vc_dir) - cls._toolset_default_cache[msvc.version] = toolset_default - - return toolset_default - - @classmethod - def _msvc_version_toolset_vcvars(cls, msvc, vc_dir, toolset_version): - - if toolset_version == '14.0': - return toolset_version - - toolsets_sxs, toolsets_full = cls._msvc_version_toolsets(msvc, vc_dir) - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric == cls.VS2019.vc_buildtools_def.vc_version_numeric: - # necessary to detect toolset not found - if toolset_version == '14.28.16.8': - new_toolset_version = '14.28' - # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 - # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files - # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. - debug( - 'rewrite toolset_version=%s => toolset_version=%s', - repr(toolset_version), repr(new_toolset_version) - ) - toolset_version = new_toolset_version - - if toolset_version in toolsets_sxs: - toolset_vcvars = toolsets_sxs[toolset_version] - return toolset_vcvars - - for toolset_full in toolsets_full: - if toolset_full.startswith(toolset_version): - toolset_vcvars = toolset_full - return toolset_vcvars - - return None - - @classmethod - def _msvc_script_argument_toolset_constraints(cls, msvc, toolset_version): - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2017', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( - repr(toolset_version), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) - ) - return err_msg - - m = cls.re_toolset_version.match(toolset_version) - if not m: - debug('invalid: re_toolset_version: toolset_version=%s', repr(toolset_version)) - err_msg = 'MSVC_TOOLSET_VERSION {} format is not supported'.format( - repr(toolset_version) - ) - return err_msg - - toolset_ver = m.group('version') - toolset_vernum = float(toolset_ver) - - if toolset_vernum < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: toolset version constraint: %s < %s VS2015', - repr(toolset_vernum), repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} < {} VS2015".format( - repr(toolset_version), repr(toolset_ver), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - return err_msg - - if toolset_vernum > msvc.vs_def.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: toolset version constraint: toolset %s > %s msvc', - repr(toolset_vernum), repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} > {} MSVC_VERSION".format( - repr(toolset_version), repr(toolset_ver), repr(msvc.version) - ) - return err_msg - - if toolset_vernum == cls.VS2015.vc_buildtools_def.vc_version_numeric: - # tooset = 14.0 - if cls.re_toolset_full.match(toolset_version): - if not cls.re_toolset_140.match(toolset_version): - debug( - 'invalid: toolset version 14.0 constraint: %s != 14.0', - repr(toolset_version) - ) - err_msg = "MSVC_TOOLSET_VERSION ({}) constraint violation: toolset version {} != '14.0'".format( - repr(toolset_version), repr(toolset_version) - ) - return err_msg - return None - - if cls.re_toolset_full.match(toolset_version): - debug('valid: re_toolset_full: toolset_version=%s', repr(toolset_version)) - return None - - if cls.re_toolset_sxs.match(toolset_version): - debug('valid: re_toolset_sxs: toolset_version=%s', repr(toolset_version)) - return None - - debug('invalid: method exit: toolset_version=%s', repr(toolset_version)) - err_msg = "MSVC_TOOLSET_VERSION ({}) format is not supported".format(repr(toolset_version)) - return err_msg - - @classmethod - def _msvc_script_argument_toolset(cls, env, msvc, vc_dir, arglist): - - toolset_version = env['MSVC_TOOLSET_VERSION'] - debug('MSVC_VERSION=%s, MSVC_TOOLSET_VERSION=%s', repr(msvc.version), repr(toolset_version)) - - if not toolset_version: - return None - - err_msg = cls._msvc_script_argument_toolset_constraints(msvc, toolset_version) - if err_msg: - raise MSVCArgumentError(err_msg) - - if toolset_version.startswith('14.0') and len(toolset_version) > len('14.0'): - new_toolset_version = '14.0' - debug( - 'rewrite toolset_version=%s => toolset_version=%s', - repr(toolset_version), repr(new_toolset_version) - ) - toolset_version = new_toolset_version - - toolset_vcvars = cls._msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version) - debug( - 'toolset: toolset_version=%s, toolset_vcvars=%s', - repr(toolset_version), repr(toolset_vcvars) - ) - - if not toolset_vcvars: - err_msg = "MSVC_TOOLSET_VERSION {} not found for MSVC_VERSION {}".format( - repr(toolset_version), repr(msvc.version) - ) - raise MSVCArgumentError(err_msg) - - argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_vcvars)) - arglist.append(argpair) - - return toolset_vcvars - - @classmethod - def _msvc_script_default_toolset(cls, env, msvc, vc_dir, arglist): - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: - return None - - toolset_default = cls._msvc_default_toolset(msvc, vc_dir) - if not toolset_default: - return None - - debug('MSVC_VERSION=%s, toolset_default=%s', repr(msvc.version), repr(toolset_default)) - - argpair = (cls.SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_default)) - arglist.append(argpair) - - return toolset_default - - @classmethod - def _user_script_argument_toolset(cls, env, toolset_version, user_argstr): - - matches = [m for m in cls.re_vcvars_toolset.finditer(user_argstr)] - if not matches: - return None - - if len(matches) > 1: - debug('multiple toolset version declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) - err_msg = "multiple toolset version declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) - raise MSVCArgumentError(err_msg) - - if not toolset_version: - user_toolset = matches[0].group('toolset') - return user_toolset - - env_argstr = env.get('MSVC_TOOLSET_VERSION','') - debug('multiple toolset version declarations: MSVC_TOOLSET_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - - err_msg = "multiple toolset version declarations: MSVC_TOOLSET_VERSION={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) - - raise MSVCArgumentError(err_msg) - - @classmethod - def _msvc_script_argument_spectre(cls, env, msvc, arglist): - - spectre_libs = env['MSVC_SPECTRE_LIBS'] - debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) - - if not spectre_libs: - return None - - if spectre_libs not in (True, '1'): - return None - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2017.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2017', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2017.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( - repr(spectre_libs), repr(msvc.version), repr(cls.VS2017.vc_buildtools_def.vc_version) - ) - raise MSVCArgumentError(err_msg) - - spectre_arg = 'spectre' - - # spectre libs may not be installed - argpair = (cls.SortOrder.SPECTRE, '-vcvars_spectre_libs={}'.format(spectre_arg)) - arglist.append(argpair) - - return spectre_arg - - @classmethod - def _msvc_script_argument_user(cls, env, msvc, arglist): - - # subst None -> empty string - script_args = env.subst('$MSVC_SCRIPT_ARGS') - debug('MSVC_VERSION=%s, MSVC_SCRIPT_ARGS=%s', repr(msvc.version), repr(script_args)) - - if not script_args: - return None - - if msvc.vs_def.vc_buildtools_def.vc_version_numeric < cls.VS2015.vc_buildtools_def.vc_version_numeric: - debug( - 'invalid: msvc version constraint: %s < %s VS2015', - repr(msvc.vs_def.vc_buildtools_def.vc_version_numeric), - repr(cls.VS2015.vc_buildtools_def.vc_version_numeric) - ) - err_msg = "MSVC_SCRIPT_ARGS ({}) constraint violation: MSVC_VERSION {} < {} VS2015".format( - repr(script_args), repr(msvc.version), repr(cls.VS2015.vc_buildtools_def.vc_version) - ) - raise MSVCArgumentError(err_msg) - - # user arguments are not validated - argpair = (cls.SortOrder.USER, script_args) - arglist.append(argpair) - - return script_args - - @classmethod - def _user_script_argument_spectre(cls, env, spectre, user_argstr): - - matches = [m for m in cls.re_vcvars_spectre.finditer(user_argstr)] - if not matches: - return None - - if len(matches) > 1: - debug('multiple spectre declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) - err_msg = "multiple spectre declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) - raise MSVCArgumentError(err_msg) - - if not spectre: - return None - - env_argstr = env.get('MSVC_SPECTRE_LIBS','') - debug('multiple spectre declarations: MSVC_SPECTRE_LIBS=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - - err_msg = "multiple spectre declarations: MSVC_SPECTRE_LIBS={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) - - raise MSVCArgumentError(err_msg) - - @classmethod - def msvc_script_arguments(cls, env, version, vc_dir, arg): - - arglist = [] - - msvc = cls._msvc_version(version) - - if arg: - argpair = (cls.SortOrder.ARCH, arg) - arglist.append(argpair) - - if 'MSVC_SCRIPT_ARGS' in env: - user_argstr = cls._msvc_script_argument_user(env, msvc, arglist) - else: - user_argstr = None - - if 'MSVC_UWP_APP' in env: - uwp = cls._msvc_script_argument_uwp(env, msvc, arglist) - else: - uwp = None - - if user_argstr: - cls._user_script_argument_uwp(env, uwp, user_argstr) - - platform_type = 'uwp' if uwp else 'desktop' - - if 'MSVC_SDK_VERSION' in env: - sdk_version = cls._msvc_script_argument_sdk(env, msvc, platform_type, arglist) - else: - sdk_version = None - - if user_argstr: - user_sdk = cls._user_script_argument_sdk(env, sdk_version, user_argstr) - else: - user_sdk = None - - if cls.MSVC_FORCE_DEFAULT_SDK: - if not sdk_version and not user_sdk: - sdk_version = cls._msvc_script_default_sdk(env, msvc, platform_type, arglist) - - if 'MSVC_TOOLSET_VERSION' in env: - toolset_version = cls._msvc_script_argument_toolset(env, msvc, vc_dir, arglist) - else: - toolset_version = None - - if user_argstr: - user_toolset = cls._user_script_argument_toolset(env, toolset_version, user_argstr) - else: - user_toolset = None - - if cls.MSVC_FORCE_DEFAULT_TOOLSET: - if not toolset_version and not user_toolset: - toolset_version = cls._msvc_script_default_toolset(env, msvc, vc_dir, arglist) - - if 'MSVC_SPECTRE_LIBS' in env: - spectre = cls._msvc_script_argument_spectre(env, msvc, arglist) - else: - spectre = None - - if user_argstr: - cls._user_script_argument_spectre(env, spectre, user_argstr) - - if arglist: - arglist.sort() - argstr = ' '.join([argpair[-1] for argpair in arglist]).strip() - else: - argstr = '' - - debug('arguments: %s', repr(argstr)) - return argstr - - @classmethod - def reset(cls): - debug('reset %s', cls.__name__) - cls._reset_toolset_cache() - - @classmethod - def verify(cls): - debug('verify %s', cls.__name__) - cls._verify_re_sdk_dispatch_map() - -_Dispatcher.register(_ScriptArguments) - -# internal consistency check -_Dispatcher.verify() -- cgit v0.12 From 14ee60c050d577704004ba8ade7e5347e356f9e9 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:08:34 -0400 Subject: Fix typo in module name --- SCons/Tool/MSCommon/vc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 5a27f44..2e5d542 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1107,7 +1107,7 @@ def msvc_setup_env_once(env, tool=None): msg = "No versions of the MSVC compiler were found.\n" \ " Visual Studio C/C++ compilers may not be set correctly.\n" \ " Requested tool(s) are: {}".format(req_tools) - MSVC.Notfound.policy_handler(env, msg) + MSVC.NotFound.policy_handler(env, msg) def msvc_find_valid_batch_script(env, version): """Find and execute appropriate batch script to set up build env. -- cgit v0.12 From 9619adbcf75cf9f6c851e577f49d92b021012de3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 17:50:03 -0400 Subject: Cleanup MSCommon/vc imports and move Dispatcher imports and registration --- SCons/Tool/MSCommon/MSVC/Config.py | 2 +- SCons/Tool/MSCommon/MSVC/Dispatcher.py | 1 + SCons/Tool/MSCommon/MSVC/NotFound.py | 3 ++- SCons/Tool/MSCommon/MSVC/Registry.py | 3 ++- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 3 ++- SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py | 2 +- SCons/Tool/MSCommon/MSVC/WinSDK.py | 3 ++- SCons/Tool/MSCommon/MSVC/__init__.py | 7 +------ SCons/Tool/MSCommon/__init__.py | 6 ------ SCons/Tool/MSCommon/vc.py | 6 ++++++ 10 files changed, 18 insertions(+), 18 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py index 476dcb3..8f3a2cc 100644 --- a/SCons/Tool/MSCommon/MSVC/Config.py +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -30,9 +30,9 @@ from collections import ( ) from . import Dispatcher - Dispatcher.register_modulename(__name__) + UNDEFINED = object() BOOLEAN_SYMBOLS = {} diff --git a/SCons/Tool/MSCommon/MSVC/Dispatcher.py b/SCons/Tool/MSCommon/MSVC/Dispatcher.py index ebcd704..0b216ca 100644 --- a/SCons/Tool/MSCommon/MSVC/Dispatcher.py +++ b/SCons/Tool/MSCommon/MSVC/Dispatcher.py @@ -31,6 +31,7 @@ from ..common import ( debug, ) + _refs = [] def register_class(ref): diff --git a/SCons/Tool/MSCommon/MSVC/NotFound.py b/SCons/Tool/MSCommon/MSVC/NotFound.py index 7abe5ad..6ade285 100644 --- a/SCons/Tool/MSCommon/MSVC/NotFound.py +++ b/SCons/Tool/MSCommon/MSVC/NotFound.py @@ -36,15 +36,16 @@ from ..common import ( debug, ) -from . import Dispatcher from . import Config from .Exceptions import ( MSVCVersionNotFound, ) +from . import Dispatcher Dispatcher.register_modulename(__name__) + _MSVC_NOTFOUND_POLICY_DEF = Config.MSVC_NOTFOUND_POLICY_INTERNAL['warning'] def _msvc_notfound_policy_lookup(symbol): diff --git a/SCons/Tool/MSCommon/MSVC/Registry.py b/SCons/Tool/MSCommon/MSVC/Registry.py index 848b125..f9a5eb2 100644 --- a/SCons/Tool/MSCommon/MSVC/Registry.py +++ b/SCons/Tool/MSCommon/MSVC/Registry.py @@ -39,11 +39,12 @@ from .. common import ( read_reg, ) -from . import Dispatcher from . import Util +from . import Dispatcher Dispatcher.register_modulename(__name__) + def read_value(hkey, subkey_valname): try: rval = read_reg(subkey_valname, hkroot=hkey) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 324f8be..1b02396 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -37,7 +37,6 @@ from ..common import ( debug, ) -from . import Dispatcher from . import Util from . import Config from . import WinSDK @@ -47,8 +46,10 @@ from .Exceptions import ( MSVCArgumentError, ) +from . import Dispatcher Dispatcher.register_modulename(__name__) + # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') re_sdk_version_81 = re.compile(r'^8[.]1$') diff --git a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py index 8a79007..8b9faa9 100644 --- a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py +++ b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py @@ -44,9 +44,9 @@ from .. common import ( ) from . import Dispatcher - Dispatcher.register_modulename(__name__) + class _Data: separator = r';' diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py index e95c72e..bd8d9b0 100644 --- a/SCons/Tool/MSCommon/MSVC/WinSDK.py +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -31,7 +31,6 @@ from ..common import ( debug, ) -from . import Dispatcher from . import Util from . import Config from . import Registry @@ -40,8 +39,10 @@ from .Exceptions import ( MSVCInternalError, ) +from . import Dispatcher Dispatcher.register_modulename(__name__) + def _new_sdk_map(): sdk_map = { 'desktop': [], diff --git a/SCons/Tool/MSCommon/MSVC/__init__.py b/SCons/Tool/MSCommon/MSVC/__init__.py index afd993f..1341415 100644 --- a/SCons/Tool/MSCommon/MSVC/__init__.py +++ b/SCons/Tool/MSCommon/MSVC/__init__.py @@ -28,8 +28,6 @@ Functions for Microsoft Visual C/C++. from . import Exceptions from . import Util -from . import Dispatcher as _Dispatcher - from . import Config from . import Registry from . import SetupEnvDefault @@ -37,10 +35,7 @@ from . import NotFound from . import WinSDK from . import ScriptArguments -from .NotFound import ( - set_msvc_notfound_policy, - get_msvc_notfound_policy, -) +from . import Dispatcher as _Dispatcher def reset(): _Dispatcher.reset() diff --git a/SCons/Tool/MSCommon/__init__.py b/SCons/Tool/MSCommon/__init__.py index 9f35e94..7900a87 100644 --- a/SCons/Tool/MSCommon/__init__.py +++ b/SCons/Tool/MSCommon/__init__.py @@ -32,18 +32,12 @@ import SCons.Util from SCons.Tool.MSCommon.sdk import mssdk_exists, mssdk_setup_env -from SCons.Tool.MSCommon.MSVC import ( - set_msvc_notfound_policy, - get_msvc_notfound_policy, -) - from SCons.Tool.MSCommon.vc import ( msvc_exists, msvc_setup_env_tool, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere, - get_msvc_sdk_versions, ) from SCons.Tool.MSCommon.vs import ( diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 2e5d542..a2e8e42 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -61,10 +61,16 @@ from .common import CONFIG_CACHE, debug from .sdk import get_installed_sdks from . import MSVC + from .MSVC.Exceptions import ( VisualCException ) +from .MSVC.NotFound import ( + set_msvc_notfound_policy, + get_msvc_notfound_policy, +) + class UnsupportedVersion(VisualCException): pass -- cgit v0.12 From a7c56cebc24fd5aaa2dd05a1751412120c946835 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 18:36:54 -0400 Subject: Fix msvc notfound policy module path for test --- test/fixture/no_msvc/no_msvcs_sconstruct_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixture/no_msvc/no_msvcs_sconstruct_version.py b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py index f5cabf7..4c76d44 100644 --- a/test/fixture/no_msvc/no_msvcs_sconstruct_version.py +++ b/test/fixture/no_msvc/no_msvcs_sconstruct_version.py @@ -10,7 +10,7 @@ for key in SCons.Tool.MSCommon.vc._VCVER_TO_PRODUCT_DIR: SCons.Tool.MSCommon.vc.find_vc_pdir_vswhere = DummyVsWhere -SCons.Tool.MSCommon.set_msvc_notfound_policy('error') +SCons.Tool.MSCommon.vc.set_msvc_notfound_policy('error') env = SCons.Environment.Environment(MSVC_VERSION='14.3') -- cgit v0.12 From f6a7b846bba8477ca8221edd2b76dc3d1d842439 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 20 Jun 2022 21:48:57 -0400 Subject: Add global for cache reset (classmethod to module omission) and remove duplicate import --- SCons/Tool/MSCommon/MSVC/Registry.py | 2 -- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 2 ++ SCons/Tool/MSCommon/MSVC/WinSDK.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Registry.py b/SCons/Tool/MSCommon/MSVC/Registry.py index f9a5eb2..492f3d0 100644 --- a/SCons/Tool/MSCommon/MSVC/Registry.py +++ b/SCons/Tool/MSCommon/MSVC/Registry.py @@ -30,8 +30,6 @@ import os from SCons.Util import ( HKEY_LOCAL_MACHINE, HKEY_CURRENT_USER, - HKEY_LOCAL_MACHINE, - HKEY_CURRENT_USER, ) from .. common import ( diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 1b02396..d0fc6ef 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -373,6 +373,8 @@ _toolset_version_cache = {} _toolset_default_cache = {} def _reset_toolset_cache(): + global _toolset_version_cache + global _toolset_default_cache debug('reset: toolset cache') _toolset_version_cache = {} _toolset_default_cache = {} diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py index bd8d9b0..8338c27 100644 --- a/SCons/Tool/MSCommon/MSVC/WinSDK.py +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -152,6 +152,8 @@ _sdk_map_cache = {} _sdk_cache = {} def _reset_sdk_cache(): + global _sdk_map_cache + global _sdk_cache debug('') _sdk_map_cache = {} _sdk_cache = {} -- cgit v0.12 From dd328cff200935a7f570396f06b93a3da82278d7 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 08:06:01 -0400 Subject: Suppress sider imported but unused for namespace. Restrict MSVC_UWP_APP boolean symbols accepted. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 13 ++++++++----- SCons/Tool/MSCommon/MSVC/__init__.py | 16 ++++++++-------- SCons/Tool/MSCommon/vc.py | 4 ++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index d0fc6ef..a64d9f4 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -50,6 +50,9 @@ from . import Dispatcher Dispatcher.register_modulename(__name__) +# MSVC_UWP_APP argument: boolean True +_UWP_ARGUMENT_BOOLEAN_TRUE = (True, '1') + # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') re_sdk_version_81 = re.compile(r'^8[.]1$') @@ -91,10 +94,10 @@ re_vcvars_toolset = re.compile(r'(?:(?(?:[-]{1,2}|[/])vc re_vcvars_spectre = re.compile(r'(?:(?(?:[-]{1,2}|[/])vcvars_spectre_libs[=](?P\S*))(?:(?!\S)|$)',re.IGNORECASE) # Force default sdk argument -MSVC_FORCE_DEFAULT_SDK = False +_MSVC_FORCE_DEFAULT_SDK = False # Force default toolset argument -MSVC_FORCE_DEFAULT_TOOLSET = False +_MSVC_FORCE_DEFAULT_TOOLSET = False # MSVC batch file arguments: # @@ -148,7 +151,7 @@ def _msvc_script_argument_uwp(env, msvc, arglist): if not uwp_app: return None - if uwp_app not in Config.BOOLEAN_SYMBOLS[True]: + if uwp_app not in _UWP_ARGUMENT_BOOLEAN_TRUE: return None if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: @@ -691,7 +694,7 @@ def msvc_script_arguments(env, version, vc_dir, arg): else: user_sdk = None - if MSVC_FORCE_DEFAULT_SDK: + if _MSVC_FORCE_DEFAULT_SDK: if not sdk_version and not user_sdk: sdk_version = _msvc_script_default_sdk(env, msvc, platform_type, arglist) @@ -705,7 +708,7 @@ def msvc_script_arguments(env, version, vc_dir, arg): else: user_toolset = None - if MSVC_FORCE_DEFAULT_TOOLSET: + if _MSVC_FORCE_DEFAULT_TOOLSET: if not toolset_version and not user_toolset: toolset_version = _msvc_script_default_toolset(env, msvc, vc_dir, arglist) diff --git a/SCons/Tool/MSCommon/MSVC/__init__.py b/SCons/Tool/MSCommon/MSVC/__init__.py index 1341415..c07e849 100644 --- a/SCons/Tool/MSCommon/MSVC/__init__.py +++ b/SCons/Tool/MSCommon/MSVC/__init__.py @@ -25,15 +25,15 @@ Functions for Microsoft Visual C/C++. """ -from . import Exceptions -from . import Util +from . import Exceptions # noqa: F401 +from . import Util # noqa: F401 -from . import Config -from . import Registry -from . import SetupEnvDefault -from . import NotFound -from . import WinSDK -from . import ScriptArguments +from . import Config # noqa: F401 +from . import Registry # noqa: F401 +from . import SetupEnvDefault # noqa: F401 +from . import NotFound # noqa: F401 +from . import WinSDK # noqa: F401 +from . import ScriptArguments # noqa: F401 from . import Dispatcher as _Dispatcher diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index a2e8e42..2f1ec11 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -67,8 +67,8 @@ from .MSVC.Exceptions import ( ) from .MSVC.NotFound import ( - set_msvc_notfound_policy, - get_msvc_notfound_policy, + set_msvc_notfound_policy, # noqa: F401 + get_msvc_notfound_policy, # noqa: F401 ) class UnsupportedVersion(VisualCException): -- cgit v0.12 From 5dd220bf7d625771acc7f7ca476275795029b560 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 08:59:21 -0400 Subject: Add internal, undocumented SCONS_CACHE_MSVC_FORCE_DEFAULTS environment variable to force default SDK and toolset arguments. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index a64d9f4..8bef3f5 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -34,6 +34,7 @@ from collections import ( ) from ..common import ( + CONFIG_CACHE, debug, ) @@ -50,6 +51,13 @@ from . import Dispatcher Dispatcher.register_modulename(__name__) +# Force default SDK and toolset arguments in cache +_SCONS_CACHE_MSVC_FORCE_DEFAULTS = False +if CONFIG_CACHE: + # SCONS_CACHE_MSVC_FORCE_DEFAULTS is internal and not documented. + if os.environ.get('SCONS_CACHE_MSVC_FORCE_DEFAULTS') in Config.BOOLEAN_SYMBOLS[True]: + _SCONS_CACHE_MSVC_FORCE_DEFAULTS = True + # MSVC_UWP_APP argument: boolean True _UWP_ARGUMENT_BOOLEAN_TRUE = (True, '1') @@ -99,6 +107,20 @@ _MSVC_FORCE_DEFAULT_SDK = False # Force default toolset argument _MSVC_FORCE_DEFAULT_TOOLSET = False +def _msvc_force_default_sdk(force=True): + global _MSVC_FORCE_DEFAULT_SDK + _MSVC_FORCE_DEFAULT_SDK = force + debug('_MSVC_FORCE_DEFAULT_SDK=%s', repr(force)) + +def _msvc_force_default_toolset(force=True): + global _MSVC_FORCE_DEFAULT_TOOLSET + _MSVC_FORCE_DEFAULT_TOOLSET = force + debug('_MSVC_FORCE_DEFAULT_TOOLSET=%s', repr(force)) + +if _SCONS_CACHE_MSVC_FORCE_DEFAULTS: + _msvc_force_default_sdk(True) + _msvc_force_default_toolset(True) + # MSVC batch file arguments: # # VS2022: UWP, SDK, TOOLSET, SPECTRE -- cgit v0.12 From ac9b54756cedd01c36bfbf1bca7d59f92fd08f15 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:08:53 -0400 Subject: Consider MSVC_TOOLSET_VERSION specification intent to use msvc tools. Update boolean symbols accepted for uwp and spectre. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 9 +++++---- SCons/Tool/MSCommon/vc.py | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 8bef3f5..2a94650 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -58,8 +58,9 @@ if CONFIG_CACHE: if os.environ.get('SCONS_CACHE_MSVC_FORCE_DEFAULTS') in Config.BOOLEAN_SYMBOLS[True]: _SCONS_CACHE_MSVC_FORCE_DEFAULTS = True -# MSVC_UWP_APP argument: boolean True -_UWP_ARGUMENT_BOOLEAN_TRUE = (True, '1') +# Script argument: boolean True +_ARGUMENT_BOOLEAN_TRUE_LEGACY = (True, '1') # MSVC_UWP_APP +_ARGUMENT_BOOLEAN_TRUE = (True,) # TODO: verify SDK 10 version folder names 10.0.XXXXX.0 {1,3} last? re_sdk_version_100 = re.compile(r'^10[.][0-9][.][0-9]{5}[.][0-9]{1}$') @@ -173,7 +174,7 @@ def _msvc_script_argument_uwp(env, msvc, arglist): if not uwp_app: return None - if uwp_app not in _UWP_ARGUMENT_BOOLEAN_TRUE: + if uwp_app not in _ARGUMENT_BOOLEAN_TRUE_LEGACY: return None if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: @@ -610,7 +611,7 @@ def _msvc_script_argument_spectre(env, msvc, arglist): if not spectre_libs: return None - if spectre_libs not in (True, '1'): + if spectre_libs not in _ARGUMENT_BOOLEAN_TRUE: return None if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 2f1ec11..dc97e2f 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1300,12 +1300,20 @@ def msvc_setup_env_user(env=None): if env: # Intent is to use msvc tools: - # MSVC_VERSION or MSVS_VERSION: defined and is True - # MSVC_USE_SCRIPT: defined and (is string or is False) - # MSVC_USE_SETTINGS: defined and is not None + # MSVC_VERSION: defined and evaluates True + # MSVS_VERSION: defined and evaluates True + # MSVC_TOOLSET_VERSION: defined and evaluates True + # MSVC_USE_SCRIPT: defined and (is string or evaluates False) + # MSVC_USE_SETTINGS: defined and is not None + + # Arguments possibly present but not considered: + # MSVC_SDK_VERSION + # MSVC_UWP_APP + # MSVC_SPECTRE_LIBS + # MSVC_SCRIPT_ARGS # defined and is True - for key in ['MSVC_VERSION', 'MSVS_VERSION']: + for key in ['MSVC_VERSION', 'MSVS_VERSION', 'MSVC_TOOLSET_VERSION']: if key in env and env[key]: rval = True debug('key=%s, return=%s', repr(key), rval) -- cgit v0.12 From 377e8152bbd9cc4556318283c845dd66defe2d8c Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:11:29 -0400 Subject: Comment out BatchFileExecutionWarning definition and invocation. --- SCons/Tool/MSCommon/vc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index dc97e2f..7b6034e 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -95,8 +95,8 @@ class MSVCScriptNotFound(VisualCException): class MSVCUseSettingsError(VisualCException): pass -class BatchFileExecutionWarning(SCons.Warnings.WarningOnByDefault): - pass +#class BatchFileExecutionWarning(SCons.Warnings.WarningOnByDefault): +# pass # Dict to 'canonalize' the arch @@ -1055,7 +1055,7 @@ def script_env(script, args=None): # detected errors, cl.exe on path debug('script=%s args=%s errors=%s', repr(script), repr(args), script_errmsg) # This may be a bad idea (scons environment != vs cmdline environment) - SCons.Warnings.warn(BatchFileExecutionWarning, script_errmsg) + #SCons.Warnings.warn(BatchFileExecutionWarning, script_errmsg) # TODO: errlog/errstr should be added to cache and warning moved to call site # once we updated cache, give a chance to write out if user wanted -- cgit v0.12 From 016e7f7bdc6b901916da08a361b5cca8c24ee600 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:26:36 -0400 Subject: Update flake8 F401 placement --- SCons/Tool/MSCommon/vc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 7b6034e..f34e9e0 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -67,9 +67,9 @@ from .MSVC.Exceptions import ( ) from .MSVC.NotFound import ( - set_msvc_notfound_policy, # noqa: F401 - get_msvc_notfound_policy, # noqa: F401 -) + set_msvc_notfound_policy, + get_msvc_notfound_policy, +) # noqa: F401 class UnsupportedVersion(VisualCException): pass -- cgit v0.12 From 56bd50d3a15cbe3c82e84bf703a3f4bf1e5615d5 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:32:31 -0400 Subject: Add comment and import one-by-one for msvc notfound policy and flake8 F401 --- SCons/Tool/MSCommon/vc.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index f34e9e0..56f21e8 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -66,10 +66,9 @@ from .MSVC.Exceptions import ( VisualCException ) -from .MSVC.NotFound import ( - set_msvc_notfound_policy, - get_msvc_notfound_policy, -) # noqa: F401 +# msvc test(s) expect avaiable via vc +from .MSVC.NotFound import set_msvc_notfound_policy # noqa: F401 +from .MSVC.NotFound import get_msvc_notfound_policy # noqa: F401 class UnsupportedVersion(VisualCException): pass -- cgit v0.12 From ad7a59d1621d4154aeb51ba402f77aa18d65e21d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 10:38:05 -0400 Subject: Fix sider issue --- SCons/Tool/MSCommon/vc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 56f21e8..5130727 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -66,7 +66,7 @@ from .MSVC.Exceptions import ( VisualCException ) -# msvc test(s) expect avaiable via vc +# msvc test(s) expect notfound policy available via vc from .MSVC.NotFound import set_msvc_notfound_policy # noqa: F401 from .MSVC.NotFound import get_msvc_notfound_policy # noqa: F401 -- cgit v0.12 From a5938cce33f77237e45ad86a3f43e58f65ba74d3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 11:22:29 -0400 Subject: Move SCONS_CACHE_MSVC_FORCE_DEFAULTS environment variable query to MSCommon and set boolean if active. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 18 +++++++----------- SCons/Tool/MSCommon/common.py | 5 +++++ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 2a94650..aa96946 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -34,7 +34,7 @@ from collections import ( ) from ..common import ( - CONFIG_CACHE, + CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS, debug, ) @@ -51,13 +51,6 @@ from . import Dispatcher Dispatcher.register_modulename(__name__) -# Force default SDK and toolset arguments in cache -_SCONS_CACHE_MSVC_FORCE_DEFAULTS = False -if CONFIG_CACHE: - # SCONS_CACHE_MSVC_FORCE_DEFAULTS is internal and not documented. - if os.environ.get('SCONS_CACHE_MSVC_FORCE_DEFAULTS') in Config.BOOLEAN_SYMBOLS[True]: - _SCONS_CACHE_MSVC_FORCE_DEFAULTS = True - # Script argument: boolean True _ARGUMENT_BOOLEAN_TRUE_LEGACY = (True, '1') # MSVC_UWP_APP _ARGUMENT_BOOLEAN_TRUE = (True,) @@ -118,9 +111,12 @@ def _msvc_force_default_toolset(force=True): _MSVC_FORCE_DEFAULT_TOOLSET = force debug('_MSVC_FORCE_DEFAULT_TOOLSET=%s', repr(force)) -if _SCONS_CACHE_MSVC_FORCE_DEFAULTS: - _msvc_force_default_sdk(True) - _msvc_force_default_toolset(True) +def msvc_force_default_arguments(force=True): + _msvc_force_default_sdk(force) + _msvc_force_default_toolset(force) + +if CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS: + msvc_force_default_arguments(force=True) # MSVC batch file arguments: # diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index c9f07f5..da8fd55 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -102,6 +102,11 @@ CONFIG_CACHE = os.environ.get('SCONS_CACHE_MSVC_CONFIG') if CONFIG_CACHE in ('1', 'true', 'True'): CONFIG_CACHE = os.path.join(os.path.expanduser('~'), 'scons_msvc_cache.json') +# SCONS_CACHE_MSVC_FORCE_DEFAULTS is internal-use so undocumented. +CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS = False +if CONFIG_CACHE: + if os.environ.get('SCONS_CACHE_MSVC_FORCE_DEFAULTS') in ('1', 'true', 'True'): + CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS = True def read_script_env_cache(): """ fetch cached msvc env vars if requested, else return empty dict """ -- cgit v0.12 From 52e349c07058d6b7b2b5eb46ae6371427bee9849 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 12:37:03 -0400 Subject: Remove debug messages that by default are noisy. --- SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py | 2 -- SCons/Tool/MSCommon/vc.py | 7 ++----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py index 8b9faa9..e1c05bc 100644 --- a/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py +++ b/SCons/Tool/MSCommon/MSVC/SetupEnvDefault.py @@ -73,7 +73,6 @@ def _initialize(env, msvc_exists_func): debug('msvc default:msvc_installed=%s', _Data.msvc_installed) def register_tool(env, tool, msvc_exists_func): - debug('msvc default:tool=%s', tool) if _Data.need_init: _initialize(env, msvc_exists_func) if _Data.msvc_installed: @@ -90,7 +89,6 @@ def register_tool(env, tool, msvc_exists_func): debug('msvc default:tool=%s, msvc_tools=%s', tool, _Data.msvc_tools) def register_setup(env, msvc_exists_func): - debug('msvc default') if _Data.need_init: _initialize(env, msvc_exists_func) _Data.n_setup += 1 diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 5130727..7ea3583 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1102,7 +1102,6 @@ def msvc_setup_env_once(env, tool=None): has_run = False if not has_run: - debug('tool=%s', repr(tool)) MSVC.SetupEnvDefault.register_setup(env, msvc_exists) msvc_setup_env(env) env["MSVC_SETUP_RUN"] = True @@ -1285,13 +1284,13 @@ def msvc_setup_env(env): SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) def msvc_exists(env=None, version=None): - debug('version=%s', repr(version)) vcs = get_installed_vcs(env) if version is None: rval = len(vcs) > 0 else: rval = version in vcs - debug('version=%s, return=%s', repr(version), rval) + if not rval: + debug('version=%s, return=%s', repr(version), rval) return rval def msvc_setup_env_user(env=None): @@ -1336,14 +1335,12 @@ def msvc_setup_env_user(env=None): return rval def msvc_setup_env_tool(env=None, version=None, tool=None): - debug('tool=%s, version=%s', repr(tool), repr(version)) MSVC.SetupEnvDefault.register_tool(env, tool, msvc_exists) rval = False if not rval and msvc_exists(env, version): rval = True if not rval and msvc_setup_env_user(env): rval = True - debug('tool=%s, version=%s, return=%s', repr(tool), repr(version), rval) return rval def get_msvc_sdk_versions(msvc_version=None, msvc_uwp_app=False): -- cgit v0.12 From 7683a2f958287ceee257b0449eed767b0dd275e5 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 21 Jun 2022 12:46:14 -0400 Subject: Remove MSVC_TOOLSET_VERSION from intent to use msvc tools (attached to default version or specific version). --- SCons/Tool/MSCommon/vc.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 7ea3583..675b8d0 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1304,14 +1304,8 @@ def msvc_setup_env_user(env=None): # MSVC_USE_SCRIPT: defined and (is string or evaluates False) # MSVC_USE_SETTINGS: defined and is not None - # Arguments possibly present but not considered: - # MSVC_SDK_VERSION - # MSVC_UWP_APP - # MSVC_SPECTRE_LIBS - # MSVC_SCRIPT_ARGS - # defined and is True - for key in ['MSVC_VERSION', 'MSVS_VERSION', 'MSVC_TOOLSET_VERSION']: + for key in ['MSVC_VERSION', 'MSVS_VERSION']: if key in env and env[key]: rval = True debug('key=%s, return=%s', repr(key), rval) -- cgit v0.12 From 98cf01fd1434463a7af5b3fe5375a9772882dcd2 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 22 Jun 2022 16:53:53 -0400 Subject: Reorder function declarations --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 46 ++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index aa96946..e56dd4a 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -629,6 +629,29 @@ def _msvc_script_argument_spectre(env, msvc, arglist): return spectre_arg +def _user_script_argument_spectre(env, spectre, user_argstr): + + matches = [m for m in re_vcvars_spectre.finditer(user_argstr)] + if not matches: + return None + + if len(matches) > 1: + debug('multiple spectre declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) + err_msg = "multiple spectre declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) + raise MSVCArgumentError(err_msg) + + if not spectre: + return None + + env_argstr = env.get('MSVC_SPECTRE_LIBS','') + debug('multiple spectre declarations: MSVC_SPECTRE_LIBS=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) + + err_msg = "multiple spectre declarations: MSVC_SPECTRE_LIBS={} and MSVC_SCRIPT_ARGS={}".format( + repr(env_argstr), repr(user_argstr) + ) + + raise MSVCArgumentError(err_msg) + def _msvc_script_argument_user(env, msvc, arglist): # subst None -> empty string @@ -655,29 +678,6 @@ def _msvc_script_argument_user(env, msvc, arglist): return script_args -def _user_script_argument_spectre(env, spectre, user_argstr): - - matches = [m for m in re_vcvars_spectre.finditer(user_argstr)] - if not matches: - return None - - if len(matches) > 1: - debug('multiple spectre declarations: MSVC_SCRIPT_ARGS=%s', repr(user_argstr)) - err_msg = "multiple spectre declarations: MSVC_SCRIPT_ARGS={}".format(repr(user_argstr)) - raise MSVCArgumentError(err_msg) - - if not spectre: - return None - - env_argstr = env.get('MSVC_SPECTRE_LIBS','') - debug('multiple spectre declarations: MSVC_SPECTRE_LIBS=%s, MSVC_SCRIPT_ARGS=%s', repr(env_argstr), repr(user_argstr)) - - err_msg = "multiple spectre declarations: MSVC_SPECTRE_LIBS={} and MSVC_SCRIPT_ARGS={}".format( - repr(env_argstr), repr(user_argstr) - ) - - raise MSVCArgumentError(err_msg) - def msvc_script_arguments(env, version, vc_dir, arg): arglist = [] -- cgit v0.12 From 1de714ae3085d9fd07cb8837f523037885271aa8 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 23 Jun 2022 18:58:38 -0400 Subject: Construction variable documentation additions and modifications. --- SCons/Tool/msvc.xml | 553 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 542 insertions(+), 11 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index e8df128..6663aab 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -481,9 +481,24 @@ env = Environment(MSVC_VERSION='8.0', MSVC_USE_SETTINGS=msvc_use_settings) -Note: the dictionary content requirements are based on the internal msvc implementation and -therefore may change at any time. The burden is on the user to ensure the dictionary contents -are minimally sufficient to ensure successful builds. +Important usage details: + + + +MSVC_USE_SETTINGS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The dictionary content requirements are based on the internal msvc implementation and +therefore may change at any time. + +The burden is on the user to ensure the dictionary contents are minimally sufficient to +ensure successful builds. + + + @@ -510,21 +525,65 @@ are minimally sufficient to ensure successful builds. -Build libraries for a Universal Windows Platform (UWP) Application. +Build with the Universal Windows Platform (UWP) application Visual C++ libraries. -If &cv-MSVC_UWP_APP; is set, the Visual C++ environment will be set up to point +The valid values for MSVC_UWP_APP are True, +'1', False, '0', +or None. + + + +When MSVC_UWP_APP is enabled (i.e., True or +'1'), the Visual C++ environment will be set up to point to the Windows Store compatible libraries and Visual C++ runtimes. In doing so, any libraries that are built will be able to be used in a UWP App and published to the Windows Store. -This flag will only have an effect with Visual Studio 2015 or later. -This variable must be passed as an argument to the Environment() -constructor; setting it later has no effect. + + + + + +An exception is raised when any of the following conditions are satisfied: + + +MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. + + +MSVC_UWP_APP is enabled and a UWP argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via MSVC_UWP_APP +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + -Valid values are '1' or '0' +Example - A Visual Studio 2022 build for the Universal Windows Platform: + +env = Environment(MSVC_VERSION='14.3', MSVC_UWP_APP=True) + + + + +Important usage details: + + + +MSVC_UWP_APP must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The existence of the UWP libraries is not verified when MSVC_UWP_APP is enabled +which could result in build failures. + +The burden is on the user to ensure the requisite UWP libraries are installed. + + + @@ -582,7 +641,7 @@ and also before &f-link-env-Tool; is called to ininitialize any of those tools: -Specify the scons behavior when the Microsoft Visual C/C++ compiler is not detected. +Specify the &scons; behavior when the Microsoft Visual C/C++ compiler is not detected. @@ -590,7 +649,7 @@ Specify the scons behavior when the Microsoft Visual C/C++ compiler is not detec when the requested msvc version is not detected. - + The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; behavior are: @@ -664,13 +723,485 @@ A non-default tool list is specified that does not contain any of the msvc tools +Important usage details: + + + +MSVC_NOTFOUND_POLICY must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + + + When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue subject to the conditions listed above. The default &scons; behavior may change in the future. + + + + + + +Pass user-defined arguments to the Visual C++ batch file determined via autodetection. + + + +MSVC_SCRIPT_ARGS is available for msvc batch file arguments that do not have first-class support +via construction variables or when there is an issue with the appropriate construction variable validation. +When available, it is recommended to use the appropriate construction variables (e.g., &cv-link-MSVC_TOOLSET_VERSION;) +rather than MSVC_SCRIPT_ARGS arguments. + + + +The valid values for MSVC_SCRIPT_ARGS are: None, a string, +or a list of strings. + + + +The MSVC_SCRIPT_ARGS value is converted to a scalar string (i.e., "flattened"). +The resulting scalar string, if not empty, is passed as an argument to the msvc batch file determined +via autodetection subject to the validation conditions listed below. + + + +MSVC_SCRIPT_ARGS is ignored when the value is None and when the +result from argument conversion is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SCRIPT_ARGS is specified for Visual Studio 2013 and earlier. + + + +Multiple SDK version arguments (e.g., '10.0.20348.0') are specified +in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_SDK_VERSION; is specified and an SDK version argument +(e.g., '10.0.20348.0') is specified in MSVC_SCRIPT_ARGS. +Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and MSVC_SCRIPT_ARGS +are not allowed. + + + +Multiple toolset version arguments (e.g., '-vcvars_ver=14.29') +are specified in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_TOOLSET_VERSION; is specified and a toolset version argument +(e.g., '-vcvars_ver=14.29') is specified in MSVC_SCRIPT_ARGS. +Multiple toolset version declarations via &cv-link-MSVC_TOOLSET_VERSION; and +MSVC_SCRIPT_ARGS are not allowed. + + + +Multiple spectre library arguments (e.g., '-vcvars_spectre_libs=spectre') +are specified in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument +(e.g., '-vcvars_spectre_libs=spectre') is specified in +MSVC_SCRIPT_ARGS. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; +and MSVC_SCRIPT_ARGS are not allowed. + + + +Multiple UWP arguments (e.g., uwp or store) are specified +in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_UWP_APP; is enabled and a UWP argument (e.g., uwp or +store) is specified in MSVC_SCRIPT_ARGS. Multiple UWP declarations +via &cv-link-MSVC_UWP_APP; and MSVC_SCRIPT_ARGS are not allowed. + + + + + + +Example 1 - A Visual Studio 2022 build with an SDK version and a toolset version +specified with a string argument: + +env = Environment(MSVC_VERSION='14.3', MSVC_SCRIPT_ARGS='10.0.20348.0 -vcvars_ver=14.29.30133') + + + + +Example 2 - A Visual Studio 2022 build with an SDK version and a toolset version +specified with a list argument: + +env = Environment(MSVC_VERSION='14.3', MSVC_SCRIPT_ARGS=['10.0.20348.0', '-vcvars_ver=14.29.30133']) + + + + +Important usage details: + + + +MSVC_SCRIPT_ARGS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + +Other than checking for multiple declarations as described above, MSVC_SCRIPT_ARGS arguments +are not validated. + + + + +Erroneous, inconsistent, and/or version incompatible MSVC_SCRIPT_ARGS arguments are likely +to result in build failures for reasons that are not readily apparent and may be difficult to diagnose. + +The burden is on the user to ensure that the arguments provided to the msvc batch file are valid, consistent +and compatible with the version of msvc selected. + + + + + + + +Build with a specific version of the Microsoft Software Development Kit (SDK). + + + +The valid values for MSVC_SDK_VERSION are: None +or a string containing the requested SDK version (e.g., '10.0.20348.0'). + + + +MSVC_SDK_VERSION is ignored when the value is None and when +the value is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SDK_VERSION is specified for Visual Studio 2013 and earlier. + + + +MSVC_SDK_VERSION is specified and an SDK version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via MSVC_SDK_VERSION +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + +The MSVC_SDK_VERSION specified does not match any of the supported formats: + + +'10.0.XXXXX.Y' [SDK 10.0] + + +'8.1' [SDK 8.1] + + + + + +The system folder for the corresponding MSVC_SDK_VERSION version is not found. +The requested SDK version does not appear to be installed. + + + +The MSVC_SDK_VERSION version does not appear to support the requested platform +type (i.e., UWP or Desktop). The requested SDK version +platform type components do not appear to be installed. + + + + + + +Example 1 - A Visual Studio 2022 build with a specific Windows SDK version: + +env = Environment(MSVC_VERSION='14.3', MSVC_SDK_VERSION='10.0.20348.0') + + + + +Example 2 - A Visual Studio 2022 build with a specific SDK version for the Universal Windows Platform: + +env = Environment(MSVC_VERSION='14.3', MSVC_SDK_VERSION='10.0.20348.0', MSVC_UWP_APP=True) + + + + +Important usage details: + + + +MSVC_SDK_VERSION must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +Should a SDK 10.0 version be installed that does not follow the naming scheme above, the +SDK version will need to be specified via &cv-link-MSVC_SCRIPT_ARGS; until the version number +validation format can be extended. + + + + +Should an exception be raised indicating that the SDK version is not found, verify that +the requested SDK version is installed with the necessary platform type components. + + + +There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' +when using the v141 build tools and the target architecture is ARM64. +Should build failures arise with this combination of settings, MSVC_SDK_VERSION may be +employed to specify a different SDK version for the build. + + + + + + + + + + + +Build with a specific Visual C++ toolset version. + + + +Specifying MSVC_TOOLSET_VERSION does not affect the autodetection and selection +of msvc instances. The MSVC_TOOLSET_VERSION is applied after +an msvc instance is selected. This could be the default version of msvc if &cv-link-MSVC_VERSION; +is not specified. + + + +The valid values for MSVC_TOOLSET_VERSION are: None +or a string containing the requested toolset version (e.g., '14.29'). + + + +MSVC_TOOLSET_VERSION is ignored when the value is None and when +the value is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_TOOLSET_VERSION is specified for Visual Studio 2015 and earlier. + + + +MSVC_TOOLSET_VERSION is specified and a toolset version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via MSVC_TOOLSET_VERSION +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + + +The MSVC_TOOLSET_VERSION specified does not match any of the supported formats: + + + + + +'XX.Y' + + + +'XX.YY' + + + +'XX.YY.ZZZZZ' + + + +'XX.YY.Z' to 'XX.YY.ZZZZ' + +[&scons; extension not directly supported by the msvc batch files and may be removed in the future] + + + + +'XX.YY.ZZ.N' [SxS format] + + + +'XX.YY.ZZ.NN' [SxS format] + + + + + + + +The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified +is for Visual Studio 2013 and earlier (e.g., '12.0'). + + + +The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified +is greater than the msvc version selected (e.g., '99.0'). + + + +A system folder for the corresponding MSVC_TOOLSET_VERSION version is not found. +The requested toolset version does not appear to be installed. + + + + + + +Toolset selection details: + + + +SxS version numbers are not always in three dot format (e.g., 'XX.YY.ZZ.NN') as shown above. + +In Visual Studio 2022 for example, 14.16 is an SxS toolset version that is directly +mapped to toolset version 14.16.27023. + + + +When MSVC_TOOLSET_VERSION is not an SxS version number or a full toolset version number: +the first toolset version, ranked in descending order, that matches the MSVC_TOOLSET_VERSION +prefix is selected. + + + +When MSVC_TOOLSET_VERSION is specified using the major msvc version prefix +(i.e., 'XX.Y') and the major msvc version is that of the latest release of +Visual Studio, the selected toolset version may not be the same as the default Visual C++ toolset version. + +In the latest release of Visual Studio, the default Visual C++ toolset version is not necessarily the +toolset with the largest version number. + + + + + + +Example environment constructor invocations with toolset version specifications: + +Environment(MSVC_TOOLSET_VERSION='14.2') +Environment(MSVC_TOOLSET_VERSION='14.29') +Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') +Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') + + + + +Important usage details: + + + +MSVC_TOOLSET_VERSION must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The existence of the toolset host architecture and target architecture folders are not verified +when MSVC_TOOLSET_VERSION is specified which could result in build failures. + +The burden is on the user to ensure the requisite toolset target architecture build tools are installed. + + + + + + + + + + + +Build with the spectre-mitigated Visual C++ libraries. + + + +The valid values for MSVC_SPECTRE_LIBS are: True, +False, or None. + + + +When MSVC_SPECTRE_LIBS is enabled (i.e., True), +the Visual C++ environment will include the paths to the spectre-mitigated implementations +of the Microsoft Visual C++ libraries. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SPECTRE_LIBS is enabled for Visual Studio 2015 and earlier. + + + +MSVC_SPECTRE_LIBS is enabled and a spectre library argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via MSVC_SPECTRE_LIBS +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + + + + +Example - A Visual Studio 2022 build with spectre mitigated Visual C++ libraries: + +env = Environment(MSVC_VERSION='14.3', MSVC_SPECTRE_LIBS=True) + + + + +Important usage details: + + + +MSVC_SPECTRE_LIBS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + +Additional compiler switches (e.g., /Qspectre) are necessary for including +spectre mitigations when building user artifacts. Refer to the Visual Studio documentation for +details. + + + + +The existence of the spectre mitigations libraries is not verified when MSVC_SPECTRE_LIBS +is enabled which could result in build failures. + +The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. + + + + + + + -- cgit v0.12 From f99f7cab7bc3af5ef18e3189f531572df4558fe4 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Thu, 23 Jun 2022 18:59:01 -0400 Subject: Construction variable documentation additions and modifications. --- doc/generated/variables.gen | 558 +++++++++++++++++++++++++++++++++++++++++++- doc/generated/variables.mod | 8 + 2 files changed, 555 insertions(+), 11 deletions(-) diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index 996e35a..f1b3fff 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -4686,7 +4686,7 @@ will be compiled separately. MSVC_NOTFOUND_POLICY -Specify the scons behavior when the Microsoft Visual C/C++ compiler is not detected. +Specify the &scons; behavior when the Microsoft Visual C/C++ compiler is not detected. @@ -4694,7 +4694,7 @@ Specify the scons behavior when the Microsoft Visual C/C++ compiler is not detec when the requested msvc version is not detected. - + The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; behavior are: @@ -4768,10 +4768,487 @@ A non-default tool list is specified that does not contain any of the msvc tools +Important usage details: + + + +MSVC_NOTFOUND_POLICY must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + + + When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue subject to the conditions listed above. The default &scons; behavior may change in the future. + + + + + MSVC_SCRIPT_ARGS + + +Pass user-defined arguments to the Visual C++ batch file determined via autodetection. + + + +MSVC_SCRIPT_ARGS is available for msvc batch file arguments that do not have first-class support +via construction variables or when there is an issue with the appropriate construction variable validation. +When available, it is recommended to use the appropriate construction variables (e.g., &cv-link-MSVC_TOOLSET_VERSION;) +rather than MSVC_SCRIPT_ARGS arguments. + + + +The valid values for MSVC_SCRIPT_ARGS are: None, a string, +or a list of strings. + + + +The MSVC_SCRIPT_ARGS value is converted to a scalar string (i.e., "flattened"). +The resulting scalar string, if not empty, is passed as an argument to the msvc batch file determined +via autodetection subject to the validation conditions listed below. + + + +MSVC_SCRIPT_ARGS is ignored when the value is None and when the +result from argument conversion is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SCRIPT_ARGS is specified for Visual Studio 2013 and earlier. + + + +Multiple SDK version arguments (e.g., '10.0.20348.0') are specified +in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_SDK_VERSION; is specified and an SDK version argument +(e.g., '10.0.20348.0') is specified in MSVC_SCRIPT_ARGS. +Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and MSVC_SCRIPT_ARGS +are not allowed. + + + +Multiple toolset version arguments (e.g., '-vcvars_ver=14.29') +are specified in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_TOOLSET_VERSION; is specified and a toolset version argument +(e.g., '-vcvars_ver=14.29') is specified in MSVC_SCRIPT_ARGS. +Multiple toolset version declarations via &cv-link-MSVC_TOOLSET_VERSION; and +MSVC_SCRIPT_ARGS are not allowed. + + + +Multiple spectre library arguments (e.g., '-vcvars_spectre_libs=spectre') +are specified in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument +(e.g., '-vcvars_spectre_libs=spectre') is specified in +MSVC_SCRIPT_ARGS. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; +and MSVC_SCRIPT_ARGS are not allowed. + + + +Multiple UWP arguments (e.g., uwp or store) are specified +in MSVC_SCRIPT_ARGS. + + + +&cv-link-MSVC_UWP_APP; is enabled and a UWP argument (e.g., uwp or +store) is specified in MSVC_SCRIPT_ARGS. Multiple UWP declarations +via &cv-link-MSVC_UWP_APP; and MSVC_SCRIPT_ARGS are not allowed. + + + + + + +Example 1 - A Visual Studio 2022 build with an SDK version and a toolset version +specified with a string argument: + +env = Environment(MSVC_VERSION='14.3', MSVC_SCRIPT_ARGS='10.0.20348.0 -vcvars_ver=14.29.30133') + + + + +Example 2 - A Visual Studio 2022 build with an SDK version and a toolset version +specified with a list argument: + +env = Environment(MSVC_VERSION='14.3', MSVC_SCRIPT_ARGS=['10.0.20348.0', '-vcvars_ver=14.29.30133']) + + + + +Important usage details: + + + +MSVC_SCRIPT_ARGS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + +Other than checking for multiple declarations as described above, MSVC_SCRIPT_ARGS arguments +are not validated. + + + + +Erroneous, inconsistent, and/or version incompatible MSVC_SCRIPT_ARGS arguments are likely +to result in build failures for reasons that are not readily apparent and may be difficult to diagnose. + +The burden is on the user to ensure that the arguments provided to the msvc batch file are valid, consistent +and compatible with the version of msvc selected. + + + + + + + + + + MSVC_SDK_VERSION + + +Build with a specific version of the Microsoft Software Development Kit (SDK). + + + +The valid values for MSVC_SDK_VERSION are: None +or a string containing the requested SDK version (e.g., '10.0.20348.0'). + + + +MSVC_SDK_VERSION is ignored when the value is None and when +the value is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SDK_VERSION is specified for Visual Studio 2013 and earlier. + + + +MSVC_SDK_VERSION is specified and an SDK version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via MSVC_SDK_VERSION +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + +The MSVC_SDK_VERSION specified does not match any of the supported formats: + + +'10.0.XXXXX.Y' [SDK 10.0] + + +'8.1' [SDK 8.1] + + + + + +The system folder for the corresponding MSVC_SDK_VERSION version is not found. +The requested SDK version does not appear to be installed. + + + +The MSVC_SDK_VERSION version does not appear to support the requested platform +type (i.e., UWP or Desktop). The requested SDK version +platform type components do not appear to be installed. + + + + + + +Example 1 - A Visual Studio 2022 build with a specific Windows SDK version: + +env = Environment(MSVC_VERSION='14.3', MSVC_SDK_VERSION='10.0.20348.0') + + + + +Example 2 - A Visual Studio 2022 build with a specific SDK version for the Universal Windows Platform: + +env = Environment(MSVC_VERSION='14.3', MSVC_SDK_VERSION='10.0.20348.0', MSVC_UWP_APP=True) + + + + +Important usage details: + + + +MSVC_SDK_VERSION must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +Should a SDK 10.0 version be installed that does not follow the naming scheme above, the +SDK version will need to be specified via &cv-link-MSVC_SCRIPT_ARGS; until the version number +validation format can be extended. + + + + +Should an exception be raised indicating that the SDK version is not found, verify that +the requested SDK version is installed with the necessary platform type components. + + + +There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' +when using the v141 build tools and the target architecture is ARM64. +Should build failures arise with this combination of settings, MSVC_SDK_VERSION may be +employed to specify a different SDK version for the build. + + + + + + + + + + MSVC_SPECTRE_LIBS + + +Build with the spectre-mitigated Visual C++ libraries. + + + +The valid values for MSVC_SPECTRE_LIBS are: True, +False, or None. + + + +When MSVC_SPECTRE_LIBS is enabled (i.e., True), +the Visual C++ environment will include the paths to the spectre-mitigated implementations +of the Microsoft Visual C++ libraries. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_SPECTRE_LIBS is enabled for Visual Studio 2015 and earlier. + + + +MSVC_SPECTRE_LIBS is enabled and a spectre library argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via MSVC_SPECTRE_LIBS +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + + + + +Example - A Visual Studio 2022 build with spectre mitigated Visual C++ libraries: + +env = Environment(MSVC_VERSION='14.3', MSVC_SPECTRE_LIBS=True) + + + + +Important usage details: + + + +MSVC_SPECTRE_LIBS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + +Additional compiler switches (e.g., /Qspectre) are necessary for including +spectre mitigations when building user artifacts. Refer to the Visual Studio documentation for +details. + + + + +The existence of the spectre mitigations libraries is not verified when MSVC_SPECTRE_LIBS +is enabled which could result in build failures. + +The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. + + + + + + + + + + MSVC_TOOLSET_VERSION + + +Build with a specific Visual C++ toolset version. + + + +Specifying MSVC_TOOLSET_VERSION does not affect the autodetection and selection +of msvc instances. The MSVC_TOOLSET_VERSION is applied after +an msvc instance is selected. This could be the default version of msvc if &cv-link-MSVC_VERSION; +is not specified. + + + +The valid values for MSVC_TOOLSET_VERSION are: None +or a string containing the requested toolset version (e.g., '14.29'). + + + +MSVC_TOOLSET_VERSION is ignored when the value is None and when +the value is an empty string. The validation conditions below do not apply. + + + +An exception is raised when any of the following conditions are satisfied: + + + +MSVC_TOOLSET_VERSION is specified for Visual Studio 2015 and earlier. + + + +MSVC_TOOLSET_VERSION is specified and a toolset version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via MSVC_TOOLSET_VERSION +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + + +The MSVC_TOOLSET_VERSION specified does not match any of the supported formats: + + + + + +'XX.Y' + + + +'XX.YY' + + + +'XX.YY.ZZZZZ' + + + +'XX.YY.Z' to 'XX.YY.ZZZZ' + +[&scons; extension not directly supported by the msvc batch files and may be removed in the future] + + + + +'XX.YY.ZZ.N' [SxS format] + + + +'XX.YY.ZZ.NN' [SxS format] + + + + + + + +The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified +is for Visual Studio 2013 and earlier (e.g., '12.0'). + + + +The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified +is greater than the msvc version selected (e.g., '99.0'). + + + +A system folder for the corresponding MSVC_TOOLSET_VERSION version is not found. +The requested toolset version does not appear to be installed. + + + + + + +Toolset selection details: + + + +SxS version numbers are not always in three dot format (e.g., 'XX.YY.ZZ.NN') as shown above. + +In Visual Studio 2022 for example, 14.16 is an SxS toolset version that is directly +mapped to toolset version 14.16.27023. + + + +When MSVC_TOOLSET_VERSION is not an SxS version number or a full toolset version number: +the first toolset version, ranked in descending order, that matches the MSVC_TOOLSET_VERSION +prefix is selected. + + + +When MSVC_TOOLSET_VERSION is specified using the major msvc version prefix +(i.e., 'XX.Y') and the major msvc version is that of the latest release of +Visual Studio, the selected toolset version may not be the same as the default Visual C++ toolset version. + +In the latest release of Visual Studio, the default Visual C++ toolset version is not necessarily the +toolset with the largest version number. + + + + + + +Example environment constructor invocations with toolset version specifications: + +Environment(MSVC_TOOLSET_VERSION='14.2') +Environment(MSVC_TOOLSET_VERSION='14.29') +Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') +Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') + + + + +Important usage details: + + + +MSVC_TOOLSET_VERSION must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The existence of the toolset host architecture and target architecture folders are not verified +when MSVC_TOOLSET_VERSION is specified which could result in build failures. + +The burden is on the user to ensure the requisite toolset target architecture build tools are installed. + + + + @@ -4879,9 +5356,24 @@ env = Environment(MSVC_VERSION='8.0', MSVC_USE_SETTINGS=msvc_use_settings) -Note: the dictionary content requirements are based on the internal msvc implementation and -therefore may change at any time. The burden is on the user to ensure the dictionary contents -are minimally sufficient to ensure successful builds. +Important usage details: + + + +MSVC_USE_SETTINGS must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The dictionary content requirements are based on the internal msvc implementation and +therefore may change at any time. + +The burden is on the user to ensure the dictionary contents are minimally sufficient to +ensure successful builds. + + + @@ -4891,21 +5383,65 @@ are minimally sufficient to ensure successful builds. MSVC_UWP_APP -Build libraries for a Universal Windows Platform (UWP) Application. +Build with the Universal Windows Platform (UWP) application Visual C++ libraries. + + + +The valid values for MSVC_UWP_APP are True, +'1', False, '0', +or None. -If &cv-MSVC_UWP_APP; is set, the Visual C++ environment will be set up to point +When MSVC_UWP_APP is enabled (i.e., True or +'1'), the Visual C++ environment will be set up to point to the Windows Store compatible libraries and Visual C++ runtimes. In doing so, any libraries that are built will be able to be used in a UWP App and published to the Windows Store. -This flag will only have an effect with Visual Studio 2015 or later. -This variable must be passed as an argument to the Environment() -constructor; setting it later has no effect. + + -Valid values are '1' or '0' +An exception is raised when any of the following conditions are satisfied: + + +MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. + + +MSVC_UWP_APP is enabled and a UWP argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via MSVC_UWP_APP +and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + + + + + +Example - A Visual Studio 2022 build for the Universal Windows Platform: + +env = Environment(MSVC_VERSION='14.3', MSVC_UWP_APP=True) + + + + +Important usage details: + + + +MSVC_UWP_APP must be passed as an argument to the Environment() constructor; +setting it later has no effect. + + + + +The existence of the UWP libraries is not verified when MSVC_UWP_APP is enabled +which could result in build failures. + +The burden is on the user to ensure the requisite UWP libraries are installed. + + + diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index c5cd0cd..d3d29f2 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -321,6 +321,10 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MSSDK_VERSION"> $MSVC_BATCH"> $MSVC_NOTFOUND_POLICY"> +$MSVC_SCRIPT_ARGS"> +$MSVC_SDK_VERSION"> +$MSVC_SPECTRE_LIBS"> +$MSVC_TOOLSET_VERSION"> $MSVC_USE_SCRIPT"> $MSVC_USE_SCRIPT_ARGS"> $MSVC_USE_SETTINGS"> @@ -985,6 +989,10 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $MSSDK_VERSION"> $MSVC_BATCH"> $MSVC_NOTFOUND_POLICY"> +$MSVC_SCRIPT_ARGS"> +$MSVC_SDK_VERSION"> +$MSVC_SPECTRE_LIBS"> +$MSVC_TOOLSET_VERSION"> $MSVC_USE_SCRIPT"> $MSVC_USE_SCRIPT_ARGS"> $MSVC_USE_SETTINGS"> -- cgit v0.12 From 623cdba27514874cd1af6e09911153885574113d Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Fri, 24 Jun 2022 19:21:22 -0400 Subject: Documentation updates [ci skip] --- SCons/Tool/msvc.xml | 199 ++++++++++++++++++++++++++------------------ doc/generated/variables.gen | 199 ++++++++++++++++++++++++++------------------ 2 files changed, 234 insertions(+), 164 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 6663aab..de82db0 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -349,8 +349,15 @@ Sets the preferred version of Microsoft Visual C/C++ to use. If &cv-MSVC_VERSION; is not set, SCons will (by default) select the latest version of Visual C/C++ installed on your system. If the specified version isn't installed, tool initialization will fail. -This variable must be passed as an argument to the &f-link-Environment; -constructor; setting it later has no effect. + + + +&cv-MSVC_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_VERSION; must be set before the first msvc tool is +loaded into the environment. @@ -410,7 +417,7 @@ is, if you are sure everything is set correctly already and you don't want &SCons; to change anything. -&cv-MSVC_USE_SCRIPT; overrides &cv-link-MSVC_VERSION; and &cv-link-TARGET_ARCH;. +&cv-MSVC_USE_SCRIPT; ignores &cv-link-MSVC_VERSION; and &cv-link-TARGET_ARCH;. @@ -485,8 +492,12 @@ Important usage details: -MSVC_USE_SETTINGS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_USE_SETTINGS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_USE_SETTINGS; must be set before the first msvc tool is +loaded into the environment. @@ -529,13 +540,13 @@ Build with the Universal Windows Platform (UWP) application Visual C++ libraries -The valid values for MSVC_UWP_APP are True, +The valid values for &cv-MSVC_UWP_APP; are: True, '1', False, '0', or None. -When MSVC_UWP_APP is enabled (i.e., True or +When &cv-MSVC_UWP_APP; is enabled (i.e., True or '1'), the Visual C++ environment will be set up to point to the Windows Store compatible libraries and Visual C++ runtimes. In doing so, any libraries that are built will be able to be used in a UWP App and published @@ -549,11 +560,11 @@ constructor; setting it later has no effect. --> An exception is raised when any of the following conditions are satisfied: -MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. +&cv-MSVC_UWP_APP; is enabled for Visual Studio 2013 and earlier. -MSVC_UWP_APP is enabled and a UWP argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via MSVC_UWP_APP +&cv-MSVC_UWP_APP; is enabled and a UWP argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via &cv-MSVC_UWP_APP; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. @@ -571,13 +582,17 @@ Important usage details: -MSVC_UWP_APP must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_UWP_APP; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_UWP_APP; must be set before the first msvc tool is +loaded into the environment. -The existence of the UWP libraries is not verified when MSVC_UWP_APP is enabled +The existence of the UWP libraries is not verified when &cv-MSVC_UWP_APP; is enabled which could result in build failures. The burden is on the user to ensure the requisite UWP libraries are installed. @@ -645,12 +660,12 @@ Specify the &scons; behavior when the Microsoft Visual C/C++ compiler is not det - The MSVC_NOTFOUND_POLICY specifies the &scons; behavior when no msvc versions are detected or + The &cv-MSVC_NOTFOUND_POLICY; specifies the &scons; behavior when no msvc versions are detected or when the requested msvc version is not detected. -The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; behavior are: +The valid values for &cv-MSVC_NOTFOUND_POLICY; and the corresponding &scons; behavior are: @@ -691,7 +706,7 @@ Note: in addition to the camel case values shown above, lower case and upper cas -The MSVC_NOTFOUND_POLICY is applied when any of the following conditions are satisfied: +The &cv-MSVC_NOTFOUND_POLICY; is applied when any of the following conditions are satisfied: &cv-MSVC_VERSION; is specified, the default tools list is implicitly defined (i.e., the tools list is not specified), @@ -708,7 +723,7 @@ A non-default tools list is specified that contains one or more of the msvc tool -The MSVC_NOTFOUND_POLICY is ignored when any of the following conditions are satisfied: +The &cv-MSVC_NOTFOUND_POLICY; is ignored when any of the following conditions are satisfied: &cv-MSVC_VERSION; is not specified and the default tools list is implicitly defined (i.e., the tools list is not specified). @@ -727,15 +742,19 @@ Important usage details: -MSVC_NOTFOUND_POLICY must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_NOTFOUND_POLICY; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_NOTFOUND_POLICY; must be set before the first msvc tool is +loaded into the environment. -When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue +When &cv-MSVC_NOTFOUND_POLICY; is not specified, the default &scons; behavior is to issue a warning and continue subject to the conditions listed above. The default &scons; behavior may change in the future. @@ -749,25 +768,25 @@ Pass user-defined arguments to the Visual C++ batch file determined via autodete -MSVC_SCRIPT_ARGS is available for msvc batch file arguments that do not have first-class support +&cv-MSVC_SCRIPT_ARGS; is available for msvc batch file arguments that do not have first-class support via construction variables or when there is an issue with the appropriate construction variable validation. When available, it is recommended to use the appropriate construction variables (e.g., &cv-link-MSVC_TOOLSET_VERSION;) -rather than MSVC_SCRIPT_ARGS arguments. +rather than &cv-MSVC_SCRIPT_ARGS; arguments. -The valid values for MSVC_SCRIPT_ARGS are: None, a string, +The valid values for &cv-MSVC_SCRIPT_ARGS; are: None, a string, or a list of strings. -The MSVC_SCRIPT_ARGS value is converted to a scalar string (i.e., "flattened"). +The &cv-MSVC_SCRIPT_ARGS; value is converted to a scalar string (i.e., "flattened"). The resulting scalar string, if not empty, is passed as an argument to the msvc batch file determined via autodetection subject to the validation conditions listed below. -MSVC_SCRIPT_ARGS is ignored when the value is None and when the +&cv-MSVC_SCRIPT_ARGS; is ignored when the value is None and when the result from argument conversion is an empty string. The validation conditions below do not apply. @@ -776,54 +795,54 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SCRIPT_ARGS is specified for Visual Studio 2013 and earlier. +&cv-MSVC_SCRIPT_ARGS; is specified for Visual Studio 2013 and earlier. Multiple SDK version arguments (e.g., '10.0.20348.0') are specified -in MSVC_SCRIPT_ARGS. +in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_SDK_VERSION; is specified and an SDK version argument -(e.g., '10.0.20348.0') is specified in MSVC_SCRIPT_ARGS. -Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and MSVC_SCRIPT_ARGS +(e.g., '10.0.20348.0') is specified in &cv-MSVC_SCRIPT_ARGS;. +Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and &cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple toolset version arguments (e.g., '-vcvars_ver=14.29') -are specified in MSVC_SCRIPT_ARGS. +are specified in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_TOOLSET_VERSION; is specified and a toolset version argument -(e.g., '-vcvars_ver=14.29') is specified in MSVC_SCRIPT_ARGS. +(e.g., '-vcvars_ver=14.29') is specified in &cv-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via &cv-link-MSVC_TOOLSET_VERSION; and -MSVC_SCRIPT_ARGS are not allowed. +&cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple spectre library arguments (e.g., '-vcvars_spectre_libs=spectre') -are specified in MSVC_SCRIPT_ARGS. +are specified in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument (e.g., '-vcvars_spectre_libs=spectre') is specified in -MSVC_SCRIPT_ARGS. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; -and MSVC_SCRIPT_ARGS are not allowed. +&cv-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; +and &cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple UWP arguments (e.g., uwp or store) are specified -in MSVC_SCRIPT_ARGS. +in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_UWP_APP; is enabled and a UWP argument (e.g., uwp or -store) is specified in MSVC_SCRIPT_ARGS. Multiple UWP declarations -via &cv-link-MSVC_UWP_APP; and MSVC_SCRIPT_ARGS are not allowed. +store) is specified in &cv-MSVC_SCRIPT_ARGS;. Multiple UWP declarations +via &cv-link-MSVC_UWP_APP; and &cv-MSVC_SCRIPT_ARGS; are not allowed. @@ -850,18 +869,22 @@ Important usage details: -MSVC_SCRIPT_ARGS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SCRIPT_ARGS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SCRIPT_ARGS; must be set before the first msvc tool is +loaded into the environment. -Other than checking for multiple declarations as described above, MSVC_SCRIPT_ARGS arguments +Other than checking for multiple declarations as described above, &cv-MSVC_SCRIPT_ARGS; arguments are not validated. -Erroneous, inconsistent, and/or version incompatible MSVC_SCRIPT_ARGS arguments are likely +Erroneous, inconsistent, and/or version incompatible &cv-MSVC_SCRIPT_ARGS; arguments are likely to result in build failures for reasons that are not readily apparent and may be difficult to diagnose. The burden is on the user to ensure that the arguments provided to the msvc batch file are valid, consistent @@ -881,12 +904,12 @@ Build with a specific version of the Microsoft Software Development Kit (SDK). -The valid values for MSVC_SDK_VERSION are: None -or a string containing the requested SDK version (e.g., '10.0.20348.0'). +The valid values for &cv-MSVC_SDK_VERSION; are: None +or a string containing the requested SDK version (e.g., '10.0.20348.0'). -MSVC_SDK_VERSION is ignored when the value is None and when +&cv-MSVC_SDK_VERSION; is ignored when the value is None and when the value is an empty string. The validation conditions below do not apply. @@ -895,17 +918,17 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SDK_VERSION is specified for Visual Studio 2013 and earlier. +&cv-MSVC_SDK_VERSION; is specified for Visual Studio 2013 and earlier. -MSVC_SDK_VERSION is specified and an SDK version argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via MSVC_SDK_VERSION +&cv-MSVC_SDK_VERSION; is specified and an SDK version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via &cv-MSVC_SDK_VERSION; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. -The MSVC_SDK_VERSION specified does not match any of the supported formats: +The &cv-MSVC_SDK_VERSION; specified does not match any of the supported formats: '10.0.XXXXX.Y' [SDK 10.0] @@ -917,12 +940,12 @@ The MSVC_SDK_VERSION specified does not match any of the supporte -The system folder for the corresponding MSVC_SDK_VERSION version is not found. +The system folder for the corresponding &cv-MSVC_SDK_VERSION; version is not found. The requested SDK version does not appear to be installed. -The MSVC_SDK_VERSION version does not appear to support the requested platform +The &cv-MSVC_SDK_VERSION; version does not appear to support the requested platform type (i.e., UWP or Desktop). The requested SDK version platform type components do not appear to be installed. @@ -949,8 +972,12 @@ Important usage details: -MSVC_SDK_VERSION must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SDK_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SDK_VERSION; must be set before the first msvc tool is +loaded into the environment. @@ -969,7 +996,7 @@ the requested SDK version is installed with the necessary platform type componen There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' when using the v141 build tools and the target architecture is ARM64. -Should build failures arise with this combination of settings, MSVC_SDK_VERSION may be +Should build failures arise with this combination of settings, &cv-MSVC_SDK_VERSION; may be employed to specify a different SDK version for the build. @@ -986,19 +1013,19 @@ Build with a specific Visual C++ toolset version. -Specifying MSVC_TOOLSET_VERSION does not affect the autodetection and selection -of msvc instances. The MSVC_TOOLSET_VERSION is applied after +Specifying &cv-MSVC_TOOLSET_VERSION; does not affect the autodetection and selection +of msvc instances. The &cv-MSVC_TOOLSET_VERSION; is applied after an msvc instance is selected. This could be the default version of msvc if &cv-link-MSVC_VERSION; is not specified. -The valid values for MSVC_TOOLSET_VERSION are: None -or a string containing the requested toolset version (e.g., '14.29'). +The valid values for &cv-MSVC_TOOLSET_VERSION; are: None +or a string containing the requested toolset version (e.g., '14.29'). -MSVC_TOOLSET_VERSION is ignored when the value is None and when +&cv-MSVC_TOOLSET_VERSION; is ignored when the value is None and when the value is an empty string. The validation conditions below do not apply. @@ -1007,18 +1034,18 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_TOOLSET_VERSION is specified for Visual Studio 2015 and earlier. +&cv-MSVC_TOOLSET_VERSION; is specified for Visual Studio 2015 and earlier. -MSVC_TOOLSET_VERSION is specified and a toolset version argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via MSVC_TOOLSET_VERSION +&cv-MSVC_TOOLSET_VERSION; is specified and a toolset version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via &cv-MSVC_TOOLSET_VERSION; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. -The MSVC_TOOLSET_VERSION specified does not match any of the supported formats: +The &cv-MSVC_TOOLSET_VERSION; specified does not match any of the supported formats: @@ -1055,17 +1082,17 @@ The MSVC_TOOLSET_VERSION specified does not match any of the supp -The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified -is for Visual Studio 2013 and earlier (e.g., '12.0'). +The major msvc version prefix (i.e., 'XX.Y') of the &cv-MSVC_TOOLSET_VERSION; specified +is for Visual Studio 2013 and earlier (e.g., '12.0'). -The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified -is greater than the msvc version selected (e.g., '99.0'). +The major msvc version prefix (i.e., 'XX.Y') of the &cv-MSVC_TOOLSET_VERSION; specified +is greater than the msvc version selected (e.g., '99.0'). -A system folder for the corresponding MSVC_TOOLSET_VERSION version is not found. +A system folder for the corresponding &cv-MSVC_TOOLSET_VERSION; version is not found. The requested toolset version does not appear to be installed. @@ -1084,13 +1111,13 @@ mapped to toolset version 14.16.27023. -When MSVC_TOOLSET_VERSION is not an SxS version number or a full toolset version number: -the first toolset version, ranked in descending order, that matches the MSVC_TOOLSET_VERSION +When &cv-MSVC_TOOLSET_VERSION; is not an SxS version number or a full toolset version number: +the first toolset version, ranked in descending order, that matches the &cv-MSVC_TOOLSET_VERSION; prefix is selected. -When MSVC_TOOLSET_VERSION is specified using the major msvc version prefix +When &cv-MSVC_TOOLSET_VERSION; is specified using the major msvc version prefix (i.e., 'XX.Y') and the major msvc version is that of the latest release of Visual Studio, the selected toolset version may not be the same as the default Visual C++ toolset version. @@ -1116,14 +1143,18 @@ Important usage details: -MSVC_TOOLSET_VERSION must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_TOOLSET_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_TOOLSET_VERSION; must be set before the first msvc tool is +loaded into the environment. The existence of the toolset host architecture and target architecture folders are not verified -when MSVC_TOOLSET_VERSION is specified which could result in build failures. +when &cv-MSVC_TOOLSET_VERSION; is specified which could result in build failures. The burden is on the user to ensure the requisite toolset target architecture build tools are installed. @@ -1141,12 +1172,12 @@ Build with the spectre-mitigated Visual C++ libraries. -The valid values for MSVC_SPECTRE_LIBS are: True, +The valid values for &cv-MSVC_SPECTRE_LIBS; are: True, False, or None. -When MSVC_SPECTRE_LIBS is enabled (i.e., True), +When &cv-MSVC_SPECTRE_LIBS; is enabled (i.e., True), the Visual C++ environment will include the paths to the spectre-mitigated implementations of the Microsoft Visual C++ libraries. @@ -1156,12 +1187,12 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SPECTRE_LIBS is enabled for Visual Studio 2015 and earlier. +&cv-MSVC_SPECTRE_LIBS; is enabled for Visual Studio 2015 and earlier. -MSVC_SPECTRE_LIBS is enabled and a spectre library argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via MSVC_SPECTRE_LIBS +&cv-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via &cv-MSVC_SPECTRE_LIBS; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. @@ -1180,8 +1211,12 @@ Important usage details: -MSVC_SPECTRE_LIBS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SPECTRE_LIBS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SPECTRE_LIBS; must be set before the first msvc tool is +loaded into the environment. @@ -1192,7 +1227,7 @@ details. -The existence of the spectre mitigations libraries is not verified when MSVC_SPECTRE_LIBS +The existence of the spectre mitigations libraries is not verified when &cv-MSVC_SPECTRE_LIBS; is enabled which could result in build failures. The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index f1b3fff..a0c2dc4 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -4690,12 +4690,12 @@ Specify the &scons; behavior when the Microsoft Visual C/C++ compiler is not det - The MSVC_NOTFOUND_POLICY specifies the &scons; behavior when no msvc versions are detected or + The &cv-MSVC_NOTFOUND_POLICY; specifies the &scons; behavior when no msvc versions are detected or when the requested msvc version is not detected. -The valid values for MSVC_NOTFOUND_POLICY and the corresponding &scons; behavior are: +The valid values for &cv-MSVC_NOTFOUND_POLICY; and the corresponding &scons; behavior are: @@ -4736,7 +4736,7 @@ Note: in addition to the camel case values shown above, lower case and upper cas -The MSVC_NOTFOUND_POLICY is applied when any of the following conditions are satisfied: +The &cv-MSVC_NOTFOUND_POLICY; is applied when any of the following conditions are satisfied: &cv-MSVC_VERSION; is specified, the default tools list is implicitly defined (i.e., the tools list is not specified), @@ -4753,7 +4753,7 @@ A non-default tools list is specified that contains one or more of the msvc tool -The MSVC_NOTFOUND_POLICY is ignored when any of the following conditions are satisfied: +The &cv-MSVC_NOTFOUND_POLICY; is ignored when any of the following conditions are satisfied: &cv-MSVC_VERSION; is not specified and the default tools list is implicitly defined (i.e., the tools list is not specified). @@ -4772,15 +4772,19 @@ Important usage details: -MSVC_NOTFOUND_POLICY must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_NOTFOUND_POLICY; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_NOTFOUND_POLICY; must be set before the first msvc tool is +loaded into the environment. -When MSVC_NOTFOUND_POLICY is not specified, the default &scons; behavior is to issue a warning and continue +When &cv-MSVC_NOTFOUND_POLICY; is not specified, the default &scons; behavior is to issue a warning and continue subject to the conditions listed above. The default &scons; behavior may change in the future. @@ -4795,25 +4799,25 @@ Pass user-defined arguments to the Visual C++ batch file determined via autodete -MSVC_SCRIPT_ARGS is available for msvc batch file arguments that do not have first-class support +&cv-MSVC_SCRIPT_ARGS; is available for msvc batch file arguments that do not have first-class support via construction variables or when there is an issue with the appropriate construction variable validation. When available, it is recommended to use the appropriate construction variables (e.g., &cv-link-MSVC_TOOLSET_VERSION;) -rather than MSVC_SCRIPT_ARGS arguments. +rather than &cv-MSVC_SCRIPT_ARGS; arguments. -The valid values for MSVC_SCRIPT_ARGS are: None, a string, +The valid values for &cv-MSVC_SCRIPT_ARGS; are: None, a string, or a list of strings. -The MSVC_SCRIPT_ARGS value is converted to a scalar string (i.e., "flattened"). +The &cv-MSVC_SCRIPT_ARGS; value is converted to a scalar string (i.e., "flattened"). The resulting scalar string, if not empty, is passed as an argument to the msvc batch file determined via autodetection subject to the validation conditions listed below. -MSVC_SCRIPT_ARGS is ignored when the value is None and when the +&cv-MSVC_SCRIPT_ARGS; is ignored when the value is None and when the result from argument conversion is an empty string. The validation conditions below do not apply. @@ -4822,54 +4826,54 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SCRIPT_ARGS is specified for Visual Studio 2013 and earlier. +&cv-MSVC_SCRIPT_ARGS; is specified for Visual Studio 2013 and earlier. Multiple SDK version arguments (e.g., '10.0.20348.0') are specified -in MSVC_SCRIPT_ARGS. +in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_SDK_VERSION; is specified and an SDK version argument -(e.g., '10.0.20348.0') is specified in MSVC_SCRIPT_ARGS. -Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and MSVC_SCRIPT_ARGS +(e.g., '10.0.20348.0') is specified in &cv-MSVC_SCRIPT_ARGS;. +Multiple SDK version declarations via &cv-link-MSVC_SDK_VERSION; and &cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple toolset version arguments (e.g., '-vcvars_ver=14.29') -are specified in MSVC_SCRIPT_ARGS. +are specified in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_TOOLSET_VERSION; is specified and a toolset version argument -(e.g., '-vcvars_ver=14.29') is specified in MSVC_SCRIPT_ARGS. +(e.g., '-vcvars_ver=14.29') is specified in &cv-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via &cv-link-MSVC_TOOLSET_VERSION; and -MSVC_SCRIPT_ARGS are not allowed. +&cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple spectre library arguments (e.g., '-vcvars_spectre_libs=spectre') -are specified in MSVC_SCRIPT_ARGS. +are specified in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument (e.g., '-vcvars_spectre_libs=spectre') is specified in -MSVC_SCRIPT_ARGS. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; -and MSVC_SCRIPT_ARGS are not allowed. +&cv-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via &cv-link-MSVC_SPECTRE_LIBS; +and &cv-MSVC_SCRIPT_ARGS; are not allowed. Multiple UWP arguments (e.g., uwp or store) are specified -in MSVC_SCRIPT_ARGS. +in &cv-MSVC_SCRIPT_ARGS;. &cv-link-MSVC_UWP_APP; is enabled and a UWP argument (e.g., uwp or -store) is specified in MSVC_SCRIPT_ARGS. Multiple UWP declarations -via &cv-link-MSVC_UWP_APP; and MSVC_SCRIPT_ARGS are not allowed. +store) is specified in &cv-MSVC_SCRIPT_ARGS;. Multiple UWP declarations +via &cv-link-MSVC_UWP_APP; and &cv-MSVC_SCRIPT_ARGS; are not allowed. @@ -4896,18 +4900,22 @@ Important usage details: -MSVC_SCRIPT_ARGS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SCRIPT_ARGS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SCRIPT_ARGS; must be set before the first msvc tool is +loaded into the environment. -Other than checking for multiple declarations as described above, MSVC_SCRIPT_ARGS arguments +Other than checking for multiple declarations as described above, &cv-MSVC_SCRIPT_ARGS; arguments are not validated. -Erroneous, inconsistent, and/or version incompatible MSVC_SCRIPT_ARGS arguments are likely +Erroneous, inconsistent, and/or version incompatible &cv-MSVC_SCRIPT_ARGS; arguments are likely to result in build failures for reasons that are not readily apparent and may be difficult to diagnose. The burden is on the user to ensure that the arguments provided to the msvc batch file are valid, consistent @@ -4928,12 +4936,12 @@ Build with a specific version of the Microsoft Software Development Kit (SDK). -The valid values for MSVC_SDK_VERSION are: None -or a string containing the requested SDK version (e.g., '10.0.20348.0'). +The valid values for &cv-MSVC_SDK_VERSION; are: None +or a string containing the requested SDK version (e.g., '10.0.20348.0'). -MSVC_SDK_VERSION is ignored when the value is None and when +&cv-MSVC_SDK_VERSION; is ignored when the value is None and when the value is an empty string. The validation conditions below do not apply. @@ -4942,17 +4950,17 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SDK_VERSION is specified for Visual Studio 2013 and earlier. +&cv-MSVC_SDK_VERSION; is specified for Visual Studio 2013 and earlier. -MSVC_SDK_VERSION is specified and an SDK version argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via MSVC_SDK_VERSION +&cv-MSVC_SDK_VERSION; is specified and an SDK version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple SDK version declarations via &cv-MSVC_SDK_VERSION; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. -The MSVC_SDK_VERSION specified does not match any of the supported formats: +The &cv-MSVC_SDK_VERSION; specified does not match any of the supported formats: '10.0.XXXXX.Y' [SDK 10.0] @@ -4964,12 +4972,12 @@ The MSVC_SDK_VERSION specified does not match any of the supporte -The system folder for the corresponding MSVC_SDK_VERSION version is not found. +The system folder for the corresponding &cv-MSVC_SDK_VERSION; version is not found. The requested SDK version does not appear to be installed. -The MSVC_SDK_VERSION version does not appear to support the requested platform +The &cv-MSVC_SDK_VERSION; version does not appear to support the requested platform type (i.e., UWP or Desktop). The requested SDK version platform type components do not appear to be installed. @@ -4996,8 +5004,12 @@ Important usage details: -MSVC_SDK_VERSION must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SDK_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SDK_VERSION; must be set before the first msvc tool is +loaded into the environment. @@ -5016,7 +5028,7 @@ the requested SDK version is installed with the necessary platform type componen There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' when using the v141 build tools and the target architecture is ARM64. -Should build failures arise with this combination of settings, MSVC_SDK_VERSION may be +Should build failures arise with this combination of settings, &cv-MSVC_SDK_VERSION; may be employed to specify a different SDK version for the build. @@ -5034,12 +5046,12 @@ Build with the spectre-mitigated Visual C++ libraries. -The valid values for MSVC_SPECTRE_LIBS are: True, +The valid values for &cv-MSVC_SPECTRE_LIBS; are: True, False, or None. -When MSVC_SPECTRE_LIBS is enabled (i.e., True), +When &cv-MSVC_SPECTRE_LIBS; is enabled (i.e., True), the Visual C++ environment will include the paths to the spectre-mitigated implementations of the Microsoft Visual C++ libraries. @@ -5049,12 +5061,12 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_SPECTRE_LIBS is enabled for Visual Studio 2015 and earlier. +&cv-MSVC_SPECTRE_LIBS; is enabled for Visual Studio 2015 and earlier. -MSVC_SPECTRE_LIBS is enabled and a spectre library argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via MSVC_SPECTRE_LIBS +&cv-MSVC_SPECTRE_LIBS; is enabled and a spectre library argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple spectre library declarations via &cv-MSVC_SPECTRE_LIBS; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. @@ -5073,8 +5085,12 @@ Important usage details: -MSVC_SPECTRE_LIBS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_SPECTRE_LIBS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_SPECTRE_LIBS; must be set before the first msvc tool is +loaded into the environment. @@ -5085,7 +5101,7 @@ details. -The existence of the spectre mitigations libraries is not verified when MSVC_SPECTRE_LIBS +The existence of the spectre mitigations libraries is not verified when &cv-MSVC_SPECTRE_LIBS; is enabled which could result in build failures. The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. @@ -5105,19 +5121,19 @@ Build with a specific Visual C++ toolset version. -Specifying MSVC_TOOLSET_VERSION does not affect the autodetection and selection -of msvc instances. The MSVC_TOOLSET_VERSION is applied after +Specifying &cv-MSVC_TOOLSET_VERSION; does not affect the autodetection and selection +of msvc instances. The &cv-MSVC_TOOLSET_VERSION; is applied after an msvc instance is selected. This could be the default version of msvc if &cv-link-MSVC_VERSION; is not specified. -The valid values for MSVC_TOOLSET_VERSION are: None -or a string containing the requested toolset version (e.g., '14.29'). +The valid values for &cv-MSVC_TOOLSET_VERSION; are: None +or a string containing the requested toolset version (e.g., '14.29'). -MSVC_TOOLSET_VERSION is ignored when the value is None and when +&cv-MSVC_TOOLSET_VERSION; is ignored when the value is None and when the value is an empty string. The validation conditions below do not apply. @@ -5126,18 +5142,18 @@ An exception is raised when any of the following conditions are satisfied: -MSVC_TOOLSET_VERSION is specified for Visual Studio 2015 and earlier. +&cv-MSVC_TOOLSET_VERSION; is specified for Visual Studio 2015 and earlier. -MSVC_TOOLSET_VERSION is specified and a toolset version argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via MSVC_TOOLSET_VERSION +&cv-MSVC_TOOLSET_VERSION; is specified and a toolset version argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple toolset version declarations via &cv-MSVC_TOOLSET_VERSION; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. -The MSVC_TOOLSET_VERSION specified does not match any of the supported formats: +The &cv-MSVC_TOOLSET_VERSION; specified does not match any of the supported formats: @@ -5174,17 +5190,17 @@ The MSVC_TOOLSET_VERSION specified does not match any of the supp -The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified -is for Visual Studio 2013 and earlier (e.g., '12.0'). +The major msvc version prefix (i.e., 'XX.Y') of the &cv-MSVC_TOOLSET_VERSION; specified +is for Visual Studio 2013 and earlier (e.g., '12.0'). -The major msvc version prefix (i.e., 'XX.Y') of the MSVC_TOOLSET_VERSION specified -is greater than the msvc version selected (e.g., '99.0'). +The major msvc version prefix (i.e., 'XX.Y') of the &cv-MSVC_TOOLSET_VERSION; specified +is greater than the msvc version selected (e.g., '99.0'). -A system folder for the corresponding MSVC_TOOLSET_VERSION version is not found. +A system folder for the corresponding &cv-MSVC_TOOLSET_VERSION; version is not found. The requested toolset version does not appear to be installed. @@ -5203,13 +5219,13 @@ mapped to toolset version 14.16.27023. -When MSVC_TOOLSET_VERSION is not an SxS version number or a full toolset version number: -the first toolset version, ranked in descending order, that matches the MSVC_TOOLSET_VERSION +When &cv-MSVC_TOOLSET_VERSION; is not an SxS version number or a full toolset version number: +the first toolset version, ranked in descending order, that matches the &cv-MSVC_TOOLSET_VERSION; prefix is selected. -When MSVC_TOOLSET_VERSION is specified using the major msvc version prefix +When &cv-MSVC_TOOLSET_VERSION; is specified using the major msvc version prefix (i.e., 'XX.Y') and the major msvc version is that of the latest release of Visual Studio, the selected toolset version may not be the same as the default Visual C++ toolset version. @@ -5235,14 +5251,18 @@ Important usage details: -MSVC_TOOLSET_VERSION must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_TOOLSET_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_TOOLSET_VERSION; must be set before the first msvc tool is +loaded into the environment. The existence of the toolset host architecture and target architecture folders are not verified -when MSVC_TOOLSET_VERSION is specified which could result in build failures. +when &cv-MSVC_TOOLSET_VERSION; is specified which could result in build failures. The burden is on the user to ensure the requisite toolset target architecture build tools are installed. @@ -5283,7 +5303,7 @@ is, if you are sure everything is set correctly already and you don't want &SCons; to change anything. -&cv-MSVC_USE_SCRIPT; overrides &cv-link-MSVC_VERSION; and &cv-link-TARGET_ARCH;. +&cv-MSVC_USE_SCRIPT; ignores &cv-link-MSVC_VERSION; and &cv-link-TARGET_ARCH;. @@ -5360,8 +5380,12 @@ Important usage details: -MSVC_USE_SETTINGS must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_USE_SETTINGS; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_USE_SETTINGS; must be set before the first msvc tool is +loaded into the environment. @@ -5387,13 +5411,13 @@ Build with the Universal Windows Platform (UWP) application Visual C++ libraries -The valid values for MSVC_UWP_APP are True, +The valid values for &cv-MSVC_UWP_APP; are: True, '1', False, '0', or None. -When MSVC_UWP_APP is enabled (i.e., True or +When &cv-MSVC_UWP_APP; is enabled (i.e., True or '1'), the Visual C++ environment will be set up to point to the Windows Store compatible libraries and Visual C++ runtimes. In doing so, any libraries that are built will be able to be used in a UWP App and published @@ -5407,11 +5431,11 @@ constructor; setting it later has no effect. --> An exception is raised when any of the following conditions are satisfied: -MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. +&cv-MSVC_UWP_APP; is enabled for Visual Studio 2013 and earlier. -MSVC_UWP_APP is enabled and a UWP argument is specified in -&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via MSVC_UWP_APP +&cv-MSVC_UWP_APP; is enabled and a UWP argument is specified in +&cv-link-MSVC_SCRIPT_ARGS;. Multiple UWP declarations via &cv-MSVC_UWP_APP; and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. @@ -5429,13 +5453,17 @@ Important usage details: -MSVC_UWP_APP must be passed as an argument to the Environment() constructor; -setting it later has no effect. +&cv-MSVC_UWP_APP; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_UWP_APP; must be set before the first msvc tool is +loaded into the environment. -The existence of the UWP libraries is not verified when MSVC_UWP_APP is enabled +The existence of the UWP libraries is not verified when &cv-MSVC_UWP_APP; is enabled which could result in build failures. The burden is on the user to ensure the requisite UWP libraries are installed. @@ -5458,8 +5486,15 @@ Sets the preferred version of Microsoft Visual C/C++ to use. If &cv-MSVC_VERSION; is not set, SCons will (by default) select the latest version of Visual C/C++ installed on your system. If the specified version isn't installed, tool initialization will fail. -This variable must be passed as an argument to the &f-link-Environment; -constructor; setting it later has no effect. + + + +&cv-MSVC_VERSION; must be passed as an argument to the &f-link-Environment; +constructor when an msvc tool (e.g., &t-link-msvc;, &t-link-msvs;, etc.) is +loaded via the default tools list or via a tools list passed to the +&f-link-Environment; constructor. +Otherwise, &cv-MSVC_VERSION; must be set before the first msvc tool is +loaded into the environment. -- cgit v0.12 From 56d8f264280d47bafa19ce982641191bfca5cad2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 25 Jun 2022 10:56:42 -0400 Subject: updated blurb in CHANGES.txt and RELEASE.txt to indicate more specifically what's fixed. Address a few lint issues in the lex.py file --- CHANGES.txt | 5 ++--- RELEASE.txt | 5 +++-- SCons/Tool/lex.py | 13 ++++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c986c1e..87766c3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -135,9 +135,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. to connect to the server during start up. - - Updated lex emitter to respect escaped spaces when climbing out of a the SConscript dir. - Previously if the build/source directories absolute path included a space the lex emitter - would fail to use the correct paths. + - lex: Fixed an issue with the lex tool where file arguments specified to either "--header-file=" + or "--tables-file=" which included a space in the path to the file would be processed incorrectly - Ninja: added option "--skip-ninja-regen" to enable skipping regeneration of the ninja file if scons can determine the ninja file doesnot need to be regenerated, which will also skip restarting the scons daemon. Note this option is could result in incorrect rebuilds diff --git a/RELEASE.txt b/RELEASE.txt index da580b8..fce184f 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -111,9 +111,9 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY if scons can determine the ninja file doesnot need to be regenerated, which will also skip restarting the scons daemon. Note this option is could result in incorrect rebuilds if scons Glob or scons generated files are used in ninja build target's command lines. + FIXES ----- - - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) - Ninja: Fix issue where Configure files weren't being properly processed when build run @@ -156,7 +156,8 @@ FIXES - The system environment variable names imported for MSVC 7.0 and 6.0 were updated to be consistent with the variables names defined by their respective installers. This fixes an error caused when bypassing MSVC detection by specifying the MSVC 7.0 batch file directly. -- Updated lex emitter to respect escaped spaces when climbing out of a the SConscript dir +- lex: Fixed an issue with the lex tool where file arguments specified to either "--header-file=" + or "--tables-file=" which included a space in the path to the file would be processed incorrectly - Suppress issuing a warning when there are no installed Visual Studio instances for the default tools configuration (issue #2813). When msvc is the default compiler because there are no compilers installed, a build may fail due to the cl.exe command not being recognized. At diff --git a/SCons/Tool/lex.py b/SCons/Tool/lex.py index c33d0fa..96f9bcb 100644 --- a/SCons/Tool/lex.py +++ b/SCons/Tool/lex.py @@ -46,6 +46,7 @@ if sys.platform == 'win32': else: BINS = ["flex", "lex"] + def lexEmitter(target, source, env): sourceBase, sourceExt = os.path.splitext(SCons.Util.to_String(source[0])) @@ -56,18 +57,19 @@ def lexEmitter(target, source, env): # files generated by flex. # Different options that are used to trigger the creation of extra files. - fileGenOptions = ["--header-file=", "--tables-file="] + file_gen_options = ["--header-file=", "--tables-file="] lexflags = env.subst_list("$LEXFLAGS", target=target, source=source) for option in SCons.Util.CLVar(lexflags): - for fileGenOption in fileGenOptions: + for fileGenOption in file_gen_options: l = len(fileGenOption) if option[:l] == fileGenOption: # A file generating option is present, so add the # file name to the target list. - fileName = option[l:].strip() - target.append(fileName) - return (target, source) + file_name = option[l:].strip() + target.append(file_name) + return target, source + def get_lex_path(env, append_paths=False): """ @@ -128,6 +130,7 @@ def generate(env): env["LEX"] = env.Detect(BINS) env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" + def exists(env): if sys.platform == 'win32': return get_lex_path(env) -- cgit v0.12 From 9b7ba04890c6a88b49ff282659199183dda996f9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 25 Jun 2022 14:06:38 -0400 Subject: Minor edits. Link to Action Objects instead of just referencing it, and removed extraneous '(see)' --- SCons/Environment.xml | 2 +- doc/man/scons.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 23fba62..485fe39 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -1646,7 +1646,7 @@ and then executed. Any additional arguments to &f-Execute; are passed on to the &f-link-Action; factory function which actually creates the Action object -(see the manpage section "Action Objects" +(see the manpage section Action Objects for a description). Example: diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 8545cb7..ef96d28 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -6076,7 +6076,7 @@ keyword arguments may not both be supplied in a single call to &f-Action; Printing of action strings is affected by the setting of -&cv-link-PRINT_CMD_LINE_FUNC; (see). +&cv-link-PRINT_CMD_LINE_FUNC;. Examples: -- cgit v0.12 From 773f267b2ec42b4e629d01bab51f69d49d1605c8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 25 Jun 2022 15:31:38 -0400 Subject: minor edits plus adding a warning about using chdir --- doc/man/scons.xml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 2e7784c..19f5eb1 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -2635,7 +2635,7 @@ often (though not always) by tools which determine whether the external dependencies for the builder are satisfied, and which perform the necessary setup (see Tools). -Builers are attached to a &consenv; as methods. +Builders are attached to a &consenv; as methods. The available builder methods are registered as key-value pairs in the &cv-link-BUILDERS; attribute of the &consenv;, @@ -2694,8 +2694,8 @@ env.Program('bar', source='bar.c foo.c'.split()) -Sources and targets can be specified as a a scalar or as a list, -either of strings or nodes (more on nodes below). +Sources and targets can be specified as a scalar or as a list, +composed of either strings or nodes (more on nodes below). When specifying path strings, &Python; follows the POSIX pathname convention: if a string begins with the operating system pathname separator @@ -2806,7 +2806,7 @@ env.Program('build/prog', ['f1.c', 'f2.c'], srcdir='src') The optional parse_flags -keyword argument causes behavior similarl to the +keyword argument causes behavior similar to the &f-link-env-MergeFlags; method, where the argument value is broken into individual settings and merged into the appropriate &consvars;. @@ -2841,6 +2841,23 @@ and evaluates true, then &scons; will change to the target file's directory. + + +Python only keeps one current directory +location even if there are multiple threads. +This means that use of the +chdir +argument +will +not +work with the SCons + +option, +because individual worker threads spawned +by SCons interfere with each other +when they start changing directory. + + # scons will change to the "sub" subdirectory # before executing the "cp" command. @@ -2950,7 +2967,7 @@ which causes it to produce a linker map file in addition to the executable file actually being linked. If the &b-link-Program; builder's emitter is configured to add this mapfile if the option is set, -then two targets will be returned when you only asked for one. +then two targets will be returned when you only provided for one. -- cgit v0.12 From 607d719e44d58de52f261b95a6c7c8a4dfa5b225 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sat, 25 Jun 2022 19:18:59 -0400 Subject: Minor documentation update based on stress tests [ci skip] --- SCons/Tool/msvc.xml | 10 ++++++---- doc/generated/variables.gen | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index de82db0..ab02252 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -994,10 +994,12 @@ the requested SDK version is installed with the necessary platform type componen -There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' -when using the v141 build tools and the target architecture is ARM64. -Should build failures arise with this combination of settings, &cv-MSVC_SDK_VERSION; may be -employed to specify a different SDK version for the build. +There is a known issue with the Microsoft libraries when the target architecture is +ARM64 and a Windows 11 SDK (version '10.0.22000.0' and later) is used +with the v141 build tools and older v142 toolsets +(versions '14.28.29333' and earlier). Should build failures arise with these combinations +of settings due to unresolved symbols in the Microsoft libraries, &cv-MSVC_SDK_VERSION; may be employed to +specify a Windows 10 SDK (e.g., '10.0.20348.0') for the build. diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index a0c2dc4..c34b70c 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -5026,10 +5026,12 @@ the requested SDK version is installed with the necessary platform type componen -There is a known issue with the Microsoft libraries for SDK version '10.0.22000.0' -when using the v141 build tools and the target architecture is ARM64. -Should build failures arise with this combination of settings, &cv-MSVC_SDK_VERSION; may be -employed to specify a different SDK version for the build. +There is a known issue with the Microsoft libraries when the target architecture is +ARM64 and a Windows 11 SDK (version '10.0.22000.0' and later) is used +with the v141 build tools and older v142 toolsets +(versions '14.28.29333' and earlier). Should build failures arise with these combinations +of settings due to unresolved symbols in the Microsoft libraries, &cv-MSVC_SDK_VERSION; may be employed to +specify a Windows 10 SDK (e.g., '10.0.20348.0') for the build. -- cgit v0.12 From d98f0faf2c06cda35b3da02a245ad78a32f245a7 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 26 Jun 2022 09:12:39 -0400 Subject: Update CHANGES.txt and RELEASE.txt --- CHANGES.txt | 23 ++++++++++++++++++++++- RELEASE.txt | 24 +++++++++++++++++++++--- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bea5838..c6c6732 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -31,8 +31,29 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER determination when configuring the build environment. This could lead to build failures when only an MSVC Express instance is installed and the MSVC version is not explicitly specified (issue #2668 and issue #2697). - - Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler + - Added MSVC_USE_SETTINGS construction variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. + - Added MSVC_SDK_VERSION construction variable which allows building with a specific Microsoft + SDK version. This variable is used with the msvc batch file determined via autodetection subject + to validation constraints. Refer to the documentation for additional requirements and validation + details. + - Added MSVC_TOOLSET_VERSION construction variable which allows building with a specific toolset + version. This variable is used with the msvc batch file determined via autodetection subject to + validation constraints. This variable does not affect the autodetection and selection of msvc + instances. The toolset version is applied after an msvc instance is selected. This could be the + default version of msvc. Refer to the documentation for additional requirements and validation + details. Addresses issue #3265, issue #3664, and pull request #4149. + - Added MSVC_SPECTRE_LIBS construction variable which allows building with spectre-mitigated + Visual C++ libraries. This variable is used with the msvc batch file determined via autodetection + subject to validation constraints. Refer to the documentation for additional requirements and + validation details. + - Added MSVC_SCRIPT_ARGS construction variable which specifies command line arguments that are + passed to the msvc batch file determined via autodetection subject to validation constraints. + Refer to the documentation for additional requirements and validation details. Addresses + enhancement issue #4106. + - An exception is raised when MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. + Previous behavior was to silently ignore MSVC_UWP_APP when enabled for Visual Studio 2013 + and earlier. Refer to the documentation for additional requirements and validation details. - The imported system environment variable names for MSVC 7.0 and 6.0 have been changed to the names set by their respective installers. Prior to this change, bypassing MSVC detection by specifying the MSVC 7.0 batch file directly would fail due to using an erroneous environment diff --git a/RELEASE.txt b/RELEASE.txt index c86c1d5..b244fda 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -23,8 +23,23 @@ NEW FUNCTIONALITY variables). This allows the user to customize how (for example) PATH is constructed. Note that these are called for every build command run by SCons. It could have considerable performance impact if not used carefully. -- Added MSVC_USE_SETTINGS variable to pass a dictionary to configure the msvc compiler +- Added MSVC_USE_SETTINGS construction variable to pass a dictionary to configure the msvc compiler system environment as an alternative to bypassing Visual Studio autodetection entirely. +- Added MSVC_SDK_VERSION construction variable which allows building with a specific Microsoft + SDK version. This variable is used with the msvc batch file determined via autodetection. Refer + to the documentation for additional requirements and validation details. +- Added MSVC_TOOLSET_VERSION construction variable which allows building with a specific toolset + version. This variable is used with the msvc batch file determined via autodetection. This + variable does not affect the autodetection and selection of msvc instances. The toolset version + is applied after an msvc instance is selected. This could be the default version of msvc. Refer + to the documentation for additional requirements and validation details. Addresses issue #3265, + issue #3664, and pull request #4149. +- Added MSVC_SPECTRE_LIBS construction variable which allows building with spectre-mitigated + Visual C++ libraries. This variable is used with the msvc batch file determined via autodetection. + Refer to the documentation for additional requirements and validation details. +- Added MSVC_SCRIPT_ARGS construction variable which specifies command line arguments that are + passed to the msvc batch file determined via autodetection. Refer to the documentation for + additional requirements and validation details. Addresses enhancement issue #4106. - Ninja: Added new alias "shutdown-ninja-scons-daemon" to allow ninja to shutdown the daemon. Also added cleanup to test framework to kill ninja scons daemons and clean ip daemon logs. NOTE: Test for this requires python psutil module. It will be skipped if not present. @@ -96,8 +111,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY automatically executes ninja. - Add JavaScanner to include JAVACLASSPATH as a dependency when using the Java tool. - The build argument (i.e., x86) is no longer passed to the MSVC 6.0 to 7.1 batch - files. This may improve the effectiveness of the internal msvc cache when using - MSVC detection and when bypassing MSVC detection as the MSVC 6.0 to 7.1 batch files + files. This may improve the effectiveness of the internal msvc cache when using + MSVC detection and when bypassing MSVC detection as the MSVC 6.0 to 7.1 batch files do not expect any arguments. - Propagate the OS and windir environment variables from the system environment to the msvc environment. The OS and windir environment variables are used in the MSVC 6.0 batch file @@ -109,6 +124,9 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY require delayed expansion to be enabled which is currently not supported and is typically not enabled by default on the host system. The batch files may also require environment variables that are not included by default in the msvc environment. +- An exception is raised when MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. + Previous behavior was to silently ignore MSVC_UWP_APP when enabled for Visual Studio 2013 + and earlier. Refer to the documentation for additional requirements and validation details. - Ninja: added option "--skip-ninja-regen" to enable skipping regeneration of the ninja file if scons can determine the ninja file doesnot need to be regenerated, which will also skip restarting the scons daemon. Note this option is could result in incorrect rebuilds -- cgit v0.12 From 7a38a4b6547c3776b20f4b1435773a7458a8ebd8 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 26 Jun 2022 10:54:38 -0400 Subject: Set global lxml etree XSLT maximum traversal depth. Update generated documentation artifacts. --- SCons/Tool/docbook/__init__.py | 12 +++++ doc/generated/functions.gen | 52 +++++++++++---------- doc/generated/tools.gen | 17 ++++--- doc/generated/variables.gen | 103 +++++++++++++++++++++++++++-------------- doc/generated/variables.mod | 2 + 5 files changed, 117 insertions(+), 69 deletions(-) diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 4c3f60c..5cf5e61 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -66,6 +66,18 @@ re_manvolnum = re.compile(r"([^<]*)") re_refname = re.compile(r"([^<]*)") # +# lxml etree XSLT global max traversal depth +# + +lmxl_xslt_global_max_depth = 3100 + +if has_lxml and lmxl_xslt_global_max_depth: + def __lxml_xslt_set_global_max_depth(max_depth): + from lxml import etree + etree.XSLT.set_global_max_depth(max_depth) + __lxml_xslt_set_global_max_depth(lmxl_xslt_global_max_depth) + +# # Helper functions # def __extend_targets_sources(target, source): diff --git a/doc/generated/functions.gen b/doc/generated/functions.gen index a2d9acd..efc4c9e 100644 --- a/doc/generated/functions.gen +++ b/doc/generated/functions.gen @@ -1611,45 +1611,49 @@ See the manpage section "Construction Environments" for more details. - Execute(action, [strfunction, varlist]) - env.Execute(action, [strfunction, varlist]) + Execute(action, [actionargs ...]) + env.Execute(action, [actionargs ...]) -Executes an Action object. -The specified +Executes an Action. action may be an Action object -(see manpage section "Action Objects" -for an explanation of behavior), or it may be a command-line string, list of commands, or executable &Python; function, -each of which will be converted +each of which will first be converted into an Action object and then executed. Any additional arguments to &f-Execute; -(strfunction, varlist) are passed on to the &f-link-Action; factory function -which actually creates the Action object. -The exit value of the command -or return value of the &Python; function -will be returned. +which actually creates the Action object +(see the manpage section Action Objects +for a description). Example: + + + +Execute(Copy('file.out', 'file.in')) + + +&f-Execute; performs its action immediately, +as part of the SConscript-reading phase. +There are no sources or targets declared in an +&f-Execute; call, so any objects it manipulates +will not be tracked as part of the &SCons; dependency graph. +In the example above, neither +file.out nor +file.in will be tracked objects. -Note that +&f-Execute; returns the exit value of the command +or return value of the &Python; function. &scons; -will print an error message if the executed +prints an error message if the executed action -fails--that is, -exits with or returns a non-zero value. -&scons; -will +fails (exits with or returns a non-zero value), +however it does not, -however, -automatically terminate the build -if the specified -action -fails. +automatically terminate the build for such a failure. If you want the build to stop in response to a failed &f-Execute; call, @@ -1657,8 +1661,6 @@ you must explicitly check for a non-zero return value: -Execute(Copy('file.out', 'file.in')) - if Execute("mkdir sub/dir/ectory"): # The mkdir failed, don't try to build. Exit(1) diff --git a/doc/generated/tools.gen b/doc/generated/tools.gen index ecc301a..35ded17 100644 --- a/doc/generated/tools.gen +++ b/doc/generated/tools.gen @@ -389,35 +389,35 @@ Sets construction variables for the dvips utility. Set construction variables for generic POSIX Fortran 03 compilers. -Sets: &cv-link-F03;, &cv-link-F03COM;, &cv-link-F03FLAGS;, &cv-link-F03PPCOM;, &cv-link-SHF03;, &cv-link-SHF03COM;, &cv-link-SHF03FLAGS;, &cv-link-SHF03PPCOM;, &cv-link-_F03INCFLAGS;.Uses: &cv-link-F03COMSTR;, &cv-link-F03PPCOMSTR;, &cv-link-SHF03COMSTR;, &cv-link-SHF03PPCOMSTR;. +Sets: &cv-link-F03;, &cv-link-F03COM;, &cv-link-F03FLAGS;, &cv-link-F03PPCOM;, &cv-link-SHF03;, &cv-link-SHF03COM;, &cv-link-SHF03FLAGS;, &cv-link-SHF03PPCOM;, &cv-link-_F03INCFLAGS;.Uses: &cv-link-F03COMSTR;, &cv-link-F03PPCOMSTR;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-SHF03COMSTR;, &cv-link-SHF03PPCOMSTR;. f08 Set construction variables for generic POSIX Fortran 08 compilers. -Sets: &cv-link-F08;, &cv-link-F08COM;, &cv-link-F08FLAGS;, &cv-link-F08PPCOM;, &cv-link-SHF08;, &cv-link-SHF08COM;, &cv-link-SHF08FLAGS;, &cv-link-SHF08PPCOM;, &cv-link-_F08INCFLAGS;.Uses: &cv-link-F08COMSTR;, &cv-link-F08PPCOMSTR;, &cv-link-SHF08COMSTR;, &cv-link-SHF08PPCOMSTR;. +Sets: &cv-link-F08;, &cv-link-F08COM;, &cv-link-F08FLAGS;, &cv-link-F08PPCOM;, &cv-link-SHF08;, &cv-link-SHF08COM;, &cv-link-SHF08FLAGS;, &cv-link-SHF08PPCOM;, &cv-link-_F08INCFLAGS;.Uses: &cv-link-F08COMSTR;, &cv-link-F08PPCOMSTR;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-SHF08COMSTR;, &cv-link-SHF08PPCOMSTR;. f77 Set construction variables for generic POSIX Fortran 77 compilers. -Sets: &cv-link-F77;, &cv-link-F77COM;, &cv-link-F77FILESUFFIXES;, &cv-link-F77FLAGS;, &cv-link-F77PPCOM;, &cv-link-F77PPFILESUFFIXES;, &cv-link-FORTRAN;, &cv-link-FORTRANCOM;, &cv-link-FORTRANFLAGS;, &cv-link-SHF77;, &cv-link-SHF77COM;, &cv-link-SHF77FLAGS;, &cv-link-SHF77PPCOM;, &cv-link-SHFORTRAN;, &cv-link-SHFORTRANCOM;, &cv-link-SHFORTRANFLAGS;, &cv-link-SHFORTRANPPCOM;, &cv-link-_F77INCFLAGS;.Uses: &cv-link-F77COMSTR;, &cv-link-F77PPCOMSTR;, &cv-link-FORTRANCOMSTR;, &cv-link-FORTRANPPCOMSTR;, &cv-link-SHF77COMSTR;, &cv-link-SHF77PPCOMSTR;, &cv-link-SHFORTRANCOMSTR;, &cv-link-SHFORTRANPPCOMSTR;. +Sets: &cv-link-F77;, &cv-link-F77COM;, &cv-link-F77FILESUFFIXES;, &cv-link-F77FLAGS;, &cv-link-F77PPCOM;, &cv-link-F77PPFILESUFFIXES;, &cv-link-FORTRAN;, &cv-link-FORTRANCOM;, &cv-link-FORTRANFLAGS;, &cv-link-SHF77;, &cv-link-SHF77COM;, &cv-link-SHF77FLAGS;, &cv-link-SHF77PPCOM;, &cv-link-SHFORTRAN;, &cv-link-SHFORTRANCOM;, &cv-link-SHFORTRANFLAGS;, &cv-link-SHFORTRANPPCOM;, &cv-link-_F77INCFLAGS;.Uses: &cv-link-F77COMSTR;, &cv-link-F77PPCOMSTR;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-FORTRANCOMSTR;, &cv-link-FORTRANFLAGS;, &cv-link-FORTRANPPCOMSTR;, &cv-link-SHF77COMSTR;, &cv-link-SHF77PPCOMSTR;, &cv-link-SHFORTRANCOMSTR;, &cv-link-SHFORTRANFLAGS;, &cv-link-SHFORTRANPPCOMSTR;. f90 Set construction variables for generic POSIX Fortran 90 compilers. -Sets: &cv-link-F90;, &cv-link-F90COM;, &cv-link-F90FLAGS;, &cv-link-F90PPCOM;, &cv-link-SHF90;, &cv-link-SHF90COM;, &cv-link-SHF90FLAGS;, &cv-link-SHF90PPCOM;, &cv-link-_F90INCFLAGS;.Uses: &cv-link-F90COMSTR;, &cv-link-F90PPCOMSTR;, &cv-link-SHF90COMSTR;, &cv-link-SHF90PPCOMSTR;. +Sets: &cv-link-F90;, &cv-link-F90COM;, &cv-link-F90FLAGS;, &cv-link-F90PPCOM;, &cv-link-SHF90;, &cv-link-SHF90COM;, &cv-link-SHF90FLAGS;, &cv-link-SHF90PPCOM;, &cv-link-_F90INCFLAGS;.Uses: &cv-link-F90COMSTR;, &cv-link-F90PPCOMSTR;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-SHF90COMSTR;, &cv-link-SHF90PPCOMSTR;. f95 Set construction variables for generic POSIX Fortran 95 compilers. -Sets: &cv-link-F95;, &cv-link-F95COM;, &cv-link-F95FLAGS;, &cv-link-F95PPCOM;, &cv-link-SHF95;, &cv-link-SHF95COM;, &cv-link-SHF95FLAGS;, &cv-link-SHF95PPCOM;, &cv-link-_F95INCFLAGS;.Uses: &cv-link-F95COMSTR;, &cv-link-F95PPCOMSTR;, &cv-link-SHF95COMSTR;, &cv-link-SHF95PPCOMSTR;. +Sets: &cv-link-F95;, &cv-link-F95COM;, &cv-link-F95FLAGS;, &cv-link-F95PPCOM;, &cv-link-SHF95;, &cv-link-SHF95COM;, &cv-link-SHF95FLAGS;, &cv-link-SHF95PPCOM;, &cv-link-_F95INCFLAGS;.Uses: &cv-link-F95COMSTR;, &cv-link-F95PPCOMSTR;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-SHF95COMSTR;, &cv-link-SHF95PPCOMSTR;. fortran @@ -437,10 +437,8 @@ Set construction variables for the &gXX; C++ compiler. g77 Set construction variables for the &g77; Fortran compiler. -Calls the &t-f77; Tool module -to set variables. - +Sets: &cv-link-F77;, &cv-link-F77COM;, &cv-link-F77FILESUFFIXES;, &cv-link-F77PPCOM;, &cv-link-F77PPFILESUFFIXES;, &cv-link-FORTRAN;, &cv-link-FORTRANCOM;, &cv-link-FORTRANPPCOM;, &cv-link-SHF77;, &cv-link-SHF77COM;, &cv-link-SHF77FLAGS;, &cv-link-SHF77PPCOM;, &cv-link-SHFORTRAN;, &cv-link-SHFORTRANCOM;, &cv-link-SHFORTRANFLAGS;, &cv-link-SHFORTRANPPCOM;.Uses: &cv-link-F77FLAGS;, &cv-link-FORTRANCOMMONFLAGS;, &cv-link-FORTRANFLAGS;. gas @@ -516,7 +514,8 @@ environment: gfortran -Sets construction variables for the GNU F95/F2003 GNU compiler. +Sets construction variables for the GNU Fortran compiler. +Calls the &t-link-fortran; Tool module to set variables. Sets: &cv-link-F77;, &cv-link-F90;, &cv-link-F95;, &cv-link-FORTRAN;, &cv-link-SHF77;, &cv-link-SHF77FLAGS;, &cv-link-SHF90;, &cv-link-SHF90FLAGS;, &cv-link-SHF95;, &cv-link-SHF95FLAGS;, &cv-link-SHFORTRAN;, &cv-link-SHFORTRANFLAGS;. diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index c34b70c..32db05e 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -2682,6 +2682,17 @@ in the &cv-link-FORTRANFLAGS;, + + + FORTRANCOMMONFLAGS + + +General user-specified options that are passed to the Fortran compiler. +Similar to &cv-link-FORTRANFLAGS;, +but this variable is applied to all dialects. + + + FORTRANCOMSTR @@ -2709,7 +2720,8 @@ default, this is ['.f', '.for', '.ftn'] FORTRANFLAGS -General user-specified options that are passed to the Fortran compiler. +General user-specified options for the FORTRAN dialect +that are passed to the Fortran compiler. Note that this variable does not contain @@ -6578,44 +6590,65 @@ A Python function used to print the command lines as they are executed or options or their equivalents). -The function should take four arguments: +The function must accept four arguments: s, -the command being executed (a string), target, -the target being built (file node, list, or string name(s)), +source and +env. +s +is a string showing the command being executed, +target, +is the target being built (file node, list, or string name(s)), source, -the source(s) used (file node, list, or string name(s)), and -env, -the environment being used. +is the source(s) used (file node, list, or string name(s)), +and env +is the environment being used. -The function must do the printing itself. The default implementation, -used if this variable is not set or is None, is: +The function must do the printing itself. +The default implementation, +used if this variable is not set or is None, +is to just print the string, as in: def print_cmd_line(s, target, source, env): - sys.stdout.write(s + "\n") + sys.stdout.write(s + "\n") -Here's an example of a more interesting function: +Here is an example of a more interesting function: def print_cmd_line(s, target, source, env): - sys.stdout.write("Building %s -> %s...\n" % - (' and '.join([str(x) for x in source]), - ' and '.join([str(x) for x in target]))) -env=Environment(PRINT_CMD_LINE_FUNC=print_cmd_line) -env.Program('foo', 'foo.c') + sys.stdout.write( + "Building %s -> %s...\n" + % ( + ' and '.join([str(x) for x in source]), + ' and '.join([str(x) for x in target]), + ) + ) + +env = Environment(PRINT_CMD_LINE_FUNC=print_cmd_line) +env.Program('foo', ['foo.c', 'bar.c']) -This just prints "Building targetname from sourcename..." instead -of the actual commands. -Such a function could also log the actual commands to a log file, -for example. +This prints: + + + +... +scons: Building targets ... +Building bar.c -> bar.o... +Building foo.c -> foo.o... +Building foo.o and bar.o -> foo... +scons: done building targets. + + + +Another example could be a function that logs the actual commands to a file. @@ -7587,9 +7620,9 @@ targets. def custom_shell_env(env, target, source, shell_env): """customize shell_env if desired""" if str(target[0]) == 'special_target': - shell_env['SPECIAL_VAR'] = env.subst('SOME_VAR', target=target, source=source) + shell_env['SPECIAL_VAR'] = env.subst('SOME_VAR', target=target, source=source) return shell_env - + env["SHELL_ENV_GENERATORS"] = [custom_shell_env] @@ -7663,7 +7696,7 @@ Options that are passed to the Fortran 03 compiler to generated shared-library objects. You only need to set &cv-link-SHF03FLAGS; if you need to define specific user options for Fortran 03 files. -You should normally set the &cv-link-SHFORTRANFLAGS; variable, +You should normally set the &cv-link-FORTRANCOMMONFLAGS; variable, which specifies the user-specified options passed to the default Fortran compiler for all Fortran versions. @@ -7751,7 +7784,7 @@ Options that are passed to the Fortran 08 compiler to generated shared-library objects. You only need to set &cv-link-SHF08FLAGS; if you need to define specific user options for Fortran 08 files. -You should normally set the &cv-link-SHFORTRANFLAGS; variable, +You should normally set the &cv-link-FORTRANCOMMONFLAGS; variable, which specifies the user-specified options passed to the default Fortran compiler for all Fortran versions. @@ -7839,7 +7872,7 @@ Options that are passed to the Fortran 77 compiler to generated shared-library objects. You only need to set &cv-link-SHF77FLAGS; if you need to define specific user options for Fortran 77 files. -You should normally set the &cv-link-SHFORTRANFLAGS; variable, +You should normally set the &cv-link-FORTRANCOMMONFLAGS; variable, which specifies the user-specified options passed to the default Fortran compiler for all Fortran versions. @@ -7927,7 +7960,7 @@ Options that are passed to the Fortran 90 compiler to generated shared-library objects. You only need to set &cv-link-SHF90FLAGS; if you need to define specific user options for Fortran 90 files. -You should normally set the &cv-link-SHFORTRANFLAGS; variable, +You should normally set the &cv-link-FORTRANCOMMONFLAGS; variable, which specifies the user-specified options passed to the default Fortran compiler for all Fortran versions. @@ -8015,7 +8048,7 @@ Options that are passed to the Fortran 95 compiler to generated shared-library objects. You only need to set &cv-link-SHF95FLAGS; if you need to define specific user options for Fortran 95 files. -You should normally set the &cv-link-SHFORTRANFLAGS; variable, +You should normally set the &cv-link-FORTRANCOMMONFLAGS; variable, which specifies the user-specified options passed to the default Fortran compiler for all Fortran versions. @@ -8389,7 +8422,7 @@ which would be a symlink and point to libtest.so.0.1.2 A command interpreter function that will be called to execute command line -strings. The function must expect the following arguments: +strings. The function must accept five arguments: @@ -8397,18 +8430,18 @@ def spawn(shell, escape, cmd, args, env): -sh -is a string naming the shell program to use. +shell +is a string naming the shell program to use, escape is a function that can be called to escape shell special characters in -the command line. +the command line, cmd -is the path to the command to be executed. +is the path to the command to be executed, args -is the arguments to the command. +holds the arguments to the command and env -is a dictionary of the environment variables -in which the command should be executed. +is a dictionary of environment variables +defining the execution environment in which the command should be executed. diff --git a/doc/generated/variables.mod b/doc/generated/variables.mod index d3d29f2..cc51043 100644 --- a/doc/generated/variables.mod +++ b/doc/generated/variables.mod @@ -186,6 +186,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $File"> $FORTRAN"> $FORTRANCOM"> +$FORTRANCOMMONFLAGS"> $FORTRANCOMSTR"> $FORTRANFILESUFFIXES"> $FORTRANFLAGS"> @@ -854,6 +855,7 @@ THIS IS AN AUTOMATICALLY-GENERATED FILE. DO NOT EDIT. $File"> $FORTRAN"> $FORTRANCOM"> +$FORTRANCOMMONFLAGS"> $FORTRANCOMSTR"> $FORTRANFILESUFFIXES"> $FORTRANFLAGS"> -- cgit v0.12 From 20a092ffda8d2492a5745b5aa93c48c14d0d4e76 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Sun, 26 Jun 2022 12:26:12 -0400 Subject: Add blurb for additional MSVC_UWP_APP values accepted. --- CHANGES.txt | 1 + RELEASE.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index c6c6732..381b061 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -54,6 +54,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - An exception is raised when MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. Previous behavior was to silently ignore MSVC_UWP_APP when enabled for Visual Studio 2013 and earlier. Refer to the documentation for additional requirements and validation details. + MSVC_UWP_APP was extended to accept True, False, and None in addition to '1' and '0'. - The imported system environment variable names for MSVC 7.0 and 6.0 have been changed to the names set by their respective installers. Prior to this change, bypassing MSVC detection by specifying the MSVC 7.0 batch file directly would fail due to using an erroneous environment diff --git a/RELEASE.txt b/RELEASE.txt index b244fda..12dec62 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -127,6 +127,7 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - An exception is raised when MSVC_UWP_APP is enabled for Visual Studio 2013 and earlier. Previous behavior was to silently ignore MSVC_UWP_APP when enabled for Visual Studio 2013 and earlier. Refer to the documentation for additional requirements and validation details. + MSVC_UWP_APP was extended to accept True, False, and None in addition to '1' and '0'. - Ninja: added option "--skip-ninja-regen" to enable skipping regeneration of the ninja file if scons can determine the ninja file doesnot need to be regenerated, which will also skip restarting the scons daemon. Note this option is could result in incorrect rebuilds -- cgit v0.12 From 2e2b4640b09a94b0aa2b2f90581295e28ba83650 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 27 Jun 2022 10:40:55 -0400 Subject: Rework SxS toolset version support and vcvars bug fix handling. Update MSVC_TOOLSET_VERSION documentation. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 81 ++++++++++++++++++++++------- SCons/Tool/msvc.xml | 39 +++++++++----- doc/generated/variables.gen | 39 +++++++++----- 3 files changed, 113 insertions(+), 46 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index e56dd4a..6a4ce3e 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -86,9 +86,22 @@ re_toolset_140 = re.compile(r'''^(?: (?:14[.]0{2}[.]0{1,5}) # 14.00.0 - 14.00.00000 )$''', re.VERBOSE) -# valid SxS formats will be matched with re_toolset_full: match 3 '.' format +# SxS toolset version: MM.mm.VV.vv format re_toolset_sxs = re.compile(r'^[1-9][0-9][.][0-9]{2}[.][0-9]{2}[.][0-9]{1,2}$') +# SxS version bugfix +_msvc_sxs_bugfix_folder = set() +_msvc_sxs_bugfix_version = {} + +for msvc_version, sxs_version, sxs_bugfix in [ + # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 + # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files + # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. + ('14.2', '14.28.16.8', '14.28') +]: + _msvc_sxs_bugfix_folder.add((msvc_version, sxs_bugfix)) + _msvc_sxs_bugfix_version[(msvc_version, sxs_version)] = sxs_bugfix + # MSVC_SCRIPT_ARGS re_vcvars_uwp = re.compile(r'(?:(?(?:uwp|store))(?:(?!\S)|$)',re.IGNORECASE) re_vcvars_sdk = re.compile(r'(?:(?(?:[1-9][0-9]*[.]\S*))(?:(?!\S)|$)',re.IGNORECASE) @@ -327,14 +340,42 @@ def _msvc_read_toolset_file(msvc, filename): debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) return toolset_version +def _msvc_sxs_toolset_folder(msvc, sxs_folder): + + if re_toolset_sxs.match(sxs_folder): + return sxs_folder + + key = (msvc.vs_def.vc_buildtools_def.vc_version, sxs_folder) + if key in _msvc_sxs_bugfix_folder: + return sxs_folder + + debug('sxs folder: ignore version=%s', repr(sxs_folder)) + return None + +def _msvc_sxs_toolset_version(msvc, sxs_toolset): + + if not re_toolset_sxs.match(sxs_toolset): + return None, False + + key = (msvc.vs_def.vc_buildtools_def.vc_version, sxs_toolset) + sxs_bugfix = _msvc_sxs_bugfix_version.get(key) + if not sxs_bugfix: + return sxs_toolset, False + + debug('sxs bugfix: version=%s => version=%s', repr(sxs_toolset), repr(sxs_bugfix)) + return sxs_bugfix, True + def _msvc_read_toolset_folders(msvc, vc_dir): toolsets_sxs = {} toolsets_full = [] build_dir = os.path.join(vc_dir, "Auxiliary", "Build") - sxs_toolsets = [f.name for f in os.scandir(build_dir) if f.is_dir()] - for sxs_toolset in sxs_toolsets: + sxs_folders = [f.name for f in os.scandir(build_dir) if f.is_dir()] + for sxs_folder in sxs_folders: + sxs_toolset = _msvc_sxs_toolset_folder(msvc, sxs_folder) + if not sxs_toolset: + continue filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) filepath = os.path.join(build_dir, sxs_toolset, filename) debug('sxs toolset: check file=%s', repr(filepath)) @@ -428,23 +469,27 @@ def _msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version): toolsets_sxs, toolsets_full = _msvc_version_toolsets(msvc, vc_dir) - if msvc.vs_def.vc_buildtools_def.vc_version_numeric == VS2019.vc_buildtools_def.vc_version_numeric: - # necessary to detect toolset not found - if toolset_version == '14.28.16.8': - new_toolset_version = '14.28' - # VS2019\Common7\Tools\vsdevcmd\ext\vcvars.bat AzDO Bug#1293526 - # special handling of the 16.8 SxS toolset, use VC\Auxiliary\Build\14.28 directory and SxS files - # if SxS version 14.28 not present/installed, fallback selection of toolset VC\Tools\MSVC\14.28.nnnnn. - debug( - 'rewrite toolset_version=%s => toolset_version=%s', - repr(toolset_version), repr(new_toolset_version) - ) - toolset_version = new_toolset_version - - if toolset_version in toolsets_sxs: - toolset_vcvars = toolsets_sxs[toolset_version] + if toolset_version in toolsets_full: + # full toolset version provided + toolset_vcvars = toolset_version return toolset_vcvars + sxs_toolset, sxs_isbugfix = _msvc_sxs_toolset_version(msvc, toolset_version) + if sxs_toolset: + # SxS version provided + sxs_version = toolsets_sxs.get(sxs_toolset, None) + if sxs_version: + # SxS full toolset version + if sxs_version in toolsets_full: + toolset_vcvars = sxs_version + return toolset_vcvars + return None + # SxS version file missing + if not sxs_isbugfix: + return None + # SxS version bugfix: check toolset version + toolset_version = sxs_toolset + for toolset_full in toolsets_full: if toolset_full.startswith(toolset_version): toolset_vcvars = toolset_full diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index ab02252..11c6478 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -1106,13 +1106,6 @@ Toolset selection details: -SxS version numbers are not always in three dot format (e.g., 'XX.YY.ZZ.NN') as shown above. - -In Visual Studio 2022 for example, 14.16 is an SxS toolset version that is directly -mapped to toolset version 14.16.27023. - - - When &cv-MSVC_TOOLSET_VERSION; is not an SxS version number or a full toolset version number: the first toolset version, ranked in descending order, that matches the &cv-MSVC_TOOLSET_VERSION; prefix is selected. @@ -1131,13 +1124,31 @@ toolset with the largest version number. -Example environment constructor invocations with toolset version specifications: - -Environment(MSVC_TOOLSET_VERSION='14.2') -Environment(MSVC_TOOLSET_VERSION='14.29') -Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') -Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') - +Example 1 - A default Visual Studio build with a partial toolset version specified: + +env = Environment(MSVC_TOOLSET_VERSION='14.2') + + + + +Example 2 - A default Visual Studio build with a partial toolset version specified: + +env = Environment(MSVC_TOOLSET_VERSION='14.29') + + + + +Example 3 - A Visual Studio 2022 build with a full toolset version specified: + +env = Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') + + + + +Example 4 - A Visual Studio 2022 build with an SxS toolset version specified: + +env = Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') + diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index 32db05e..d5980ff 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -5226,13 +5226,6 @@ Toolset selection details: -SxS version numbers are not always in three dot format (e.g., 'XX.YY.ZZ.NN') as shown above. - -In Visual Studio 2022 for example, 14.16 is an SxS toolset version that is directly -mapped to toolset version 14.16.27023. - - - When &cv-MSVC_TOOLSET_VERSION; is not an SxS version number or a full toolset version number: the first toolset version, ranked in descending order, that matches the &cv-MSVC_TOOLSET_VERSION; prefix is selected. @@ -5251,13 +5244,31 @@ toolset with the largest version number. -Example environment constructor invocations with toolset version specifications: - -Environment(MSVC_TOOLSET_VERSION='14.2') -Environment(MSVC_TOOLSET_VERSION='14.29') -Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') -Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') - +Example 1 - A default Visual Studio build with a partial toolset version specified: + +env = Environment(MSVC_TOOLSET_VERSION='14.2') + + + + +Example 2 - A default Visual Studio build with a partial toolset version specified: + +env = Environment(MSVC_TOOLSET_VERSION='14.29') + + + + +Example 3 - A Visual Studio 2022 build with a full toolset version specified: + +env = Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.30133') + + + + +Example 4 - A Visual Studio 2022 build with an SxS toolset version specified: + +env = Environment(MSVC_VERSION='14.3', MSVC_TOOLSET_VERSION='14.29.16.11') + -- cgit v0.12 From b94e801aba5a6864644a156b5260ce6d721f4fc3 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 27 Jun 2022 14:36:02 -0400 Subject: Move verify invocation to last line of vc.py. Add verify method to Config module to check that all _VCVER versions are defined locally. Update module docstrings for Dispatcher and MSVC init. --- SCons/Tool/MSCommon/MSVC/Config.py | 15 +++++++++++++++ SCons/Tool/MSCommon/MSVC/Dispatcher.py | 25 ++++++++++++++++++++++--- SCons/Tool/MSCommon/MSVC/__init__.py | 14 ++++++++++++-- SCons/Tool/MSCommon/vc.py | 3 +++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py index 8f3a2cc..a4cb874 100644 --- a/SCons/Tool/MSCommon/MSVC/Config.py +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -29,6 +29,12 @@ from collections import ( namedtuple, ) +from . import Util + +from .Exceptions import ( + MSVCInternalError, +) + from . import Dispatcher Dispatcher.register_modulename(__name__) @@ -282,3 +288,12 @@ for policy_value, policy_symbol_list in [ MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol] = policy_def MSVC_NOTFOUND_POLICY_EXTERNAL[policy_symbol.upper()] = policy_def +def verify(): + from .. import vc + for msvc_version in vc._VCVER: + vc_version = Util.get_version_prefix(msvc_version) + if vc_version in MSVC_VERSION_INTERNAL: + continue + err_msg = 'vc_version {} not in MSVC_VERSION_INTERNAL'.format(repr(vc_version)) + raise MSVCInternalError(err_msg) + diff --git a/SCons/Tool/MSCommon/MSVC/Dispatcher.py b/SCons/Tool/MSCommon/MSVC/Dispatcher.py index 0b216ca..dab1e15 100644 --- a/SCons/Tool/MSCommon/MSVC/Dispatcher.py +++ b/SCons/Tool/MSCommon/MSVC/Dispatcher.py @@ -23,6 +23,25 @@ """ Internal method dispatcher for Microsoft Visual C/C++. + +MSVC modules can register their module (register_modulename) and individual +classes (register_class) with the method dispatcher during initialization. MSVC +modules tend to be registered immediately after the Dispatcher import near the +top of the file. Methods in the MSVC modules can be invoked indirectly without +having to hard-code the method calls effectively decoupling the upstream module +with the downstream modules: + +The reset method dispatches calls to all registered objects with a reset method +and/or a _reset method. The reset methods are used to restore data structures +to their initial state for testing purposes. Typically, this involves clearing +cached values. + +The verify method dispatches calls to all registered objects with a verify +method and/or a _verify method. The verify methods are used to check that +initialized data structures distributed across multiple modules are internally +consistent. An exception is raised when a verification constraint violation +is detected. Typically, this verifies that initialized dictionaries support +all of the requisite keys as new versions are added. """ import sys @@ -34,13 +53,13 @@ from ..common import ( _refs = [] -def register_class(ref): - _refs.append(ref) - def register_modulename(modname): module = sys.modules[modname] _refs.append(module) +def register_class(ref): + _refs.append(ref) + def reset(): debug('') for ref in _refs: diff --git a/SCons/Tool/MSCommon/MSVC/__init__.py b/SCons/Tool/MSCommon/MSVC/__init__.py index c07e849..b0ef5dd 100644 --- a/SCons/Tool/MSCommon/MSVC/__init__.py +++ b/SCons/Tool/MSCommon/MSVC/__init__.py @@ -23,6 +23,16 @@ """ Functions for Microsoft Visual C/C++. + +The reset method is used to restore MSVC module data structures to their +initial state for testing purposes. + +The verify method is used as a sanity check that MSVC module data structures +are internally consistent. + +Currently: +* reset is invoked from reset_installed_vcs in the vc module. +* verify is invoked from the last line in the vc module. """ from . import Exceptions # noqa: F401 @@ -40,6 +50,6 @@ from . import Dispatcher as _Dispatcher def reset(): _Dispatcher.reset() -#reset() # testing -_Dispatcher.verify() +def verify(): + _Dispatcher.verify() diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 675b8d0..7c65879 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1352,3 +1352,6 @@ def get_msvc_sdk_versions(msvc_version=None, msvc_uwp_app=False): rval = MSVC.WinSDK.get_msvc_sdk_version_list(msvc_version, msvc_uwp_app) return rval +# internal consistency check (should be last) +MSVC.verify() + -- cgit v0.12 From 90922c1195eef8d75664d59ed8668fd8e5679390 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Mon, 27 Jun 2022 15:09:13 -0400 Subject: Add docstrings to MSVC/Util.py methods. --- SCons/Tool/MSCommon/MSVC/Util.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py index 15abdcd..ba70b24 100644 --- a/SCons/Tool/MSCommon/MSVC/Util.py +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -29,6 +29,16 @@ import os import re def listdir_dirs(p): + """Get a list of qualified subdirectory paths from a windows path. + + Args: + p: str + windows path + + Returns: + List[str]: list of qualified subdirectory paths + + """ dirs = [] for dir_name in os.listdir(p): dir_path = os.path.join(p, dir_name) @@ -37,6 +47,16 @@ def listdir_dirs(p): return dirs def process_path(p): + """Normalize a windows path. + + Args: + p: str + windows path + + Returns: + str: normalized windows path + + """ if p: p = os.path.normpath(p) p = os.path.realpath(p) @@ -46,6 +66,16 @@ def process_path(p): re_version_prefix = re.compile(r'^(?P[0-9.]+).*') def get_version_prefix(version): + """Get the version number prefix from a string. + + Args: + version: str + version string + + Returns: + str: the version number prefix + + """ m = re_version_prefix.match(version) if m: rval = m.group('version') -- cgit v0.12 From d5a2a52087c968058fff3f2cc983f75953235e1a Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 28 Jun 2022 17:16:49 -0400 Subject: Additional validation for MSVC_SDK_VERSION and MSVC_SPECTRE_LIBS. Adjust documentation. Add additional exceptions for SDK version not found, toolset version not found, and spectre libraries not found. Add data structure for platform type. --- SCons/Tool/MSCommon/MSVC/Config.py | 27 +++ SCons/Tool/MSCommon/MSVC/Exceptions.py | 11 +- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 262 ++++++++++++++++++++++------ SCons/Tool/MSCommon/MSVC/Util.py | 20 +++ SCons/Tool/MSCommon/MSVC/WinSDK.py | 39 +++-- SCons/Tool/msvc.xml | 17 +- doc/generated/variables.gen | 17 +- 7 files changed, 314 insertions(+), 79 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py index a4cb874..3b71cd8 100644 --- a/SCons/Tool/MSCommon/MSVC/Config.py +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -55,6 +55,33 @@ for bool, symbol_list, symbol_case_list in [ for symbol in BOOLEAN_SYMBOLS[bool]: BOOLEAN_EXTERNAL[symbol] = bool +MSVC_PLATFORM_DEFINITION = namedtuple('MSVCPlatform', [ + 'vc_platform', + 'is_uwp', +]) + +MSVC_PLATFORM_DEFINITION_LIST = [] + +MSVC_PLATFORM_INTERNAL = {} +MSVC_PLATFORM_EXTERNAL = {} + +for vc_platform, is_uwp in [ + ('Desktop', False), + ('UWP', True), +]: + + vc_platform_def = MSVC_PLATFORM_DEFINITION( + vc_platform = vc_platform, + is_uwp = is_uwp, + ) + + MSVC_PLATFORM_DEFINITION_LIST.append(vc_platform_def) + + MSVC_PLATFORM_INTERNAL[vc_platform] = vc_platform_def + + for symbol in [vc_platform, vc_platform.lower(), vc_platform.upper()]: + MSVC_PLATFORM_EXTERNAL[symbol] = vc_platform_def + MSVC_RUNTIME_DEFINITION = namedtuple('MSVCRuntime', [ 'vc_runtime', 'vc_runtime_numeric', diff --git a/SCons/Tool/MSCommon/MSVC/Exceptions.py b/SCons/Tool/MSCommon/MSVC/Exceptions.py index 7a61ec5..015fede 100644 --- a/SCons/Tool/MSCommon/MSVC/Exceptions.py +++ b/SCons/Tool/MSCommon/MSVC/Exceptions.py @@ -28,10 +28,19 @@ Exceptions for Microsoft Visual C/C++. class VisualCException(Exception): pass +class MSVCInternalError(VisualCException): + pass + class MSVCVersionNotFound(VisualCException): pass -class MSVCInternalError(VisualCException): +class MSVCSDKVersionNotFound(VisualCException): + pass + +class MSVCToolsetVersionNotFound(VisualCException): + pass + +class MSVCSpectreLibsNotFound(VisualCException): pass class MSVCArgumentError(VisualCException): diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 6a4ce3e..33e13fd 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -44,6 +44,9 @@ from . import WinSDK from .Exceptions import ( MSVCInternalError, + MSVCSDKVersionNotFound, + MSVCToolsetVersionNotFound, + MSVCSpectreLibsNotFound, MSVCArgumentError, ) @@ -131,6 +134,18 @@ def msvc_force_default_arguments(force=True): if CONFIG_CACHE_FORCE_DEFAULT_ARGUMENTS: msvc_force_default_arguments(force=True) +# UWP SDK 8.1 and SDK 10: +# +# https://stackoverflow.com/questions/46659238/build-windows-app-compatible-for-8-1-and-10 +# VS2019 - UWP (Except for Win10Mobile) +# VS2017 - UWP +# VS2015 - UWP, Win8.1 StoreApp, WP8/8.1 StoreApp +# VS2013 - Win8/8.1 StoreApp, WP8/8.1 StoreApp + +# SPECTRE LIBS (msvc documentation): +# "There are no versions of Spectre-mitigated libraries for Universal Windows (UWP) apps or +# components. App-local deployment of such libraries isn't possible." + # MSVC batch file arguments: # # VS2022: UWP, SDK, TOOLSET, SPECTRE @@ -159,7 +174,7 @@ VS2017 = Config.MSVS_VERSION_INTERNAL['2017'] VS2015 = Config.MSVS_VERSION_INTERNAL['2015'] MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ - 'version', # fully qualified msvc version (e.g., '14.1Exp') + 'version', # full version (e.g., '14.1Exp', '14.32.31326') 'vs_def', ]) @@ -175,6 +190,18 @@ def _msvc_version(version): return version_args +def _toolset_version(version): + + verstr = Util.get_msvc_version_prefix(version) + vs_def = Config.MSVC_VERSION_INTERNAL[verstr] + + version_args = MSVC_VERSION_ARGS_DEFINITION( + version = version, + vs_def = vs_def, + ) + + return version_args + def _msvc_script_argument_uwp(env, msvc, arglist): uwp_app = env['MSVC_UWP_APP'] @@ -252,12 +279,38 @@ def _msvc_script_argument_sdk_constraints(msvc, sdk_version): err_msg = "MSVC_SDK_VERSION ({}) is not supported".format(repr(sdk_version)) return err_msg -def _msvc_script_argument_sdk(env, msvc, platform_type, arglist): +def _msvc_script_argument_sdk_platform_constraints(msvc, toolset, sdk_version, platform_def): + + if sdk_version == '8.1' and platform_def.is_uwp: + + vs_def = toolset.vs_def if toolset else msvc.vs_def + + if vs_def.vc_buildtools_def.vc_version_numeric > VS2015.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: uwp/store SDK 8.1 msvc_version constraint: %s > %s VS2015', + repr(vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2015.vc_buildtools_def.vc_version_numeric) + ) + if toolset and toolset.vs_def != msvc.vs_def: + err_msg = "MSVC_SDK_VERSION ({}) and platform type ({}) constraint violation: toolset version {} > {} VS2015".format( + repr(sdk_version), repr(platform_def.vc_platform), + repr(toolset.version), repr(VS2015.vc_buildtools_def.vc_version) + ) + else: + err_msg = "MSVC_SDK_VERSION ({}) and platform type ({}) constraint violation: MSVC_VERSION {} > {} VS2015".format( + repr(sdk_version), repr(platform_def.vc_platform), + repr(msvc.version), repr(VS2015.vc_buildtools_def.vc_version) + ) + return err_msg + + return None + +def _msvc_script_argument_sdk(env, msvc, toolset, platform_def, arglist): sdk_version = env['MSVC_SDK_VERSION'] debug( 'MSVC_VERSION=%s, MSVC_SDK_VERSION=%s, platform_type=%s', - repr(msvc.version), repr(sdk_version), repr(platform_type) + repr(msvc.version), repr(sdk_version), repr(platform_def.vc_platform) ) if not sdk_version: @@ -267,12 +320,16 @@ def _msvc_script_argument_sdk(env, msvc, platform_type, arglist): if err_msg: raise MSVCArgumentError(err_msg) - sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) + sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_def) if sdk_version not in sdk_list: err_msg = "MSVC_SDK_VERSION {} not found for platform type {}".format( - repr(sdk_version), repr(platform_type) + repr(sdk_version), repr(platform_def.vc_platform) ) + raise MSVCSDKVersionNotFound(err_msg) + + err_msg = _msvc_script_argument_sdk_platform_constraints(msvc, toolset, sdk_version, platform_def) + if err_msg: raise MSVCArgumentError(err_msg) argpair = (SortOrder.SDK, sdk_version) @@ -280,12 +337,12 @@ def _msvc_script_argument_sdk(env, msvc, platform_type, arglist): return sdk_version -def _msvc_script_default_sdk(env, msvc, platform_type, arglist): +def _msvc_script_default_sdk(env, msvc, platform_def, arglist): if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2015.vc_buildtools_def.vc_version_numeric: return None - sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_type) + sdk_list = WinSDK.get_sdk_version_list(msvc.vs_def.vc_sdk_versions, platform_def) if not len(sdk_list): return None @@ -293,7 +350,7 @@ def _msvc_script_default_sdk(env, msvc, platform_type, arglist): debug( 'MSVC_VERSION=%s, sdk_default=%s, platform_type=%s', - repr(msvc.version), repr(sdk_default), repr(platform_type) + repr(msvc.version), repr(sdk_default), repr(platform_def.vc_platform) ) argpair = (SortOrder.SDK, sdk_default) @@ -597,8 +654,9 @@ def _msvc_script_argument_toolset(env, msvc, vc_dir, arglist): err_msg = "MSVC_TOOLSET_VERSION {} not found for MSVC_VERSION {}".format( repr(toolset_version), repr(msvc.version) ) - raise MSVCArgumentError(err_msg) + raise MSVCToolsetVersionNotFound(err_msg) + # toolset may not be installed for host/target argpair = (SortOrder.TOOLSET, '-vcvars_ver={}'.format(toolset_vcvars)) arglist.append(argpair) @@ -644,16 +702,7 @@ def _user_script_argument_toolset(env, toolset_version, user_argstr): raise MSVCArgumentError(err_msg) -def _msvc_script_argument_spectre(env, msvc, arglist): - - spectre_libs = env['MSVC_SPECTRE_LIBS'] - debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) - - if not spectre_libs: - return None - - if spectre_libs not in _ARGUMENT_BOOLEAN_TRUE: - return None +def _msvc_script_argument_spectre_constraints(msvc, toolset, spectre_libs, platform_def): if msvc.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: debug( @@ -664,11 +713,63 @@ def _msvc_script_argument_spectre(env, msvc, arglist): err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: MSVC_VERSION {} < {} VS2017".format( repr(spectre_libs), repr(msvc.version), repr(VS2017.vc_buildtools_def.vc_version) ) + return err_msg + + if toolset: + if toolset.vs_def.vc_buildtools_def.vc_version_numeric < VS2017.vc_buildtools_def.vc_version_numeric: + debug( + 'invalid: toolset version constraint: %s < %s VS2017', + repr(toolset.vs_def.vc_buildtools_def.vc_version_numeric), + repr(VS2017.vc_buildtools_def.vc_version_numeric) + ) + err_msg = "MSVC_SPECTRE_LIBS ({}) constraint violation: toolset version {} < {} VS2017".format( + repr(spectre_libs), repr(toolset.version), repr(VS2017.vc_buildtools_def.vc_version) + ) + return err_msg + + + if platform_def.is_uwp: + debug( + 'invalid: spectre_libs=%s and platform_type=%s', + repr(spectre_libs), repr(platform_def.vc_platform) + ) + err_msg = "MSVC_SPECTRE_LIBS ({}) are not supported for platform type ({})".format( + repr(spectre_libs), repr(platform_def.vc_platform) + ) + return err_msg + + return None + +def _msvc_script_argument_spectre(env, msvc, vc_dir, toolset, platform_def, arglist): + + spectre_libs = env['MSVC_SPECTRE_LIBS'] + debug('MSVC_VERSION=%s, MSVC_SPECTRE_LIBS=%s', repr(msvc.version), repr(spectre_libs)) + + if not spectre_libs: + return None + + if spectre_libs not in _ARGUMENT_BOOLEAN_TRUE: + return None + + err_msg = _msvc_script_argument_spectre_constraints(msvc, toolset, spectre_libs, platform_def) + if err_msg: raise MSVCArgumentError(err_msg) + if toolset: + spectre_dir = os.path.join(vc_dir, "Tools", "MSVC", toolset.version, "lib", "spectre") + if not os.path.exists(spectre_dir): + debug( + 'spectre libs: msvc_version=%s, toolset_version=%s, spectre_dir=%s', + repr(msvc.version), repr(toolset.version), repr(spectre_dir) + ) + err_msg = "Spectre libraries not found for MSVC_VERSION {} toolset version {}".format( + repr(msvc.version), repr(toolset.version) + ) + raise MSVCSpectreLibsNotFound(err_msg) + spectre_arg = 'spectre' - # spectre libs may not be installed + # spectre libs may not be installed for host/target argpair = (SortOrder.SPECTRE, '-vcvars_spectre_libs={}'.format(spectre_arg)) arglist.append(argpair) @@ -723,66 +824,111 @@ def _msvc_script_argument_user(env, msvc, arglist): return script_args +def _msvc_process_construction_variables(env): + + for cache_variable in [ + _MSVC_FORCE_DEFAULT_TOOLSET, + _MSVC_FORCE_DEFAULT_SDK, + ]: + if cache_variable: + return True + + for env_variable in [ + 'MSVC_UWP_APP', + 'MSVC_TOOLSET_VERSION', + 'MSVC_SDK_VERSION', + 'MSVC_SPECTRE_LIBS', + ]: + if env.get(env_variable, None) != None: + return True + + return False + def msvc_script_arguments(env, version, vc_dir, arg): arglist = [] - msvc = _msvc_version(version) - if arg: argpair = (SortOrder.ARCH, arg) arglist.append(argpair) + msvc = _msvc_version(version) + if 'MSVC_SCRIPT_ARGS' in env: user_argstr = _msvc_script_argument_user(env, msvc, arglist) else: user_argstr = None - if 'MSVC_UWP_APP' in env: - uwp = _msvc_script_argument_uwp(env, msvc, arglist) - else: - uwp = None + if _msvc_process_construction_variables(env): - if user_argstr: - _user_script_argument_uwp(env, uwp, user_argstr) + # MSVC_UWP_APP - platform_type = 'uwp' if uwp else 'desktop' + if 'MSVC_UWP_APP' in env: + uwp = _msvc_script_argument_uwp(env, msvc, arglist) + else: + uwp = None - if 'MSVC_SDK_VERSION' in env: - sdk_version = _msvc_script_argument_sdk(env, msvc, platform_type, arglist) - else: - sdk_version = None + if user_argstr: + _user_script_argument_uwp(env, uwp, user_argstr) - if user_argstr: - user_sdk = _user_script_argument_sdk(env, sdk_version, user_argstr) - else: - user_sdk = None + is_uwp = True if uwp else False + platform_def = WinSDK.get_msvc_platform(is_uwp) - if _MSVC_FORCE_DEFAULT_SDK: - if not sdk_version and not user_sdk: - sdk_version = _msvc_script_default_sdk(env, msvc, platform_type, arglist) + # MSVC_TOOLSET_VERSION - if 'MSVC_TOOLSET_VERSION' in env: - toolset_version = _msvc_script_argument_toolset(env, msvc, vc_dir, arglist) - else: - toolset_version = None + if 'MSVC_TOOLSET_VERSION' in env: + toolset_version = _msvc_script_argument_toolset(env, msvc, vc_dir, arglist) + else: + toolset_version = None - if user_argstr: - user_toolset = _user_script_argument_toolset(env, toolset_version, user_argstr) - else: - user_toolset = None + if user_argstr: + user_toolset = _user_script_argument_toolset(env, toolset_version, user_argstr) + else: + user_toolset = None - if _MSVC_FORCE_DEFAULT_TOOLSET: if not toolset_version and not user_toolset: - toolset_version = _msvc_script_default_toolset(env, msvc, vc_dir, arglist) - - if 'MSVC_SPECTRE_LIBS' in env: - spectre = _msvc_script_argument_spectre(env, msvc, arglist) - else: - spectre = None - - if user_argstr: - _user_script_argument_spectre(env, spectre, user_argstr) + default_toolset = _msvc_script_default_toolset(env, msvc, vc_dir, arglist) + else: + default_toolset = None + + if _MSVC_FORCE_DEFAULT_TOOLSET: + if default_toolset: + toolset_version = default_toolset + + if user_toolset: + toolset = None + elif toolset_version: + toolset = _toolset_version(toolset_version) + elif default_toolset: + toolset = _toolset_version(default_toolset) + else: + toolset = None + + # MSVC_SDK_VERSION + + if 'MSVC_SDK_VERSION' in env: + sdk_version = _msvc_script_argument_sdk(env, msvc, toolset, platform_def, arglist) + else: + sdk_version = None + + if user_argstr: + user_sdk = _user_script_argument_sdk(env, sdk_version, user_argstr) + else: + user_sdk = None + + if _MSVC_FORCE_DEFAULT_SDK: + if not sdk_version and not user_sdk: + sdk_version = _msvc_script_default_sdk(env, msvc, platform_def, arglist) + + # MSVC_SPECTRE_LIBS + + if 'MSVC_SPECTRE_LIBS' in env: + spectre = _msvc_script_argument_spectre(env, msvc, vc_dir, toolset, platform_def, arglist) + else: + spectre = None + + if user_argstr: + _user_script_argument_spectre(env, spectre, user_argstr) if arglist: arglist.sort() diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py index ba70b24..ed87f9d 100644 --- a/SCons/Tool/MSCommon/MSVC/Util.py +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -83,3 +83,23 @@ def get_version_prefix(version): rval = '' return rval +re_msvc_version_prefix = re.compile(r'^(?P[1-9][0-9]?[.][0-9]).*') + +def get_msvc_version_prefix(version): + """Get the msvc version number prefix from a string. + + Args: + version: str + version string + + Returns: + str: the msvc version number prefix + + """ + m = re_msvc_version_prefix.match(version) + if m: + rval = m.group('version') + else: + rval = '' + return rval + diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py index 8338c27..42526c2 100644 --- a/SCons/Tool/MSCommon/MSVC/WinSDK.py +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -43,10 +43,13 @@ from . import Dispatcher Dispatcher.register_modulename(__name__) +_DESKTOP = Config.MSVC_PLATFORM_INTERNAL['Desktop'] +_UWP = Config.MSVC_PLATFORM_INTERNAL['UWP'] + def _new_sdk_map(): sdk_map = { - 'desktop': [], - 'uwp': [], + _DESKTOP.vc_platform: [], + _UWP.vc_platform: [], } return sdk_map @@ -84,20 +87,20 @@ def _sdk_10_layout(version): if not os.path.exists(sdk_inc_path): continue - for platform_type, sdk_inc_file in [ - ('desktop', 'winsdkver.h'), - ('uwp', 'windows.h'), + for vc_platform, sdk_inc_file in [ + (_DESKTOP.vc_platform, 'winsdkver.h'), + (_UWP.vc_platform, 'windows.h'), ]: if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): continue - key = (version_nbr, platform_type) + key = (version_nbr, vc_platform) if key in sdk_version_platform_seen: continue sdk_version_platform_seen.add(key) - sdk_map[platform_type].append(version_nbr) + sdk_map[vc_platform].append(version_nbr) for key, val in sdk_map.items(): val.sort(reverse=True) @@ -128,20 +131,20 @@ def _sdk_81_layout(version): if not os.path.exists(sdk_inc_path): continue - for platform_type, sdk_inc_file in [ - ('desktop', 'winsdkver.h'), - ('uwp', 'windows.h'), + for vc_platform, sdk_inc_file in [ + (_DESKTOP.vc_platform, 'winsdkver.h'), + (_UWP.vc_platform, 'windows.h'), ]: if not os.path.exists(os.path.join(sdk_inc_path, sdk_inc_file)): continue - key = (version_nbr, platform_type) + key = (version_nbr, vc_platform) if key in sdk_version_platform_seen: continue sdk_version_platform_seen.add(key) - sdk_map[platform_type].append(version_nbr) + sdk_map[vc_platform].append(version_nbr) for key, val in sdk_map.items(): val.sort(reverse=True) @@ -218,9 +221,13 @@ def _sdk_map(version_list): _sdk_cache[key] = sdk_map return sdk_map -def get_sdk_version_list(version_list, platform_type): +def get_msvc_platform(is_uwp=False): + platform_def = _UWP if is_uwp else _DESKTOP + return platform_def + +def get_sdk_version_list(version_list, platform_def): sdk_map = _sdk_map(version_list) - sdk_list = sdk_map.get(platform_type, []) + sdk_list = sdk_map.get(platform_def.vc_platform, []) return sdk_list def get_msvc_sdk_version_list(msvc_version, msvc_uwp_app=False): @@ -235,8 +242,8 @@ def get_msvc_sdk_version_list(msvc_version, msvc_uwp_app=False): return sdk_versions is_uwp = True if msvc_uwp_app in Config.BOOLEAN_SYMBOLS[True] else False - platform_type = 'uwp' if is_uwp else 'desktop' - sdk_list = get_sdk_version_list(vs_def.vc_sdk_versions, platform_type) + platform_def = get_msvc_platform(is_uwp) + sdk_list = get_sdk_version_list(vs_def.vc_sdk_versions, platform_def) sdk_versions.extend(sdk_list) debug('sdk_versions=%s', repr(sdk_versions)) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 11c6478..9297100 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -950,6 +950,13 @@ type (i.e., UWP or Desktop). The requeste platform type components do not appear to be installed. + +The &cv-MSVC_SDK_VERSION; version is 8.1, the platform type is +UWP, and the build tools selected are from Visual Studio 2017 +and later (i.e., &cv-link-MSVC_VERSION; must be '14.0' or &cv-link-MSVC_TOOLSET_VERSION; +must be '14.0'). + + @@ -1209,6 +1216,12 @@ An exception is raised when any of the following conditions are satisfied: and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + +&cv-MSVC_SPECTRE_LIBS; is enabled and the platform type is UWP. There +are no spectre-mitigated libraries for Universal Windows Platform (UWP) applications or +components. + + @@ -1240,8 +1253,8 @@ details. -The existence of the spectre mitigations libraries is not verified when &cv-MSVC_SPECTRE_LIBS; -is enabled which could result in build failures. +The existence of the spectre libraries host architecture and target architecture folders are not +verified when &cv-MSVC_SPECTRE_LIBS; is enabled which could result in build failures. The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. diff --git a/doc/generated/variables.gen b/doc/generated/variables.gen index d5980ff..80d5b18 100644 --- a/doc/generated/variables.gen +++ b/doc/generated/variables.gen @@ -4994,6 +4994,13 @@ type (i.e., UWP or Desktop). The requeste platform type components do not appear to be installed. + +The &cv-MSVC_SDK_VERSION; version is 8.1, the platform type is +UWP, and the build tools selected are from Visual Studio 2017 +and later (i.e., &cv-link-MSVC_VERSION; must be '14.0' or &cv-link-MSVC_TOOLSET_VERSION; +must be '14.0'). + + @@ -5084,6 +5091,12 @@ An exception is raised when any of the following conditions are satisfied: and &cv-link-MSVC_SCRIPT_ARGS; are not allowed. + +&cv-MSVC_SPECTRE_LIBS; is enabled and the platform type is UWP. There +are no spectre-mitigated libraries for Universal Windows Platform (UWP) applications or +components. + + @@ -5115,8 +5128,8 @@ details. -The existence of the spectre mitigations libraries is not verified when &cv-MSVC_SPECTRE_LIBS; -is enabled which could result in build failures. +The existence of the spectre libraries host architecture and target architecture folders are not +verified when &cv-MSVC_SPECTRE_LIBS; is enabled which could result in build failures. The burden is on the user to ensure the requisite libraries with spectre mitigations are installed. -- cgit v0.12 From f500d2e6d12d50feec93517c077194adc9a31306 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Tue, 28 Jun 2022 17:26:56 -0400 Subject: Fix sider issue. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 33e13fd..556cae8 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -839,7 +839,7 @@ def _msvc_process_construction_variables(env): 'MSVC_SDK_VERSION', 'MSVC_SPECTRE_LIBS', ]: - if env.get(env_variable, None) != None: + if env.get(env_variable, None) is not None: return True return False -- cgit v0.12 From 8dd2c436317301067f2637829572c36499e24318 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 29 Jun 2022 06:58:30 -0400 Subject: Use msvc version prefix instead of version prefix for internal dictionary lookup. --- SCons/Tool/MSCommon/MSVC/Config.py | 2 +- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 2 +- SCons/Tool/MSCommon/MSVC/WinSDK.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Config.py b/SCons/Tool/MSCommon/MSVC/Config.py index 3b71cd8..fa7dbc1 100644 --- a/SCons/Tool/MSCommon/MSVC/Config.py +++ b/SCons/Tool/MSCommon/MSVC/Config.py @@ -318,7 +318,7 @@ for policy_value, policy_symbol_list in [ def verify(): from .. import vc for msvc_version in vc._VCVER: - vc_version = Util.get_version_prefix(msvc_version) + vc_version = Util.get_msvc_version_prefix(msvc_version) if vc_version in MSVC_VERSION_INTERNAL: continue err_msg = 'vc_version {} not in MSVC_VERSION_INTERNAL'.format(repr(vc_version)) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 556cae8..56a4676 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -180,7 +180,7 @@ MSVC_VERSION_ARGS_DEFINITION = namedtuple('MSVCVersionArgsDefinition', [ def _msvc_version(version): - verstr = Util.get_version_prefix(version) + verstr = Util.get_msvc_version_prefix(version) vs_def = Config.MSVC_VERSION_INTERNAL[verstr] version_args = MSVC_VERSION_ARGS_DEFINITION( diff --git a/SCons/Tool/MSCommon/MSVC/WinSDK.py b/SCons/Tool/MSCommon/MSVC/WinSDK.py index 42526c2..b17f850 100644 --- a/SCons/Tool/MSCommon/MSVC/WinSDK.py +++ b/SCons/Tool/MSCommon/MSVC/WinSDK.py @@ -235,7 +235,7 @@ def get_msvc_sdk_version_list(msvc_version, msvc_uwp_app=False): sdk_versions = [] - verstr = Util.get_version_prefix(msvc_version) + verstr = Util.get_msvc_version_prefix(msvc_version) vs_def = Config.MSVC_VERSION_EXTERNAL.get(verstr, None) if not vs_def: debug('vs_def is not defined') -- cgit v0.12 From a63c1cfdeffbcf7e9214702df4f93e5f015ead10 Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 29 Jun 2022 15:20:44 -0400 Subject: Add 14.0 toolset registry check as done in msvc vsvars140.bat. --- SCons/Tool/MSCommon/MSVC/Registry.py | 3 +++ SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 19 ++++++++++++++++++- SCons/Tool/MSCommon/vc.py | 1 - 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/Registry.py b/SCons/Tool/MSCommon/MSVC/Registry.py index 492f3d0..9ffa01e 100644 --- a/SCons/Tool/MSCommon/MSVC/Registry.py +++ b/SCons/Tool/MSCommon/MSVC/Registry.py @@ -107,3 +107,6 @@ def windows_kit_query_paths(version): q = windows_kits(version) return microsoft_query_paths(q) +def vstudio_sxs_vc7(version): + return '\\'.join([r'VisualStudio\SxS\VC7', version]) + diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 56a4676..4db478a 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -40,6 +40,7 @@ from ..common import ( from . import Util from . import Config +from . import Registry from . import WinSDK from .Exceptions import ( @@ -491,13 +492,16 @@ def _msvc_read_toolset_default(msvc, vc_dir): _toolset_version_cache = {} _toolset_default_cache = {} +_toolset_have140_cache = None def _reset_toolset_cache(): global _toolset_version_cache global _toolset_default_cache + global _toolset_have140_cache debug('reset: toolset cache') _toolset_version_cache = {} _toolset_default_cache = {} + _toolset_have140_cache = None def _msvc_version_toolsets(msvc, vc_dir): @@ -519,10 +523,23 @@ def _msvc_default_toolset(msvc, vc_dir): return toolset_default +def _msvc_have140_toolset(): + global _toolset_have140_cache + + if _toolset_have140_cache is None: + suffix = Registry.vstudio_sxs_vc7('14.0') + vcinstalldirs = [record[0] for record in Registry.microsoft_query_paths(suffix)] + debug('vc140 toolset: paths=%s', repr(vcinstalldirs)) + _toolset_have140_cache = True if vcinstalldirs else False + + return _toolset_have140_cache + def _msvc_version_toolset_vcvars(msvc, vc_dir, toolset_version): if toolset_version == '14.0': - return toolset_version + if _msvc_have140_toolset(): + return toolset_version + return None toolsets_sxs, toolsets_full = _msvc_version_toolsets(msvc, vc_dir) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 7c65879..502a71f 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -1300,7 +1300,6 @@ def msvc_setup_env_user(env=None): # Intent is to use msvc tools: # MSVC_VERSION: defined and evaluates True # MSVS_VERSION: defined and evaluates True - # MSVC_TOOLSET_VERSION: defined and evaluates True # MSVC_USE_SCRIPT: defined and (is string or evaluates False) # MSVC_USE_SETTINGS: defined and is not None -- cgit v0.12 From fa7f870b7768fae625396c0a0dfface959fee65f Mon Sep 17 00:00:00 2001 From: Joseph Brill <48932340+jcbrill@users.noreply.github.com> Date: Wed, 29 Jun 2022 20:44:23 -0400 Subject: Fix Util.py docstring for listdir_dirs. Minor update to reading sxs toolsets and toolset folders. --- SCons/Tool/MSCommon/MSVC/ScriptArguments.py | 64 ++++++++++++++--------------- SCons/Tool/MSCommon/MSVC/Util.py | 26 +++++++----- 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py index 4db478a..14c8c6c 100644 --- a/SCons/Tool/MSCommon/MSVC/ScriptArguments.py +++ b/SCons/Tool/MSCommon/MSVC/ScriptArguments.py @@ -398,16 +398,16 @@ def _msvc_read_toolset_file(msvc, filename): debug('IndexError: msvc_version=%s, filename=%s', repr(msvc.version), repr(filename)) return toolset_version -def _msvc_sxs_toolset_folder(msvc, sxs_folder): +def _msvc_sxs_toolset_folder(msvc, sxs_version): - if re_toolset_sxs.match(sxs_folder): - return sxs_folder + if re_toolset_sxs.match(sxs_version): + return sxs_version - key = (msvc.vs_def.vc_buildtools_def.vc_version, sxs_folder) + key = (msvc.vs_def.vc_buildtools_def.vc_version, sxs_version) if key in _msvc_sxs_bugfix_folder: - return sxs_folder + return sxs_version - debug('sxs folder: ignore version=%s', repr(sxs_folder)) + debug('sxs folder: ignore version=%s', repr(sxs_version)) return None def _msvc_sxs_toolset_version(msvc, sxs_toolset): @@ -429,35 +429,35 @@ def _msvc_read_toolset_folders(msvc, vc_dir): toolsets_full = [] build_dir = os.path.join(vc_dir, "Auxiliary", "Build") - sxs_folders = [f.name for f in os.scandir(build_dir) if f.is_dir()] - for sxs_folder in sxs_folders: - sxs_toolset = _msvc_sxs_toolset_folder(msvc, sxs_folder) - if not sxs_toolset: - continue - filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_toolset) - filepath = os.path.join(build_dir, sxs_toolset, filename) - debug('sxs toolset: check file=%s', repr(filepath)) - if os.path.exists(filepath): - toolset_version = _msvc_read_toolset_file(msvc, filepath) - if not toolset_version: + if os.path.exists(build_dir): + for sxs_version, sxs_path in Util.listdir_dirs(build_dir): + sxs_version = _msvc_sxs_toolset_folder(msvc, sxs_version) + if not sxs_version: continue - toolsets_sxs[sxs_toolset] = toolset_version - debug( - 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', - repr(msvc.version), repr(sxs_toolset), toolset_version - ) + filename = 'Microsoft.VCToolsVersion.{}.txt'.format(sxs_version) + filepath = os.path.join(sxs_path, filename) + debug('sxs toolset: check file=%s', repr(filepath)) + if os.path.exists(filepath): + toolset_version = _msvc_read_toolset_file(msvc, filepath) + if not toolset_version: + continue + toolsets_sxs[sxs_version] = toolset_version + debug( + 'sxs toolset: msvc_version=%s, sxs_version=%s, toolset_version=%s', + repr(msvc.version), repr(sxs_version), repr(toolset_version) + ) toolset_dir = os.path.join(vc_dir, "Tools", "MSVC") - toolsets = [f.name for f in os.scandir(toolset_dir) if f.is_dir()] - for toolset_version in toolsets: - binpath = os.path.join(toolset_dir, toolset_version, "bin") - debug('toolset: check binpath=%s', repr(binpath)) - if os.path.exists(binpath): - toolsets_full.append(toolset_version) - debug( - 'toolset: msvc_version=%s, toolset_version=%s', - repr(msvc.version), repr(toolset_version) - ) + if os.path.exists(toolset_dir): + for toolset_version, toolset_path in Util.listdir_dirs(toolset_dir): + binpath = os.path.join(toolset_path, "bin") + debug('toolset: check binpath=%s', repr(binpath)) + if os.path.exists(binpath): + toolsets_full.append(toolset_version) + debug( + 'toolset: msvc_version=%s, toolset_version=%s', + repr(msvc.version), repr(toolset_version) + ) toolsets_full.sort(reverse=True) debug('msvc_version=%s, toolsets=%s', repr(msvc.version), repr(toolsets_full)) diff --git a/SCons/Tool/MSCommon/MSVC/Util.py b/SCons/Tool/MSCommon/MSVC/Util.py index ed87f9d..f1983ba 100644 --- a/SCons/Tool/MSCommon/MSVC/Util.py +++ b/SCons/Tool/MSCommon/MSVC/Util.py @@ -29,14 +29,17 @@ import os import re def listdir_dirs(p): - """Get a list of qualified subdirectory paths from a windows path. + """ + Return a list of tuples for each subdirectory of the given directory path. + Each tuple is comprised of the subdirectory name and the qualified subdirectory path. + Assumes the given directory path exists and is a directory. Args: p: str - windows path + directory path Returns: - List[str]: list of qualified subdirectory paths + list[tuple[str,str]]: a list of tuples """ dirs = [] @@ -47,14 +50,15 @@ def listdir_dirs(p): return dirs def process_path(p): - """Normalize a windows path. + """ + Normalize a system path Args: p: str - windows path + system path Returns: - str: normalized windows path + str: normalized system path """ if p: @@ -66,11 +70,12 @@ def process_path(p): re_version_prefix = re.compile(r'^(?P[0-9.]+).*') def get_version_prefix(version): - """Get the version number prefix from a string. + """ + Get the version number prefix from a string. Args: version: str - version string + version specification Returns: str: the version number prefix @@ -86,11 +91,12 @@ def get_version_prefix(version): re_msvc_version_prefix = re.compile(r'^(?P[1-9][0-9]?[.][0-9]).*') def get_msvc_version_prefix(version): - """Get the msvc version number prefix from a string. + """ + Get the msvc version number prefix from a string. Args: version: str - version string + version specification Returns: str: the msvc version number prefix -- cgit v0.12 From 3fc497c7775f572a32056a1c6b52ee4592c4bced Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 26 Jun 2022 13:07:57 -0600 Subject: Improvements to lex and yacc tools The mocked tools mylex.py and myyacc.py now understand the file-generation options, and generate a dummy file with predictable contents, for checking. This allows more testing of the path through the SCons support for these two without needing live commands. New tests added which invoke the file-generation options, and make sure the extra files are created, and that SCons detects and tracks the added targets. Work is done in a subdirectory, which exposes some existing known inconsistent behavior (the regular generated file goes in the subdir per the LEXCOM and YACCOM generated line, while the ones generated from commandline options go in the topdir) - but we're going to allow that behavior to continue for backwards compat. Same fix applied to yacc tool that PR #4168 did for lex - do subst_list() instead of subst() to preserve spaces in paths. That fix left the lex tool unable to pass the new test, as it could not see the individual arguments in the FLAGS variable, which was solved by indexing into the subst'd list so we can iterate over the args again. Test and tool cleanup; add DefaultEnvironment calls, etc. Note this mentions, but does not address the problem described in issue 4154. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + SCons/Tool/__init__.py | 8 +-- SCons/Tool/lex.py | 58 ++++++++++++--------- SCons/Tool/lex.xml | 35 ++++++++++++- SCons/Tool/yacc.py | 73 +++++++++++++++----------- SCons/Tool/yacc.xml | 70 ++++++++++++++++++++++--- doc/scons.mod | 1 - test/LEX/FLEXFLAGS.py | 93 ++++++++++++++++++++++++++++++++++ test/LEX/LEX.py | 1 + test/LEX/LEXCOM.py | 20 ++++---- test/LEX/LEXCOMSTR.py | 22 ++++---- test/LEX/LEXFLAGS.py | 10 ++-- test/LEX/lex_headerfile.py | 4 ++ test/LEX/live.py | 28 ++++------ test/LEX/live_mingw.py | 4 +- test/LEX/no_lex.py | 12 ++--- test/YACC/BISONFLAGS.py | 92 +++++++++++++++++++++++++++++++++ test/YACC/YACC-fixture/myyacc.py | 12 +++-- test/YACC/YACC.py | 36 +++++++------ test/YACC/YACCCOM.py | 20 ++++---- test/YACC/YACCCOMSTR.py | 22 ++++---- test/YACC/YACCFLAGS-fixture/myyacc.py | 78 ++++++++++++++++++++++------ test/YACC/YACCFLAGS.py | 31 ++++++------ test/YACC/YACCHFILESUFFIX.py | 33 ++++++------ test/YACC/YACCHXXFILESUFFIX.py | 33 ++++++------ test/YACC/YACCVCGFILESUFFIX.py | 33 ++++++------ test/YACC/live-check-output-cleaned.py | 13 +++-- test/YACC/live.py | 21 ++++---- test/fixture/mylex.py | 31 +++++++++++- testing/framework/TestSCons.py | 33 ++++++++---- 30 files changed, 668 insertions(+), 261 deletions(-) create mode 100644 test/LEX/FLEXFLAGS.py create mode 100644 test/YACC/BISONFLAGS.py diff --git a/CHANGES.txt b/CHANGES.txt index bea5838..6188863 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -202,6 +202,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER but was not applied to other dialects, and e2e tests explicitly checked that FORTRANFLAGS did not propagate outside the FORTRAN dialect, so the conclusion is that behavior is intentional (issue #2257) + - Improvements to lex and yacc tools: better documentation of + extra-file options, add test for extra-file behavior. From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py index 8e4a22d..afc579f 100644 --- a/SCons/Tool/__init__.py +++ b/SCons/Tool/__init__.py @@ -36,6 +36,7 @@ tool specifications. import sys import os import importlib.util +from typing import Optional import SCons.Builder import SCons.Errors @@ -829,7 +830,7 @@ def tool_list(platform, env): return [x for x in tools if x] -def find_program_path(env, key_program, default_paths=None, add_path=False) -> str: +def find_program_path(env, key_program, default_paths=None, add_path=False) -> Optional[str]: """ Find the location of a tool using various means. @@ -849,6 +850,8 @@ def find_program_path(env, key_program, default_paths=None, add_path=False) -> s # Then in the OS path path = SCons.Util.WhereIs(key_program) if path: + if add_path: + env.AppendENVPath('PATH', os.path.dirname(path)) return path # Finally, add the defaults and check again. @@ -864,8 +867,7 @@ def find_program_path(env, key_program, default_paths=None, add_path=False) -> s # leave that to the caller, unless add_path is true. env['ENV']['PATH'] = save_path if path and add_path: - bin_dir = os.path.dirname(path) - env.AppendENVPath('PATH', bin_dir) + env.AppendENVPath('PATH', os.path.dirname(path)) return path diff --git a/SCons/Tool/lex.py b/SCons/Tool/lex.py index 96f9bcb..262fe25 100644 --- a/SCons/Tool/lex.py +++ b/SCons/Tool/lex.py @@ -23,6 +23,9 @@ """Tool-specific initialization for lex. +This tool should support multiple lex implementations, +but is in actuality biased towards GNU Flex. + There normally shouldn't be any need to import this module directly. It will usually be imported through the generic SCons.Tool.Tool() selection method. @@ -30,14 +33,17 @@ selection method. import os.path import sys +from typing import Optional import SCons.Action import SCons.Tool -import SCons.Util import SCons.Warnings from SCons.Platform.mingw import MINGW_DEFAULT_PATHS from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS from SCons.Platform.win32 import CHOCO_DEFAULT_PATH +from SCons.Util import CLVar, to_String + +DEFAULT_PATHS = CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR") @@ -47,20 +53,25 @@ else: BINS = ["flex", "lex"] -def lexEmitter(target, source, env): - sourceBase, sourceExt = os.path.splitext(SCons.Util.to_String(source[0])) +def lexEmitter(target, source, env) -> tuple: + """Adds extra files generated by lex program to target list.""" + sourceBase, sourceExt = os.path.splitext(to_String(source[0])) if sourceExt == ".lm": # If using Objective-C target = [sourceBase + ".m"] # the extension is ".m". - # This emitter essentially tries to add to the target all extra - # files generated by flex. - - # Different options that are used to trigger the creation of extra files. + # With --header-file and ----tables-file, the file to write is defined + # by the option argument. Extract this and include in the list of targets. + # NOTE: a filename passed to the command this way is not modified by SCons, + # and so will be interpreted relative to the project top directory at + # execution time, while the name added to the target list will be + # interpreted relative to the SConscript directory - a possibile mismatch. + # + # These are GNU flex-only options. + # TODO: recognize --outfile also? file_gen_options = ["--header-file=", "--tables-file="] - lexflags = env.subst_list("$LEXFLAGS", target=target, source=source) - for option in SCons.Util.CLVar(lexflags): + for option in lexflags[0]: for fileGenOption in file_gen_options: l = len(fileGenOption) if option[:l] == fileGenOption: @@ -68,28 +79,29 @@ def lexEmitter(target, source, env): # file name to the target list. file_name = option[l:].strip() target.append(file_name) + return target, source -def get_lex_path(env, append_paths=False): +def get_lex_path(env, append_paths=False) -> Optional[str]: """ - Find the path to the lex tool, searching several possible names + Returns the path to the lex tool, searching several possible names. - Only called in the Windows case, so the default_path - can be Windows-specific + Only called in the Windows case, so the `default_path` argument to + :func:`find_program_path` can be Windows-specific. - :param env: current construction environment - :param append_paths: if set, add the path to the tool to PATH - :return: path to lex tool, if found + Args: + env: current construction environment + append_paths: if set, add the path to the tool to PATH """ for prog in BINS: bin_path = SCons.Tool.find_program_path( env, prog, - default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS ) + default_paths=DEFAULT_PATHS, + add_path=append_paths, + ) if bin_path: - if append_paths: - env.AppendENVPath('PATH', os.path.dirname(bin_path)) return bin_path SCons.Warnings.warn( @@ -98,7 +110,7 @@ def get_lex_path(env, append_paths=False): ) -def generate(env): +def generate(env) -> None: """Add Builders and construction variables for lex to an Environment.""" c_file, cxx_file = SCons.Tool.createCFileBuilders(env) @@ -117,21 +129,21 @@ def generate(env): cxx_file.add_action(".ll", LexAction) cxx_file.add_emitter(".ll", lexEmitter) - env["LEXFLAGS"] = SCons.Util.CLVar("") + env["LEXFLAGS"] = CLVar("") if sys.platform == 'win32': # ignore the return - we do not need the full path here _ = get_lex_path(env, append_paths=True) env["LEX"] = env.Detect(BINS) if not env.get("LEXUNISTD"): - env["LEXUNISTD"] = SCons.Util.CLVar("") + env["LEXUNISTD"] = CLVar("") env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET" else: env["LEX"] = env.Detect(BINS) env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET" -def exists(env): +def exists(env) -> Optional[str]: if sys.platform == 'win32': return get_lex_path(env) else: diff --git a/SCons/Tool/lex.xml b/SCons/Tool/lex.xml index 5afb754..8622ced 100644 --- a/SCons/Tool/lex.xml +++ b/SCons/Tool/lex.xml @@ -1,6 +1,28 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/c-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/c-hl.xml index 1503dd1..44ab02d 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/c-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/c-hl.xml @@ -1,101 +1,101 @@ - - - - - /** - */ - - - - /// - - - - /* - */ - - // - - - # - \ - - - - " - \ - - - ' - \ - - - 0x - ul - lu - u - l - - - - . - - e - ul - lu - u - f - l - - - - auto - _Bool - break - case - char - _Complex - const - continue - default - do - double - else - enum - extern - float - for - goto - if - _Imaginary - inline - int - long - register - restrict - return - short - signed - sizeof - static - struct - switch - typedef - union - unsigned - void - volatile - while - + + + + + /** + */ + + + + /// + + + + /* + */ + + // + + + # + \ + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + l + + + + auto + _Bool + break + case + char + _Complex + const + continue + default + do + double + else + enum + extern + float + for + goto + if + _Imaginary + inline + int + long + register + restrict + return + short + signed + sizeof + static + struct + switch + typedef + union + unsigned + void + volatile + while + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/common.xsl b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/common.xsl index e9b5650..1742377 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/common.xsl +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/common.xsl @@ -1,120 +1,120 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - unprocessed xslthl style: - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + unprocessed xslthl style: + + + + + + + + + + + + + + + + + + + + diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/cpp-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/cpp-hl.xml index db57d5e..2213b7c 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/cpp-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/cpp-hl.xml @@ -1,150 +1,150 @@ - - - - - /** - */ - - - - /// - - - - /* - */ - - // - - - # - \ - - - - " - \ - - - ' - \ - - - 0x - ul - lu - u - l - - - - . - - e - ul - lu - u - f - l - - - - - auto - _Bool - break - case - char - _Complex - const - continue - default - do - double - else - enum - extern - float - for - goto - if - _Imaginary - inline - int - long - register - restrict - return - short - signed - sizeof - static - struct - switch - typedef - union - unsigned - void - volatile - while - - asm - dynamic_cast - namespace - reinterpret_cast - try - bool - explicit - new - static_cast - typeid - catch - false - operator - template - typename - class - friend - private - this - using - const_cast - inline - public - throw - virtual - delete - mutable - protected - true - wchar_t - + + + + + /** + */ + + + + /// + + + + /* + */ + + // + + + # + \ + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + l + + + + + auto + _Bool + break + case + char + _Complex + const + continue + default + do + double + else + enum + extern + float + for + goto + if + _Imaginary + inline + int + long + register + restrict + return + short + signed + sizeof + static + struct + switch + typedef + union + unsigned + void + volatile + while + + asm + dynamic_cast + namespace + reinterpret_cast + try + bool + explicit + new + static_cast + typeid + catch + false + operator + template + typename + class + friend + private + this + using + const_cast + inline + public + throw + virtual + delete + mutable + protected + true + wchar_t + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/csharp-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/csharp-hl.xml index 99c2e3e..8a8a76d 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/csharp-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/csharp-hl.xml @@ -1,187 +1,187 @@ - - - - - /** - */ - - - - /// - - - - /* - */ - - // - - - [ - ] - ( - ) - - - - # - \ - - - - - @" - " - \ - - - - " - \ - - - ' - \ - - - 0x - ul - lu - u - l - - - - . - - e - ul - lu - u - f - d - m - l - - - - abstract - as - base - bool - break - byte - case - catch - char - checked - class - const - continue - decimal - default - delegate - do - double - else - enum - event - explicit - extern - false - finally - fixed - float - for - foreach - goto - if - implicit - in - int - interface - internal - is - lock - long - namespace - new - null - object - operator - out - override - params - private - protected - public - readonly - ref - return - sbyte - sealed - short - sizeof - stackalloc - static - string - struct - switch - this - throw - true - try - typeof - uint - ulong - unchecked - unsafe - ushort - using - virtual - void - volatile - while - - - - add - alias - get - global - partial - remove - set - value - where - yield - + + + + + /** + */ + + + + /// + + + + /* + */ + + // + + + [ + ] + ( + ) + + + + # + \ + + + + + @" + " + \ + + + + " + \ + + + ' + \ + + + 0x + ul + lu + u + l + + + + . + + e + ul + lu + u + f + d + m + l + + + + abstract + as + base + bool + break + byte + case + catch + char + checked + class + const + continue + decimal + default + delegate + do + double + else + enum + event + explicit + extern + false + finally + fixed + float + for + foreach + goto + if + implicit + in + int + interface + internal + is + lock + long + namespace + new + null + object + operator + out + override + params + private + protected + public + readonly + ref + return + sbyte + sealed + short + sizeof + stackalloc + static + string + struct + switch + this + throw + true + try + typeof + uint + ulong + unchecked + unsafe + ushort + using + virtual + void + volatile + while + + + + add + alias + get + global + partial + remove + set + value + where + yield + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/delphi-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/delphi-hl.xml index d5b4d1a..2a45d29 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/delphi-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/delphi-hl.xml @@ -1,200 +1,200 @@ - - - - - - {$ - } - - - - - (*$ - ) - - - - { - } - - - (* - *) - - // - - ' - - - - #$ - - - - - # - - - - - $ - - - - . - e - - - - - and - else - inherited - packed - then - array - end - initialization - procedure - threadvar - as - except - inline - program - to - asm - exports - interface - property - try - begin - file - is - raise - type - case - final - label - record - unit - class - finalization - library - repeat - unsafe - const - finally - mod - resourcestring - until - constructor - for - nil - sealed - uses - destructor - function - not - set - var - dispinterface - goto - object - shl - while - div - if - of - shr - with - do - implementation - or - static - xor - downto - in - out - string - - - at - on - - - absolute - dynamic - local - platform - requires - abstract - export - message - private - resident - assembler - external - name - protected - safecall - automated - far - near - public - stdcall - cdecl - forward - nodefault - published - stored - contains - implements - overload - read - varargs - default - index - override - readonly - virtual - deprecated - inline - package - register - write - dispid - library - pascal - reintroduce - writeonly - - + + + + + + {$ + } + + + + + (*$ + ) + + + + { + } + + + (* + *) + + // + + ' + + + + #$ + + + + + # + + + + + $ + + + + . + e + + + + + and + else + inherited + packed + then + array + end + initialization + procedure + threadvar + as + except + inline + program + to + asm + exports + interface + property + try + begin + file + is + raise + type + case + final + label + record + unit + class + finalization + library + repeat + unsafe + const + finally + mod + resourcestring + until + constructor + for + nil + sealed + uses + destructor + function + not + set + var + dispinterface + goto + object + shl + while + div + if + of + shr + with + do + implementation + or + static + xor + downto + in + out + string + + + at + on + + + absolute + dynamic + local + platform + requires + abstract + export + message + private + resident + assembler + external + name + protected + safecall + automated + far + near + public + stdcall + cdecl + forward + nodefault + published + stored + contains + implements + overload + read + varargs + default + index + override + readonly + virtual + deprecated + inline + package + register + write + dispid + library + pascal + reintroduce + writeonly + + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ini-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ini-hl.xml index 8a938f3..84be9c6 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ini-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ini-hl.xml @@ -1,45 +1,45 @@ - - - - ; - - - ^(\[.+\]\s*)$ - - MULTILINE - - - - ^(.+)(?==) - - MULTILINE - + + + + ; + + + ^(\[.+\]\s*)$ + + MULTILINE + + + + ^(.+)(?==) + + MULTILINE + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/java-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/java-hl.xml index 672d518..d499d83 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/java-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/java-hl.xml @@ -1,117 +1,117 @@ - - - - - /** - */ - - - - /* - */ - - // - - " - \ - - - ' - \ - - - @ - ( - ) - - - 0x - - - - . - e - f - d - l - - - - abstract - boolean - break - byte - case - catch - char - class - const - continue - default - do - double - else - extends - final - finally - float - for - goto - if - implements - import - instanceof - int - interface - long - native - new - package - private - protected - public - return - short - static - strictfp - super - switch - synchronized - this - throw - throws - transient - try - void - volatile - while - + + + + + /** + */ + + + + /* + */ + + // + + " + \ + + + ' + \ + + + @ + ( + ) + + + 0x + + + + . + e + f + d + l + + + + abstract + boolean + break + byte + case + catch + char + class + const + continue + default + do + double + else + extends + final + finally + float + for + goto + if + implements + import + instanceof + int + interface + long + native + new + package + private + protected + public + return + short + static + strictfp + super + switch + synchronized + this + throw + throws + transient + try + void + volatile + while + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/javascript-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/javascript-hl.xml index 08c90ba..5749b88 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/javascript-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/javascript-hl.xml @@ -1,147 +1,147 @@ - - - - - /* - */ - - // - - " - \ - - - ' - \ - - - 0x - - - - . - e - - - - break - case - catch - continue - default - delete - do - else - finally - for - function - if - in - instanceof - new - return - switch - this - throw - try - typeof - var - void - while - with - - abstract - boolean - byte - char - class - const - debugger - double - enum - export - extends - final - float - goto - implements - import - int - interface - long - native - package - private - protected - public - short - static - super - synchronized - throws - transient - volatile - - - prototype - - Array - Boolean - Date - Error - EvalError - Function - Math - Number - Object - RangeError - ReferenceError - RegExp - String - SyntaxError - TypeError - URIError - - decodeURI - decodeURIComponent - encodeURI - encodeURIComponent - eval - isFinite - isNaN - parseFloat - parseInt - - Infinity - NaN - undefined - + + + + + /* + */ + + // + + " + \ + + + ' + \ + + + 0x + + + + . + e + + + + break + case + catch + continue + default + delete + do + else + finally + for + function + if + in + instanceof + new + return + switch + this + throw + try + typeof + var + void + while + with + + abstract + boolean + byte + char + class + const + debugger + double + enum + export + extends + final + float + goto + implements + import + int + interface + long + native + package + private + protected + public + short + static + super + synchronized + throws + transient + volatile + + + prototype + + Array + Boolean + Date + Error + EvalError + Function + Math + Number + Object + RangeError + ReferenceError + RegExp + String + SyntaxError + TypeError + URIError + + decodeURI + decodeURIComponent + encodeURI + encodeURIComponent + eval + isFinite + isNaN + parseFloat + parseInt + + Infinity + NaN + undefined + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/m2-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/m2-hl.xml index b145f74..a3ef314 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/m2-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/m2-hl.xml @@ -1,90 +1,90 @@ - - - - - (* - *) - - - " - - - ' - - - . - e - - - - and - array - begin - by - case - const - definition - div - do - else - elsif - end - exit - export - for - from - if - implementation - import - in - loop - mod - module - not - of - or - pointer - procedure - qualified - record - repeat - return - set - then - to - type - until - var - while - with - - + + + + + (* + *) + + + " + + + ' + + + . + e + + + + and + array + begin + by + case + const + definition + div + do + else + elsif + end + exit + export + for + from + if + implementation + import + in + loop + mod + module + not + of + or + pointer + procedure + qualified + record + repeat + return + set + then + to + type + until + var + while + with + + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/myxml-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/myxml-hl.xml index afa4be7..efa90d7 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/myxml-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/myxml-hl.xml @@ -1,116 +1,116 @@ - - - - - - - - A - ABBR - ACRONYM - ADDRESS - APPLET - AREA - B - BASE - BASEFONT - BDO - BIG - BLOCKQUOTE - BODY - BR - BUTTON - CAPTION - CENTER - CITE - CODE - COL - COLGROUP - DD - DEL - DFN - DIR - DIV - DL - DT - EM - FIELDSET - FONT - FORM - FRAME - FRAMESET - H1 - H2 - H3 - H4 - H5 - H6 - HEAD - HR - HTML - I - IFRAME - IMG - INPUT - INS - ISINDEX - KBD - LABEL - LEGEND - LI - LINK - MAP - MENU - META - NOFRAMES - NOSCRIPT - OBJECT - OL - OPTGROUP - OPTION - P - PARAM - PRE - Q - S - SAMP - SCRIPT - SELECT - SMALL - SPAN - STRIKE - STRONG - STYLE - SUB - SUP - TABLE - TBODY - TD - TEXTAREA - TFOOT - TH - THEAD - TITLE - TR - TT - U - UL - VAR - XMP - - - - - xsl: - - - + + + + + + + + A + ABBR + ACRONYM + ADDRESS + APPLET + AREA + B + BASE + BASEFONT + BDO + BIG + BLOCKQUOTE + BODY + BR + BUTTON + CAPTION + CENTER + CITE + CODE + COL + COLGROUP + DD + DEL + DFN + DIR + DIV + DL + DT + EM + FIELDSET + FONT + FORM + FRAME + FRAMESET + H1 + H2 + H3 + H4 + H5 + H6 + HEAD + HR + HTML + I + IFRAME + IMG + INPUT + INS + ISINDEX + KBD + LABEL + LEGEND + LI + LINK + MAP + MENU + META + NOFRAMES + NOSCRIPT + OBJECT + OL + OPTGROUP + OPTION + P + PARAM + PRE + Q + S + SAMP + SCRIPT + SELECT + SMALL + SPAN + STRIKE + STRONG + STYLE + SUB + SUP + TABLE + TBODY + TD + TEXTAREA + TFOOT + TH + THEAD + TITLE + TR + TT + U + UL + VAR + XMP + + + + + xsl: + + + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/perl-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/perl-hl.xml index da1924a..23fdfd8 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/perl-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/perl-hl.xml @@ -1,120 +1,120 @@ - - - - # - - << - ' - " - - - - " - \ - - - ' - \ - - - - 0x - - - - . - - - - - if - unless - while - until - foreach - else - elsif - for - when - default - given - - caller - continue - die - do - dump - eval - exit - goto - last - next - redo - return - sub - wantarray - - caller - import - local - my - package - use - - do - import - no - package - require - use - - bless - dbmclose - dbmopen - package - ref - tie - tied - untie - use - - and - or - not - eq - ne - lt - gt - le - ge - cmp - + + + + # + + << + ' + " + + + + " + \ + + + ' + \ + + + + 0x + + + + . + + + + + if + unless + while + until + foreach + else + elsif + for + when + default + given + + caller + continue + die + do + dump + eval + exit + goto + last + next + redo + return + sub + wantarray + + caller + import + local + my + package + use + + do + import + no + package + require + use + + bless + dbmclose + dbmopen + package + ref + tie + tied + untie + use + + and + or + not + eq + ne + lt + gt + le + ge + cmp + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/php-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/php-hl.xml index 4a70225..0daa348 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/php-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/php-hl.xml @@ -1,149 +1,149 @@ - - - - - /** - */ - - - - /// - - - - /* - */ - - // - # - - " - \ - - - - ' - \ - - - - <<< - - - 0x - - - - . - e - - - - and - or - xor - __FILE__ - exception - __LINE__ - array - as - break - case - class - const - continue - declare - default - die - do - echo - else - elseif - empty - enddeclare - endfor - endforeach - endif - endswitch - endwhile - eval - exit - extends - for - foreach - function - global - if - include - include_once - isset - list - new - print - require - require_once - return - static - switch - unset - use - var - while - __FUNCTION__ - __CLASS__ - __METHOD__ - final - php_user_filter - interface - implements - extends - public - private - protected - abstract - clone - try - catch - throw - cfunction - old_function - true - false - - - - - ?> - <?php - <?= - - + + + + + /** + */ + + + + /// + + + + /* + */ + + // + # + + " + \ + + + + ' + \ + + + + <<< + + + 0x + + + + . + e + + + + and + or + xor + __FILE__ + exception + __LINE__ + array + as + break + case + class + const + continue + declare + default + die + do + echo + else + elseif + empty + enddeclare + endfor + endforeach + endif + endswitch + endwhile + eval + exit + extends + for + foreach + function + global + if + include + include_once + isset + list + new + print + require + require_once + return + static + switch + unset + use + var + while + __FUNCTION__ + __CLASS__ + __METHOD__ + final + php_user_filter + interface + implements + extends + public + private + protected + abstract + clone + try + catch + throw + cfunction + old_function + true + false + + + + + ?> + <?php + <?= + + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/python-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/python-hl.xml index 791bc7a..1b1f087 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/python-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/python-hl.xml @@ -1,100 +1,100 @@ - - - - - - @ - ( - ) - - # - - """ - - - - ''' - - - - " - \ - - - ' - \ - - - 0x - l - - - - . - - e - l - - - - and - del - from - not - while - as - elif - global - or - with - assert - else - if - pass - yield - break - except - import - print - class - exec - in - raise - continue - finally - is - return - def - for - lambda - try - + + + + + + @ + ( + ) + + # + + """ + + + + ''' + + + + " + \ + + + ' + \ + + + 0x + l + + + + . + + e + l + + + + and + del + from + not + while + as + elif + global + or + with + assert + else + if + pass + yield + break + except + import + print + class + exec + in + raise + continue + finally + is + return + def + for + lambda + try + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ruby-hl.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ruby-hl.xml index 78189b0..2f74352 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ruby-hl.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/ruby-hl.xml @@ -1,109 +1,109 @@ - - - - # - - << - - - - " - \ - - - %Q{ - } - \ - - - %/ - / - \ - - - ' - \ - - - %q{ - } - \ - - - 0x - - - - . - e - - - - alias - and - BEGIN - begin - break - case - class - def - defined - do - else - elsif - END - end - ensure - false - for - if - in - module - next - nil - not - or - redo - rescue - retry - return - self - super - then - true - undef - unless - until - when - while - yield - + + + + # + + << + + + + " + \ + + + %Q{ + } + \ + + + %/ + / + \ + + + ' + \ + + + %q{ + } + \ + + + 0x + + + + . + e + + + + alias + and + BEGIN + begin + break + case + class + def + defined + do + else + elsif + END + end + ensure + false + for + if + in + module + next + nil + not + or + redo + rescue + retry + return + self + super + then + true + undef + unless + until + when + while + yield + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/xslthl-config.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/xslthl-config.xml index b24e469..910289a 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/xslthl-config.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/highlighting/xslthl-config.xml @@ -1,46 +1,46 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/html/highlight.xsl b/SCons/Tool/docbook/docbook-xsl-1.76.1/html/highlight.xsl index f7307a4..d6fc969 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/html/highlight.xsl +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/html/highlight.xsl @@ -1,86 +1,86 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/params/bibliography.style.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/params/bibliography.style.xml index fa44582..363e980 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/params/bibliography.style.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/params/bibliography.style.xml @@ -1,35 +1,35 @@ - - -bibliography.style -list -normal -iso690 - - -bibliography.style -Style used for formatting of biblioentries. - - - - -normal - - - -Description - -Currently only normal and -iso690 styles are supported. - -In order to use ISO690 style to the full extent you might need -to use additional markup described on the -following WiKi page. - - - + + +bibliography.style +list +normal +iso690 + + +bibliography.style +Style used for formatting of biblioentries. + + + + +normal + + + +Description + +Currently only normal and +iso690 styles are supported. + +In order to use ISO690 style to the full extent you might need +to use additional markup described on the +following WiKi page. + + + diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/params/highlight.source.xml b/SCons/Tool/docbook/docbook-xsl-1.76.1/params/highlight.source.xml index 41d7b2f..c4eb1a8 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/params/highlight.source.xml +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/params/highlight.source.xml @@ -1,82 +1,82 @@ - - -highlight.source -boolean - - -highlight.source -Should the content of programlisting -be syntactically highlighted? - - - - - - - - -Description - -When this parameter is non-zero, the stylesheets will try to do syntax highlighting of the -content of programlisting elements. You specify the language for each programlisting -by using the language attribute. The highlight.default.language -parameter can be used to specify the language for programlistings without a language -attribute. Syntax highlighting also works for screen and synopsis elements. - -The actual highlighting work is done by the XSLTHL extension module. This is an external Java library that has to be -downloaded separately (see below). - - -In order to use this extension, you must - -add xslthl-2.x.x.jar to your Java classpath. The latest version is available -from the XSLT syntax highlighting project -at SourceForge. - - -use a customization layer in which you import one of the following stylesheet modules: - - - html/highlight.xsl - - - - xhtml/highlight.xsl - - - - xhtml-1_1/highlight.xsl - - - - fo/highlight.xsl - - - - - -let either the xslthl.config Java system property or the -highlight.xslthl.config parameter point to the configuration file for syntax -highlighting (using URL syntax). DocBook XSL comes with a ready-to-use configuration file, -highlighting/xslthl-config.xml. - - - -The extension works with Saxon 6.5.x and Xalan-J. (Saxon 8.5 or later is also supported, but since it is -an XSLT 2.0 processor it is not guaranteed to work with DocBook XSL in all circumstances.) - -The following is an example of a Saxon 6 command adapted for syntax highlighting, to be used on Windows: - - -java -cp c:/Java/saxon.jar;c:/Java/xslthl-2.0.1.jar --Dxslthl.config=file:///c:/docbook-xsl/highlighting/xslthl-config.xml com.icl.saxon.StyleSheet --o test.html test.xml myhtml.xsl - - - - + + +highlight.source +boolean + + +highlight.source +Should the content of programlisting +be syntactically highlighted? + + + + + + + + +Description + +When this parameter is non-zero, the stylesheets will try to do syntax highlighting of the +content of programlisting elements. You specify the language for each programlisting +by using the language attribute. The highlight.default.language +parameter can be used to specify the language for programlistings without a language +attribute. Syntax highlighting also works for screen and synopsis elements. + +The actual highlighting work is done by the XSLTHL extension module. This is an external Java library that has to be +downloaded separately (see below). + + +In order to use this extension, you must + +add xslthl-2.x.x.jar to your Java classpath. The latest version is available +from the XSLT syntax highlighting project +at SourceForge. + + +use a customization layer in which you import one of the following stylesheet modules: + + + html/highlight.xsl + + + + xhtml/highlight.xsl + + + + xhtml-1_1/highlight.xsl + + + + fo/highlight.xsl + + + + + +let either the xslthl.config Java system property or the +highlight.xslthl.config parameter point to the configuration file for syntax +highlighting (using URL syntax). DocBook XSL comes with a ready-to-use configuration file, +highlighting/xslthl-config.xml. + + + +The extension works with Saxon 6.5.x and Xalan-J. (Saxon 8.5 or later is also supported, but since it is +an XSLT 2.0 processor it is not guaranteed to work with DocBook XSL in all circumstances.) + +The following is an example of a Saxon 6 command adapted for syntax highlighting, to be used on Windows: + + +java -cp c:/Java/saxon.jar;c:/Java/xslthl-2.0.1.jar +-Dxslthl.config=file:///c:/docbook-xsl/highlighting/xslthl-config.xml com.icl.saxon.StyleSheet +-o test.html test.xml myhtml.xsl + + + + diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.async.js b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.async.js index 2597dde..c91a9c6 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.async.js +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.async.js @@ -1,72 +1,72 @@ -/* - * Async Treeview 0.1 - Lazy-loading extension for Treeview - * - * http://bassistance.de/jquery-plugins/jquery-plugin-treeview/ - * - * Copyright (c) 2007 Jörn Zaefferer - * - * Dual licensed under the MIT and GPL licenses: - * http://www.opensource.org/licenses/mit-license.php - * http://www.gnu.org/licenses/gpl.html - * - * Revision: $Id$ - * - */ - -;(function($) { - -function load(settings, root, child, container) { - $.getJSON(settings.url, {root: root}, function(response) { - function createNode(parent) { - var current = $("
  • ").attr("id", this.id || "").html("" + this.text + "").appendTo(parent); - if (this.classes) { - current.children("span").addClass(this.classes); - } - if (this.expanded) { - current.addClass("open"); - } - if (this.hasChildren || this.children && this.children.length) { - var branch = $("
      ").appendTo(current); - if (this.hasChildren) { - current.addClass("hasChildren"); - createNode.call({ - text:"placeholder", - id:"placeholder", - children:[] - }, branch); - } - if (this.children && this.children.length) { - $.each(this.children, createNode, [branch]) - } - } - } - $.each(response, createNode, [child]); - $(container).treeview({add: child}); - }); -} - -var proxied = $.fn.treeview; -$.fn.treeview = function(settings) { - if (!settings.url) { - return proxied.apply(this, arguments); - } - var container = this; - load(settings, "source", this, container); - var userToggle = settings.toggle; - return proxied.call(this, $.extend({}, settings, { - collapsed: true, - toggle: function() { - var $this = $(this); - if ($this.hasClass("hasChildren")) { - var childList = $this.removeClass("hasChildren").find("ul"); - childList.empty(); - load(settings, this.id, childList, container); - } - if (userToggle) { - userToggle.apply(this, arguments); - } - } - })); -}; - +/* + * Async Treeview 0.1 - Lazy-loading extension for Treeview + * + * http://bassistance.de/jquery-plugins/jquery-plugin-treeview/ + * + * Copyright (c) 2007 Jörn Zaefferer + * + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + * + * Revision: $Id$ + * + */ + +;(function($) { + +function load(settings, root, child, container) { + $.getJSON(settings.url, {root: root}, function(response) { + function createNode(parent) { + var current = $("
    • ").attr("id", this.id || "").html("" + this.text + "").appendTo(parent); + if (this.classes) { + current.children("span").addClass(this.classes); + } + if (this.expanded) { + current.addClass("open"); + } + if (this.hasChildren || this.children && this.children.length) { + var branch = $("
        ").appendTo(current); + if (this.hasChildren) { + current.addClass("hasChildren"); + createNode.call({ + text:"placeholder", + id:"placeholder", + children:[] + }, branch); + } + if (this.children && this.children.length) { + $.each(this.children, createNode, [branch]) + } + } + } + $.each(response, createNode, [child]); + $(container).treeview({add: child}); + }); +} + +var proxied = $.fn.treeview; +$.fn.treeview = function(settings) { + if (!settings.url) { + return proxied.apply(this, arguments); + } + var container = this; + load(settings, "source", this, container); + var userToggle = settings.toggle; + return proxied.call(this, $.extend({}, settings, { + collapsed: true, + toggle: function() { + var $this = $(this); + if ($this.hasClass("hasChildren")) { + var childList = $this.removeClass("hasChildren").find("ul"); + childList.empty(); + load(settings, this.id, childList, container); + } + if (userToggle) { + userToggle.apply(this, arguments); + } + } + })); +}; + })(jQuery); \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.css b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.css index dbf425b..d7e2c00 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.css +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/common/jquery/treeview/jquery.treeview.css @@ -1,85 +1,85 @@ -.treeview, .treeview ul { - padding: 0; - margin: 0; - list-style: none; -} - -.treeview ul { - background-color: white; - margin-top: 4px; -} - -.treeview .hitarea { - background: url(images/treeview-default.gif) -64px -25px no-repeat; - height: 16px; - width: 16px; - margin-left: -16px; - float: left; - cursor: pointer; -} -/* fix for IE6 */ -* html .hitarea { - display: inline; - float:none; -} - -.treeview li { - margin: 0; - padding: 3px 0 3px 16px; -} - -.treeview a.selected { - background-color: #eee; -} - -#treecontrol { margin: 1em 0; display: none; } - -.treeview .hover { color: red; cursor: pointer; } - -.treeview li { background: url(images/treeview-default-line.gif) 0 0 no-repeat; } -.treeview li.collapsable, .treeview li.expandable { background-position: 0 -176px; } - -.treeview .expandable-hitarea { background-position: -80px -3px; } - -.treeview li.last { background-position: 0 -1766px } -.treeview li.lastCollapsable, .treeview li.lastExpandable { background-image: url(images/treeview-default.gif); } -.treeview li.lastCollapsable { background-position: 0 -111px } -.treeview li.lastExpandable { background-position: -32px -67px } - -.treeview div.lastCollapsable-hitarea, .treeview div.lastExpandable-hitarea { background-position: 0; } - -.treeview-red li { background-image: url(images/treeview-red-line.gif); } -.treeview-red .hitarea, .treeview-red li.lastCollapsable, .treeview-red li.lastExpandable { background-image: url(images/treeview-red.gif); } - -.treeview-black li { background-image: url(images/treeview-black-line.gif); } -.treeview-black .hitarea, .treeview-black li.lastCollapsable, .treeview-black li.lastExpandable { background-image: url(images/treeview-black.gif); } - -.treeview-gray li { background-image: url(images/treeview-gray-line.gif); } -.treeview-gray .hitarea, .treeview-gray li.lastCollapsable, .treeview-gray li.lastExpandable { background-image: url(images/treeview-gray.gif); } - -.treeview-famfamfam li { background-image: url(images/treeview-famfamfam-line.gif); } -.treeview-famfamfam .hitarea, .treeview-famfamfam li.lastCollapsable, .treeview-famfamfam li.lastExpandable { background-image: url(images/treeview-famfamfam.gif); } - - -.filetree li { padding: 3px 0 2px 16px; } -.filetree span.folder, .filetree span.file { padding: 1px 0 1px 16px; display: block; } -.filetree span.folder { background: url(images/folder.gif) 0 0 no-repeat; } -.filetree li.expandable span.folder { background: url(images/folder-closed.gif) 0 0 no-repeat; } -.filetree span.file { background: url(images/file.gif) 0 0 no-repeat; } - -html, body {height:100%; margin: 0; padding: 0; } - -/* -html>body { - font-size: 16px; - font-size: 68.75%; -} Reset Base Font Size */ - /* -body { - font-family: Verdana, helvetica, arial, sans-serif; - font-size: 68.75%; - background: #fff; - color: #333; -} */ - +.treeview, .treeview ul { + padding: 0; + margin: 0; + list-style: none; +} + +.treeview ul { + background-color: white; + margin-top: 4px; +} + +.treeview .hitarea { + background: url(images/treeview-default.gif) -64px -25px no-repeat; + height: 16px; + width: 16px; + margin-left: -16px; + float: left; + cursor: pointer; +} +/* fix for IE6 */ +* html .hitarea { + display: inline; + float:none; +} + +.treeview li { + margin: 0; + padding: 3px 0 3px 16px; +} + +.treeview a.selected { + background-color: #eee; +} + +#treecontrol { margin: 1em 0; display: none; } + +.treeview .hover { color: red; cursor: pointer; } + +.treeview li { background: url(images/treeview-default-line.gif) 0 0 no-repeat; } +.treeview li.collapsable, .treeview li.expandable { background-position: 0 -176px; } + +.treeview .expandable-hitarea { background-position: -80px -3px; } + +.treeview li.last { background-position: 0 -1766px } +.treeview li.lastCollapsable, .treeview li.lastExpandable { background-image: url(images/treeview-default.gif); } +.treeview li.lastCollapsable { background-position: 0 -111px } +.treeview li.lastExpandable { background-position: -32px -67px } + +.treeview div.lastCollapsable-hitarea, .treeview div.lastExpandable-hitarea { background-position: 0; } + +.treeview-red li { background-image: url(images/treeview-red-line.gif); } +.treeview-red .hitarea, .treeview-red li.lastCollapsable, .treeview-red li.lastExpandable { background-image: url(images/treeview-red.gif); } + +.treeview-black li { background-image: url(images/treeview-black-line.gif); } +.treeview-black .hitarea, .treeview-black li.lastCollapsable, .treeview-black li.lastExpandable { background-image: url(images/treeview-black.gif); } + +.treeview-gray li { background-image: url(images/treeview-gray-line.gif); } +.treeview-gray .hitarea, .treeview-gray li.lastCollapsable, .treeview-gray li.lastExpandable { background-image: url(images/treeview-gray.gif); } + +.treeview-famfamfam li { background-image: url(images/treeview-famfamfam-line.gif); } +.treeview-famfamfam .hitarea, .treeview-famfamfam li.lastCollapsable, .treeview-famfamfam li.lastExpandable { background-image: url(images/treeview-famfamfam.gif); } + + +.filetree li { padding: 3px 0 2px 16px; } +.filetree span.folder, .filetree span.file { padding: 1px 0 1px 16px; display: block; } +.filetree span.folder { background: url(images/folder.gif) 0 0 no-repeat; } +.filetree li.expandable span.folder { background: url(images/folder-closed.gif) 0 0 no-repeat; } +.filetree span.file { background: url(images/file.gif) 0 0 no-repeat; } + +html, body {height:100%; margin: 0; padding: 0; } + +/* +html>body { + font-size: 16px; + font-size: 68.75%; +} Reset Base Font Size */ + /* +body { + font-family: Verdana, helvetica, arial, sans-serif; + font-size: 68.75%; + background: #fff; + color: #333; +} */ + a img { border: none; } \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/content/search/nwSearchFnt.js b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/content/search/nwSearchFnt.js index 0111559..0c4336d 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/content/search/nwSearchFnt.js +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/docs/content/search/nwSearchFnt.js @@ -1,513 +1,513 @@ -/*---------------------------------------------------------------------------- - * JavaScript for webhelp search - *---------------------------------------------------------------------------- - This file is part of the webhelpsearch plugin for DocBook WebHelp - Copyright (c) 2007-2008 NexWave Solutions All Rights Reserved. - www.nexwave.biz Nadege Quaine - http://kasunbg.blogspot.com/ Kasun Gajasinghe - */ - -//string initialization -var htmlfileList = "htmlFileList.js"; -var htmlfileinfoList = "htmlFileInfoList.js"; -var useCJKTokenizing = false; - -/* Cette fonction verifie la validite de la recherche entrre par l utilisateur */ -function Verifie(ditaSearch_Form) { - - // Check browser compatibitily - if (navigator.userAgent.indexOf("Konquerer") > -1) { - - alert(txt_browser_not_supported); - return; - } - - - var expressionInput = document.ditaSearch_Form.textToSearch.value; - //Set a cookie to store the searched keywords - $.cookie('textToSearch', expressionInput); - - - if (expressionInput.length < 1) { - - // expression is invalid - alert(txt_enter_at_least_1_char); - // reactive la fenetre de search (utile car cadres) - document.ditaSearch_Form.textToSearch.focus(); - } - else { - - // Effectuer la recherche - Effectuer_recherche(expressionInput); - - // reactive la fenetre de search (utile car cadres) - document.ditaSearch_Form.textToSearch.focus(); - } -} - -var stemQueryMap = new Array(); // A hashtable which maps stems to query words - -/* This function parses the search expression, loads the indices and displays the results*/ -function Effectuer_recherche(expressionInput) { - - /* Display a waiting message */ - //DisplayWaitingMessage(); - - /*data initialisation*/ - var searchFor = ""; // expression en lowercase et sans les caracte res speciaux - //w = new Object(); // hashtable, key=word, value = list of the index of the html files - scriptLetterTab = new Scriptfirstchar(); // Array containing the first letter of each word to look for - var wordsList = new Array(); // Array with the words to look for - var finalWordsList = new Array(); // Array with the words to look for after removing spaces - var linkTab = new Array(); - var fileAndWordList = new Array(); - var txt_wordsnotfound = ""; - - - /*nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux*/ - searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "= 0; i--) { - if (fileAndWordList[i] != undefined) { - linkTab.push("

        " + txt_results_for + " " + "" + fileAndWordList[i][0].motslisteDisplay + "" + "

        "); - - linkTab.push("
          "); - for (t in fileAndWordList[i]) { - //DEBUG: alert(": "+ fileAndWordList[i][t].filenb+" " +fileAndWordList[i][t].motsliste); - //linkTab.push("
        • "+fl[fileAndWordList[i][t].filenb]+"
          • "); - var tempInfo = fil[fileAndWordList[i][t].filenb]; - var pos1 = tempInfo.indexOf("@@@"); - var pos2 = tempInfo.lastIndexOf("@@@"); - var tempPath = tempInfo.substring(0, pos1); - var tempTitle = tempInfo.substring(pos1 + 3, pos2); - var tempShortdesc = tempInfo.substring(pos2 + 3, tempInfo.length); - - //file:///home/kasun/docbook/WEBHELP/webhelp-draft-output-format-idea/src/main/resources/web/webhelp/installation.html - var linkString = "
        • " + tempTitle + ""; - // var linkString = "
        • " + tempTitle + ""; - if ((tempShortdesc != "null")) { - linkString += "\n
          " + tempShortdesc + "
          "; - } - linkString += "
          • "; - linkTab.push(linkString); - } - linkTab.push("
        "); - } - } - } - - var results = ""; - if (linkTab.length > 0) { - /*writeln ("

        " + txt_results_for + " " + "" + cleanwordsList + "" + "
        "+"

        ");*/ - results = "

        "; - //write("

          "); - for (t in linkTab) { - results += linkTab[t].toString(); - } - results += "

          "; - } else { - results = "

          " + "Your search returned no results for " + "" + txt_wordsnotfound + "" + "

          "; - } - //alert(results); - document.getElementById('searchResults').innerHTML = results; -} - -function tokenize(wordsList){ - var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces - var cleanwordsList = new Array(); // Array with the words to look for - for(var j in wordsList){ - var word = wordsList[j]; - if(typeof stemmer != "undefined" ){ - stemQueryMap[stemmer(word)] = word; - } else { - stemQueryMap[word] = word; - } - } - //stemmedWordsList is the stemmed list of words separated by spaces. - for (var t in wordsList) { - wordsList[t] = wordsList[t].replace(/(%22)|^-/g, ""); - if (wordsList[t] != "%20") { - scriptLetterTab.add(wordsList[t].charAt(0)); - cleanwordsList.push(wordsList[t]); - } - } - - if(typeof stemmer != "undefined" ){ - //Do the stemming using Porter's stemming algorithm - for (var i = 0; i < cleanwordsList.length; i++) { - var stemWord = stemmer(cleanwordsList[i]); - stemmedWordsList.push(stemWord); - } - } else { - stemmedWordsList = cleanwordsList; - } - return stemmedWordsList; -} - -//Invoker of CJKTokenizer class methods. -function cjkTokenize(wordsList){ - var allTokens= new Array(); - var notCJKTokens= new Array(); - var j=0; - for(j=0;j"; - return this.input.substring(this.offset,this.offset+2); - } - - function getAllTokens(){ - while(this.incrementToken()){ - var tmp = this.tokenize(); - this.tokens.push(tmp); - } - return this.unique(this.tokens); -// document.getElementById("content").innerHTML += tokens+" "; -// document.getElementById("content").innerHTML += "
          dada"+sortedTokens+" "; -// console.log(tokens.length+"dsdsds"); - /*for(i=0;i t2.length) { - return 1; - } else { - return -1; - } - //return t1.length - t2.length); +/*---------------------------------------------------------------------------- + * JavaScript for webhelp search + *---------------------------------------------------------------------------- + This file is part of the webhelpsearch plugin for DocBook WebHelp + Copyright (c) 2007-2008 NexWave Solutions All Rights Reserved. + www.nexwave.biz Nadege Quaine + http://kasunbg.blogspot.com/ Kasun Gajasinghe + */ + +//string initialization +var htmlfileList = "htmlFileList.js"; +var htmlfileinfoList = "htmlFileInfoList.js"; +var useCJKTokenizing = false; + +/* Cette fonction verifie la validite de la recherche entrre par l utilisateur */ +function Verifie(ditaSearch_Form) { + + // Check browser compatibitily + if (navigator.userAgent.indexOf("Konquerer") > -1) { + + alert(txt_browser_not_supported); + return; + } + + + var expressionInput = document.ditaSearch_Form.textToSearch.value; + //Set a cookie to store the searched keywords + $.cookie('textToSearch', expressionInput); + + + if (expressionInput.length < 1) { + + // expression is invalid + alert(txt_enter_at_least_1_char); + // reactive la fenetre de search (utile car cadres) + document.ditaSearch_Form.textToSearch.focus(); + } + else { + + // Effectuer la recherche + Effectuer_recherche(expressionInput); + + // reactive la fenetre de search (utile car cadres) + document.ditaSearch_Form.textToSearch.focus(); + } +} + +var stemQueryMap = new Array(); // A hashtable which maps stems to query words + +/* This function parses the search expression, loads the indices and displays the results*/ +function Effectuer_recherche(expressionInput) { + + /* Display a waiting message */ + //DisplayWaitingMessage(); + + /*data initialisation*/ + var searchFor = ""; // expression en lowercase et sans les caracte res speciaux + //w = new Object(); // hashtable, key=word, value = list of the index of the html files + scriptLetterTab = new Scriptfirstchar(); // Array containing the first letter of each word to look for + var wordsList = new Array(); // Array with the words to look for + var finalWordsList = new Array(); // Array with the words to look for after removing spaces + var linkTab = new Array(); + var fileAndWordList = new Array(); + var txt_wordsnotfound = ""; + + + /*nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux*/ + searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "= 0; i--) { + if (fileAndWordList[i] != undefined) { + linkTab.push("

          " + txt_results_for + " " + "" + fileAndWordList[i][0].motslisteDisplay + "" + "

          "); + + linkTab.push("
            "); + for (t in fileAndWordList[i]) { + //DEBUG: alert(": "+ fileAndWordList[i][t].filenb+" " +fileAndWordList[i][t].motsliste); + //linkTab.push("
          • "+fl[fileAndWordList[i][t].filenb]+"
            • "); + var tempInfo = fil[fileAndWordList[i][t].filenb]; + var pos1 = tempInfo.indexOf("@@@"); + var pos2 = tempInfo.lastIndexOf("@@@"); + var tempPath = tempInfo.substring(0, pos1); + var tempTitle = tempInfo.substring(pos1 + 3, pos2); + var tempShortdesc = tempInfo.substring(pos2 + 3, tempInfo.length); + + //file:///home/kasun/docbook/WEBHELP/webhelp-draft-output-format-idea/src/main/resources/web/webhelp/installation.html + var linkString = "
          • " + tempTitle + ""; + // var linkString = "
          • " + tempTitle + ""; + if ((tempShortdesc != "null")) { + linkString += "\n
            " + tempShortdesc + "
            "; + } + linkString += "
            • "; + linkTab.push(linkString); + } + linkTab.push("
          "); + } + } + } + + var results = ""; + if (linkTab.length > 0) { + /*writeln ("

          " + txt_results_for + " " + "" + cleanwordsList + "" + "
          "+"

          ");*/ + results = "

          "; + //write("

            "); + for (t in linkTab) { + results += linkTab[t].toString(); + } + results += "

            "; + } else { + results = "

            " + "Your search returned no results for " + "" + txt_wordsnotfound + "" + "

            "; + } + //alert(results); + document.getElementById('searchResults').innerHTML = results; +} + +function tokenize(wordsList){ + var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces + var cleanwordsList = new Array(); // Array with the words to look for + for(var j in wordsList){ + var word = wordsList[j]; + if(typeof stemmer != "undefined" ){ + stemQueryMap[stemmer(word)] = word; + } else { + stemQueryMap[word] = word; + } + } + //stemmedWordsList is the stemmed list of words separated by spaces. + for (var t in wordsList) { + wordsList[t] = wordsList[t].replace(/(%22)|^-/g, ""); + if (wordsList[t] != "%20") { + scriptLetterTab.add(wordsList[t].charAt(0)); + cleanwordsList.push(wordsList[t]); + } + } + + if(typeof stemmer != "undefined" ){ + //Do the stemming using Porter's stemming algorithm + for (var i = 0; i < cleanwordsList.length; i++) { + var stemWord = stemmer(cleanwordsList[i]); + stemmedWordsList.push(stemWord); + } + } else { + stemmedWordsList = cleanwordsList; + } + return stemmedWordsList; +} + +//Invoker of CJKTokenizer class methods. +function cjkTokenize(wordsList){ + var allTokens= new Array(); + var notCJKTokens= new Array(); + var j=0; + for(j=0;j"; + return this.input.substring(this.offset,this.offset+2); + } + + function getAllTokens(){ + while(this.incrementToken()){ + var tmp = this.tokenize(); + this.tokens.push(tmp); + } + return this.unique(this.tokens); +// document.getElementById("content").innerHTML += tokens+" "; +// document.getElementById("content").innerHTML += "
            dada"+sortedTokens+" "; +// console.log(tokens.length+"dsdsds"); + /*for(i=0;i t2.length) { + return 1; + } else { + return -1; + } + //return t1.length - t2.length); } \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.async.js b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.async.js index 2597dde..c91a9c6 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.async.js +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.async.js @@ -1,72 +1,72 @@ -/* - * Async Treeview 0.1 - Lazy-loading extension for Treeview - * - * http://bassistance.de/jquery-plugins/jquery-plugin-treeview/ - * - * Copyright (c) 2007 Jörn Zaefferer - * - * Dual licensed under the MIT and GPL licenses: - * http://www.opensource.org/licenses/mit-license.php - * http://www.gnu.org/licenses/gpl.html - * - * Revision: $Id$ - * - */ - -;(function($) { - -function load(settings, root, child, container) { - $.getJSON(settings.url, {root: root}, function(response) { - function createNode(parent) { - var current = $("
          • ").attr("id", this.id || "").html("" + this.text + "").appendTo(parent); - if (this.classes) { - current.children("span").addClass(this.classes); - } - if (this.expanded) { - current.addClass("open"); - } - if (this.hasChildren || this.children && this.children.length) { - var branch = $("
              ").appendTo(current); - if (this.hasChildren) { - current.addClass("hasChildren"); - createNode.call({ - text:"placeholder", - id:"placeholder", - children:[] - }, branch); - } - if (this.children && this.children.length) { - $.each(this.children, createNode, [branch]) - } - } - } - $.each(response, createNode, [child]); - $(container).treeview({add: child}); - }); -} - -var proxied = $.fn.treeview; -$.fn.treeview = function(settings) { - if (!settings.url) { - return proxied.apply(this, arguments); - } - var container = this; - load(settings, "source", this, container); - var userToggle = settings.toggle; - return proxied.call(this, $.extend({}, settings, { - collapsed: true, - toggle: function() { - var $this = $(this); - if ($this.hasClass("hasChildren")) { - var childList = $this.removeClass("hasChildren").find("ul"); - childList.empty(); - load(settings, this.id, childList, container); - } - if (userToggle) { - userToggle.apply(this, arguments); - } - } - })); -}; - +/* + * Async Treeview 0.1 - Lazy-loading extension for Treeview + * + * http://bassistance.de/jquery-plugins/jquery-plugin-treeview/ + * + * Copyright (c) 2007 Jörn Zaefferer + * + * Dual licensed under the MIT and GPL licenses: + * http://www.opensource.org/licenses/mit-license.php + * http://www.gnu.org/licenses/gpl.html + * + * Revision: $Id$ + * + */ + +;(function($) { + +function load(settings, root, child, container) { + $.getJSON(settings.url, {root: root}, function(response) { + function createNode(parent) { + var current = $("
            • ").attr("id", this.id || "").html("" + this.text + "").appendTo(parent); + if (this.classes) { + current.children("span").addClass(this.classes); + } + if (this.expanded) { + current.addClass("open"); + } + if (this.hasChildren || this.children && this.children.length) { + var branch = $("
                ").appendTo(current); + if (this.hasChildren) { + current.addClass("hasChildren"); + createNode.call({ + text:"placeholder", + id:"placeholder", + children:[] + }, branch); + } + if (this.children && this.children.length) { + $.each(this.children, createNode, [branch]) + } + } + } + $.each(response, createNode, [child]); + $(container).treeview({add: child}); + }); +} + +var proxied = $.fn.treeview; +$.fn.treeview = function(settings) { + if (!settings.url) { + return proxied.apply(this, arguments); + } + var container = this; + load(settings, "source", this, container); + var userToggle = settings.toggle; + return proxied.call(this, $.extend({}, settings, { + collapsed: true, + toggle: function() { + var $this = $(this); + if ($this.hasClass("hasChildren")) { + var childList = $this.removeClass("hasChildren").find("ul"); + childList.empty(); + load(settings, this.id, childList, container); + } + if (userToggle) { + userToggle.apply(this, arguments); + } + } + })); +}; + })(jQuery); \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.css b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.css index dbf425b..d7e2c00 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.css +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/common/jquery/treeview/jquery.treeview.css @@ -1,85 +1,85 @@ -.treeview, .treeview ul { - padding: 0; - margin: 0; - list-style: none; -} - -.treeview ul { - background-color: white; - margin-top: 4px; -} - -.treeview .hitarea { - background: url(images/treeview-default.gif) -64px -25px no-repeat; - height: 16px; - width: 16px; - margin-left: -16px; - float: left; - cursor: pointer; -} -/* fix for IE6 */ -* html .hitarea { - display: inline; - float:none; -} - -.treeview li { - margin: 0; - padding: 3px 0 3px 16px; -} - -.treeview a.selected { - background-color: #eee; -} - -#treecontrol { margin: 1em 0; display: none; } - -.treeview .hover { color: red; cursor: pointer; } - -.treeview li { background: url(images/treeview-default-line.gif) 0 0 no-repeat; } -.treeview li.collapsable, .treeview li.expandable { background-position: 0 -176px; } - -.treeview .expandable-hitarea { background-position: -80px -3px; } - -.treeview li.last { background-position: 0 -1766px } -.treeview li.lastCollapsable, .treeview li.lastExpandable { background-image: url(images/treeview-default.gif); } -.treeview li.lastCollapsable { background-position: 0 -111px } -.treeview li.lastExpandable { background-position: -32px -67px } - -.treeview div.lastCollapsable-hitarea, .treeview div.lastExpandable-hitarea { background-position: 0; } - -.treeview-red li { background-image: url(images/treeview-red-line.gif); } -.treeview-red .hitarea, .treeview-red li.lastCollapsable, .treeview-red li.lastExpandable { background-image: url(images/treeview-red.gif); } - -.treeview-black li { background-image: url(images/treeview-black-line.gif); } -.treeview-black .hitarea, .treeview-black li.lastCollapsable, .treeview-black li.lastExpandable { background-image: url(images/treeview-black.gif); } - -.treeview-gray li { background-image: url(images/treeview-gray-line.gif); } -.treeview-gray .hitarea, .treeview-gray li.lastCollapsable, .treeview-gray li.lastExpandable { background-image: url(images/treeview-gray.gif); } - -.treeview-famfamfam li { background-image: url(images/treeview-famfamfam-line.gif); } -.treeview-famfamfam .hitarea, .treeview-famfamfam li.lastCollapsable, .treeview-famfamfam li.lastExpandable { background-image: url(images/treeview-famfamfam.gif); } - - -.filetree li { padding: 3px 0 2px 16px; } -.filetree span.folder, .filetree span.file { padding: 1px 0 1px 16px; display: block; } -.filetree span.folder { background: url(images/folder.gif) 0 0 no-repeat; } -.filetree li.expandable span.folder { background: url(images/folder-closed.gif) 0 0 no-repeat; } -.filetree span.file { background: url(images/file.gif) 0 0 no-repeat; } - -html, body {height:100%; margin: 0; padding: 0; } - -/* -html>body { - font-size: 16px; - font-size: 68.75%; -} Reset Base Font Size */ - /* -body { - font-family: Verdana, helvetica, arial, sans-serif; - font-size: 68.75%; - background: #fff; - color: #333; -} */ - +.treeview, .treeview ul { + padding: 0; + margin: 0; + list-style: none; +} + +.treeview ul { + background-color: white; + margin-top: 4px; +} + +.treeview .hitarea { + background: url(images/treeview-default.gif) -64px -25px no-repeat; + height: 16px; + width: 16px; + margin-left: -16px; + float: left; + cursor: pointer; +} +/* fix for IE6 */ +* html .hitarea { + display: inline; + float:none; +} + +.treeview li { + margin: 0; + padding: 3px 0 3px 16px; +} + +.treeview a.selected { + background-color: #eee; +} + +#treecontrol { margin: 1em 0; display: none; } + +.treeview .hover { color: red; cursor: pointer; } + +.treeview li { background: url(images/treeview-default-line.gif) 0 0 no-repeat; } +.treeview li.collapsable, .treeview li.expandable { background-position: 0 -176px; } + +.treeview .expandable-hitarea { background-position: -80px -3px; } + +.treeview li.last { background-position: 0 -1766px } +.treeview li.lastCollapsable, .treeview li.lastExpandable { background-image: url(images/treeview-default.gif); } +.treeview li.lastCollapsable { background-position: 0 -111px } +.treeview li.lastExpandable { background-position: -32px -67px } + +.treeview div.lastCollapsable-hitarea, .treeview div.lastExpandable-hitarea { background-position: 0; } + +.treeview-red li { background-image: url(images/treeview-red-line.gif); } +.treeview-red .hitarea, .treeview-red li.lastCollapsable, .treeview-red li.lastExpandable { background-image: url(images/treeview-red.gif); } + +.treeview-black li { background-image: url(images/treeview-black-line.gif); } +.treeview-black .hitarea, .treeview-black li.lastCollapsable, .treeview-black li.lastExpandable { background-image: url(images/treeview-black.gif); } + +.treeview-gray li { background-image: url(images/treeview-gray-line.gif); } +.treeview-gray .hitarea, .treeview-gray li.lastCollapsable, .treeview-gray li.lastExpandable { background-image: url(images/treeview-gray.gif); } + +.treeview-famfamfam li { background-image: url(images/treeview-famfamfam-line.gif); } +.treeview-famfamfam .hitarea, .treeview-famfamfam li.lastCollapsable, .treeview-famfamfam li.lastExpandable { background-image: url(images/treeview-famfamfam.gif); } + + +.filetree li { padding: 3px 0 2px 16px; } +.filetree span.folder, .filetree span.file { padding: 1px 0 1px 16px; display: block; } +.filetree span.folder { background: url(images/folder.gif) 0 0 no-repeat; } +.filetree li.expandable span.folder { background: url(images/folder-closed.gif) 0 0 no-repeat; } +.filetree span.file { background: url(images/file.gif) 0 0 no-repeat; } + +html, body {height:100%; margin: 0; padding: 0; } + +/* +html>body { + font-size: 16px; + font-size: 68.75%; +} Reset Base Font Size */ + /* +body { + font-family: Verdana, helvetica, arial, sans-serif; + font-size: 68.75%; + background: #fff; + color: #333; +} */ + a img { border: none; } \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/en-us.props b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/en-us.props index da284ce..f7ed270 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/en-us.props +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/en-us.props @@ -1,45 +1,45 @@ -DEF01=this -DEF02=is -DEF03=the -DEF04=in -DEF05=i -DEF06=on -DEF07=a -DEF08=about -DEF09=an -DEF10=are -DEF11=as -DEF12=at -DEF13=be -DEF14=by -DEF15=com -DEF16=de -DEF17=en -DEF18=for -DEF19=from -DEF20=how -DEF21=it -DEF22=la -DEF23=of -DEF24=on -DEF25=or -DEF26=that -DEF27=to -DEF28=was -DEF29=what -DEF30=when -DEF31=where -DEF32=who -DEF33=will -DEF34=with -DEF35=und -DEF36=Next -DEF37=Prev -DEF38=Home -DEF39=Motive -DEF40=Inc -DEF41=Copyright -DEF42=All -DEF43=rights -DEF44=reserved +DEF01=this +DEF02=is +DEF03=the +DEF04=in +DEF05=i +DEF06=on +DEF07=a +DEF08=about +DEF09=an +DEF10=are +DEF11=as +DEF12=at +DEF13=be +DEF14=by +DEF15=com +DEF16=de +DEF17=en +DEF18=for +DEF19=from +DEF20=how +DEF21=it +DEF22=la +DEF23=of +DEF24=on +DEF25=or +DEF26=that +DEF27=to +DEF28=was +DEF29=what +DEF30=when +DEF31=where +DEF32=who +DEF33=will +DEF34=with +DEF35=und +DEF36=Next +DEF37=Prev +DEF38=Home +DEF39=Motive +DEF40=Inc +DEF41=Copyright +DEF42=All +DEF43=rights +DEF44=reserved DEF45=Up \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/es-es.props b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/es-es.props index fb73bdc..b1d0b40 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/es-es.props +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/es-es.props @@ -1,179 +1,179 @@ -DEF01=un -DEF02=una -DEF03=unas -DEF04=unos -DEF05=uno -DEF06=sobre -DEF07=todo -DEF08=también -DEF09=tras -DEF10=otro -DEF11=algún -DEF12=alguno -DEF13=alguna -DEF14=algunos -DEF15=algunas -DEF16=ser -DEF17=es -DEF18=soy -DEF19=eres -DEF20=somos -DEF21=sois -DEF22=estoy -DEF23=esta -DEF24=estamos -DEF25=estais -DEF26=estan -DEF27=como -DEF28=en -DEF29=para -DEF30=atras -DEF31=porque -DEF32=por -DEF33=estado -DEF34=estaba -DEF35=ante -DEF36=antes -DEF37=siendo -DEF38=ambos -DEF39=pero -DEF40=por -DEF41=poder -DEF42=puede -DEF43=puedo -DEF44=podemos -DEF45=podeis -DEF46=pueden -DEF47=fui -DEF48=fue -DEF49=fuimos -DEF50=fueron -DEF51=hacer -DEF52=hago -DEF53=hace -DEF54=hacemos -DEF55=haceis -DEF56=hacen -DEF57=cada -DEF58=fin -DEF59=incluso -DEF60=primero -DEF61=desde -DEF62=conseguir -DEF63=consigo -DEF64=consigue -DEF65=consigues -DEF66=conseguimos -DEF67=consiguen -DEF68=ir -DEF69=voy -DEF70=va -DEF71=vamos -DEF72=vais -DEF73=van -DEF74=vaya -DEF75=gueno -DEF76=ha -DEF77=tener -DEF78=tengo -DEF79=tiene -DEF80=tenemos -DEF81=teneis -DEF82=tienen -DEF83=el -DEF84=la -DEF85=lo -DEF86=las -DEF87=los -DEF88=su -DEF89=aqui -DEF90=mio -DEF91=tuyo -DEF92=ellos -DEF93=ellas -DEF94=nos -DEF95=nosotros -DEF96=vosotros -DEF97=vosotras -DEF98=si -DEF99=dentro -DEF100=solo -DEF101=solamente -DEF102=saber -DEF103=sabes -DEF104=sabe -DEF105=sabemos -DEF106=sabeis -DEF107=saben -DEF108=ultimo -DEF109=largo -DEF110=bastante -DEF111=haces -DEF112=muchos -DEF113=aquellos -DEF114=aquellas -DEF115=sus -DEF116=entonces -DEF117=tiempo -DEF118=verdad -DEF119=verdadero -DEF120=verdadera -DEF121=cierto -DEF122=ciertos -DEF123=cierta -DEF124=ciertas -DEF125=intentar -DEF126=intento -DEF127=intenta -DEF128=intentas -DEF129=intentamos -DEF130=intentais -DEF131=intentan -DEF132=dos -DEF133=bajo -DEF134=arriba -DEF135=encima -DEF136=usar -DEF137=uso -DEF138=usas -DEF139=usa -DEF140=usamos -DEF141=usais -DEF142=usan -DEF143=emplear -DEF144=empleo -DEF145=empleas -DEF146=emplean -DEF147=ampleamos -DEF148=empleais -DEF149=valor -DEF150=muy -DEF151=era -DEF152=eras -DEF153=eramos -DEF154=eran -DEF155=modo -DEF156=bien -DEF157=cual -DEF158=cuando -DEF159=donde -DEF160=mientras -DEF161=quien -DEF162=con -DEF163=entre -DEF164=sin -DEF165=trabajo -DEF166=trabajar -DEF167=trabajas -DEF168=trabaja -DEF169=trabajamos -DEF170=trabajais -DEF171=trabajan -DEF172=podria -DEF173=podrias -DEF174=podriamos -DEF175=podrian -DEF176=podriais -DEF177=yo -DEF178=aquel +DEF01=un +DEF02=una +DEF03=unas +DEF04=unos +DEF05=uno +DEF06=sobre +DEF07=todo +DEF08=también +DEF09=tras +DEF10=otro +DEF11=algún +DEF12=alguno +DEF13=alguna +DEF14=algunos +DEF15=algunas +DEF16=ser +DEF17=es +DEF18=soy +DEF19=eres +DEF20=somos +DEF21=sois +DEF22=estoy +DEF23=esta +DEF24=estamos +DEF25=estais +DEF26=estan +DEF27=como +DEF28=en +DEF29=para +DEF30=atras +DEF31=porque +DEF32=por +DEF33=estado +DEF34=estaba +DEF35=ante +DEF36=antes +DEF37=siendo +DEF38=ambos +DEF39=pero +DEF40=por +DEF41=poder +DEF42=puede +DEF43=puedo +DEF44=podemos +DEF45=podeis +DEF46=pueden +DEF47=fui +DEF48=fue +DEF49=fuimos +DEF50=fueron +DEF51=hacer +DEF52=hago +DEF53=hace +DEF54=hacemos +DEF55=haceis +DEF56=hacen +DEF57=cada +DEF58=fin +DEF59=incluso +DEF60=primero +DEF61=desde +DEF62=conseguir +DEF63=consigo +DEF64=consigue +DEF65=consigues +DEF66=conseguimos +DEF67=consiguen +DEF68=ir +DEF69=voy +DEF70=va +DEF71=vamos +DEF72=vais +DEF73=van +DEF74=vaya +DEF75=gueno +DEF76=ha +DEF77=tener +DEF78=tengo +DEF79=tiene +DEF80=tenemos +DEF81=teneis +DEF82=tienen +DEF83=el +DEF84=la +DEF85=lo +DEF86=las +DEF87=los +DEF88=su +DEF89=aqui +DEF90=mio +DEF91=tuyo +DEF92=ellos +DEF93=ellas +DEF94=nos +DEF95=nosotros +DEF96=vosotros +DEF97=vosotras +DEF98=si +DEF99=dentro +DEF100=solo +DEF101=solamente +DEF102=saber +DEF103=sabes +DEF104=sabe +DEF105=sabemos +DEF106=sabeis +DEF107=saben +DEF108=ultimo +DEF109=largo +DEF110=bastante +DEF111=haces +DEF112=muchos +DEF113=aquellos +DEF114=aquellas +DEF115=sus +DEF116=entonces +DEF117=tiempo +DEF118=verdad +DEF119=verdadero +DEF120=verdadera +DEF121=cierto +DEF122=ciertos +DEF123=cierta +DEF124=ciertas +DEF125=intentar +DEF126=intento +DEF127=intenta +DEF128=intentas +DEF129=intentamos +DEF130=intentais +DEF131=intentan +DEF132=dos +DEF133=bajo +DEF134=arriba +DEF135=encima +DEF136=usar +DEF137=uso +DEF138=usas +DEF139=usa +DEF140=usamos +DEF141=usais +DEF142=usan +DEF143=emplear +DEF144=empleo +DEF145=empleas +DEF146=emplean +DEF147=ampleamos +DEF148=empleais +DEF149=valor +DEF150=muy +DEF151=era +DEF152=eras +DEF153=eramos +DEF154=eran +DEF155=modo +DEF156=bien +DEF157=cual +DEF158=cuando +DEF159=donde +DEF160=mientras +DEF161=quien +DEF162=con +DEF163=entre +DEF164=sin +DEF165=trabajo +DEF166=trabajar +DEF167=trabajas +DEF168=trabaja +DEF169=trabajamos +DEF170=trabajais +DEF171=trabajan +DEF172=podria +DEF173=podrias +DEF174=podriamos +DEF175=podrian +DEF176=podriais +DEF177=yo +DEF178=aquel DEF179=qué \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/nwSearchFnt.js b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/nwSearchFnt.js index 0111559..0c4336d 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/nwSearchFnt.js +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/nwSearchFnt.js @@ -1,513 +1,513 @@ -/*---------------------------------------------------------------------------- - * JavaScript for webhelp search - *---------------------------------------------------------------------------- - This file is part of the webhelpsearch plugin for DocBook WebHelp - Copyright (c) 2007-2008 NexWave Solutions All Rights Reserved. - www.nexwave.biz Nadege Quaine - http://kasunbg.blogspot.com/ Kasun Gajasinghe - */ - -//string initialization -var htmlfileList = "htmlFileList.js"; -var htmlfileinfoList = "htmlFileInfoList.js"; -var useCJKTokenizing = false; - -/* Cette fonction verifie la validite de la recherche entrre par l utilisateur */ -function Verifie(ditaSearch_Form) { - - // Check browser compatibitily - if (navigator.userAgent.indexOf("Konquerer") > -1) { - - alert(txt_browser_not_supported); - return; - } - - - var expressionInput = document.ditaSearch_Form.textToSearch.value; - //Set a cookie to store the searched keywords - $.cookie('textToSearch', expressionInput); - - - if (expressionInput.length < 1) { - - // expression is invalid - alert(txt_enter_at_least_1_char); - // reactive la fenetre de search (utile car cadres) - document.ditaSearch_Form.textToSearch.focus(); - } - else { - - // Effectuer la recherche - Effectuer_recherche(expressionInput); - - // reactive la fenetre de search (utile car cadres) - document.ditaSearch_Form.textToSearch.focus(); - } -} - -var stemQueryMap = new Array(); // A hashtable which maps stems to query words - -/* This function parses the search expression, loads the indices and displays the results*/ -function Effectuer_recherche(expressionInput) { - - /* Display a waiting message */ - //DisplayWaitingMessage(); - - /*data initialisation*/ - var searchFor = ""; // expression en lowercase et sans les caracte res speciaux - //w = new Object(); // hashtable, key=word, value = list of the index of the html files - scriptLetterTab = new Scriptfirstchar(); // Array containing the first letter of each word to look for - var wordsList = new Array(); // Array with the words to look for - var finalWordsList = new Array(); // Array with the words to look for after removing spaces - var linkTab = new Array(); - var fileAndWordList = new Array(); - var txt_wordsnotfound = ""; - - - /*nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux*/ - searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "= 0; i--) { - if (fileAndWordList[i] != undefined) { - linkTab.push("

                " + txt_results_for + " " + "" + fileAndWordList[i][0].motslisteDisplay + "" + "

                "); - - linkTab.push("
                  "); - for (t in fileAndWordList[i]) { - //DEBUG: alert(": "+ fileAndWordList[i][t].filenb+" " +fileAndWordList[i][t].motsliste); - //linkTab.push("
                • "+fl[fileAndWordList[i][t].filenb]+"
                  • "); - var tempInfo = fil[fileAndWordList[i][t].filenb]; - var pos1 = tempInfo.indexOf("@@@"); - var pos2 = tempInfo.lastIndexOf("@@@"); - var tempPath = tempInfo.substring(0, pos1); - var tempTitle = tempInfo.substring(pos1 + 3, pos2); - var tempShortdesc = tempInfo.substring(pos2 + 3, tempInfo.length); - - //file:///home/kasun/docbook/WEBHELP/webhelp-draft-output-format-idea/src/main/resources/web/webhelp/installation.html - var linkString = "
                • " + tempTitle + ""; - // var linkString = "
                • " + tempTitle + ""; - if ((tempShortdesc != "null")) { - linkString += "\n
                  " + tempShortdesc + "
                  "; - } - linkString += "
                  • "; - linkTab.push(linkString); - } - linkTab.push("
                "); - } - } - } - - var results = ""; - if (linkTab.length > 0) { - /*writeln ("

                " + txt_results_for + " " + "" + cleanwordsList + "" + "
                "+"

                ");*/ - results = "

                "; - //write("

                  "); - for (t in linkTab) { - results += linkTab[t].toString(); - } - results += "

                  "; - } else { - results = "

                  " + "Your search returned no results for " + "" + txt_wordsnotfound + "" + "

                  "; - } - //alert(results); - document.getElementById('searchResults').innerHTML = results; -} - -function tokenize(wordsList){ - var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces - var cleanwordsList = new Array(); // Array with the words to look for - for(var j in wordsList){ - var word = wordsList[j]; - if(typeof stemmer != "undefined" ){ - stemQueryMap[stemmer(word)] = word; - } else { - stemQueryMap[word] = word; - } - } - //stemmedWordsList is the stemmed list of words separated by spaces. - for (var t in wordsList) { - wordsList[t] = wordsList[t].replace(/(%22)|^-/g, ""); - if (wordsList[t] != "%20") { - scriptLetterTab.add(wordsList[t].charAt(0)); - cleanwordsList.push(wordsList[t]); - } - } - - if(typeof stemmer != "undefined" ){ - //Do the stemming using Porter's stemming algorithm - for (var i = 0; i < cleanwordsList.length; i++) { - var stemWord = stemmer(cleanwordsList[i]); - stemmedWordsList.push(stemWord); - } - } else { - stemmedWordsList = cleanwordsList; - } - return stemmedWordsList; -} - -//Invoker of CJKTokenizer class methods. -function cjkTokenize(wordsList){ - var allTokens= new Array(); - var notCJKTokens= new Array(); - var j=0; - for(j=0;j"; - return this.input.substring(this.offset,this.offset+2); - } - - function getAllTokens(){ - while(this.incrementToken()){ - var tmp = this.tokenize(); - this.tokens.push(tmp); - } - return this.unique(this.tokens); -// document.getElementById("content").innerHTML += tokens+" "; -// document.getElementById("content").innerHTML += "
                  dada"+sortedTokens+" "; -// console.log(tokens.length+"dsdsds"); - /*for(i=0;i t2.length) { - return 1; - } else { - return -1; - } - //return t1.length - t2.length); +/*---------------------------------------------------------------------------- + * JavaScript for webhelp search + *---------------------------------------------------------------------------- + This file is part of the webhelpsearch plugin for DocBook WebHelp + Copyright (c) 2007-2008 NexWave Solutions All Rights Reserved. + www.nexwave.biz Nadege Quaine + http://kasunbg.blogspot.com/ Kasun Gajasinghe + */ + +//string initialization +var htmlfileList = "htmlFileList.js"; +var htmlfileinfoList = "htmlFileInfoList.js"; +var useCJKTokenizing = false; + +/* Cette fonction verifie la validite de la recherche entrre par l utilisateur */ +function Verifie(ditaSearch_Form) { + + // Check browser compatibitily + if (navigator.userAgent.indexOf("Konquerer") > -1) { + + alert(txt_browser_not_supported); + return; + } + + + var expressionInput = document.ditaSearch_Form.textToSearch.value; + //Set a cookie to store the searched keywords + $.cookie('textToSearch', expressionInput); + + + if (expressionInput.length < 1) { + + // expression is invalid + alert(txt_enter_at_least_1_char); + // reactive la fenetre de search (utile car cadres) + document.ditaSearch_Form.textToSearch.focus(); + } + else { + + // Effectuer la recherche + Effectuer_recherche(expressionInput); + + // reactive la fenetre de search (utile car cadres) + document.ditaSearch_Form.textToSearch.focus(); + } +} + +var stemQueryMap = new Array(); // A hashtable which maps stems to query words + +/* This function parses the search expression, loads the indices and displays the results*/ +function Effectuer_recherche(expressionInput) { + + /* Display a waiting message */ + //DisplayWaitingMessage(); + + /*data initialisation*/ + var searchFor = ""; // expression en lowercase et sans les caracte res speciaux + //w = new Object(); // hashtable, key=word, value = list of the index of the html files + scriptLetterTab = new Scriptfirstchar(); // Array containing the first letter of each word to look for + var wordsList = new Array(); // Array with the words to look for + var finalWordsList = new Array(); // Array with the words to look for after removing spaces + var linkTab = new Array(); + var fileAndWordList = new Array(); + var txt_wordsnotfound = ""; + + + /*nqu: expressionInput, la recherche est lower cased, plus remplacement des char speciaux*/ + searchFor = expressionInput.toLowerCase().replace(/<\//g, "_st_").replace(/\$_/g, "_di_").replace(/\.|%2C|%3B|%21|%3A|@|\/|\*/g, " ").replace(/(%20)+/g, " ").replace(/_st_/g, "= 0; i--) { + if (fileAndWordList[i] != undefined) { + linkTab.push("

                  " + txt_results_for + " " + "" + fileAndWordList[i][0].motslisteDisplay + "" + "

                  "); + + linkTab.push("
                    "); + for (t in fileAndWordList[i]) { + //DEBUG: alert(": "+ fileAndWordList[i][t].filenb+" " +fileAndWordList[i][t].motsliste); + //linkTab.push("
                  • "+fl[fileAndWordList[i][t].filenb]+"
                    • "); + var tempInfo = fil[fileAndWordList[i][t].filenb]; + var pos1 = tempInfo.indexOf("@@@"); + var pos2 = tempInfo.lastIndexOf("@@@"); + var tempPath = tempInfo.substring(0, pos1); + var tempTitle = tempInfo.substring(pos1 + 3, pos2); + var tempShortdesc = tempInfo.substring(pos2 + 3, tempInfo.length); + + //file:///home/kasun/docbook/WEBHELP/webhelp-draft-output-format-idea/src/main/resources/web/webhelp/installation.html + var linkString = "
                  • " + tempTitle + ""; + // var linkString = "
                  • " + tempTitle + ""; + if ((tempShortdesc != "null")) { + linkString += "\n
                    " + tempShortdesc + "
                    "; + } + linkString += "
                    • "; + linkTab.push(linkString); + } + linkTab.push("
                  "); + } + } + } + + var results = ""; + if (linkTab.length > 0) { + /*writeln ("

                  " + txt_results_for + " " + "" + cleanwordsList + "" + "
                  "+"

                  ");*/ + results = "

                  "; + //write("

                    "); + for (t in linkTab) { + results += linkTab[t].toString(); + } + results += "

                    "; + } else { + results = "

                    " + "Your search returned no results for " + "" + txt_wordsnotfound + "" + "

                    "; + } + //alert(results); + document.getElementById('searchResults').innerHTML = results; +} + +function tokenize(wordsList){ + var stemmedWordsList = new Array(); // Array with the words to look for after removing spaces + var cleanwordsList = new Array(); // Array with the words to look for + for(var j in wordsList){ + var word = wordsList[j]; + if(typeof stemmer != "undefined" ){ + stemQueryMap[stemmer(word)] = word; + } else { + stemQueryMap[word] = word; + } + } + //stemmedWordsList is the stemmed list of words separated by spaces. + for (var t in wordsList) { + wordsList[t] = wordsList[t].replace(/(%22)|^-/g, ""); + if (wordsList[t] != "%20") { + scriptLetterTab.add(wordsList[t].charAt(0)); + cleanwordsList.push(wordsList[t]); + } + } + + if(typeof stemmer != "undefined" ){ + //Do the stemming using Porter's stemming algorithm + for (var i = 0; i < cleanwordsList.length; i++) { + var stemWord = stemmer(cleanwordsList[i]); + stemmedWordsList.push(stemWord); + } + } else { + stemmedWordsList = cleanwordsList; + } + return stemmedWordsList; +} + +//Invoker of CJKTokenizer class methods. +function cjkTokenize(wordsList){ + var allTokens= new Array(); + var notCJKTokens= new Array(); + var j=0; + for(j=0;j"; + return this.input.substring(this.offset,this.offset+2); + } + + function getAllTokens(){ + while(this.incrementToken()){ + var tmp = this.tokenize(); + this.tokens.push(tmp); + } + return this.unique(this.tokens); +// document.getElementById("content").innerHTML += tokens+" "; +// document.getElementById("content").innerHTML += "
                    dada"+sortedTokens+" "; +// console.log(tokens.length+"dsdsds"); + /*for(i=0;i t2.length) { + return 1; + } else { + return -1; + } + //return t1.length - t2.length); } \ No newline at end of file diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/punctuation.props b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/punctuation.props index d3e3fcd..edb3413 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/punctuation.props +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/webhelp/template/content/search/punctuation.props @@ -1,31 +1,31 @@ -Punct01=\\u3002 -Punct02=\\u3003 -Punct03=\\u300C -Punct04=\\u300D -Punct05=\\u300E -Punct06=\\u300F -Punct07=\\u301D -Punct08=\\u301E -Punct09=\\u301F -Punct10=\\u309B -Punct11=\\u2018 -Punct12=\\u2019 -Punct13=\\u201A -Punct14=\\u201C -Punct15=\\u201D -Punct16=\\u201E -Punct17=\\u2032 -Punct18=\\u2033 -Punct19=\\u2035 -Punct20=\\u2039 -Punct21=\\u203A -Punct22=\\u201E -Punct23=\\u00BB -Punct24=\\u00AB -Punct25=© -Punct26=’ -Punct27=\\u00A0 -Punct28=\\u2014 - - - +Punct01=\\u3002 +Punct02=\\u3003 +Punct03=\\u300C +Punct04=\\u300D +Punct05=\\u300E +Punct06=\\u300F +Punct07=\\u301D +Punct08=\\u301E +Punct09=\\u301F +Punct10=\\u309B +Punct11=\\u2018 +Punct12=\\u2019 +Punct13=\\u201A +Punct14=\\u201C +Punct15=\\u201D +Punct16=\\u201E +Punct17=\\u2032 +Punct18=\\u2033 +Punct19=\\u2035 +Punct20=\\u2039 +Punct21=\\u203A +Punct22=\\u201E +Punct23=\\u00BB +Punct24=\\u00AB +Punct25=© +Punct26=’ +Punct27=\\u00A0 +Punct28=\\u2014 + + + diff --git a/SCons/Tool/docbook/docs/html.xsl b/SCons/Tool/docbook/docs/html.xsl index f53e1b3..407e18d 100644 --- a/SCons/Tool/docbook/docs/html.xsl +++ b/SCons/Tool/docbook/docs/html.xsl @@ -1,55 +1,55 @@ - - - - - - - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - + + + + + + + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + diff --git a/SCons/Tool/docbook/docs/pdf.xsl b/SCons/Tool/docbook/docs/pdf.xsl index 25b4433..8703ced 100644 --- a/SCons/Tool/docbook/docs/pdf.xsl +++ b/SCons/Tool/docbook/docs/pdf.xsl @@ -1,62 +1,62 @@ - - - - - - - - - - -0pt - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - - - - - + + + + + + + + + + +0pt + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + + + + + diff --git a/doc/design/chtml.xsl b/doc/design/chtml.xsl index 1051247..6d3dfde 100644 --- a/doc/design/chtml.xsl +++ b/doc/design/chtml.xsl @@ -1,57 +1,57 @@ - - - - - - - - - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - + + + + + + + + + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + diff --git a/doc/design/html.xsl b/doc/design/html.xsl index c50e17e..0cd9f9b 100644 --- a/doc/design/html.xsl +++ b/doc/design/html.xsl @@ -1,151 +1,151 @@ - - - - - - - - - - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - - - - - - - - - - - - - - - - - - - <xsl:copy-of select="$title"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + + + + + + + + + + + + + + + + + + + <xsl:copy-of select="$title"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/design/pdf.xsl b/doc/design/pdf.xsl index faea7fc..a398e5c 100644 --- a/doc/design/pdf.xsl +++ b/doc/design/pdf.xsl @@ -1,73 +1,73 @@ - - - - - - - - - - - - -0pt - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - bold - - - - - - - - - - - + + + + + + + + + + + + +0pt + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + bold + + + + + + + + + + + diff --git a/doc/reference/chtml.xsl b/doc/reference/chtml.xsl index f0c9474..c2e7404 100644 --- a/doc/reference/chtml.xsl +++ b/doc/reference/chtml.xsl @@ -1,57 +1,57 @@ - - - - - - - - - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - + + + + + + + + + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + diff --git a/doc/reference/html.xsl b/doc/reference/html.xsl index c50e17e..0cd9f9b 100644 --- a/doc/reference/html.xsl +++ b/doc/reference/html.xsl @@ -1,151 +1,151 @@ - - - - - - - - - - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - - - - - - - - - - - - - - - - - - - <xsl:copy-of select="$title"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + + + + + + + + + + + + + + + + + + + <xsl:copy-of select="$title"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/reference/pdf.xsl b/doc/reference/pdf.xsl index faea7fc..a398e5c 100644 --- a/doc/reference/pdf.xsl +++ b/doc/reference/pdf.xsl @@ -1,73 +1,73 @@ - - - - - - - - - - - - -0pt - - - - -/appendix toc,title -article/appendix nop -/article toc,title -book toc,title,figure,table,example,equation -/chapter toc,title -part toc,title -/preface toc,title -reference toc,title -/sect1 toc -/sect2 toc -/sect3 toc -/sect4 toc -/sect5 toc -/section toc -set toc,title - - - - bold - - - - - - - - - - - + + + + + + + + + + + + +0pt + + + + +/appendix toc,title +article/appendix nop +/article toc,title +book toc,title,figure,table,example,equation +/chapter toc,title +part toc,title +/preface toc,title +reference toc,title +/sect1 toc +/sect2 toc +/sect3 toc +/sect4 toc +/sect5 toc +/section toc +set toc,title + + + + bold + + + + + + + + + + + diff --git a/scripts/scons.bat b/scripts/scons.bat index cf7a5ea..a91cd5d 100644 --- a/scripts/scons.bat +++ b/scripts/scons.bat @@ -1,37 +1,37 @@ -@REM __COPYRIGHT__ -@echo off -set SCONS_ERRORLEVEL= -if "%OS%" == "Windows_NT" goto WinNT - -@REM for 9x/Me you better not have more than 9 args -python -c "from os.path import join; import sys; sys.path = [ join(sys.prefix, 'Lib', 'site-packages', 'scons-__VERSION__'), join(sys.prefix, 'Lib', 'site-packages', 'scons'), join(sys.prefix, 'scons-__VERSION__'), join(sys.prefix, 'scons')] + sys.path; import SCons.Script; SCons.Script.main()" %* -@REM no way to set exit status of this script for 9x/Me -goto endscons - -@REM Credit where credit is due: we return the exit code despite our -@REM use of setlocal+endlocal using a technique from Bear's Journal: -@REM http://code-bear.com/bearlog/2007/06/01/getting-the-exit-code-from-a-batch-file-that-is-run-from-a-python-program/ - -:WinNT -setlocal -@REM ensure the script will be executed with the Python it was installed for -pushd %~dp0.. -set path=%~dp0;%CD%;%path% -popd -@REM try the script named as the .bat file in current dir, then in Scripts subdir -set scriptname=%~dp0%~n0.py -if not exist "%scriptname%" set scriptname=%~dp0Scripts\%~n0.py -@REM Handle when running from wheel where the script has no .py extension -if not exist "%scriptname%" set scriptname=%~dp0%~n0 -python "%scriptname%" %* -endlocal & set SCONS_ERRORLEVEL=%ERRORLEVEL% - -if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto returncode -if errorlevel 9009 echo you do not have python in your PATH -goto endscons - -:returncode -exit /B %SCONS_ERRORLEVEL% - -:endscons -call :returncode %SCONS_ERRORLEVEL% +@REM __COPYRIGHT__ +@echo off +set SCONS_ERRORLEVEL= +if "%OS%" == "Windows_NT" goto WinNT + +@REM for 9x/Me you better not have more than 9 args +python -c "from os.path import join; import sys; sys.path = [ join(sys.prefix, 'Lib', 'site-packages', 'scons-__VERSION__'), join(sys.prefix, 'Lib', 'site-packages', 'scons'), join(sys.prefix, 'scons-__VERSION__'), join(sys.prefix, 'scons')] + sys.path; import SCons.Script; SCons.Script.main()" %* +@REM no way to set exit status of this script for 9x/Me +goto endscons + +@REM Credit where credit is due: we return the exit code despite our +@REM use of setlocal+endlocal using a technique from Bear's Journal: +@REM http://code-bear.com/bearlog/2007/06/01/getting-the-exit-code-from-a-batch-file-that-is-run-from-a-python-program/ + +:WinNT +setlocal +@REM ensure the script will be executed with the Python it was installed for +pushd %~dp0.. +set path=%~dp0;%CD%;%path% +popd +@REM try the script named as the .bat file in current dir, then in Scripts subdir +set scriptname=%~dp0%~n0.py +if not exist "%scriptname%" set scriptname=%~dp0Scripts\%~n0.py +@REM Handle when running from wheel where the script has no .py extension +if not exist "%scriptname%" set scriptname=%~dp0%~n0 +python "%scriptname%" %* +endlocal & set SCONS_ERRORLEVEL=%ERRORLEVEL% + +if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto returncode +if errorlevel 9009 echo you do not have python in your PATH +goto endscons + +:returncode +exit /B %SCONS_ERRORLEVEL% + +:endscons +call :returncode %SCONS_ERRORLEVEL% diff --git a/test/Batch/changed_sources_main.cpp b/test/Batch/changed_sources_main.cpp index 038fea3..810625a 100644 --- a/test/Batch/changed_sources_main.cpp +++ b/test/Batch/changed_sources_main.cpp @@ -1,7 +1,7 @@ - -#include - -int main() -{ - std::cout << "Hello, world!\n"; -} + +#include + +int main() +{ + std::cout << "Hello, world!\n"; +} diff --git a/test/Dir/PyPackageDir/PyPackageDir.py b/test/Dir/PyPackageDir/PyPackageDir.py index ca3bd01..b430ea0 100644 --- a/test/Dir/PyPackageDir/PyPackageDir.py +++ b/test/Dir/PyPackageDir/PyPackageDir.py @@ -1,54 +1,54 @@ -#!/usr/bin/env python -# -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import TestSCons - -test = TestSCons.TestSCons() - -test.dir_fixture('image') - -test.run(arguments = '.', stdout = """\ -scons: Reading SConscript files ... -Test identification of directory for a given python package -testmod1 -. -submod1 -submod1/submod2 -Test parameter substitution -submod1/submod2 -submod1/submod2 -scons: done reading SConscript files. -scons: Building targets ... -scons: `.' is up to date. -scons: done building targets. -""") - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import TestSCons + +test = TestSCons.TestSCons() + +test.dir_fixture('image') + +test.run(arguments = '.', stdout = """\ +scons: Reading SConscript files ... +Test identification of directory for a given python package +testmod1 +. +submod1 +submod1/submod2 +Test parameter substitution +submod1/submod2 +submod1/submod2 +scons: done reading SConscript files. +scons: Building targets ... +scons: `.' is up to date. +scons: done building targets. +""") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Dir/PyPackageDir/image/SConstruct b/test/Dir/PyPackageDir/image/SConstruct index 7e841ac..aceb245 100644 --- a/test/Dir/PyPackageDir/image/SConstruct +++ b/test/Dir/PyPackageDir/image/SConstruct @@ -1,31 +1,31 @@ -import sys, os - -oldsyspath = sys.path -dir_path = Dir('.').srcnode().abspath -dir_path = os.path.join(dir_path, 'syspath') -sys.path.append(dir_path) - -def TestPyPackageDir(env, modname): - packagepath = env.PyPackageDir(modname).abspath - # Convert from an absolute path back to a relative one for testing - commonprefix = os.path.commonprefix([dir_path, packagepath]) - relpath = os.path.relpath(packagepath, commonprefix) - relpath = relpath.replace(os.sep, '/') - print(relpath) - -DefaultEnvironment(tools=[]) - -print("Test identification of directory for a given python package") -env = Environment(tools=[]) -TestPyPackageDir(env, 'testmod1') -TestPyPackageDir(env, 'testmod2') -TestPyPackageDir(env, 'submod1.testmod3') -TestPyPackageDir(env, 'submod1.submod2.testmod4') - -print("Test parameter substitution") -env = Environment(tools=[], FOO = 'submod1.submod2.testmod4') -TestPyPackageDir(env, '${FOO}') -env = Environment(tools=[], FOO = 'submod1.submod2', BAR = 'testmod4') -TestPyPackageDir(env, '${FOO}.${BAR}') - -sys.path = oldsyspath +import sys, os + +oldsyspath = sys.path +dir_path = Dir('.').srcnode().abspath +dir_path = os.path.join(dir_path, 'syspath') +sys.path.append(dir_path) + +def TestPyPackageDir(env, modname): + packagepath = env.PyPackageDir(modname).abspath + # Convert from an absolute path back to a relative one for testing + commonprefix = os.path.commonprefix([dir_path, packagepath]) + relpath = os.path.relpath(packagepath, commonprefix) + relpath = relpath.replace(os.sep, '/') + print(relpath) + +DefaultEnvironment(tools=[]) + +print("Test identification of directory for a given python package") +env = Environment(tools=[]) +TestPyPackageDir(env, 'testmod1') +TestPyPackageDir(env, 'testmod2') +TestPyPackageDir(env, 'submod1.testmod3') +TestPyPackageDir(env, 'submod1.submod2.testmod4') + +print("Test parameter substitution") +env = Environment(tools=[], FOO = 'submod1.submod2.testmod4') +TestPyPackageDir(env, '${FOO}') +env = Environment(tools=[], FOO = 'submod1.submod2', BAR = 'testmod4') +TestPyPackageDir(env, '${FOO}.${BAR}') + +sys.path = oldsyspath diff --git a/test/MSVC/PCH-source.py b/test/MSVC/PCH-source.py index ccab66d..a51dd33 100644 --- a/test/MSVC/PCH-source.py +++ b/test/MSVC/PCH-source.py @@ -1,102 +1,102 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Test use of pre-compiled headers when the source .cpp file shows -up in both the env.PCH() and the env.Program() source list. - -Issue 2505: http://github.com/SCons/scons/issues/2505 -""" - -import TestSCons - -test = TestSCons.TestSCons() - -test.skip_if_not_msvc() - -test.write('SConstruct', """\ -env = Environment(tools=['msvc', 'mslink']) -env['PCH'] = env.PCH('Source1.cpp')[0] -env['PCHSTOP'] = 'Header1.hpp' -env.Program('foo', ['foo.cpp', 'Source2.cpp', 'Source1.cpp']) -""" % locals()) - -test.write('Header1.hpp', r""" -""") - -test.write('Source1.cpp', r""" -#include - -#include "Header1.hpp" - -void -Source1(void) { - printf("Source1.cpp\n"); -} -""") - -test.write('Source2.cpp', r""" -#include - -#include "Header1.hpp" - -void -Source2(void) { - printf("Source2.cpp\n"); -} -""") - -test.write('foo.cpp', r""" -#include - -#include "Header1.hpp" - -void Source1(void); -void Source2(void); - -int -main(int argc, char *argv[]) -{ - Source1(); - Source2(); - printf("foo.cpp\n"); -} -""") - -test.run(arguments = ".") - -test.run(program=test.workpath('foo'+TestSCons._exe), - stdout="Source1.cpp\nSource2.cpp\nfoo.cpp\n") - - - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test use of pre-compiled headers when the source .cpp file shows +up in both the env.PCH() and the env.Program() source list. + +Issue 2505: http://github.com/SCons/scons/issues/2505 +""" + +import TestSCons + +test = TestSCons.TestSCons() + +test.skip_if_not_msvc() + +test.write('SConstruct', """\ +env = Environment(tools=['msvc', 'mslink']) +env['PCH'] = env.PCH('Source1.cpp')[0] +env['PCHSTOP'] = 'Header1.hpp' +env.Program('foo', ['foo.cpp', 'Source2.cpp', 'Source1.cpp']) +""" % locals()) + +test.write('Header1.hpp', r""" +""") + +test.write('Source1.cpp', r""" +#include + +#include "Header1.hpp" + +void +Source1(void) { + printf("Source1.cpp\n"); +} +""") + +test.write('Source2.cpp', r""" +#include + +#include "Header1.hpp" + +void +Source2(void) { + printf("Source2.cpp\n"); +} +""") + +test.write('foo.cpp', r""" +#include + +#include "Header1.hpp" + +void Source1(void); +void Source2(void); + +int +main(int argc, char *argv[]) +{ + Source1(); + Source2(); + printf("foo.cpp\n"); +} +""") + +test.run(arguments = ".") + +test.run(program=test.workpath('foo'+TestSCons._exe), + stdout="Source1.cpp\nSource2.cpp\nfoo.cpp\n") + + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/MSVC/embed-manifest.py b/test/MSVC/embed-manifest.py index c0f8408..b4f5314 100644 --- a/test/MSVC/embed-manifest.py +++ b/test/MSVC/embed-manifest.py @@ -1,94 +1,94 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Verify that manifest files get embedded correctly in EXEs and DLLs -""" - -import TestSCons - -_exe = TestSCons._exe -_dll = TestSCons._dll -_lib = TestSCons._lib - -test = TestSCons.TestSCons() - -test.skip_if_not_msvc() - -test.write('SConstruct', """\ -env=Environment(WINDOWS_EMBED_MANIFEST=True) -env.Append(CCFLAGS = '/MD') -env.Append(LINKFLAGS = '/MANIFEST') -env.Append(SHLINKFLAGS = '/MANIFEST') -exe=env.Program('test.cpp') -dll=env.SharedLibrary('testdll.cpp') -env.Command('exe-extracted.manifest', exe, - '$MT /nologo -inputresource:${SOURCE};1 -out:${TARGET}') -env.Command('dll-extracted.manifest', dll, - '$MT /nologo -inputresource:${SOURCE};2 -out:${TARGET}') -env2=Environment(WINDOWS_EMBED_MANIFEST=True) # no /MD here -env2.Program('test-nomanifest', env2.Object('test-nomanifest', 'test.cpp')) -""") - -test.write('test.cpp', """\ -#include -#include -int -main(int argc, char *argv) -{ - printf("test.cpp\\n"); - exit (0); -} -""") - -test.write('testdll.cpp', """\ -#include -#include - -__declspec(dllexport) int -testdll(int argc, char *argv) -{ - printf("testdll.cpp\\n"); - return 0; -} -""") - -test.run(arguments='.') - -test.must_exist('test%s' % _exe) -test.must_exist('test%s.manifest' % _exe) -test.must_contain('exe-extracted.manifest', '', mode='r') -test.must_exist('testdll%s' % _dll) -test.must_exist('testdll%s.manifest' % _dll) -test.must_contain('dll-extracted.manifest', '', mode='r') - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify that manifest files get embedded correctly in EXEs and DLLs +""" + +import TestSCons + +_exe = TestSCons._exe +_dll = TestSCons._dll +_lib = TestSCons._lib + +test = TestSCons.TestSCons() + +test.skip_if_not_msvc() + +test.write('SConstruct', """\ +env=Environment(WINDOWS_EMBED_MANIFEST=True) +env.Append(CCFLAGS = '/MD') +env.Append(LINKFLAGS = '/MANIFEST') +env.Append(SHLINKFLAGS = '/MANIFEST') +exe=env.Program('test.cpp') +dll=env.SharedLibrary('testdll.cpp') +env.Command('exe-extracted.manifest', exe, + '$MT /nologo -inputresource:${SOURCE};1 -out:${TARGET}') +env.Command('dll-extracted.manifest', dll, + '$MT /nologo -inputresource:${SOURCE};2 -out:${TARGET}') +env2=Environment(WINDOWS_EMBED_MANIFEST=True) # no /MD here +env2.Program('test-nomanifest', env2.Object('test-nomanifest', 'test.cpp')) +""") + +test.write('test.cpp', """\ +#include +#include +int +main(int argc, char *argv) +{ + printf("test.cpp\\n"); + exit (0); +} +""") + +test.write('testdll.cpp', """\ +#include +#include + +__declspec(dllexport) int +testdll(int argc, char *argv) +{ + printf("testdll.cpp\\n"); + return 0; +} +""") + +test.run(arguments='.') + +test.must_exist('test%s' % _exe) +test.must_exist('test%s.manifest' % _exe) +test.must_contain('exe-extracted.manifest', '', mode='r') +test.must_exist('testdll%s' % _dll) +test.must_exist('testdll%s.manifest' % _dll) +test.must_contain('dll-extracted.manifest', '', mode='r') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/MSVS/CPPPATH-Dirs.py b/test/MSVS/CPPPATH-Dirs.py index 53c7a8b..3883331 100644 --- a/test/MSVS/CPPPATH-Dirs.py +++ b/test/MSVS/CPPPATH-Dirs.py @@ -1,94 +1,94 @@ - -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -""" -Test that MSVS generation works when CPPPATH contains Dir nodes. -Also make sure changing CPPPATH causes rebuild. -""" - -import os -import sys - -import TestSConsMSVS - -test = TestSConsMSVS.TestSConsMSVS() - -if sys.platform != 'win32': - msg = "Skipping Visual Studio test on non-Windows platform '%s'\n" % sys.platform - test.skip_test(msg) - -import SCons.Tool.MSCommon as msc -if not msc.msvs_exists(): - msg = "No MSVS toolchain found...skipping test\n" - test.skip_test(msg) - -SConscript_contents = """\ -env = Environment() - -sources = ['main.cpp'] - -program = env.Program(target = 'hello', source = sources) - -if ARGUMENTS.get('moreincludes'): - env.AppendUnique(CPPPATH = [env.Dir('.'), env.Dir('myincludes')]) -else: - env.AppendUnique(CPPPATH = [env.Dir('.')]) - -env.MSVSProject(target = 'Hello' + env['MSVSPROJECTSUFFIX'], - srcs = sources, - buildtarget = program, - variant = 'Release') -""" - -test.write('SConstruct', SConscript_contents) - -test.write('main.cpp', """\ -#include -int main(void) { - printf("hello, world!\\n"); -} -""") - -test.run() - -if not os.path.exists(test.workpath('Hello.vcproj')) and \ - not os.path.exists(test.workpath('Hello.vcxproj')): - test.fail_test("Failed to create Visual Studio project Hello.vcproj or Hello.vcxproj") -test.must_exist(test.workpath('Hello.sln')) -# vcproj = test.read('Test.vcproj', 'r') - -test.run(arguments='moreincludes=1') -test.must_not_contain_any_line(test.stdout(), ['is up to date']) -test.must_contain_all_lines(test.stdout(), ['Adding', 'Hello']) - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: + +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test that MSVS generation works when CPPPATH contains Dir nodes. +Also make sure changing CPPPATH causes rebuild. +""" + +import os +import sys + +import TestSConsMSVS + +test = TestSConsMSVS.TestSConsMSVS() + +if sys.platform != 'win32': + msg = "Skipping Visual Studio test on non-Windows platform '%s'\n" % sys.platform + test.skip_test(msg) + +import SCons.Tool.MSCommon as msc +if not msc.msvs_exists(): + msg = "No MSVS toolchain found...skipping test\n" + test.skip_test(msg) + +SConscript_contents = """\ +env = Environment() + +sources = ['main.cpp'] + +program = env.Program(target = 'hello', source = sources) + +if ARGUMENTS.get('moreincludes'): + env.AppendUnique(CPPPATH = [env.Dir('.'), env.Dir('myincludes')]) +else: + env.AppendUnique(CPPPATH = [env.Dir('.')]) + +env.MSVSProject(target = 'Hello' + env['MSVSPROJECTSUFFIX'], + srcs = sources, + buildtarget = program, + variant = 'Release') +""" + +test.write('SConstruct', SConscript_contents) + +test.write('main.cpp', """\ +#include +int main(void) { + printf("hello, world!\\n"); +} +""") + +test.run() + +if not os.path.exists(test.workpath('Hello.vcproj')) and \ + not os.path.exists(test.workpath('Hello.vcxproj')): + test.fail_test("Failed to create Visual Studio project Hello.vcproj or Hello.vcxproj") +test.must_exist(test.workpath('Hello.sln')) +# vcproj = test.read('Test.vcproj', 'r') + +test.run(arguments='moreincludes=1') +test.must_not_contain_any_line(test.stdout(), ['is up to date']) +test.must_contain_all_lines(test.stdout(), ['Adding', 'Hello']) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/NoClean.py b/test/NoClean.py index fe6a2bc..1cf9577 100644 --- a/test/NoClean.py +++ b/test/NoClean.py @@ -1,97 +1,97 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -# -# This test ensures that NoClean works correctly, even when it's applied to -# a single target in the return list of an multi-target Builder. -# -import TestSCons - -test = TestSCons.TestSCons() - -test.write('SConstruct', """ -def action(target, source, env): - for t in target: - with open(t.get_internal_path(), 'w'): - pass -Command('1.out', 'SConstruct', action) -NoClean('1.out') -""") - -test.write('SConstruct.force', """ -def action(target, source, env): - for t in target: - with open(t.get_internal_path(), 'w'): - pass - with open('4.out', 'w'): - pass -res = Command('3.out', 'SConstruct.force', action) -Clean('4.out', res) -NoClean('4.out') -""") - -test.write('SConstruct.multi', """ -def action(target, source, env): - for t in target: - with open(t.get_internal_path(), 'w'): - pass -Command(['5.out', '6.out'], 'SConstruct.multi', action) -NoClean('6.out') -""") - -# -# Basic check: NoClean keeps files -# -test.run() -test.run(arguments='-c') - -test.must_exist('1.out') - -# -# Check: NoClean overrides Clean -# -test.run(arguments=['-f', 'SConstruct.force']) -test.run(arguments=['-f', 'SConstruct.force', '-c']) - -test.must_not_exist('3.out') -test.must_exist('4.out') - -# -# Check: NoClean works for multi-target Builders -# -test.run(arguments=['-f', 'SConstruct.multi']) -test.run(arguments=['-f', 'SConstruct.multi', '-c']) - -test.must_not_exist('5.out') -test.must_exist('6.out') - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +# +# This test ensures that NoClean works correctly, even when it's applied to +# a single target in the return list of an multi-target Builder. +# +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """ +def action(target, source, env): + for t in target: + with open(t.get_internal_path(), 'w'): + pass +Command('1.out', 'SConstruct', action) +NoClean('1.out') +""") + +test.write('SConstruct.force', """ +def action(target, source, env): + for t in target: + with open(t.get_internal_path(), 'w'): + pass + with open('4.out', 'w'): + pass +res = Command('3.out', 'SConstruct.force', action) +Clean('4.out', res) +NoClean('4.out') +""") + +test.write('SConstruct.multi', """ +def action(target, source, env): + for t in target: + with open(t.get_internal_path(), 'w'): + pass +Command(['5.out', '6.out'], 'SConstruct.multi', action) +NoClean('6.out') +""") + +# +# Basic check: NoClean keeps files +# +test.run() +test.run(arguments='-c') + +test.must_exist('1.out') + +# +# Check: NoClean overrides Clean +# +test.run(arguments=['-f', 'SConstruct.force']) +test.run(arguments=['-f', 'SConstruct.force', '-c']) + +test.must_not_exist('3.out') +test.must_exist('4.out') + +# +# Check: NoClean works for multi-target Builders +# +test.run(arguments=['-f', 'SConstruct.multi']) +test.run(arguments=['-f', 'SConstruct.multi', '-c']) + +test.must_not_exist('5.out') +test.must_exist('6.out') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool1.py b/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool1.py index 072daf0..1c024e1 100644 --- a/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool1.py +++ b/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool2/__init__.py b/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool2/__init__.py index f4ccefe..225b9aa 100644 --- a/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool2/__init__.py +++ b/test/toolpath/nested/image/Libs/tools_example/Toolpath_TestTool2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_1.py b/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_1.py index 2a70e67..c266f89 100644 --- a/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_1.py +++ b/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_2/__init__.py b/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_2/__init__.py index 424991f..f203260 100644 --- a/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_2/__init__.py +++ b/test/toolpath/nested/image/Libs/tools_example/subdir1/Toolpath_TestTool1_2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1_2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1_2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_1.py b/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_1.py index 13d0496..b1b47a2 100644 --- a/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_1.py +++ b/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py b/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py index 3f8fd5e..407df86 100644 --- a/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py +++ b/test/toolpath/nested/image/Libs/tools_example/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2_2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2_2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/SConstruct b/test/toolpath/nested/image/SConstruct index 1187a7c..7fac870 100644 --- a/test/toolpath/nested/image/SConstruct +++ b/test/toolpath/nested/image/SConstruct @@ -1,69 +1,69 @@ -import sys, os - -toollist = ['Toolpath_TestTool1', - 'Toolpath_TestTool2', - 'subdir1.Toolpath_TestTool1_1', - 'subdir1.Toolpath_TestTool1_2', - 'subdir1.subdir2.Toolpath_TestTool2_1', - 'subdir1.subdir2.Toolpath_TestTool2_2', - ] - -print('Test where tools are located under site_scons/site_tools') -env1 = Environment(tools=toollist) -print("env1['Toolpath_TestTool1'] = %s"%env1.get('Toolpath_TestTool1')) -print("env1['Toolpath_TestTool2'] = %s"%env1.get('Toolpath_TestTool2')) -print("env1['Toolpath_TestTool1_1'] = %s"%env1.get('Toolpath_TestTool1_1')) -print("env1['Toolpath_TestTool1_2'] = %s"%env1.get('Toolpath_TestTool1_2')) -print("env1['Toolpath_TestTool2_1'] = %s"%env1.get('Toolpath_TestTool2_1')) -print("env1['Toolpath_TestTool2_2'] = %s"%env1.get('Toolpath_TestTool2_2')) - -print('Test where toolpath is set in the env constructor') -env2 = Environment(tools=toollist, toolpath=['Libs/tools_example']) -print("env2['Toolpath_TestTool1'] = %s"%env2.get('Toolpath_TestTool1')) -print("env2['Toolpath_TestTool2'] = %s"%env2.get('Toolpath_TestTool2')) -print("env2['Toolpath_TestTool1_1'] = %s"%env2.get('Toolpath_TestTool1_1')) -print("env2['Toolpath_TestTool1_2'] = %s"%env2.get('Toolpath_TestTool1_2')) -print("env2['Toolpath_TestTool2_1'] = %s"%env2.get('Toolpath_TestTool2_1')) -print("env2['Toolpath_TestTool2_2'] = %s"%env2.get('Toolpath_TestTool2_2')) - -print('Test a Clone') -base = Environment(tools=[], toolpath=['Libs/tools_example']) -derived = base.Clone(tools=['subdir1.Toolpath_TestTool1_1']) -print("derived['Toolpath_TestTool1_1'] = %s"%derived.get('Toolpath_TestTool1_1')) - - -print('Test using syspath as the toolpath') -print('Lets pretend that tools_example within Libs is actually a module installed via pip') -oldsyspath = sys.path -dir_path = Dir('.').srcnode().abspath -dir_path = os.path.join(dir_path, 'Libs') -sys.path.append(dir_path) - -searchpaths = [] -for item in sys.path: - if os.path.isdir(item): searchpaths.append(item) - -toollist = ['tools_example.Toolpath_TestTool1', - 'tools_example.Toolpath_TestTool2', - 'tools_example.subdir1.Toolpath_TestTool1_1', - 'tools_example.subdir1.Toolpath_TestTool1_2', - 'tools_example.subdir1.subdir2.Toolpath_TestTool2_1', - 'tools_example.subdir1.subdir2.Toolpath_TestTool2_2', - ] - -env3 = Environment(tools=toollist, toolpath=searchpaths) -print("env3['Toolpath_TestTool1'] = %s"%env3.get('Toolpath_TestTool1')) -print("env3['Toolpath_TestTool2'] = %s"%env3.get('Toolpath_TestTool2')) -print("env3['Toolpath_TestTool1_1'] = %s"%env3.get('Toolpath_TestTool1_1')) -print("env3['Toolpath_TestTool1_2'] = %s"%env3.get('Toolpath_TestTool1_2')) -print("env3['Toolpath_TestTool2_1'] = %s"%env3.get('Toolpath_TestTool2_1')) -print("env3['Toolpath_TestTool2_2'] = %s"%env3.get('Toolpath_TestTool2_2')) - - -print('Test using PyPackageDir') -toollist = ['Toolpath_TestTool2_1', 'Toolpath_TestTool2_2'] -env4 = Environment(tools = toollist, toolpath = [PyPackageDir('tools_example.subdir1.subdir2')]) -print("env4['Toolpath_TestTool2_1'] = %s"%env4.get('Toolpath_TestTool2_1')) -print("env4['Toolpath_TestTool2_2'] = %s"%env4.get('Toolpath_TestTool2_2')) - -sys.path = oldsyspath +import sys, os + +toollist = ['Toolpath_TestTool1', + 'Toolpath_TestTool2', + 'subdir1.Toolpath_TestTool1_1', + 'subdir1.Toolpath_TestTool1_2', + 'subdir1.subdir2.Toolpath_TestTool2_1', + 'subdir1.subdir2.Toolpath_TestTool2_2', + ] + +print('Test where tools are located under site_scons/site_tools') +env1 = Environment(tools=toollist) +print("env1['Toolpath_TestTool1'] = %s"%env1.get('Toolpath_TestTool1')) +print("env1['Toolpath_TestTool2'] = %s"%env1.get('Toolpath_TestTool2')) +print("env1['Toolpath_TestTool1_1'] = %s"%env1.get('Toolpath_TestTool1_1')) +print("env1['Toolpath_TestTool1_2'] = %s"%env1.get('Toolpath_TestTool1_2')) +print("env1['Toolpath_TestTool2_1'] = %s"%env1.get('Toolpath_TestTool2_1')) +print("env1['Toolpath_TestTool2_2'] = %s"%env1.get('Toolpath_TestTool2_2')) + +print('Test where toolpath is set in the env constructor') +env2 = Environment(tools=toollist, toolpath=['Libs/tools_example']) +print("env2['Toolpath_TestTool1'] = %s"%env2.get('Toolpath_TestTool1')) +print("env2['Toolpath_TestTool2'] = %s"%env2.get('Toolpath_TestTool2')) +print("env2['Toolpath_TestTool1_1'] = %s"%env2.get('Toolpath_TestTool1_1')) +print("env2['Toolpath_TestTool1_2'] = %s"%env2.get('Toolpath_TestTool1_2')) +print("env2['Toolpath_TestTool2_1'] = %s"%env2.get('Toolpath_TestTool2_1')) +print("env2['Toolpath_TestTool2_2'] = %s"%env2.get('Toolpath_TestTool2_2')) + +print('Test a Clone') +base = Environment(tools=[], toolpath=['Libs/tools_example']) +derived = base.Clone(tools=['subdir1.Toolpath_TestTool1_1']) +print("derived['Toolpath_TestTool1_1'] = %s"%derived.get('Toolpath_TestTool1_1')) + + +print('Test using syspath as the toolpath') +print('Lets pretend that tools_example within Libs is actually a module installed via pip') +oldsyspath = sys.path +dir_path = Dir('.').srcnode().abspath +dir_path = os.path.join(dir_path, 'Libs') +sys.path.append(dir_path) + +searchpaths = [] +for item in sys.path: + if os.path.isdir(item): searchpaths.append(item) + +toollist = ['tools_example.Toolpath_TestTool1', + 'tools_example.Toolpath_TestTool2', + 'tools_example.subdir1.Toolpath_TestTool1_1', + 'tools_example.subdir1.Toolpath_TestTool1_2', + 'tools_example.subdir1.subdir2.Toolpath_TestTool2_1', + 'tools_example.subdir1.subdir2.Toolpath_TestTool2_2', + ] + +env3 = Environment(tools=toollist, toolpath=searchpaths) +print("env3['Toolpath_TestTool1'] = %s"%env3.get('Toolpath_TestTool1')) +print("env3['Toolpath_TestTool2'] = %s"%env3.get('Toolpath_TestTool2')) +print("env3['Toolpath_TestTool1_1'] = %s"%env3.get('Toolpath_TestTool1_1')) +print("env3['Toolpath_TestTool1_2'] = %s"%env3.get('Toolpath_TestTool1_2')) +print("env3['Toolpath_TestTool2_1'] = %s"%env3.get('Toolpath_TestTool2_1')) +print("env3['Toolpath_TestTool2_2'] = %s"%env3.get('Toolpath_TestTool2_2')) + + +print('Test using PyPackageDir') +toollist = ['Toolpath_TestTool2_1', 'Toolpath_TestTool2_2'] +env4 = Environment(tools = toollist, toolpath = [PyPackageDir('tools_example.subdir1.subdir2')]) +print("env4['Toolpath_TestTool2_1'] = %s"%env4.get('Toolpath_TestTool2_1')) +print("env4['Toolpath_TestTool2_2'] = %s"%env4.get('Toolpath_TestTool2_2')) + +sys.path = oldsyspath diff --git a/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool1.py b/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool1.py index 072daf0..1c024e1 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool1.py +++ b/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool2/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool2/__init__.py index f4ccefe..225b9aa 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool2/__init__.py +++ b/test/toolpath/nested/image/site_scons/site_tools/Toolpath_TestTool2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_1.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_1.py index 2a70e67..c266f89 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_1.py +++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_2/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_2/__init__.py index 424991f..f203260 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_2/__init__.py +++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/Toolpath_TestTool1_2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool1_2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool1_2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_1.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_1.py index 13d0496..b1b47a2 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_1.py +++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py index 3f8fd5e..407df86 100644 --- a/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py +++ b/test/toolpath/nested/image/site_scons/site_tools/subdir1/subdir2/Toolpath_TestTool2_2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['Toolpath_TestTool2_2'] = 1 -def exists(env): - return 1 +def generate(env): + env['Toolpath_TestTool2_2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/relative_import/image/SConstruct b/test/toolpath/relative_import/image/SConstruct index c9b28bb..cf607d1 100644 --- a/test/toolpath/relative_import/image/SConstruct +++ b/test/toolpath/relative_import/image/SConstruct @@ -1,10 +1,10 @@ -env = Environment(tools=['TestTool1', 'TestTool1.TestTool1_2'], toolpath=['tools']) - -# Test a relative import within the root of the tools directory -print("env['TestTool1'] = %s"%env.get('TestTool1')) -print("env['TestTool1_1'] = %s"%env.get('TestTool1_1')) - -# Test a relative import within a sub dir -print("env['TestTool1_2'] = %s"%env.get('TestTool1_2')) -print("env['TestTool1_2_1'] = %s"%env.get('TestTool1_2_1')) -print("env['TestTool1_2_2'] = %s"%env.get('TestTool1_2_2')) +env = Environment(tools=['TestTool1', 'TestTool1.TestTool1_2'], toolpath=['tools']) + +# Test a relative import within the root of the tools directory +print("env['TestTool1'] = %s"%env.get('TestTool1')) +print("env['TestTool1_1'] = %s"%env.get('TestTool1_1')) + +# Test a relative import within a sub dir +print("env['TestTool1_2'] = %s"%env.get('TestTool1_2')) +print("env['TestTool1_2_1'] = %s"%env.get('TestTool1_2_1')) +print("env['TestTool1_2_2'] = %s"%env.get('TestTool1_2_2')) diff --git a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_1.py b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_1.py index 4c9a7bc..584d353 100644 --- a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_1.py +++ b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['TestTool1_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['TestTool1_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_1.py b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_1.py index e65f8cd..62d754b 100644 --- a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_1.py +++ b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_1.py @@ -1,4 +1,4 @@ -def generate(env): - env['TestTool1_2_1'] = 1 -def exists(env): - return 1 +def generate(env): + env['TestTool1_2_1'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_2/__init__.py b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_2/__init__.py index 463baee..12c6018 100644 --- a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_2/__init__.py +++ b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/TestTool1_2_2/__init__.py @@ -1,4 +1,4 @@ -def generate(env): - env['TestTool1_2_2'] = 1 -def exists(env): - return 1 +def generate(env): + env['TestTool1_2_2'] = 1 +def exists(env): + return 1 diff --git a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/__init__.py b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/__init__.py index 8bd698f..58fdc93 100644 --- a/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/__init__.py +++ b/test/toolpath/relative_import/image/tools/TestTool1/TestTool1_2/__init__.py @@ -1,11 +1,11 @@ -from . import TestTool1_2_1 -from . import TestTool1_2_2 - -def generate(env): - env['TestTool1_2'] = 1 - TestTool1_2_1.generate(env) - TestTool1_2_2.generate(env) -def exists(env): - TestTool1_2_1.exists(env) - TestTool1_2_2.exists(env) - return 1 +from . import TestTool1_2_1 +from . import TestTool1_2_2 + +def generate(env): + env['TestTool1_2'] = 1 + TestTool1_2_1.generate(env) + TestTool1_2_2.generate(env) +def exists(env): + TestTool1_2_1.exists(env) + TestTool1_2_2.exists(env) + return 1 diff --git a/test/toolpath/relative_import/image/tools/TestTool1/__init__.py b/test/toolpath/relative_import/image/tools/TestTool1/__init__.py index d5510d2..c479560 100644 --- a/test/toolpath/relative_import/image/tools/TestTool1/__init__.py +++ b/test/toolpath/relative_import/image/tools/TestTool1/__init__.py @@ -1,9 +1,9 @@ -from . import TestTool1_1 - -def generate(env): - env['TestTool1'] = 1 - # Include another tool within the same directory - TestTool1_1.generate(env) -def exists(env): - TestTool1_1.exists(env) - return 1 +from . import TestTool1_1 + +def generate(env): + env['TestTool1'] = 1 + # Include another tool within the same directory + TestTool1_1.generate(env) +def exists(env): + TestTool1_1.exists(env) + return 1 diff --git a/test/toolpath/relative_import/relative_import.py b/test/toolpath/relative_import/relative_import.py index f4362f7..43ae6cf 100644 --- a/test/toolpath/relative_import/relative_import.py +++ b/test/toolpath/relative_import/relative_import.py @@ -1,52 +1,52 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import TestSCons - -test = TestSCons.TestSCons() - -test.dir_fixture('image') - -test.run(arguments = '.', stdout = """\ -scons: Reading SConscript files ... -env['TestTool1'] = 1 -env['TestTool1_1'] = 1 -env['TestTool1_2'] = 1 -env['TestTool1_2_1'] = 1 -env['TestTool1_2_2'] = 1 -scons: done reading SConscript files. -scons: Building targets ... -scons: `.' is up to date. -scons: done building targets. -""") - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import TestSCons + +test = TestSCons.TestSCons() + +test.dir_fixture('image') + +test.run(arguments = '.', stdout = """\ +scons: Reading SConscript files ... +env['TestTool1'] = 1 +env['TestTool1_1'] = 1 +env['TestTool1_2'] = 1 +env['TestTool1_2_1'] = 1 +env['TestTool1_2_2'] = 1 +scons: done reading SConscript files. +scons: Building targets ... +scons: `.' is up to date. +scons: done building targets. +""") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 4cfbbb3dccbeff6834305e5745530e6ffd380482 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 20 Aug 2022 11:18:14 -0600 Subject: .git-blame-ignore-revs in separate commit [skip appveyor] Forgot that the commit id changes if you amend, so needed to put the previous commit - with the rest of the changes - into the ignore file as a fresh commit. Signed-off-by: Mats Wichmann --- .git-blame-ignore-revs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index e78dce9..538cb23 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,2 @@ # files reformatted from DOS line-endings -c42bc3d61d29cce6b3c7b6917a2d7b5cd33a7459 +1277d8e5ab6457ed18d291100539f31d1bdb2d7c -- cgit v0.12 From 6884a878e930c867f3946a38e3da2e930e2f2c8b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 21 Aug 2022 06:56:41 -0600 Subject: Promote sphinx-book-theme to preferred [skip appveyor] This will now be used for the API docs build. Signed-off-by: Mats Wichmann --- .github/workflows/scons-package.yml | 2 -- requirements-pkg.txt | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/scons-package.yml b/.github/workflows/scons-package.yml index ff71862..91acecb 100644 --- a/.github/workflows/scons-package.yml +++ b/.github/workflows/scons-package.yml @@ -24,8 +24,6 @@ jobs: python -m pip install --upgrade pip setuptools wheel #python -m pip install flake8 pytest if [ -f requirements-pkg.txt ]; then pip install -r requirements-pkg.txt; elif [ -f requirements.txt ]; then pip install -r requirements.txt; fi - #XXX if we keep this, merge into requirements-pkg.txt - pip install sphinx-book-theme sudo apt-get update sudo apt-get -y install docbook-xml docbook-xsl xsltproc fop docbook-xsl-doc-pdf # try to keep the texlive install as small as we can to save some time/space diff --git a/requirements-pkg.txt b/requirements-pkg.txt index 3c2c385..10b5393 100644 --- a/requirements-pkg.txt +++ b/requirements-pkg.txt @@ -9,5 +9,5 @@ readme-renderer # sphinx pinned because it has broken several times on new releases sphinx>=5.1.1 -sphinx_rtd_theme +sphinx-book-theme rst2pdf -- cgit v0.12 From f1f36b69ce5594b0e5a6d56b731a4df4e2ae5321 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 21 Aug 2022 07:46:36 -0600 Subject: Needed to turn off sphinx-rtd-theme another place [skip appveyor] Signed-off-by: Mats Wichmann --- doc/sphinx/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 7f304e0..5ad5448 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -38,7 +38,7 @@ extensions = [ 'sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.viewcode', - 'sphinx_rtd_theme', + #'sphinx_rtd_theme', 'rst2pdf.pdfbuilder', ] -- cgit v0.12 From 02d7ef2030eb6836b65a6f1e832344bd68ea33a4 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 23 Aug 2022 08:50:46 -0600 Subject: Tweak --hash-format manpage entry [skip appveyor] Previously, there are some wording problems and one paragraph was essentially a restatement of an earlier one. Also corrected the claim that --hash-format=md5 would create an unadorned .sconsign.dblite - tested to show it is actually .sconsign_md5.dblite Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index e7f6db0..44bb852 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1205,26 +1205,33 @@ be appropriate for most uses. Set the hashing algorithm used by SCons to ALGORITHM. This value determines the hashing algorithm used in generating -&contentsigs; or &CacheDir; keys. +&contentsigs;, &buildsigs; and &CacheDir; keys. -The supported list of values are: md5, sha1, and sha256. +The supported list of values are: +md5, +sha1 +and sha256. However, the Python interpreter used to run SCons must have the corresponding support available in the hashlib module to use the specified algorithm. -Specifying this value changes the name of the SConsign database. -For example, will create a SConsign -database with name .sconsign_sha256.dblite. - -If this option is not specified, a the first supported hash format found -is selected. Typically this is MD5, however, if you are on a FIPS-compliant system -and using a version of Python less than 3.9, SHA1 or SHA256 will be chosen as the default. -Python 3.9 and onwards clients will always default to MD5, even in FIPS mode, unless -otherwise specified with the option. +If this option is omitted, +the first supported hash format found is selected. +Typically this is MD5, however, on a FIPS-compliant system +using a version of Python less than 3.9, +SHA1 or SHA256 will be chosen as the default. +Python 3.9 and onwards clients will always default to MD5, even in FIPS mode. + -For MD5 databases (either explicitly specified with or -defaulted), the SConsign database is.sconsign.dblite. The newer SHA1 and -SHA256 selections meanwhile store their databases to .sconsign_algorithmname.dblite +Specifying this option changes the name of the SConsign database. +The default database is .sconsign.dblite; +in the presence of this option, +ALGORITHM will +be included in the name to indicate the difference, +even if the argument is md5. +For example, will create a SConsign +database named .sconsign_sha256.dblite. + Available since &scons; 4.2. -- cgit v0.12 From f1b8f95b6c687588b4368f31d3872916ad7eac55 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 23 Aug 2022 09:37:36 -0600 Subject: Check in forgotten entity definition [skip appveyor] Added a plural of build signature to scons.mod and forgot to check in the change to that file, so validation failed on the CI run. Signed-off-by: Mats Wichmann --- doc/scons.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/scons.mod b/doc/scons.mod index 0c1a85e..ea1decc 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -453,6 +453,7 @@ content signature"> content signatures"> build signature"> +build signatures"> true"> false"> -- cgit v0.12 From e9f7759b5571863e5bfd72c35209130a1e9f7764 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 23 Aug 2022 10:22:19 -0600 Subject: manpage hash-format - use present tense [skip appveyor] Since we're here anyway, use present tense (rather than future) for the --hash-format section. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 44bb852..6ab82d2 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1218,18 +1218,18 @@ to use the specified algorithm. If this option is omitted, the first supported hash format found is selected. Typically this is MD5, however, on a FIPS-compliant system -using a version of Python less than 3.9, -SHA1 or SHA256 will be chosen as the default. -Python 3.9 and onwards clients will always default to MD5, even in FIPS mode. +using a version of Python older than 3.9, +SHA1 or SHA256 is chosen as the default. +Python 3.9 and onwards clients always default to MD5, even in FIPS mode. Specifying this option changes the name of the SConsign database. -The default database is .sconsign.dblite; -in the presence of this option, -ALGORITHM will +The default database is .sconsign.dblite. +In the presence of this option, +ALGORITHM is be included in the name to indicate the difference, even if the argument is md5. -For example, will create a SConsign +For example, uses a SConsign database named .sconsign_sha256.dblite. -- cgit v0.12 From e1b9b1fe8759912961651bf069b13a6ec040bea6 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 23 Aug 2022 10:26:00 -0600 Subject: manpage - fix grammar problem [skip appveyor] A stray word wasn't immediately spotted as it appeared on the next line. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 6ab82d2..2155e9f 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1227,7 +1227,7 @@ Python 3.9 and onwards clients always default to MD5, even in FIPS mode. The default database is .sconsign.dblite. In the presence of this option, ALGORITHM is -be included in the name to indicate the difference, +included in the name to indicate the difference, even if the argument is md5. For example, uses a SConsign database named .sconsign_sha256.dblite. -- cgit v0.12 From 15f01c65f35af1f1defae386f1fccc3070397e00 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sun, 4 Sep 2022 18:10:00 +0300 Subject: Import functions with adding SConscript namespace --- SCons/Script/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index aed31a5..c2c5e2d 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -126,8 +126,7 @@ GetBuildFailures = Main.GetBuildFailures #profiling = Main.profiling #repositories = Main.repositories -from . import SConscript -_SConscript = SConscript +from . import SConscript as _SConscript call_stack = _SConscript.call_stack -- cgit v0.12 From 061a76ccb6bf605022292b88ffe535c75c407d82 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sun, 4 Sep 2022 18:20:33 +0300 Subject: Exec `EnsureSconsVersion` without init of default environment --- SCons/Script/SConscript.py | 8 +++++--- SCons/Script/__init__.py | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 548ef44..a85c12c 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -390,7 +390,8 @@ class SConsEnvironment(SCons.Environment.Base): in 'v_major' and 'v_minor', and 0 otherwise.""" return (major > v_major or (major == v_major and minor > v_minor)) - def _get_major_minor_revision(self, version_string): + @staticmethod + def _get_major_minor_revision(version_string): """Split a version string into major, minor and (optionally) revision parts. @@ -488,14 +489,15 @@ class SConsEnvironment(SCons.Environment.Base): def Default(self, *targets): SCons.Script._Set_Default_Targets(self, targets) - def EnsureSConsVersion(self, major, minor, revision=0): + @staticmethod + def EnsureSConsVersion(major, minor, revision=0): """Exit abnormally if the SCons version is not late enough.""" # split string to avoid replacement during build process if SCons.__version__ == '__' + 'VERSION__': SCons.Warnings.warn(SCons.Warnings.DevelopmentVersionWarning, "EnsureSConsVersion is ignored for development version") return - scons_ver = self._get_major_minor_revision(SCons.__version__) + scons_ver = SConsEnvironment._get_major_minor_revision(SCons.__version__) if scons_ver < (major, minor, revision): if revision: scons_ver_string = '%d.%d.%d' % (major, minor, revision) diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index c2c5e2d..029dc8e 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -287,14 +287,17 @@ def Variables(files=None, args=ARGUMENTS): return SCons.Variables.Variables(files, args) -# The list of global functions to add to the SConscript name space -# that end up calling corresponding methods or Builders in the +# Adding global functions to the SConscript name space. +# +# Static functions that do not use state in DefaultEnvironment(). +EnsureSConsVersion = _SConscript.SConsEnvironment.EnsureSConsVersion + +# Functions that end up calling methods or Builders in the # DefaultEnvironment(). GlobalDefaultEnvironmentFunctions = [ # Methods from the SConsEnvironment class, above. 'Default', 'EnsurePythonVersion', - 'EnsureSConsVersion', 'Exit', 'Export', 'GetLaunchDir', -- cgit v0.12 From d181182262be9b2262c781afff1f7e020f6f60de Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sun, 4 Sep 2022 19:00:36 +0300 Subject: Add CHANGES.txt entry --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 957e683..d46a5d1 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,6 +23,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Ryan Saunders - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. + From Anatoli Babenia + - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() + RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From Joseph Brill: -- cgit v0.12 From 991c022c301f2bd2aa3bc3d7b0b0bef9f2e9da82 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Sun, 4 Sep 2022 19:14:14 +0300 Subject: Make EnsurePythonVersion static too --- CHANGES.txt | 3 ++- SCons/Script/SConscript.py | 3 ++- SCons/Script/__init__.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d46a5d1..d26b78c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -24,7 +24,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. From Anatoli Babenia - - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() + - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and + EnsurePythonVersion(). RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index a85c12c..99bb84d 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -507,7 +507,8 @@ class SConsEnvironment(SCons.Environment.Base): (scons_ver_string, SCons.__version__)) sys.exit(2) - def EnsurePythonVersion(self, major, minor): + @staticmethod + def EnsurePythonVersion(major, minor): """Exit abnormally if the Python version is not late enough.""" if sys.version_info < (major, minor): v = sys.version.split()[0] diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index 029dc8e..0250bd1 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -291,13 +291,13 @@ def Variables(files=None, args=ARGUMENTS): # # Static functions that do not use state in DefaultEnvironment(). EnsureSConsVersion = _SConscript.SConsEnvironment.EnsureSConsVersion +EnsurePythonVersion = _SConscript.SConsEnvironment.EnsurePythonVersion # Functions that end up calling methods or Builders in the # DefaultEnvironment(). GlobalDefaultEnvironmentFunctions = [ # Methods from the SConsEnvironment class, above. 'Default', - 'EnsurePythonVersion', 'Exit', 'Export', 'GetLaunchDir', -- cgit v0.12 From bcef632d5748f5afb66856e00e89ad7961e41491 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 6 Sep 2022 23:23:56 +0300 Subject: CHANGES.txt is sorted by last names --- CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index d26b78c..4f4b1e1 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,10 @@ NOTE: 4.3.0 now requires Python 3.6.0 and above. Python 3.5.x is no longer suppo RELEASE VERSION/DATE TO BE FILLED IN LATER + From Anatoli Babenia + - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and + EnsurePythonVersion(). + From William Deegan: - Added ValidateOptions() which will check that all command line options are in either those specified by SCons itself, or by AddOption() in SConstruct/SConscript. It should @@ -23,10 +27,6 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Ryan Saunders - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. - From Anatoli Babenia - - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and - EnsurePythonVersion(). - RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From Joseph Brill: -- cgit v0.12 From 66ccceb2e2ee98ae7c20a589bef8ae1f36fd6ea0 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Tue, 6 Sep 2022 23:25:37 +0300 Subject: Nitpick : at the end of last name lines --- CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4f4b1e1..8e32f3d 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,7 +9,7 @@ NOTE: 4.3.0 now requires Python 3.6.0 and above. Python 3.5.x is no longer suppo RELEASE VERSION/DATE TO BE FILLED IN LATER - From Anatoli Babenia + From Anatoli Babenia: - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and EnsurePythonVersion(). @@ -21,10 +21,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). - From Flaviu Tamas + From Flaviu Tamas: - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. - From Ryan Saunders + From Ryan Saunders: - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 -- cgit v0.12 From 5299c8fd6fd899cabfcf86063586bf31801ef298 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Wed, 7 Sep 2022 05:48:05 +0300 Subject: Update RELEASE.txt --- RELEASE.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/RELEASE.txt b/RELEASE.txt index 3eff8ad..93d47f1 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -29,6 +29,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY --------------------------------------- - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. +- Calling EnsureSConsVersion() and EnsurePythonVersion() won't initialize + DefaultEnvironment anymore. FIXES ----- -- cgit v0.12 From f80095b86330115c0daadf22452f08935f02b76c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Sep 2022 16:43:09 -0600 Subject: Handle a list as source in Copy Copy (in the copy_func()) unconditionally converts the source argument to a string. This breaks the case where it was passed as a list - it should remain a list with the members converted to string, rather than being a single string. We assume a list source is supported, since the code just a few lines lower down detects a list and iterates through it. A test is added in test/Copy-Action.py to try to detect this case - it fails before the change and passes after. With the change to make a list argument work, found that the message printed by the matching strfunction (originally a lambda) needed the same adjustment, if the caller did not supply a list that was already strings (e.g. a list of Nodes, as from calling Glob()). Fixed up. At the same time, did some fiddling with the other Action Functions. Chmod had not been adjusted to emit the new Python standard for octal values (0o755 instead of 0755) - the change is only for the message, not for the function, but required aligning the test with the new format. Added some docstrings. Fixes #3009 Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 +++ RELEASE.txt | 13 ++++++ SCons/Defaults.py | 120 +++++++++++++++++++++++++++++++++---------------- doc/man/scons.xml | 22 ++++++--- doc/user/factories.xml | 4 +- test/Chmod.py | 63 +++++++++++++++----------- test/Copy-Action.py | 6 +++ 7 files changed, 161 insertions(+), 74 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 8e32f3d..e895b29 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -27,6 +27,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Ryan Saunders: - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. + From Mats Wichmann: + - A list argument as the source to the Copy() action function is now + correctly handled by converting elements to string. Copy bails if + asked to copy a list to an existing non-directory destination. + Both the implementation and the strfunction which prints the progress + message were adjusted. Fixes #3009. + RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From Joseph Brill: diff --git a/RELEASE.txt b/RELEASE.txt index 93d47f1..da4c7c8 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -31,6 +31,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. - Calling EnsureSConsVersion() and EnsurePythonVersion() won't initialize DefaultEnvironment anymore. +- The console message from the Chmod() action function now displays + octal modes using the modern Python syntax (0o755 rather than 0755). FIXES ----- @@ -38,6 +40,9 @@ FIXES - List fixes of outright bugs - Added missing newline to generated compilation database (compile_commands.json) +- A list argument as the source to the Copy() action function is now handled. + Both the implementation and the strfunction which prints the progress + message were adjusted. IMPROVEMENTS ------------ @@ -51,6 +56,12 @@ PACKAGING - List changes in the way SCons is packaged and/or released +- SCons now has three requirements files: requirements.txt describes + requirements to run scons; requirements-dev.txt requirements to + develop it - mainly things needed to run the testsuite; + requirements_pkg.txt are the requirements to do a full build + (including docs build) with an intent to create the packages. + DOCUMENTATION ------------- @@ -58,6 +69,8 @@ DOCUMENTATION typo fixes, even if they're mentioned in src/CHANGES.txt to give the contributor credit) +- Updated the --hash-format manpage entry. + DEVELOPMENT ----------- diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 65bd75a..218eb10 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -41,10 +41,12 @@ import SCons.Action import SCons.Builder import SCons.CacheDir import SCons.Environment +import SCons.Errors import SCons.PathList import SCons.Scanner.Dir import SCons.Subst import SCons.Tool +import SCons.Util # A placeholder for a default Environment (for fetching source files # from source code management systems and the like). This must be @@ -79,7 +81,6 @@ def DefaultEnvironment(*args, **kw): """ global _default_env if not _default_env: - import SCons.Util _default_env = SCons.Environment.Environment(*args, **kw) _default_env.Decider('content') global DefaultEnvironment @@ -157,15 +158,19 @@ LdModuleLinkAction = SCons.Action.Action("$LDMODULECOM", "$LDMODULECOMSTR") ActionFactory = SCons.Action.ActionFactory -def get_paths_str(dest): - # If dest is a list, we need to manually call str() on each element +def get_paths_str(dest) -> str: + """Generates a string from *dest* for use in a strfunction. + + If *dest* is a list, manually converts each elem to a string. + """ + def quote(arg): + return f'"{arg}"' + if SCons.Util.is_List(dest): - elem_strs = [] - for element in dest: - elem_strs.append('"' + str(element) + '"') - return '[' + ', '.join(elem_strs) + ']' + elem_strs = [quote(d) for d in dest] + return f'[{", ".join(elem_strs)}]' else: - return '"' + str(dest) + '"' + return quote(dest) permission_dic = { @@ -187,8 +192,14 @@ permission_dic = { } -def chmod_func(dest, mode): - import SCons.Util +def chmod_func(dest, mode) -> None: + """Implementation of the Chmod action function. + + *mode* can be either an integer (normally expressed in octal mode, + as in 0o755) or a string following the syntax of the POSIX chmod + command (for example "ugo+w"). The latter must be converted, since + the underlying Python only takes the numeric form. + """ from string import digits SCons.Node.FS.invalidate_node_memos(dest) if not SCons.Util.is_List(dest): @@ -231,41 +242,63 @@ def chmod_func(dest, mode): os.chmod(str(element), curr_perm & ~new_perm) -def chmod_strfunc(dest, mode): - import SCons.Util +def chmod_strfunc(dest, mode) -> str: + """strfunction for the Chmod action function.""" if not SCons.Util.is_String(mode): - return 'Chmod(%s, 0%o)' % (get_paths_str(dest), mode) + return f'Chmod({get_paths_str(dest)}, {mode:#o})' else: - return 'Chmod(%s, "%s")' % (get_paths_str(dest), str(mode)) + return f'Chmod({get_paths_str(dest)}, "{mode}")' + Chmod = ActionFactory(chmod_func, chmod_strfunc) -def copy_func(dest, src, symlinks=True): - """ - If symlinks (is true), then a symbolic link will be + +def copy_func(dest, src, symlinks=True) -> int: + """Implementation of the Copy action function. + + Copies *src* to *dest*. If *src* is a list, *dest* must be + a directory, or not exist (will be created). + + Since Python :mod:`shutil` methods, which know nothing about + SCons Nodes, will be called to perform the actual copying, + args are converted to strings first. + + If *symlinks* evaluates true, then a symbolic link will be shallow copied and recreated as a symbolic link; otherwise, copying a symbolic link will be equivalent to copying the symbolic link's final target regardless of symbolic link depth. """ dest = str(dest) - src = str(src) + src = [str(n) for n in src] if SCons.Util.is_List(src) else str(src) SCons.Node.FS.invalidate_node_memos(dest) - if SCons.Util.is_List(src) and os.path.isdir(dest): + if SCons.Util.is_List(src): + if not os.path.exists(dest): + os.makedirs(dest, exist_ok=True) + elif not os.path.isdir(dest): + # is Python's NotADirectoryError more appropriate? + raise SCons.Errors.UserError( + f'Copy() called with list src but dest "{dest}" is not a directory' + ) + for file in src: shutil.copy2(file, dest) return 0 + elif os.path.islink(src): if symlinks: - return os.symlink(os.readlink(src), dest) - else: - return copy_func(dest, os.path.realpath(src)) + os.symlink(os.readlink(src), dest) + return 0 + + return copy_func(dest, os.path.realpath(src)) + elif os.path.isfile(src): shutil.copy2(src, dest) return 0 + else: shutil.copytree(src, dest, symlinks) # copytree returns None in python2 and destination string in python3 @@ -273,13 +306,20 @@ def copy_func(dest, src, symlinks=True): return 0 -Copy = ActionFactory( - copy_func, - lambda dest, src, symlinks=True: 'Copy("%s", "%s")' % (dest, src) -) +def copy_strfunc(dest, src, symlinks=True) -> str: + """strfunction for the Copy action function.""" + return f'Copy({get_paths_str(dest)}, {get_paths_str(src)})' + + +Copy = ActionFactory(copy_func, copy_strfunc) -def delete_func(dest, must_exist=0): +def delete_func(dest, must_exist=False) -> None: + """Implementation of the Delete action function. + + Lets the Python :func:`os.unlink` raise an error if *dest* does not exist, + unless *must_exist* evaluates false (the default). + """ SCons.Node.FS.invalidate_node_memos(dest) if not SCons.Util.is_List(dest): dest = [dest] @@ -296,14 +336,16 @@ def delete_func(dest, must_exist=0): os.unlink(entry) -def delete_strfunc(dest, must_exist=0): - return 'Delete(%s)' % get_paths_str(dest) +def delete_strfunc(dest, must_exist=False) -> str: + """strfunction for the Delete action function.""" + return f'Delete({get_paths_str(dest)})' Delete = ActionFactory(delete_func, delete_strfunc) -def mkdir_func(dest): +def mkdir_func(dest) -> None: + """Implementation of the Mkdir action function.""" SCons.Node.FS.invalidate_node_memos(dest) if not SCons.Util.is_List(dest): dest = [dest] @@ -311,22 +353,23 @@ def mkdir_func(dest): os.makedirs(str(entry), exist_ok=True) -Mkdir = ActionFactory(mkdir_func, - lambda _dir: 'Mkdir(%s)' % get_paths_str(_dir)) +Mkdir = ActionFactory(mkdir_func, lambda _dir: f'Mkdir({get_paths_str(_dir)})') -def move_func(dest, src): +def move_func(dest, src) -> None: + """Implementation of the Move action function.""" SCons.Node.FS.invalidate_node_memos(dest) SCons.Node.FS.invalidate_node_memos(src) shutil.move(src, dest) -Move = ActionFactory(move_func, - lambda dest, src: 'Move("%s", "%s")' % (dest, src), - convert=str) +Move = ActionFactory( + move_func, lambda dest, src: f'Move("{dest}", "{src}")', convert=str +) -def touch_func(dest): +def touch_func(dest) -> None: + """Implementation of the Touch action function.""" SCons.Node.FS.invalidate_node_memos(dest) if not SCons.Util.is_List(dest): dest = [dest] @@ -341,8 +384,7 @@ def touch_func(dest): os.utime(file, (atime, mtime)) -Touch = ActionFactory(touch_func, - lambda file: 'Touch(%s)' % get_paths_str(file)) +Touch = ActionFactory(touch_func, lambda file: f'Touch({get_paths_str(file)})') # Internal utility functions diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 2155e9f..19dbb76 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -6438,21 +6438,26 @@ changes the permissions on the specified dest file or directory to the specified mode -which can be octal or string, similar to the bash command. +which can be octal or string, similar to the POSIX +chmod command. Examples: Execute(Chmod('file', 0o755)) -env.Command('foo.out', 'foo.in', - [Copy('$TARGET', '$SOURCE'), - Chmod('$TARGET', 0o755)]) +env.Command( + 'foo.out', + 'foo.in', + [Copy('$TARGET', '$SOURCE'), Chmod('$TARGET', 0o755)], +) Execute(Chmod('file', "ugo+w")) -env.Command('foo.out', 'foo.in', - [Copy('$TARGET', '$SOURCE'), - Chmod('$TARGET', "ugo+w")]) +env.Command( + 'foo.out', + 'foo.in', + [Copy('$TARGET', '$SOURCE'), Chmod('$TARGET', "ugo+w")], +) @@ -6473,6 +6478,9 @@ that will copy the source file or directory to the dest destination file or directory. +If src is a list, +dest must be a directory +if it already exists. Examples: diff --git a/doc/user/factories.xml b/doc/user/factories.xml index bb68504..362b6f0 100644 --- a/doc/user/factories.xml +++ b/doc/user/factories.xml @@ -190,10 +190,10 @@ touch $* # Symbolic link shallow copied as a new symbolic link: -Command("LinkIn", "LinkOut", Copy("$TARGET", "$SOURCE"[, True])) +Command("LinkIn", "LinkOut", Copy("$TARGET", "$SOURCE", symlinks=True)) # Symbolic link target copied as a file or directory: -Command("LinkIn", "FileOrDirectoryOut", Copy("$TARGET", "$SOURCE", False)) +Command("LinkIn", "FileOrDirectoryOut", Copy("$TARGET", "$SOURCE", symlinks=False)) diff --git a/test/Chmod.py b/test/Chmod.py index 64f4ed9..7af95b4 100644 --- a/test/Chmod.py +++ b/test/Chmod.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify that the Chmod() Action works. @@ -43,26 +42,36 @@ Execute(Chmod('f1', 0o666)) Execute(Chmod(('f1-File'), 0o666)) Execute(Chmod('d2', 0o777)) Execute(Chmod(Dir('d2-Dir'), 0o777)) + def cat(env, source, target): target = str(target[0]) with open(target, "wb") as f: for src in source: with open(str(src), "rb") as infp: f.write(infp.read()) + Cat = Action(cat) env = Environment() -env.Command('bar.out', 'bar.in', [Cat, - Chmod("f3", 0o666), - Chmod("d4", 0o777)]) -env = Environment(FILE = 'f5') +env.Command( + 'bar.out', + 'bar.in', + [Cat, Chmod("f3", 0o666), Chmod("d4", 0o777)], +) +env = Environment(FILE='f5') env.Command('f6.out', 'f6.in', [Chmod('$FILE', 0o666), Cat]) -env.Command('f7.out', 'f7.in', [Cat, - Chmod('Chmod-$SOURCE', 0o666), - Chmod('${TARGET}-Chmod', 0o666)]) +env.Command( + 'f7.out', + 'f7.in', + [Cat, Chmod('Chmod-$SOURCE', 0o666), Chmod('${TARGET}-Chmod', 0o666)], +) # Make sure Chmod works with a list of arguments -env = Environment(FILE = 'f9') -env.Command('f8.out', 'f8.in', [Chmod(['$FILE', File('f10')], 0o666), Cat]) +env = Environment(FILE='f9') +env.Command( + 'f8.out', + 'f8.in', + [Chmod(['$FILE', File('f10')], 0o666), Cat], +) Execute(Chmod(['d11', Dir('d12')], 0o777)) Execute(Chmod('f13', "a=r")) Execute(Chmod('f14', "ogu+w")) @@ -117,28 +126,30 @@ os.chmod(test.workpath('f15'), 0o444) os.chmod(test.workpath('d16'), 0o555) os.chmod(test.workpath('d17'), 0o555) os.chmod(test.workpath('d18'), 0o555) -expect = test.wrap_stdout(read_str = """\ -Chmod("f1", 0666) -Chmod("f1-File", 0666) -Chmod("d2", 0777) -Chmod("d2-Dir", 0777) -Chmod(["d11", "d12"], 0777) + +expect = test.wrap_stdout( + read_str = """\ +Chmod("f1", 0o666) +Chmod("f1-File", 0o666) +Chmod("d2", 0o777) +Chmod("d2-Dir", 0o777) +Chmod(["d11", "d12"], 0o777) Chmod("f13", "a=r") Chmod("f14", "ogu+w") Chmod("f15", "ug=rw, go+ rw") Chmod("d16", "0777") Chmod(["d17", "d18"], "ogu = rwx") """, - build_str = """\ + build_str = """\ cat(["bar.out"], ["bar.in"]) -Chmod("f3", 0666) -Chmod("d4", 0777) -Chmod("f5", 0666) +Chmod("f3", 0o666) +Chmod("d4", 0o777) +Chmod("f5", 0o666) cat(["f6.out"], ["f6.in"]) cat(["f7.out"], ["f7.in"]) -Chmod("Chmod-f7.in", 0666) -Chmod("f7.out-Chmod", 0666) -Chmod(["f9", "f10"], 0666) +Chmod("Chmod-f7.in", 0o666) +Chmod("f7.out-Chmod", 0o666) +Chmod(["f9", "f10"], 0o666) cat(["f8.out"], ["f8.in"]) """) test.run(options = '-n', arguments = '.', stdout = expect) diff --git a/test/Copy-Action.py b/test/Copy-Action.py index e135a9f..e2e01ef 100644 --- a/test/Copy-Action.py +++ b/test/Copy-Action.py @@ -40,6 +40,8 @@ test.write('SConstruct', """\ Execute(Copy('f1.out', 'f1.in')) Execute(Copy(File('d2.out'), 'd2.in')) Execute(Copy('d3.out', File('f3.in'))) +# Issue #3009: make sure it's not mangled if src is a list. +Execute(Copy('d7.out', Glob('f?.in'))) def cat(env, source, target): target = str(target[0]) @@ -71,6 +73,7 @@ test.subdir('d2.in') test.write(['d2.in', 'file'], "d2.in/file\n") test.write('f3.in', "f3.in\n") test.subdir('d3.out') +test.subdir('d7.out') test.write('bar.in', "bar.in\n") test.write('f4.in', "f4.in\n") test.subdir('d5.in') @@ -101,6 +104,7 @@ expect = test.wrap_stdout( Copy("f1.out", "f1.in") Copy("d2.out", "d2.in") Copy("d3.out", "f3.in") +Copy("d7.out", ["f1.in", "f3.in", "f4.in", "f6.in", "f7.in", "f8.in", "f9.in"]) """, build_str="""\ cat(["bar.out"], ["bar.in"]) @@ -123,6 +127,7 @@ test.run(options='-n', arguments='.', stdout=expect) test.must_not_exist('f1.out') test.must_not_exist('d2.out') test.must_not_exist(os.path.join('d3.out', 'f3.in')) +test.must_not_exist(os.path.join('d7.out', 'f7.in')) test.must_not_exist('f4.out') test.must_not_exist('d5.out') test.must_not_exist(os.path.join('d6.out', 'f6.in')) @@ -141,6 +146,7 @@ test.run() test.must_match('f1.out', "f1.in\n", mode='r') test.must_match(['d2.out', 'file'], "d2.in/file\n", mode='r') test.must_match(['d3.out', 'f3.in'], "f3.in\n", mode='r') +test.must_match(['d7.out', 'f7.in'], "f7.in\n", mode='r') test.must_match('f4.out', "f4.in\n", mode='r') test.must_match(['d5.out', 'file'], "d5.in/file\n", mode='r') test.must_match(['d6.out', 'f6.in'], "f6.in\n", mode='r') -- cgit v0.12 From 767cc21f9f2f75ef5cf62fc6ea8d2343530287ed Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 11 Sep 2022 17:14:25 -0600 Subject: Tweak Copy() changes: Simplify check for copy-a-list case: just try the os.makedirs, catch the exception, and raise a BuildError which seems to fit better. Add a second test for copying lists in test/Copy-Action: now tests both a list of explicit strings and a list of Nodes. Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 ++-- SCons/Defaults.py | 11 +++++------ test/Copy-Action.py | 5 +++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e895b29..addc5f7 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -29,8 +29,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Mats Wichmann: - A list argument as the source to the Copy() action function is now - correctly handled by converting elements to string. Copy bails if - asked to copy a list to an existing non-directory destination. + correctly handled by converting elements to string. Copy errors out + if asked to copy a list to an existing non-directory destination. Both the implementation and the strfunction which prints the progress message were adjusted. Fixes #3009. diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 218eb10..32b924f 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -276,14 +276,13 @@ def copy_func(dest, src, symlinks=True) -> int: SCons.Node.FS.invalidate_node_memos(dest) if SCons.Util.is_List(src): - if not os.path.exists(dest): + # this fails only if dest exists and is not a dir + try: os.makedirs(dest, exist_ok=True) - elif not os.path.isdir(dest): - # is Python's NotADirectoryError more appropriate? - raise SCons.Errors.UserError( - f'Copy() called with list src but dest "{dest}" is not a directory' + except FileExistsError: + raise SCons.Errors.BuildError( + errstr=f'Error: Copy() called with list src but "{dest}" is not a directory' ) - for file in src: shutil.copy2(file, dest) return 0 diff --git a/test/Copy-Action.py b/test/Copy-Action.py index e2e01ef..1b1356e 100644 --- a/test/Copy-Action.py +++ b/test/Copy-Action.py @@ -41,6 +41,8 @@ Execute(Copy('f1.out', 'f1.in')) Execute(Copy(File('d2.out'), 'd2.in')) Execute(Copy('d3.out', File('f3.in'))) # Issue #3009: make sure it's not mangled if src is a list. +# make sure both list-of-str and list-of-Node work +Execute(Copy('d7.out', ['f10.in', 'f11.in'])) Execute(Copy('d7.out', Glob('f?.in'))) def cat(env, source, target): @@ -104,6 +106,7 @@ expect = test.wrap_stdout( Copy("f1.out", "f1.in") Copy("d2.out", "d2.in") Copy("d3.out", "f3.in") +Copy("d7.out", ["f10.in", "f11.in"]) Copy("d7.out", ["f1.in", "f3.in", "f4.in", "f6.in", "f7.in", "f8.in", "f9.in"]) """, build_str="""\ @@ -128,6 +131,7 @@ test.must_not_exist('f1.out') test.must_not_exist('d2.out') test.must_not_exist(os.path.join('d3.out', 'f3.in')) test.must_not_exist(os.path.join('d7.out', 'f7.in')) +test.must_not_exist(os.path.join('d7.out', 'f11.in')) test.must_not_exist('f4.out') test.must_not_exist('d5.out') test.must_not_exist(os.path.join('d6.out', 'f6.in')) @@ -147,6 +151,7 @@ test.must_match('f1.out', "f1.in\n", mode='r') test.must_match(['d2.out', 'file'], "d2.in/file\n", mode='r') test.must_match(['d3.out', 'f3.in'], "f3.in\n", mode='r') test.must_match(['d7.out', 'f7.in'], "f7.in\n", mode='r') +test.must_match(['d7.out', 'f11.in'], "f11.in\n", mode='r') test.must_match('f4.out', "f4.in\n", mode='r') test.must_match(['d5.out', 'file'], "d5.in/file\n", mode='r') test.must_match(['d6.out', 'f6.in'], "f6.in\n", mode='r') -- cgit v0.12 From 7c2d4271d57d2da462859d29435740de3b412d10 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 11 Sep 2022 17:56:52 -0600 Subject: Copy(): tweak the error message for copy-a-list Signed-off-by: Mats Wichmann --- SCons/Defaults.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 32b924f..40c3e4a 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -281,7 +281,11 @@ def copy_func(dest, src, symlinks=True) -> int: os.makedirs(dest, exist_ok=True) except FileExistsError: raise SCons.Errors.BuildError( - errstr=f'Error: Copy() called with list src but "{dest}" is not a directory' + errstr=( + 'Error: Copy() called with a list of sources, ' + 'which requires target to be a directory, ' + f'but "{dest}" is not a directory.' + ) ) for file in src: shutil.copy2(file, dest) @@ -300,8 +304,6 @@ def copy_func(dest, src, symlinks=True) -> int: else: shutil.copytree(src, dest, symlinks) - # copytree returns None in python2 and destination string in python3 - # A error is raised in both cases, so we can just return 0 for success return 0 -- cgit v0.12 From a1084687b353463fd6a0caedea4b00413431cf04 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 14 Sep 2022 08:58:08 -0600 Subject: doc: change five functions to appear global-only The five functions EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir, SConscriptChdir were listed as both global and environment functions, but they do nothing in the context of an environment, so marked in the xml as "global". This only changes the presentation in the manpage & userguide appendix, not the behavior. Minor tweaks in the code around SConscriptChdir - actually use a bool True/False instead of 0/1, and added a couple of type annotations. Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 +++ RELEASE.txt | 2 ++ SCons/Environment.xml | 24 ++++++++---------------- SCons/Node/FS.py | 2 +- SCons/Node/FSTests.py | 2 +- SCons/SConf.py | 6 +++--- SCons/Script/SConscript.py | 12 ++++++------ SCons/Script/SConscript.xml | 8 ++++---- 8 files changed, 28 insertions(+), 31 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index addc5f7..1925495 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,6 +33,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER if asked to copy a list to an existing non-directory destination. Both the implementation and the strfunction which prints the progress message were adjusted. Fixes #3009. + - doc: EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and + SConscriptChdir are now listed as Global functions only; the + Environment versions still work but are not documented. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index da4c7c8..99ba7b5 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -70,6 +70,8 @@ DOCUMENTATION the contributor credit) - Updated the --hash-format manpage entry. +- EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and + SConscriptChdir are now listed as Global functions only. DEVELOPMENT ----------- diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 7a288f3..b664d60 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -2862,7 +2862,7 @@ for a complete explanation of the arguments and behavior. - + (value) @@ -2871,18 +2871,11 @@ By default, &scons; changes its working directory to the directory in which each -subsidiary SConscript file lives. +subsidiary SConscript file lives +while reading and processing that script. This behavior may be disabled -by specifying either: - - - -SConscriptChdir(0) -env.SConscriptChdir(0) - - - -in which case +by specifying an argument which +evaluates false, in which case &scons; will stay in the top-level directory while reading all SConscript files. @@ -2900,10 +2893,9 @@ Example: -env = Environment() -SConscriptChdir(0) +SConscriptChdir(False) SConscript('foo/SConscript') # will not chdir to foo -env.SConscriptChdir(1) +SConscriptChdir(True) SConscript('bar/SConscript') # will chdir to bar @@ -3485,7 +3477,7 @@ The subsidiary SConscript file must be called as if it were in variant_dir, regardless of the value of duplicate. -When calling an SConscript file, you can use the +When calling an SConscript file, you can use the exports keyword argument to pass parameters (individually or as an appropriately set up environment) so the SConscript can pick up the right settings for that variant build. diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index b4de337..7ce306f 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -1239,7 +1239,7 @@ class FS(LocalFS): else: return "" - def chdir(self, dir, change_os_dir=0): + def chdir(self, dir, change_os_dir=False): """Change the current working directory for lookups. If change_os_dir is true, we will also change the "real" cwd to match. diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index a2400f2..f3aface 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -1822,7 +1822,7 @@ class FSTestCase(_tempdirTestCase): test.write(['subdir', 'build'], "subdir/build\n") subdir = fs.Dir('subdir') - fs.chdir(subdir, change_os_dir=1) + fs.chdir(subdir, change_os_dir=True) self.fs._lookup('#build/file', subdir, SCons.Node.FS.File) def test_above_root(self): diff --git a/SCons/SConf.py b/SCons/SConf.py index 4e8d410..2a427df 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -515,7 +515,7 @@ class SConfBase: # the engine assumes the current path is the SConstruct directory ... old_fs_dir = SConfFS.getcwd() old_os_dir = os.getcwd() - SConfFS.chdir(SConfFS.Top, change_os_dir=1) + SConfFS.chdir(SConfFS.Top, change_os_dir=True) # Because we take responsibility here for writing out our # own .sconsign info (see SConfBuildTask.execute(), above), @@ -562,7 +562,7 @@ class SConfBase: finally: SConfFS.set_max_drift(save_max_drift) os.chdir(old_os_dir) - SConfFS.chdir(old_fs_dir, change_os_dir=0) + SConfFS.chdir(old_fs_dir, change_os_dir=False) if self.logstream is not None: # restore stdout / stderr sys.stdout = oldStdout @@ -772,7 +772,7 @@ class SConfBase: tb = traceback.extract_stack()[-3-self.depth] old_fs_dir = SConfFS.getcwd() - SConfFS.chdir(SConfFS.Top, change_os_dir=0) + SConfFS.chdir(SConfFS.Top, change_os_dir=False) self.logstream.write('file %s,line %d:\n\tConfigure(confdir = %s)\n' % (tb[0], tb[1], str(self.confdir)) ) SConfFS.chdir(old_fs_dir) diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 99bb84d..2b11e2f 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -57,7 +57,7 @@ GlobalDict = None global_exports = {} # chdir flag -sconscript_chdir = 1 +sconscript_chdir: bool = True def get_calling_namespaces(): """Return the locals and globals for the function that called @@ -205,7 +205,7 @@ def _SConscript(fs, *files, **kw): # Change directory to the top of the source # tree to make sure the os's cwd and the cwd of # fs match so we can open the SConscript. - fs.chdir(top, change_os_dir=1) + fs.chdir(top, change_os_dir=True) if f.rexists(): actual = f.rfile() _file_ = open(actual.get_abspath(), "rb") @@ -254,7 +254,7 @@ def _SConscript(fs, *files, **kw): # fs.chdir(), because we still need to # interpret the stuff within the SConscript file # relative to where we are logically. - fs.chdir(ldir, change_os_dir=0) + fs.chdir(ldir, change_os_dir=False) os.chdir(actual.dir.get_abspath()) # Append the SConscript directory to the beginning @@ -292,7 +292,7 @@ def _SConscript(fs, *files, **kw): if old_file is not None: call_stack[-1].globals.update({__file__:old_file}) - + else: handle_missing_SConscript(f, kw.get('must_exist', None)) @@ -306,7 +306,7 @@ def _SConscript(fs, *files, **kw): # There was no local directory, so chdir to the # Repository directory. Like above, we do this # directly. - fs.chdir(frame.prev_dir, change_os_dir=0) + fs.chdir(frame.prev_dir, change_os_dir=False) rdir = frame.prev_dir.rdir() rdir._create() # Make sure there's a directory there. try: @@ -600,7 +600,7 @@ class SConsEnvironment(SCons.Environment.Base): subst_kw['exports'] = exports return _SConscript(self.fs, *files, **subst_kw) - def SConscriptChdir(self, flag): + def SConscriptChdir(self, flag: bool) -> None: global sconscript_chdir sconscript_chdir = flag diff --git a/SCons/Script/SConscript.xml b/SCons/Script/SConscript.xml index eb52acc..7a4bc29 100644 --- a/SCons/Script/SConscript.xml +++ b/SCons/Script/SConscript.xml @@ -84,7 +84,7 @@ env.Default(hello) - + (major, minor) @@ -107,7 +107,7 @@ EnsurePythonVersion(2,2) - + (major, minor, [revision]) @@ -137,7 +137,7 @@ EnsureSConsVersion(0,96,90) - + ([value]) @@ -215,7 +215,7 @@ See the description below. - + () -- cgit v0.12 From c656c0b2885768fdf8e4d3415d94881321babf40 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Thu, 15 Sep 2022 08:35:30 +0300 Subject: Remove unused private method SConsEnvironment._exceeds_version() --- CHANGES.txt | 1 + SCons/Script/SConscript.py | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1925495..ed5fcbf 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Anatoli Babenia: - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and EnsurePythonVersion(). + - Remove unused private method SConsEnvironment._exceeds_version(). From William Deegan: - Added ValidateOptions() which will check that all command line options are in either diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index 2b11e2f..d340a91 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -385,11 +385,6 @@ class SConsEnvironment(SCons.Environment.Base): # # Private methods of an SConsEnvironment. # - def _exceeds_version(self, major, minor, v_major, v_minor): - """Return 1 if 'major' and 'minor' are greater than the version - in 'v_major' and 'v_minor', and 0 otherwise.""" - return (major > v_major or (major == v_major and minor > v_minor)) - @staticmethod def _get_major_minor_revision(version_string): """Split a version string into major, minor and (optionally) -- cgit v0.12 From 402073f4b8a4c455564862c175176b05b887b3b4 Mon Sep 17 00:00:00 2001 From: Anatoli Babenia Date: Thu, 15 Sep 2022 08:37:35 +0300 Subject: Do not initialize DefaultEnvironment on Exit(), GetLaunchDir() and SConscriptChdir() --- CHANGES.txt | 4 ++-- SCons/Script/SConscript.py | 9 ++++++--- SCons/Script/__init__.py | 11 +++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index ed5fcbf..bba6e10 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -10,8 +10,8 @@ NOTE: 4.3.0 now requires Python 3.6.0 and above. Python 3.5.x is no longer suppo RELEASE VERSION/DATE TO BE FILLED IN LATER From Anatoli Babenia: - - Do not initialize DefaultEnvironment when calling EnsureSConsVersion() and - EnsurePythonVersion(). + - Do not initialize DefaultEnvironment when calling EnsureSConsVersion(), + EnsurePythonVersion(), Exit(), GetLaunchDir() and SConscriptChdir(). - Remove unused private method SConsEnvironment._exceeds_version(). From William Deegan: diff --git a/SCons/Script/SConscript.py b/SCons/Script/SConscript.py index d340a91..b72f30e 100644 --- a/SCons/Script/SConscript.py +++ b/SCons/Script/SConscript.py @@ -510,7 +510,8 @@ class SConsEnvironment(SCons.Environment.Base): print("Python %d.%d or greater required, but you have Python %s" %(major,minor,v)) sys.exit(2) - def Exit(self, value=0): + @staticmethod + def Exit(value=0): sys.exit(value) def Export(self, *vars, **kw): @@ -518,7 +519,8 @@ class SConsEnvironment(SCons.Environment.Base): global_exports.update(compute_exports(self.Split(var))) global_exports.update(kw) - def GetLaunchDir(self): + @staticmethod + def GetLaunchDir(): global launch_dir return launch_dir @@ -595,7 +597,8 @@ class SConsEnvironment(SCons.Environment.Base): subst_kw['exports'] = exports return _SConscript(self.fs, *files, **subst_kw) - def SConscriptChdir(self, flag: bool) -> None: + @staticmethod + def SConscriptChdir(flag: bool) -> None: global sconscript_chdir sconscript_chdir = flag diff --git a/SCons/Script/__init__.py b/SCons/Script/__init__.py index 0250bd1..6cfea1b 100644 --- a/SCons/Script/__init__.py +++ b/SCons/Script/__init__.py @@ -289,22 +289,23 @@ def Variables(files=None, args=ARGUMENTS): # Adding global functions to the SConscript name space. # -# Static functions that do not use state in DefaultEnvironment(). +# Static functions that do not trigger initialization of +# DefaultEnvironment() and don't use its state. EnsureSConsVersion = _SConscript.SConsEnvironment.EnsureSConsVersion EnsurePythonVersion = _SConscript.SConsEnvironment.EnsurePythonVersion +Exit = _SConscript.SConsEnvironment.Exit +GetLaunchDir = _SConscript.SConsEnvironment.GetLaunchDir +SConscriptChdir = _SConscript.SConsEnvironment.SConscriptChdir # Functions that end up calling methods or Builders in the # DefaultEnvironment(). GlobalDefaultEnvironmentFunctions = [ # Methods from the SConsEnvironment class, above. 'Default', - 'Exit', 'Export', - 'GetLaunchDir', 'Help', 'Import', #'SConscript', is handled separately, below. - 'SConscriptChdir', # Methods from the Environment.Base class. 'AddPostAction', @@ -378,6 +379,8 @@ GlobalDefaultBuilders = [ 'Package', ] +# DefaultEnvironmentCall() initializes DefaultEnvironment() if it is not +# created yet. for name in GlobalDefaultEnvironmentFunctions + GlobalDefaultBuilders: exec ("%s = _SConscript.DefaultEnvironmentCall(%s)" % (name, repr(name))) del name -- cgit v0.12 From b06cf9abd8ba24a943fc3ec0727ecc6965a0e753 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 16 Sep 2022 09:15:09 -0600 Subject: [DOC] update SHELL_ENV_GENERATORS [skip appveyor] Signed-off-by: Mats Wichmann --- RELEASE.txt | 1 + SCons/Action.py | 39 ++++++++++++++++++++------------- SCons/Action.xml | 66 ++++++++++++++++++++++++++++---------------------------- 3 files changed, 58 insertions(+), 48 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index 99ba7b5..5d32080 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -72,6 +72,7 @@ DOCUMENTATION - Updated the --hash-format manpage entry. - EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and SConscriptChdir are now listed as Global functions only. +- Updated SHELL_ENV_GENERATORS description and added versionadded indicator. DEVELOPMENT ----------- diff --git a/SCons/Action.py b/SCons/Action.py index 6e67c7f..ccb3a2c 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -110,6 +110,7 @@ import inspect from collections import OrderedDict import SCons.Debug +import SCons.Util from SCons.Debug import logInstanceCreation import SCons.Errors import SCons.Util @@ -733,34 +734,42 @@ default_ENV = None def get_default_ENV(env): - """ - A fiddlin' little function that has an 'import SCons.Environment' which - can't be moved to the top level without creating an import loop. Since - this import creates a local variable named 'SCons', it blocks access to - the global variable, so we move it here to prevent complaints about local - variables being used uninitialized. + """Returns an execution environment. + + If there is one in *env*, just use it, else return the Default + Environment, insantiated if necessary. + + A fiddlin' little function that has an ``import SCons.Environment`` + which cannot be moved to the top level without creating an import + loop. Since this import creates a local variable named ``SCons``, + it blocks access to the global variable, so we move it here to + prevent complaints about local variables being used uninitialized. """ global default_ENV + try: return env['ENV'] except KeyError: if not default_ENV: import SCons.Environment - # This is a hideously expensive way to get a default shell + # This is a hideously expensive way to get a default execution # environment. What it really should do is run the platform # setup to get the default ENV. Fortunately, it's incredibly - # rare for an Environment not to have a shell environment, so - # we're not going to worry about it overmuch. + # rare for an Environment not to have an execution environment, + # so we're not going to worry about it overmuch. default_ENV = SCons.Environment.Environment()['ENV'] return default_ENV def _resolve_shell_env(env, target, source): - """ - First get default environment. - Then if SHELL_ENV_GENERATORS is set and is iterable, - call each callable in that list to allow it to alter - the created execution environment. + """Returns a resolved execution environment. + + First get the execution environment. Then if ``SHELL_ENV_GENERATORS`` + is set and is iterable, call each function to allow it to alter the + created execution environment, passing each the returned execution + environment from the previous call. + + .. versionadded:: 4.4 """ ENV = get_default_ENV(env) shell_gen = env.get('SHELL_ENV_GENERATORS') @@ -793,7 +802,7 @@ def _subproc(scons_env, cmd, error='ignore', **kw): if is_String(io) and io == 'devnull': kw[stream] = DEVNULL - # Figure out what shell environment to use + # Figure out what execution environment to use ENV = kw.get('env', None) if ENV is None: ENV = get_default_ENV(scons_env) diff --git a/SCons/Action.xml b/SCons/Action.xml index 209c389..becd30f 100644 --- a/SCons/Action.xml +++ b/SCons/Action.xml @@ -26,14 +26,13 @@ See its __doc__ string for a discussion of the format. -Controls whether or not SCons will +Controls whether or not &SCons; will add implicit dependencies for the commands executed to build targets. -By default, SCons will add -to each target +By default, &SCons; will add to each target an implicit dependency on the command represented by the first argument of any command line it executes (which is typically @@ -224,21 +223,39 @@ defining the execution environment in which the command should be executed. -Must be a list (or an iterable) containing functions where each function generates or -alters the environment dictionary which will be used -when executing the &cv-link-SPAWN; function. The functions will initially -be passed a reference of the current execution environment (e.g. env['ENV']), -and each called while iterating the list. Each function must return a dictionary -which will then be passed to the next function iterated. The return dictionary -should contain keys which represent the environment variables and their respective -values. +A hook allowing the execution environment to be modified prior +to the actual execution of a command line from an action +via the spawner function defined by &cv-link-SPAWN;. +Allows substitution based on targets and sources, +as well as values from the &consenv;, +adding extra environment variables, etc. + -This primary purpose of this construction variable is to give the user the ability -to substitute execution environment variables based on env, targets, and sources. -If desired, the user can completely customize the execution environment for particular -targets. + +The value must be a list (or other iterable) +of functions which each generate or +alter the execution environment dictionary. +The first function will be passed a copy of the initial execution environment +(&cv-link-ENV; in the current &consenv;); +the dictionary returned by that function is passed to the next, +until the iterable is exhausted and the result returned +for use by the command spawner. +The original execution environment is not modified. + +Each function provided in &cv-SHELL_ENV_GENERATORS; must accept four +arguments and return a dictionary: +env is the &consenv; for this action; +target is the list of targets associated with this action; +source is the list of sources associated with this action; +and shell_env is the current dictionary after iterating +any previous &cv-SHELL_ENV_GENERATORS; functions +(this can be compared to the original execution environment, +which is available as env['ENV'], to detect any changes). + + + Example: def custom_shell_env(env, target, source, shell_env): """customize shell_env if desired""" @@ -249,24 +266,7 @@ def custom_shell_env(env, target, source, shell_env): env["SHELL_ENV_GENERATORS"] = [custom_shell_env] - - env -The SCons construction environment from which the -execution environment can be derived from. - - - target -The list of targets associated with this action. - - - source -The list of sources associated with this action. - - - shell_env -The current shell_env after iterating other SHELL_ENV_GENERATORS functions. This can be compared -to the passed env['ENV'] to detect any changes. - + Available since 4.4 -- cgit v0.12 From 48d89ab8a8bf4b13b643b2e956994baac59ab5f2 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 10 Sep 2022 10:39:11 -0600 Subject: Update the Glob docs with more details Covers xml file for manpage, very minor stuff in uguide, and docstring in code. Also tweaked Ignore manpage entry, only because it was the very next description in Environment.xml and caught the eye. Signed-off-by: Mats Wichmann --- RELEASE.txt | 1 + SCons/Environment.xml | 190 ++++++++++++++++++++++++++-------------------- SCons/Node/FS.py | 66 ++++++++-------- doc/user/less-simple.xml | 18 +++-- doc/user/misc.xml | 4 +- doc/user/repositories.xml | 4 + 6 files changed, 161 insertions(+), 122 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index 99ba7b5..91d680d 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -72,6 +72,7 @@ DOCUMENTATION - Updated the --hash-format manpage entry. - EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and SConscriptChdir are now listed as Global functions only. +- Updated the docs for Glob. DEVELOPMENT ----------- diff --git a/SCons/Environment.xml b/SCons/Environment.xml index b664d60..a94d9c5 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -1901,48 +1901,81 @@ Nodes or strings representing path names. -(pattern, [ondisk, source, strings, exclude]) +(pattern, [ondisk=True, source=False, strings=False, exclude=None]) -Returns Nodes (or strings) that match the specified -pattern, -relative to the directory of the current -&SConscript; -file. +Returns a possibly empty list of Nodes (or strings) that match +pathname specification pattern. +pattern can be absolute, +top-relative, +or (most commonly) relative to the directory of the current +&SConscript; file. +&f-Glob; matches both files stored on disk and Nodes +which &SCons; already knows about, even if any corresponding +file is not currently stored on disk. The evironment method form (&f-env-Glob;) performs string substition on pattern -and returns whatever matches -the resulting expanded pattern. +and returns whatever matches the resulting expanded pattern. +The results are sorted, unlike for the similar &Python; +glob.glob function, +to ensure build order will be stable. -The specified pattern -uses Unix shell style metacharacters for matching: +can contain POSIX-style shell metacharacters for matching: + + + + + + + Pattern + Meaning + + + + + * + matches everything + + + ? + matches any single character + + + [seq] + matches any character in seq + (can be a list or a ranage). + + + [!seq] + matches any character not in seq + + + + + + +For a literal match, wrap the metacharacter in brackets to +escape the normal behavior. +For example, '[?]' matches the character +'?'. + + + +Filenames starting with a dot are specially handled - +they can only be matched by patterns that start with a dot +(or have a dot immediately following a pathname separator +character, or slash), they are not not matched by the metacharacters. +Metacharacter matches also do not span directory separators. - - * matches everything - ? matches any single character - [seq] matches any character in seq - [!seq] matches any char not in seq - - -If the first character of a filename is a dot, -it must be matched explicitly. -Character matches do -not -span directory separators. - - - -The &f-Glob; -knows about -repositories +understands repositories (see the &f-link-Repository; function) @@ -1950,8 +1983,7 @@ and source directories (see the &f-link-VariantDir; function) -and -returns a Node (or string, if so configured) +and returns a Node (or string, if so configured) match in the local (SConscript) directory if a matching Node is found anywhere in a corresponding @@ -1959,65 +1991,58 @@ repository or source directory. -The +If the optional ondisk -argument may be set to a value which evaluates -False -to disable the search for matches on disk, -thereby only returning matches among -already-configured File or Dir Nodes. -The default behavior is to -return corresponding Nodes -for any on-disk matches found. +argument evaluates false, +the search for matches on disk is disabled, +and only matches from +already-configured File or Dir Nodes are returned. -The +If the optional source -argument may be set to a value which evaluates -True -to specify that, -when the local directory is a -&f-VariantDir;, -the returned Nodes should be from the -corresponding source directory, -not the local directory. +argument evaluates true, +and the local directory is a variant directory, +then &f-Glob; returnes Nodes from +the corresponding source directory, +rather than the local directory. + -The +If the optional strings -argument may be set to a value which evaluates -True -to have the +argument evaluates true, &f-Glob; -function return strings, not Nodes, -that represent the matched files or directories. +returns matches as strings, rather than Nodes. The returned strings will be relative to the local (SConscript) directory. -(Note that This may make it easier to perform +(Note that while this may make it easier to perform arbitrary manipulation of file names, -but if the returned strings are +it loses the context &SCons; would have in the Node, +so if the returned strings are passed to a different &SConscript; file, -any Node translation will be relative -to the other +any Node translation there will be relative +to that &SConscript; directory, -not the original +not to the original &SConscript; directory.) -The +The optional exclude argument may be set to a pattern or a list of patterns -(following the same Unix shell semantics) -which must be filtered out of returned elements. -Elements matching a least one pattern of -this list will be excluded. +descibing files or directories +to filter out of the match list. +Elements matching a least one specified pattern will be excluded. +These patterns use the same syntax as for +pattern. @@ -2027,9 +2052,10 @@ Examples: Program("foo", Glob("*.c")) Zip("/tmp/everything", Glob(".??*") + Glob("*")) -sources = Glob("*.cpp", exclude=["os_*_specific_*.cpp"]) + \ - Glob( "os_%s_specific_*.cpp" % currentOS) +sources = Glob("*.cpp", exclude=["os_*_specific_*.cpp"]) \ + + Glob("os_%s_specific_*.cpp" % currentOS) + @@ -2066,24 +2092,26 @@ to call all builders. -The specified dependency file(s) -will be ignored when deciding if -the target file(s) need to be rebuilt. - - - -You can also use -&f-Ignore; -to remove a target from the default build. -In order to do this you must specify the directory the target will -be built in as the target, and the file you want to skip building -as the dependency. +Ignores dependency +when deciding if +target needs to be rebuilt. +target and +dependency +can each be a single filename or Node +or a list of filenames or Nodes. -Note that this will only remove the dependencies listed from -the files built by default. It will still be built if that -dependency is needed by another object being built. +&f-Ignore; can also be used to +remove a target from the default build +by specifying the directory the target will be built in as +target +and the file you want to skip selecting for building as +dependency. +Note that this only removes the target from +the default target selection algorithm: +if it is a dependency of another object being +built &SCons; still builds it normally. See the third and forth examples below. diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 7ce306f..f5642e2 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -2157,49 +2157,52 @@ class Dir(Base): for dirname in [n for n in names if isinstance(entries[n], Dir)]: entries[dirname].walk(func, arg) - def glob(self, pathname, ondisk=True, source=False, strings=False, exclude=None): - """ - Returns a list of Nodes (or strings) matching a specified - pathname pattern. + def glob(self, pathname, ondisk=True, source=False, strings=False, exclude=None) -> list: + """Returns a list of Nodes (or strings) matching a pathname pattern. - Pathname patterns follow UNIX shell semantics: * matches - any-length strings of any characters, ? matches any character, - and [] can enclose lists or ranges of characters. Matches do - not span directory separators. + Pathname patterns follow POSIX shell syntax:: - The matches take into account Repositories, returning local - Nodes if a corresponding entry exists in a Repository (either - an in-memory Node or something on disk). + * matches everything + ? matches any single character + [seq] matches any character in seq (ranges allowed) + [!seq] matches any char not in seq - By defafult, the glob() function matches entries that exist - on-disk, in addition to in-memory Nodes. Setting the "ondisk" - argument to False (or some other non-true value) causes the glob() - function to only match in-memory Nodes. The default behavior is - to return both the on-disk and in-memory Nodes. + The wildcard characters can be escaped by enclosing in brackets. + A leading dot is not matched by a wildcard, and needs to be + explicitly included in the pattern to be matched. Matches also + do not span directory separators. - The "source" argument, when true, specifies that corresponding - source Nodes must be returned if you're globbing in a build - directory (initialized with VariantDir()). The default behavior - is to return Nodes local to the VariantDir(). + The matches take into account Repositories, returning a local + Node if a corresponding entry exists in a Repository (either + an in-memory Node or something on disk). - The "strings" argument, when true, returns the matches as strings, - not Nodes. The strings are path names relative to this directory. + The underlying algorithm is adapted from a rather old version + of :func:`glob.glob` function in the Python standard library + (heavily modified), and uses :func:`fnmatch.fnmatch` under the covers. - The "exclude" argument, if not None, must be a pattern or a list - of patterns following the same UNIX shell semantics. - Elements matching a least one pattern of this list will be excluded - from the result. + This is the internal implementation of the external Glob API. + + Args: + pattern: pathname pattern to match. + ondisk: if false, restricts matches to in-memory Nodes. + By defafult, matches entries that exist on-disk in addition + to in-memory Nodes. + source: if true, corresponding source Nodes are returned if + globbing in a variant directory. The default behavior + is to return Nodes local to the variant directory. + strings: if true, returns the matches as strings instead of + Nodes. The strings are path names relative to this directory. + exclude: if not ``None``, must be a pattern or a list of patterns + following the same POSIX shell semantics. Elements matching at + least one pattern from *exclude* will be excluded from the result. - The underlying algorithm is adapted from the glob.glob() function - in the Python library (but heavily modified), and uses fnmatch() - under the covers. """ dirname, basename = os.path.split(pathname) if not dirname: result = self._glob1(basename, ondisk, source, strings) else: if has_glob_magic(dirname): - list = self.glob(dirname, ondisk, source, False, exclude) + list = self.glob(dirname, ondisk, source, strings=False, exclude=exclude) else: list = [self.Dir(dirname, create=True)] result = [] @@ -2226,7 +2229,8 @@ class Dir(Base): corresponding entries and returns a Node (or string) relative to the current directory if an entry is found anywhere. - TODO: handle pattern with no wildcard + TODO: handle pattern with no wildcard. Python's glob.glob uses + a separate _glob0 function to do this. """ search_dir_list = self.get_all_rdirs() for srcdir in self.srcdir_list(): diff --git a/doc/user/less-simple.xml b/doc/user/less-simple.xml index 757ffac..3eda19c 100644 --- a/doc/user/less-simple.xml +++ b/doc/user/less-simple.xml @@ -236,9 +236,10 @@ void file2() { printf("file2.c\n"); } - You can also use the &Glob; function to find all files matching a + You can also use the &f-link-Glob; function to find all files matching a certain template, using the standard shell pattern matching - characters *, ? + characters * (to match everything), + ? (to match a single character) and [abc] to match any of a, b or c. [!abc] is also supported, @@ -254,13 +255,14 @@ Program('program', Glob('*.c')) - The SCons man page has more details on using &Glob; - with variant directories - (see , below) + &f-Glob; has powerful capabilities - it matches even if the + file does not exist, but &SCons; can determine that it would + exist after a build. + You will meet it again reading about + variant directories + (see ) and repositories - (see , below), - excluding some files - and returning strings rather than Nodes. + (see ). diff --git a/doc/user/misc.xml b/doc/user/misc.xml index 1b18d2f..b1e1507 100644 --- a/doc/user/misc.xml +++ b/doc/user/misc.xml @@ -256,11 +256,11 @@ hello.c - The &FindFile; function searches for a file in a list of directories. + The &f-link-FindFile; function searches for a file in a list of directories. If there is only one directory, it can be given as a simple string. The function returns a File node if a matching file exists, or None if no file is found. - (See the documentation for the &Glob; function for an alternative way + (See the documentation for the &f-link-Glob; function for an alternative way of searching for entries in a directory.) diff --git a/doc/user/repositories.xml b/doc/user/repositories.xml index 01edec7..77d75d3 100644 --- a/doc/user/repositories.xml +++ b/doc/user/repositories.xml @@ -203,6 +203,10 @@ int main() { printf("Hello, world!\n"); } + The &f-link-Glob; function understands about repositories, + and will use the same matching algorithm as described for + explicitly-listed sources. + -- cgit v0.12 From 8d02bb357f0c97a89663d0bfee7d294dac66b398 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 25 Sep 2022 14:34:40 -0600 Subject: Fix Glob wording per review comments [skip appveyor] Signed-off-by: Mats Wichmann --- SCons/Environment.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SCons/Environment.xml b/SCons/Environment.xml index a94d9c5..2c5d563 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -1948,7 +1948,7 @@ can contain POSIX-style shell metacharacters for matching: [seq] matches any character in seq - (can be a list or a ranage). + (can be a list or a range). [!seq] @@ -1997,6 +1997,8 @@ argument evaluates false, the search for matches on disk is disabled, and only matches from already-configured File or Dir Nodes are returned. +The default is to return Nodes for +matches on disk as well. -- cgit v0.12 From 36473165f81ffd18e18d4dbdb9a1e366e9c7092f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 26 Sep 2022 08:57:19 -0600 Subject: Modernize SCONS_CACHE_MSVC_CONFIG manpage entry [skip appveyor] Mention name change; remove the wording about version changes possibly causing problems - the currrent implementation should be resilient to this. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 19dbb76..e00db3d 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -8398,29 +8398,29 @@ so the command line can be used to override (Windows only). If set, save the shell environment variables generated when setting up the Microsoft Visual C++ compiler -(and/or Build Tools) to a cache file, to give these settings, -which are relatively expensive to generate, persistence -across &scons; invocations. -Use of this option is primarily intended to aid performance -in tightly controlled Continuous Integration setups. +(and/or Build Tools) to a cache file, to give these settings +persistence across &scons; invocations. +Generating this information is relatively expensive, +so using this option may aid performance where &scons; is run often, +such as Continuous Integration setups. If set to a True-like value ("1", "true" or "True") will cache to a file named -.scons_msvc_cache.json in the user's home directory. +scons_msvc_cache.json in the user's home directory. If set to a pathname, will use that pathname for the cache. -Note: use this cache with caution as it -might be somewhat fragile: while each major toolset version -(e.g. Visual Studio 2017 vs 2019) and architecture pair will get separate -cache entries, if toolset updates cause a change -to settings within a given release series, &scons; will not -detect the change and will reuse old settings. -Remove the cache file in case of problems with this. -&scons; will ignore failures reading or writing the file -and will silently revert to non-cached behavior in such cases. - -Available since &scons; 3.1 (experimental). +Note: this implementation may still be somewhat fragile. +In case of problems, remove the cache file - recreating with +fresh info normally resolves any issues. +&SCons; ignores failures reading or writing the cache file +and will silently revert to non-cached behavior in such cases. + + +New in 3.1 (experimental). +The default cache file name was changed to +its present value in 4.4, and contents were expanded. + -- cgit v0.12 From c12f46fd9ec3fc25f11e1e9d680aace77e521c78 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 10 Oct 2022 15:57:45 -0700 Subject: removing obsolete python 2 only test --- test/print_statement.py | 56 ------------------------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 test/print_statement.py diff --git a/test/print_statement.py b/test/print_statement.py deleted file mode 100644 index 2205059..0000000 --- a/test/print_statement.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - -import sys -import TestSCons - -test = TestSCons.TestSCons() - - -test.write('SConstruct', """\ -print('python 3 style statement') -Exit(0) -""") - -test.run() - -test.write('SConstruct', """\ -print 'python 2 style statement' -Exit(0) -""") - -if sys.version_info >= (3,0): - test.skip_test('Python 2 print statement test, skipping on Python 3.\n') -else: - test.run() - -test.pass_test() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From d07f72533cec8b1a19e2909ed8e277928da9c467 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 10 Oct 2022 15:39:33 -0700 Subject: move Taskmaster and Jobs to SCons.Taskmaster --- SCons/Job.py | 439 ------------ SCons/JobTests.py | 575 ---------------- SCons/SConf.py | 4 +- SCons/Script/Main.py | 6 +- SCons/Taskmaster.py | 1059 ----------------------------- SCons/Taskmaster/Job.py | 439 ++++++++++++ SCons/Taskmaster/JobTests.py | 574 ++++++++++++++++ SCons/Taskmaster/TaskmasterTests.py | 1257 +++++++++++++++++++++++++++++++++++ SCons/Taskmaster/__init__.py | 1059 +++++++++++++++++++++++++++++ SCons/TaskmasterTests.py | 1257 ----------------------------------- 10 files changed, 3334 insertions(+), 3335 deletions(-) delete mode 100644 SCons/Job.py delete mode 100644 SCons/JobTests.py delete mode 100644 SCons/Taskmaster.py create mode 100644 SCons/Taskmaster/Job.py create mode 100644 SCons/Taskmaster/JobTests.py create mode 100644 SCons/Taskmaster/TaskmasterTests.py create mode 100644 SCons/Taskmaster/__init__.py delete mode 100644 SCons/TaskmasterTests.py diff --git a/SCons/Job.py b/SCons/Job.py deleted file mode 100644 index b398790..0000000 --- a/SCons/Job.py +++ /dev/null @@ -1,439 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Serial and Parallel classes to execute build tasks. - -The Jobs class provides a higher level interface to start, -stop, and wait on jobs. -""" - -import SCons.compat - -import os -import signal - -import SCons.Errors -import SCons.Warnings - -# The default stack size (in kilobytes) of the threads used to execute -# jobs in parallel. -# -# We use a stack size of 256 kilobytes. The default on some platforms -# is too large and prevents us from creating enough threads to fully -# parallelized the build. For example, the default stack size on linux -# is 8 MBytes. - -explicit_stack_size = None -default_stack_size = 256 - -interrupt_msg = 'Build interrupted.' - - -class InterruptState: - def __init__(self): - self.interrupted = False - - def set(self): - self.interrupted = True - - def __call__(self): - return self.interrupted - - -class Jobs: - """An instance of this class initializes N jobs, and provides - methods for starting, stopping, and waiting on all N jobs. - """ - - def __init__(self, num, taskmaster): - """ - Create 'num' jobs using the given taskmaster. - - If 'num' is 1 or less, then a serial job will be used, - otherwise a parallel job with 'num' worker threads will - be used. - - The 'num_jobs' attribute will be set to the actual number of jobs - allocated. If more than one job is requested but the Parallel - class can't do it, it gets reset to 1. Wrapping interfaces that - care should check the value of 'num_jobs' after initialization. - """ - - self.job = None - if num > 1: - stack_size = explicit_stack_size - if stack_size is None: - stack_size = default_stack_size - - try: - self.job = Parallel(taskmaster, num, stack_size) - self.num_jobs = num - except NameError: - pass - if self.job is None: - self.job = Serial(taskmaster) - self.num_jobs = 1 - - def run(self, postfunc=lambda: None): - """Run the jobs. - - postfunc() will be invoked after the jobs has run. It will be - invoked even if the jobs are interrupted by a keyboard - interrupt (well, in fact by a signal such as either SIGINT, - SIGTERM or SIGHUP). The execution of postfunc() is protected - against keyboard interrupts and is guaranteed to run to - completion.""" - self._setup_sig_handler() - try: - self.job.start() - finally: - postfunc() - self._reset_sig_handler() - - def were_interrupted(self): - """Returns whether the jobs were interrupted by a signal.""" - return self.job.interrupted() - - def _setup_sig_handler(self): - """Setup an interrupt handler so that SCons can shutdown cleanly in - various conditions: - - a) SIGINT: Keyboard interrupt - b) SIGTERM: kill or system shutdown - c) SIGHUP: Controlling shell exiting - - We handle all of these cases by stopping the taskmaster. It - turns out that it's very difficult to stop the build process - by throwing asynchronously an exception such as - KeyboardInterrupt. For example, the python Condition - variables (threading.Condition) and queues do not seem to be - asynchronous-exception-safe. It would require adding a whole - bunch of try/finally block and except KeyboardInterrupt all - over the place. - - Note also that we have to be careful to handle the case when - SCons forks before executing another process. In that case, we - want the child to exit immediately. - """ - def handler(signum, stack, self=self, parentpid=os.getpid()): - if os.getpid() == parentpid: - self.job.taskmaster.stop() - self.job.interrupted.set() - else: - os._exit(2) # pylint: disable=protected-access - - self.old_sigint = signal.signal(signal.SIGINT, handler) - self.old_sigterm = signal.signal(signal.SIGTERM, handler) - try: - self.old_sighup = signal.signal(signal.SIGHUP, handler) - except AttributeError: - pass - if (self.old_sigint is None) or (self.old_sigterm is None) or \ - (hasattr(self, "old_sighup") and self.old_sighup is None): - msg = "Overwritting previous signal handler which was not installed from Python. " + \ - "Will not be able to reinstate and so will return to default handler." - SCons.Warnings.warn(SCons.Warnings.SConsWarning, msg) - - def _reset_sig_handler(self): - """Restore the signal handlers to their previous state (before the - call to _setup_sig_handler().""" - sigint_to_use = self.old_sigint if self.old_sigint is not None else signal.SIG_DFL - sigterm_to_use = self.old_sigterm if self.old_sigterm is not None else signal.SIG_DFL - signal.signal(signal.SIGINT, sigint_to_use) - signal.signal(signal.SIGTERM, sigterm_to_use) - try: - sigterm_to_use = self.old_sighup if self.old_sighup is not None else signal.SIG_DFL - signal.signal(signal.SIGHUP, sigterm_to_use) - except AttributeError: - pass - -class Serial: - """This class is used to execute tasks in series, and is more efficient - than Parallel, but is only appropriate for non-parallel builds. Only - one instance of this class should be in existence at a time. - - This class is not thread safe. - """ - - def __init__(self, taskmaster): - """Create a new serial job given a taskmaster. - - The taskmaster's next_task() method should return the next task - that needs to be executed, or None if there are no more tasks. The - taskmaster's executed() method will be called for each task when it - is successfully executed, or failed() will be called if it failed to - execute (e.g. execute() raised an exception).""" - - self.taskmaster = taskmaster - self.interrupted = InterruptState() - - def start(self): - """Start the job. This will begin pulling tasks from the taskmaster - and executing them, and return when there are no more tasks. If a task - fails to execute (i.e. execute() raises an exception), then the job will - stop.""" - - while True: - task = self.taskmaster.next_task() - - if task is None: - break - - try: - task.prepare() - if task.needs_execute(): - task.execute() - except Exception: - if self.interrupted(): - try: - raise SCons.Errors.BuildError( - task.targets[0], errstr=interrupt_msg) - except: - task.exception_set() - else: - task.exception_set() - - # Let the failed() callback function arrange for the - # build to stop if that's appropriate. - task.failed() - else: - task.executed() - - task.postprocess() - self.taskmaster.cleanup() - - -# Trap import failure so that everything in the Job module but the -# Parallel class (and its dependent classes) will work if the interpreter -# doesn't support threads. -try: - import queue - import threading -except ImportError: - pass -else: - class Worker(threading.Thread): - """A worker thread waits on a task to be posted to its request queue, - dequeues the task, executes it, and posts a tuple including the task - and a boolean indicating whether the task executed successfully. """ - - def __init__(self, requestQueue, resultsQueue, interrupted): - super().__init__() - self.daemon = True - self.requestQueue = requestQueue - self.resultsQueue = resultsQueue - self.interrupted = interrupted - self.start() - - def run(self): - while True: - task = self.requestQueue.get() - - if task is None: - # The "None" value is used as a sentinel by - # ThreadPool.cleanup(). This indicates that there - # are no more tasks, so we should quit. - break - - try: - if self.interrupted(): - raise SCons.Errors.BuildError( - task.targets[0], errstr=interrupt_msg) - task.execute() - except: - task.exception_set() - ok = False - else: - ok = True - - self.resultsQueue.put((task, ok)) - - class ThreadPool: - """This class is responsible for spawning and managing worker threads.""" - - def __init__(self, num, stack_size, interrupted): - """Create the request and reply queues, and 'num' worker threads. - - One must specify the stack size of the worker threads. The - stack size is specified in kilobytes. - """ - self.requestQueue = queue.Queue(0) - self.resultsQueue = queue.Queue(0) - - try: - prev_size = threading.stack_size(stack_size*1024) - except AttributeError as e: - # Only print a warning if the stack size has been - # explicitly set. - if explicit_stack_size is not None: - msg = "Setting stack size is unsupported by this version of Python:\n " + \ - e.args[0] - SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) - except ValueError as e: - msg = "Setting stack size failed:\n " + str(e) - SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) - - # Create worker threads - self.workers = [] - for _ in range(num): - worker = Worker(self.requestQueue, self.resultsQueue, interrupted) - self.workers.append(worker) - - if 'prev_size' in locals(): - threading.stack_size(prev_size) - - def put(self, task): - """Put task into request queue.""" - self.requestQueue.put(task) - - def get(self): - """Remove and return a result tuple from the results queue.""" - return self.resultsQueue.get() - - def preparation_failed(self, task): - self.resultsQueue.put((task, False)) - - def cleanup(self): - """ - Shuts down the thread pool, giving each worker thread a - chance to shut down gracefully. - """ - # For each worker thread, put a sentinel "None" value - # on the requestQueue (indicating that there's no work - # to be done) so that each worker thread will get one and - # terminate gracefully. - for _ in self.workers: - self.requestQueue.put(None) - - # Wait for all of the workers to terminate. - # - # If we don't do this, later Python versions (2.4, 2.5) often - # seem to raise exceptions during shutdown. This happens - # in requestQueue.get(), as an assertion failure that - # requestQueue.not_full is notified while not acquired, - # seemingly because the main thread has shut down (or is - # in the process of doing so) while the workers are still - # trying to pull sentinels off the requestQueue. - # - # Normally these terminations should happen fairly quickly, - # but we'll stick a one-second timeout on here just in case - # someone gets hung. - for worker in self.workers: - worker.join(1.0) - self.workers = [] - - class Parallel: - """This class is used to execute tasks in parallel, and is somewhat - less efficient than Serial, but is appropriate for parallel builds. - - This class is thread safe. - """ - - def __init__(self, taskmaster, num, stack_size): - """Create a new parallel job given a taskmaster. - - The taskmaster's next_task() method should return the next - task that needs to be executed, or None if there are no more - tasks. The taskmaster's executed() method will be called - for each task when it is successfully executed, or failed() - will be called if the task failed to execute (i.e. execute() - raised an exception). - - Note: calls to taskmaster are serialized, but calls to - execute() on distinct tasks are not serialized, because - that is the whole point of parallel jobs: they can execute - multiple tasks simultaneously. """ - - self.taskmaster = taskmaster - self.interrupted = InterruptState() - self.tp = ThreadPool(num, stack_size, self.interrupted) - - self.maxjobs = num - - def start(self): - """Start the job. This will begin pulling tasks from the - taskmaster and executing them, and return when there are no - more tasks. If a task fails to execute (i.e. execute() raises - an exception), then the job will stop.""" - - jobs = 0 - - while True: - # Start up as many available tasks as we're - # allowed to. - while jobs < self.maxjobs: - task = self.taskmaster.next_task() - if task is None: - break - - try: - # prepare task for execution - task.prepare() - except: - task.exception_set() - task.failed() - task.postprocess() - else: - if task.needs_execute(): - # dispatch task - self.tp.put(task) - jobs += 1 - else: - task.executed() - task.postprocess() - - if not task and not jobs: break - - # Let any/all completed tasks finish up before we go - # back and put the next batch of tasks on the queue. - while True: - task, ok = self.tp.get() - jobs -= 1 - - if ok: - task.executed() - else: - if self.interrupted(): - try: - raise SCons.Errors.BuildError( - task.targets[0], errstr=interrupt_msg) - except: - task.exception_set() - - # Let the failed() callback function arrange - # for the build to stop if that's appropriate. - task.failed() - - task.postprocess() - - if self.tp.resultsQueue.empty(): - break - - self.tp.cleanup() - self.taskmaster.cleanup() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/JobTests.py b/SCons/JobTests.py deleted file mode 100644 index 54d7fa4..0000000 --- a/SCons/JobTests.py +++ /dev/null @@ -1,575 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import unittest -import random -import math -import sys -import time -import os - -import TestUnit - -import SCons.Job - - -def get_cpu_nums(): - # Linux, Unix and MacOS: - if hasattr( os, "sysconf" ): - if "SC_NPROCESSORS_ONLN" in os.sysconf_names: - # Linux & Unix: - ncpus = os.sysconf( "SC_NPROCESSORS_ONLN" ) - if isinstance(ncpus, int) and ncpus > 0: - return ncpus - else: # OSX: - return int(os.popen2("sysctl -n hw.ncpu")[1].read() ) - # Windows: - if "NUMBER_OF_PROCESSORS" in os.environ: - ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) - if ncpus > 0: - return ncpus - return 1 # Default - -# a large number -num_sines = 500 - -# how many parallel jobs to perform for the test -num_jobs = get_cpu_nums()*2 - -# in case we werent able to detect num cpus for this test -# just make a hardcoded suffcient large number, though not future proof -if num_jobs == 2: - num_jobs = 33 - -# how many tasks to perform for the test -num_tasks = num_jobs*5 - -class DummyLock: - """fake lock class to use if threads are not supported""" - def acquire(self): - pass - - def release(self): - pass - -class NoThreadsException(Exception): - """raised by the ParallelTestCase if threads are not supported""" - - def __str__(self): - return "the interpreter doesn't support threads" - -class Task: - """A dummy task class for testing purposes.""" - - def __init__(self, i, taskmaster): - self.i = i - self.taskmaster = taskmaster - self.was_executed = 0 - self.was_prepared = 0 - - def prepare(self): - self.was_prepared = 1 - - def _do_something(self): - pass - - def needs_execute(self): - return True - - def execute(self): - self.taskmaster.test_case.assertTrue(self.was_prepared, - "the task wasn't prepared") - - self.taskmaster.guard.acquire() - self.taskmaster.begin_list.append(self.i) - self.taskmaster.guard.release() - - # while task is executing, represent this in the parallel_list - # and then turn it off - self.taskmaster.parallel_list[self.i] = 1 - self._do_something() - self.taskmaster.parallel_list[self.i] = 0 - - # check if task was executing while another was also executing - for j in range(1, self.taskmaster.num_tasks): - if self.taskmaster.parallel_list[j + 1] == 1: - self.taskmaster.found_parallel = True - break - - self.was_executed = 1 - - self.taskmaster.guard.acquire() - self.taskmaster.end_list.append(self.i) - self.taskmaster.guard.release() - - def executed(self): - self.taskmaster.num_executed = self.taskmaster.num_executed + 1 - - self.taskmaster.test_case.assertTrue(self.was_prepared, - "the task wasn't prepared") - self.taskmaster.test_case.assertTrue(self.was_executed, - "the task wasn't really executed") - self.taskmaster.test_case.assertTrue(isinstance(self, Task), - "the task wasn't really a Task instance") - - def failed(self): - self.taskmaster.num_failed = self.taskmaster.num_failed + 1 - self.taskmaster.stop = 1 - self.taskmaster.test_case.assertTrue(self.was_prepared, - "the task wasn't prepared") - - def postprocess(self): - self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 - - def exception_set(self): - pass - -class RandomTask(Task): - def _do_something(self): - # do something that will take some random amount of time: - for i in range(random.randrange(0, 100 + num_sines, 1)): - x = math.sin(i) - time.sleep(0.01) - -class ExceptionTask: - """A dummy task class for testing purposes.""" - - def __init__(self, i, taskmaster): - self.taskmaster = taskmaster - self.was_prepared = 0 - - def prepare(self): - self.was_prepared = 1 - - def needs_execute(self): - return True - - def execute(self): - raise Exception - - def executed(self): - self.taskmaster.num_executed = self.taskmaster.num_executed + 1 - - self.taskmaster.test_case.assertTrue(self.was_prepared, - "the task wasn't prepared") - self.taskmaster.test_case.assertTrue(self.was_executed, - "the task wasn't really executed") - self.taskmaster.test_case.assertTrue(self.__class__ is Task, - "the task wasn't really a Task instance") - - def failed(self): - self.taskmaster.num_failed = self.taskmaster.num_failed + 1 - self.taskmaster.stop = 1 - self.taskmaster.test_case.assertTrue(self.was_prepared, - "the task wasn't prepared") - - def postprocess(self): - self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 - - def exception_set(self): - self.taskmaster.exception_set() - -class Taskmaster: - """A dummy taskmaster class for testing the job classes.""" - - def __init__(self, n, test_case, Task): - """n is the number of dummy tasks to perform.""" - - self.test_case = test_case - self.stop = None - self.num_tasks = n - self.num_iterated = 0 - self.num_executed = 0 - self.num_failed = 0 - self.num_postprocessed = 0 - self.parallel_list = [0] * (n+1) - self.found_parallel = False - self.Task = Task - - # 'guard' guards 'task_begin_list' and 'task_end_list' - try: - import threading - self.guard = threading.Lock() - except ImportError: - self.guard = DummyLock() - - # keep track of the order tasks are begun in - self.begin_list = [] - - # keep track of the order tasks are completed in - self.end_list = [] - - def next_task(self): - if self.stop or self.all_tasks_are_iterated(): - return None - else: - self.num_iterated = self.num_iterated + 1 - return self.Task(self.num_iterated, self) - - def all_tasks_are_executed(self): - return self.num_executed == self.num_tasks - - def all_tasks_are_iterated(self): - return self.num_iterated == self.num_tasks - - def all_tasks_are_postprocessed(self): - return self.num_postprocessed == self.num_tasks - - def tasks_were_serial(self): - """analyze the task order to see if they were serial""" - return not self.found_parallel - - def exception_set(self): - pass - - def cleanup(self): - pass - -SaveThreadPool = None -ThreadPoolCallList = [] - -class ParallelTestCase(unittest.TestCase): - def runTest(self): - """test parallel jobs""" - - try: - import threading - except ImportError: - raise NoThreadsException() - - taskmaster = Taskmaster(num_tasks, self, RandomTask) - jobs = SCons.Job.Jobs(num_jobs, taskmaster) - jobs.run() - - self.assertTrue(not taskmaster.tasks_were_serial(), - "the tasks were not executed in parallel") - self.assertTrue(taskmaster.all_tasks_are_executed(), - "all the tests were not executed") - self.assertTrue(taskmaster.all_tasks_are_iterated(), - "all the tests were not iterated over") - self.assertTrue(taskmaster.all_tasks_are_postprocessed(), - "all the tests were not postprocessed") - self.assertFalse(taskmaster.num_failed, - "some task(s) failed to execute") - - # Verify that parallel jobs will pull all of the completed tasks - # out of the queue at once, instead of one by one. We do this by - # replacing the default ThreadPool class with one that records the - # order in which tasks are put() and get() to/from the pool, and - # which sleeps a little bit before call get() to let the initial - # tasks complete and get their notifications on the resultsQueue. - - class SleepTask(Task): - def _do_something(self): - time.sleep(0.01) - - global SaveThreadPool - SaveThreadPool = SCons.Job.ThreadPool - - class WaitThreadPool(SaveThreadPool): - def put(self, task): - ThreadPoolCallList.append('put(%s)' % task.i) - return SaveThreadPool.put(self, task) - def get(self): - time.sleep(0.05) - result = SaveThreadPool.get(self) - ThreadPoolCallList.append('get(%s)' % result[0].i) - return result - - SCons.Job.ThreadPool = WaitThreadPool - - try: - taskmaster = Taskmaster(3, self, SleepTask) - jobs = SCons.Job.Jobs(2, taskmaster) - jobs.run() - - # The key here is that we get(1) and get(2) from the - # resultsQueue before we put(3), but get(1) and get(2) can - # be in either order depending on how the first two parallel - # tasks get scheduled by the operating system. - expect = [ - ['put(1)', 'put(2)', 'get(1)', 'get(2)', 'put(3)', 'get(3)'], - ['put(1)', 'put(2)', 'get(2)', 'get(1)', 'put(3)', 'get(3)'], - ] - assert ThreadPoolCallList in expect, ThreadPoolCallList - - finally: - SCons.Job.ThreadPool = SaveThreadPool - -class SerialTestCase(unittest.TestCase): - def runTest(self): - """test a serial job""" - - taskmaster = Taskmaster(num_tasks, self, RandomTask) - jobs = SCons.Job.Jobs(1, taskmaster) - jobs.run() - - self.assertTrue(taskmaster.tasks_were_serial(), - "the tasks were not executed in series") - self.assertTrue(taskmaster.all_tasks_are_executed(), - "all the tests were not executed") - self.assertTrue(taskmaster.all_tasks_are_iterated(), - "all the tests were not iterated over") - self.assertTrue(taskmaster.all_tasks_are_postprocessed(), - "all the tests were not postprocessed") - self.assertFalse(taskmaster.num_failed, - "some task(s) failed to execute") - -class NoParallelTestCase(unittest.TestCase): - def runTest(self): - """test handling lack of parallel support""" - def NoParallel(tm, num, stack_size): - raise NameError - save_Parallel = SCons.Job.Parallel - SCons.Job.Parallel = NoParallel - try: - taskmaster = Taskmaster(num_tasks, self, RandomTask) - jobs = SCons.Job.Jobs(2, taskmaster) - self.assertTrue(jobs.num_jobs == 1, - "unexpected number of jobs %d" % jobs.num_jobs) - jobs.run() - self.assertTrue(taskmaster.tasks_were_serial(), - "the tasks were not executed in series") - self.assertTrue(taskmaster.all_tasks_are_executed(), - "all the tests were not executed") - self.assertTrue(taskmaster.all_tasks_are_iterated(), - "all the tests were not iterated over") - self.assertTrue(taskmaster.all_tasks_are_postprocessed(), - "all the tests were not postprocessed") - self.assertFalse(taskmaster.num_failed, - "some task(s) failed to execute") - finally: - SCons.Job.Parallel = save_Parallel - - -class SerialExceptionTestCase(unittest.TestCase): - def runTest(self): - """test a serial job with tasks that raise exceptions""" - - taskmaster = Taskmaster(num_tasks, self, ExceptionTask) - jobs = SCons.Job.Jobs(1, taskmaster) - jobs.run() - - self.assertFalse(taskmaster.num_executed, - "a task was executed") - self.assertTrue(taskmaster.num_iterated == 1, - "exactly one task should have been iterated") - self.assertTrue(taskmaster.num_failed == 1, - "exactly one task should have failed") - self.assertTrue(taskmaster.num_postprocessed == 1, - "exactly one task should have been postprocessed") - -class ParallelExceptionTestCase(unittest.TestCase): - def runTest(self): - """test parallel jobs with tasks that raise exceptions""" - - taskmaster = Taskmaster(num_tasks, self, ExceptionTask) - jobs = SCons.Job.Jobs(num_jobs, taskmaster) - jobs.run() - - self.assertFalse(taskmaster.num_executed, - "a task was executed") - self.assertTrue(taskmaster.num_iterated >= 1, - "one or more task should have been iterated") - self.assertTrue(taskmaster.num_failed >= 1, - "one or more tasks should have failed") - self.assertTrue(taskmaster.num_postprocessed >= 1, - "one or more tasks should have been postprocessed") - -#--------------------------------------------------------------------- -# Above tested Job object with contrived Task and Taskmaster objects. -# Now test Job object with actual Task and Taskmaster objects. - -import SCons.Taskmaster -import SCons.Node -import time - -class DummyNodeInfo: - def update(self, obj): - pass - -class testnode (SCons.Node.Node): - def __init__(self): - super().__init__() - self.expect_to_be = SCons.Node.executed - self.ninfo = DummyNodeInfo() - -class goodnode (testnode): - def __init__(self): - super().__init__() - self.expect_to_be = SCons.Node.up_to_date - self.ninfo = DummyNodeInfo() - -class slowgoodnode (goodnode): - def prepare(self): - # Delay to allow scheduled Jobs to run while the dispatcher - # sleeps. Keep this short because it affects the time taken - # by this test. - time.sleep(0.15) - goodnode.prepare(self) - -class badnode (goodnode): - def __init__(self): - super().__init__() - self.expect_to_be = SCons.Node.failed - def build(self, **kw): - raise Exception('badnode exception') - -class slowbadnode (badnode): - def build(self, **kw): - # Appears to take a while to build, allowing faster builds to - # overlap. Time duration is not especially important, but if - # it is faster than slowgoodnode then these could complete - # while the scheduler is sleeping. - time.sleep(0.05) - raise Exception('slowbadnode exception') - -class badpreparenode (badnode): - def prepare(self): - raise Exception('badpreparenode exception') - -class _SConsTaskTest(unittest.TestCase): - - def _test_seq(self, num_jobs): - for node_seq in [ - [goodnode], - [badnode], - [slowbadnode], - [slowgoodnode], - [badpreparenode], - [goodnode, badnode], - [slowgoodnode, badnode], - [goodnode, slowbadnode], - [goodnode, goodnode, goodnode, slowbadnode], - [goodnode, slowbadnode, badpreparenode, slowgoodnode], - [goodnode, slowbadnode, slowgoodnode, badnode] - ]: - - self._do_test(num_jobs, node_seq) - - def _do_test(self, num_jobs, node_seq): - - testnodes = [] - for tnum in range(num_tasks): - testnodes.append(node_seq[tnum % len(node_seq)]()) - - taskmaster = SCons.Taskmaster.Taskmaster(testnodes, - tasker=SCons.Taskmaster.AlwaysTask) - - jobs = SCons.Job.Jobs(num_jobs, taskmaster) - - # Exceptions thrown by tasks are not actually propagated to - # this level, but are instead stored in the Taskmaster. - - jobs.run() - - # Now figure out if tests proceeded correctly. The first test - # that fails will shutdown the initiation of subsequent tests, - # but any tests currently queued for execution will still be - # processed, and any tests that completed before the failure - # would have resulted in new tests being queued for execution. - - # Apply the following operational heuristics of Job.py: - # 0) An initial jobset of tasks will be queued before any - # good/bad results are obtained (from "execute" of task in - # thread). - # 1) A goodnode will complete immediately on its thread and - # allow another node to be queued for execution. - # 2) A badnode will complete immediately and suppress any - # subsequent execution queuing, but all currently queued - # tasks will still be processed. - # 3) A slowbadnode will fail later. It will block slots in - # the job queue. Nodes that complete immediately will - # allow other nodes to be queued in their place, and this - # will continue until either (#2) above or until all job - # slots are filled with slowbadnode entries. - - # One approach to validating this test would be to try to - # determine exactly how many nodes executed, how many didn't, - # and the results of each, and then to assert failure on any - # mismatch (including the total number of built nodes). - # However, while this is possible to do for a single-processor - # system, it is nearly impossible to predict correctly for a - # multi-processor system and still test the characteristics of - # delayed execution nodes. Stated another way, multithreading - # is inherently non-deterministic unless you can completely - # characterize the entire system, and since that's not - # possible here, we shouldn't try. - - # Therefore, this test will simply scan the set of nodes to - # see if the node was executed or not and if it was executed - # that it obtained the expected value for that node - # (i.e. verifying we don't get failure crossovers or - # mislabelling of results). - - for N in testnodes: - state = N.get_state() - self.assertTrue(state in [SCons.Node.no_state, N.expect_to_be], - "Node %s got unexpected result: %s" % (N, state)) - - self.assertTrue([N for N in testnodes if N.get_state()], - "no nodes ran at all.") - - -class SerialTaskTest(_SConsTaskTest): - def runTest(self): - """test serial jobs with actual Taskmaster and Task""" - self._test_seq(1) - - -class ParallelTaskTest(_SConsTaskTest): - def runTest(self): - """test parallel jobs with actual Taskmaster and Task""" - self._test_seq(num_jobs) - - - -#--------------------------------------------------------------------- - -def suite(): - suite = unittest.TestSuite() - suite.addTest(ParallelTestCase()) - suite.addTest(SerialTestCase()) - suite.addTest(NoParallelTestCase()) - suite.addTest(SerialExceptionTestCase()) - suite.addTest(ParallelExceptionTestCase()) - suite.addTest(SerialTaskTest()) - suite.addTest(ParallelTaskTest()) - return suite - -if __name__ == "__main__": - runner = TestUnit.cli.get_runner() - result = runner().run(suite()) - if (len(result.failures) == 0 - and len(result.errors) == 1 - and isinstance(result.errors[0][0], SerialTestCase) - and isinstance(result.errors[0][1][0], NoThreadsException)): - sys.exit(2) - elif not result.wasSuccessful(): - sys.exit(1) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/SConf.py b/SCons/SConf.py index 2a427df..fe14a0c 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -43,7 +43,7 @@ import traceback import SCons.Action import SCons.Builder import SCons.Errors -import SCons.Job +import SCons.Taskmaster.Job import SCons.Node.FS import SCons.Taskmaster import SCons.Util @@ -551,7 +551,7 @@ class SConfBase: SConfFS.set_max_drift(0) tm = SCons.Taskmaster.Taskmaster(nodes, SConfBuildTask) # we don't want to build tests in parallel - jobs = SCons.Job.Jobs(1, tm ) + jobs = SCons.Taskmaster.Job.Jobs(1, tm) jobs.run() for n in nodes: state = n.get_state() diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 1b06a64..22042f5 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -52,7 +52,7 @@ import SCons.Debug import SCons.Defaults import SCons.Environment import SCons.Errors -import SCons.Job +import SCons.Taskmaster.Job import SCons.Node import SCons.Node.FS import SCons.Platform @@ -1134,7 +1134,7 @@ def _main(parser): SCons.Node.FS.set_duplicate(options.duplicate) fs.set_max_drift(options.max_drift) - SCons.Job.explicit_stack_size = options.stack_size + SCons.Taskmaster.Job.explicit_stack_size = options.stack_size # Hash format and chunksize are set late to support SetOption being called # in a SConscript or SConstruct file. @@ -1321,7 +1321,7 @@ def _build_targets(fs, options, targets, target_top): # to check if python configured with threads. global num_jobs num_jobs = options.num_jobs - jobs = SCons.Job.Jobs(num_jobs, taskmaster) + jobs = SCons.Taskmaster.Job.Jobs(num_jobs, taskmaster) if num_jobs > 1: msg = None if jobs.num_jobs == 1 or not python_has_threads: diff --git a/SCons/Taskmaster.py b/SCons/Taskmaster.py deleted file mode 100644 index d571795..0000000 --- a/SCons/Taskmaster.py +++ /dev/null @@ -1,1059 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Generic Taskmaster module for the SCons build engine. - -This module contains the primary interface(s) between a wrapping user -interface and the SCons build engine. There are two key classes here: - -Taskmaster - This is the main engine for walking the dependency graph and - calling things to decide what does or doesn't need to be built. - -Task - This is the base class for allowing a wrapping interface to - decide what does or doesn't actually need to be done. The - intention is for a wrapping interface to subclass this as - appropriate for different types of behavior it may need. - - The canonical example is the SCons native Python interface, - which has Task subclasses that handle its specific behavior, - like printing "'foo' is up to date" when a top-level target - doesn't need to be built, and handling the -c option by removing - targets as its "build" action. There is also a separate subclass - for suppressing this output when the -q option is used. - - The Taskmaster instantiates a Task object for each (set of) - target(s) that it decides need to be evaluated and/or built. -""" - -import sys -from abc import ABC, abstractmethod -from itertools import chain - -import SCons.Errors -import SCons.Node -import SCons.Warnings - -StateString = SCons.Node.StateString -NODE_NO_STATE = SCons.Node.no_state -NODE_PENDING = SCons.Node.pending -NODE_EXECUTING = SCons.Node.executing -NODE_UP_TO_DATE = SCons.Node.up_to_date -NODE_EXECUTED = SCons.Node.executed -NODE_FAILED = SCons.Node.failed - -print_prepare = False # set by option --debug=prepare - -# A subsystem for recording stats about how different Nodes are handled by -# the main Taskmaster loop. There's no external control here (no need for -# a --debug= option); enable it by changing the value of CollectStats. - -CollectStats = None - -class Stats: - """ - A simple class for holding statistics about the disposition of a - Node by the Taskmaster. If we're collecting statistics, each Node - processed by the Taskmaster gets one of these attached, in which case - the Taskmaster records its decision each time it processes the Node. - (Ideally, that's just once per Node.) - """ - def __init__(self): - """ - Instantiates a Taskmaster.Stats object, initializing all - appropriate counters to zero. - """ - self.considered = 0 - self.already_handled = 0 - self.problem = 0 - self.child_failed = 0 - self.not_built = 0 - self.side_effects = 0 - self.build = 0 - -StatsNodes = [] - -fmt = "%(considered)3d "\ - "%(already_handled)3d " \ - "%(problem)3d " \ - "%(child_failed)3d " \ - "%(not_built)3d " \ - "%(side_effects)3d " \ - "%(build)3d " - -def dump_stats(): - for n in sorted(StatsNodes, key=lambda a: str(a)): - print((fmt % n.attributes.stats.__dict__) + str(n)) - - -class Task(ABC): - """ SCons build engine abstract task class. - - This controls the interaction of the actual building of node - and the rest of the engine. - - This is expected to handle all of the normally-customizable - aspects of controlling a build, so any given application - *should* be able to do what it wants by sub-classing this - class and overriding methods as appropriate. If an application - needs to customize something by sub-classing Taskmaster (or - some other build engine class), we should first try to migrate - that functionality into this class. - - Note that it's generally a good idea for sub-classes to call - these methods explicitly to update state, etc., rather than - roll their own interaction with Taskmaster from scratch. - """ - def __init__(self, tm, targets, top, node): - self.tm = tm - self.targets = targets - self.top = top - self.node = node - self.exc_clear() - - def trace_message(self, method, node, description='node'): - fmt = '%-20s %s %s\n' - return fmt % (method + ':', description, self.tm.trace_node(node)) - - def display(self, message): - """ - Hook to allow the calling interface to display a message. - - This hook gets called as part of preparing a task for execution - (that is, a Node to be built). As part of figuring out what Node - should be built next, the actual target list may be altered, - along with a message describing the alteration. The calling - interface can subclass Task and provide a concrete implementation - of this method to see those messages. - """ - pass - - def prepare(self): - """ - Called just before the task is executed. - - This is mainly intended to give the target Nodes a chance to - unlink underlying files and make all necessary directories before - the Action is actually called to build the targets. - """ - global print_prepare - T = self.tm.trace - if T: T.write(self.trace_message('Task.prepare()', self.node)) - - # Now that it's the appropriate time, give the TaskMaster a - # chance to raise any exceptions it encountered while preparing - # this task. - self.exception_raise() - - if self.tm.message: - self.display(self.tm.message) - self.tm.message = None - - # Let the targets take care of any necessary preparations. - # This includes verifying that all of the necessary sources - # and dependencies exist, removing the target file(s), etc. - # - # As of April 2008, the get_executor().prepare() method makes - # sure that all of the aggregate sources necessary to build this - # Task's target(s) exist in one up-front check. The individual - # target t.prepare() methods check that each target's explicit - # or implicit dependencies exists, and also initialize the - # .sconsign info. - executor = self.targets[0].get_executor() - if executor is None: - return - executor.prepare() - for t in executor.get_action_targets(): - if print_prepare: - print("Preparing target %s..."%t) - for s in t.side_effects: - print("...with side-effect %s..."%s) - t.prepare() - for s in t.side_effects: - if print_prepare: - print("...Preparing side-effect %s..."%s) - s.prepare() - - def get_target(self): - """Fetch the target being built or updated by this task. - """ - return self.node - - @abstractmethod - def needs_execute(self): - return - - def execute(self): - """ - Called to execute the task. - - This method is called from multiple threads in a parallel build, - so only do thread safe stuff here. Do thread unsafe stuff in - prepare(), executed() or failed(). - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.execute()', self.node)) - - try: - cached_targets = [] - for t in self.targets: - if not t.retrieve_from_cache(): - break - cached_targets.append(t) - if len(cached_targets) < len(self.targets): - # Remove targets before building. It's possible that we - # partially retrieved targets from the cache, leaving - # them in read-only mode. That might cause the command - # to fail. - # - for t in cached_targets: - try: - t.fs.unlink(t.get_internal_path()) - except (IOError, OSError): - pass - self.targets[0].build() - else: - for t in cached_targets: - t.cached = 1 - except SystemExit: - exc_value = sys.exc_info()[1] - raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) - except SCons.Errors.UserError: - raise - except SCons.Errors.BuildError: - raise - except Exception as e: - buildError = SCons.Errors.convert_to_BuildError(e) - buildError.node = self.targets[0] - buildError.exc_info = sys.exc_info() - raise buildError - - def executed_without_callbacks(self): - """ - Called when the task has been successfully executed - and the Taskmaster instance doesn't want to call - the Node's callback methods. - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.executed_without_callbacks()', - self.node)) - - for t in self.targets: - if t.get_state() == NODE_EXECUTING: - for side_effect in t.side_effects: - side_effect.set_state(NODE_NO_STATE) - t.set_state(NODE_EXECUTED) - - def executed_with_callbacks(self): - """ - Called when the task has been successfully executed and - the Taskmaster instance wants to call the Node's callback - methods. - - This may have been a do-nothing operation (to preserve build - order), so we must check the node's state before deciding whether - it was "built", in which case we call the appropriate Node method. - In any event, we always call "visited()", which will handle any - post-visit actions that must take place regardless of whether - or not the target was an actual built target or a source Node. - """ - global print_prepare - T = self.tm.trace - if T: T.write(self.trace_message('Task.executed_with_callbacks()', - self.node)) - - for t in self.targets: - if t.get_state() == NODE_EXECUTING: - for side_effect in t.side_effects: - side_effect.set_state(NODE_NO_STATE) - t.set_state(NODE_EXECUTED) - if not t.cached: - t.push_to_cache() - t.built() - t.visited() - if (not print_prepare and - (not hasattr(self, 'options') or not self.options.debug_includes)): - t.release_target_info() - else: - t.visited() - - executed = executed_with_callbacks - - def failed(self): - """ - Default action when a task fails: stop the build. - - Note: Although this function is normally invoked on nodes in - the executing state, it might also be invoked on up-to-date - nodes when using Configure(). - """ - self.fail_stop() - - def fail_stop(self): - """ - Explicit stop-the-build failure. - - This sets failure status on the target nodes and all of - their dependent parent nodes. - - Note: Although this function is normally invoked on nodes in - the executing state, it might also be invoked on up-to-date - nodes when using Configure(). - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.failed_stop()', self.node)) - - # Invoke will_not_build() to clean-up the pending children - # list. - self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) - - # Tell the taskmaster to not start any new tasks - self.tm.stop() - - # We're stopping because of a build failure, but give the - # calling Task class a chance to postprocess() the top-level - # target under which the build failure occurred. - self.targets = [self.tm.current_top] - self.top = 1 - - def fail_continue(self): - """ - Explicit continue-the-build failure. - - This sets failure status on the target nodes and all of - their dependent parent nodes. - - Note: Although this function is normally invoked on nodes in - the executing state, it might also be invoked on up-to-date - nodes when using Configure(). - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.failed_continue()', self.node)) - - self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) - - def make_ready_all(self): - """ - Marks all targets in a task ready for execution. - - This is used when the interface needs every target Node to be - visited--the canonical example being the "scons -c" option. - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.make_ready_all()', self.node)) - - self.out_of_date = self.targets[:] - for t in self.targets: - t.disambiguate().set_state(NODE_EXECUTING) - for s in t.side_effects: - # add disambiguate here to mirror the call on targets above - s.disambiguate().set_state(NODE_EXECUTING) - - def make_ready_current(self): - """ - Marks all targets in a task ready for execution if any target - is not current. - - This is the default behavior for building only what's necessary. - """ - global print_prepare - T = self.tm.trace - if T: T.write(self.trace_message('Task.make_ready_current()', - self.node)) - - self.out_of_date = [] - needs_executing = False - for t in self.targets: - try: - t.disambiguate().make_ready() - is_up_to_date = not t.has_builder() or \ - (not t.always_build and t.is_up_to_date()) - except EnvironmentError as e: - raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename) - - if not is_up_to_date: - self.out_of_date.append(t) - needs_executing = True - - if needs_executing: - for t in self.targets: - t.set_state(NODE_EXECUTING) - for s in t.side_effects: - # add disambiguate here to mirror the call on targets in first loop above - s.disambiguate().set_state(NODE_EXECUTING) - else: - for t in self.targets: - # We must invoke visited() to ensure that the node - # information has been computed before allowing the - # parent nodes to execute. (That could occur in a - # parallel build...) - t.visited() - t.set_state(NODE_UP_TO_DATE) - if (not print_prepare and - (not hasattr(self, 'options') or not self.options.debug_includes)): - t.release_target_info() - - make_ready = make_ready_current - - def postprocess(self): - """ - Post-processes a task after it's been executed. - - This examines all the targets just built (or not, we don't care - if the build was successful, or even if there was no build - because everything was up-to-date) to see if they have any - waiting parent Nodes, or Nodes waiting on a common side effect, - that can be put back on the candidates list. - """ - T = self.tm.trace - if T: T.write(self.trace_message('Task.postprocess()', self.node)) - - # We may have built multiple targets, some of which may have - # common parents waiting for this build. Count up how many - # targets each parent was waiting for so we can subtract the - # values later, and so we *don't* put waiting side-effect Nodes - # back on the candidates list if the Node is also a waiting - # parent. - - targets = set(self.targets) - - pending_children = self.tm.pending_children - parents = {} - for t in targets: - # A node can only be in the pending_children set if it has - # some waiting_parents. - if t.waiting_parents: - if T: T.write(self.trace_message('Task.postprocess()', - t, - 'removing')) - pending_children.discard(t) - for p in t.waiting_parents: - parents[p] = parents.get(p, 0) + 1 - t.waiting_parents = set() - - for t in targets: - if t.side_effects is not None: - for s in t.side_effects: - if s.get_state() == NODE_EXECUTING: - s.set_state(NODE_NO_STATE) - - # The side-effects may have been transferred to - # NODE_NO_STATE by executed_with{,out}_callbacks, but was - # not taken out of the waiting parents/pending children - # data structures. Check for that now. - if s.get_state() == NODE_NO_STATE and s.waiting_parents: - pending_children.discard(s) - for p in s.waiting_parents: - parents[p] = parents.get(p, 0) + 1 - s.waiting_parents = set() - for p in s.waiting_s_e: - if p.ref_count == 0: - self.tm.candidates.append(p) - - for p, subtract in parents.items(): - p.ref_count = p.ref_count - subtract - if T: T.write(self.trace_message('Task.postprocess()', - p, - 'adjusted parent ref count')) - if p.ref_count == 0: - self.tm.candidates.append(p) - - for t in targets: - t.postprocess() - - # Exception handling subsystem. - # - # Exceptions that occur while walking the DAG or examining Nodes - # must be raised, but must be raised at an appropriate time and in - # a controlled manner so we can, if necessary, recover gracefully, - # possibly write out signature information for Nodes we've updated, - # etc. This is done by having the Taskmaster tell us about the - # exception, and letting - - def exc_info(self): - """ - Returns info about a recorded exception. - """ - return self.exception - - def exc_clear(self): - """ - Clears any recorded exception. - - This also changes the "exception_raise" attribute to point - to the appropriate do-nothing method. - """ - self.exception = (None, None, None) - self.exception_raise = self._no_exception_to_raise - - def exception_set(self, exception=None): - """ - Records an exception to be raised at the appropriate time. - - This also changes the "exception_raise" attribute to point - to the method that will, in fact - """ - if not exception: - exception = sys.exc_info() - self.exception = exception - self.exception_raise = self._exception_raise - - def _no_exception_to_raise(self): - pass - - def _exception_raise(self): - """ - Raises a pending exception that was recorded while getting a - Task ready for execution. - """ - exc = self.exc_info()[:] - try: - exc_type, exc_value, exc_traceback = exc - except ValueError: - exc_type, exc_value = exc # pylint: disable=unbalanced-tuple-unpacking - exc_traceback = None - - # raise exc_type(exc_value).with_traceback(exc_traceback) - if isinstance(exc_value, Exception): #hasattr(exc_value, 'with_traceback'): - # If exc_value is an exception, then just reraise - raise exc_value.with_traceback(exc_traceback) - else: - # else we'll create an exception using the value and raise that - raise exc_type(exc_value).with_traceback(exc_traceback) - - - # raise e.__class__, e.__class__(e), sys.exc_info()[2] - # exec("raise exc_type(exc_value).with_traceback(exc_traceback)") - - - -class AlwaysTask(Task): - def needs_execute(self): - """ - Always returns True (indicating this Task should always - be executed). - - Subclasses that need this behavior (as opposed to the default - of only executing Nodes that are out of date w.r.t. their - dependencies) can use this as follows: - - class MyTaskSubclass(SCons.Taskmaster.Task): - needs_execute = SCons.Taskmaster.AlwaysTask.needs_execute - """ - return True - -class OutOfDateTask(Task): - def needs_execute(self): - """ - Returns True (indicating this Task should be executed) if this - Task's target state indicates it needs executing, which has - already been determined by an earlier up-to-date check. - """ - return self.targets[0].get_state() == SCons.Node.executing - - -def find_cycle(stack, visited): - if stack[-1] in visited: - return None - visited.add(stack[-1]) - for n in stack[-1].waiting_parents: - stack.append(n) - if stack[0] == stack[-1]: - return stack - if find_cycle(stack, visited): - return stack - stack.pop() - return None - - -class Taskmaster: - """ - The Taskmaster for walking the dependency DAG. - """ - - def __init__(self, targets=[], tasker=None, order=None, trace=None): - self.original_top = targets - self.top_targets_left = targets[:] - self.top_targets_left.reverse() - self.candidates = [] - if tasker is None: - tasker = OutOfDateTask - self.tasker = tasker - if not order: - order = lambda l: l - self.order = order - self.message = None - self.trace = trace - self.next_candidate = self.find_next_candidate - self.pending_children = set() - - def find_next_candidate(self): - """ - Returns the next candidate Node for (potential) evaluation. - - The candidate list (really a stack) initially consists of all of - the top-level (command line) targets provided when the Taskmaster - was initialized. While we walk the DAG, visiting Nodes, all the - children that haven't finished processing get pushed on to the - candidate list. Each child can then be popped and examined in - turn for whether *their* children are all up-to-date, in which - case a Task will be created for their actual evaluation and - potential building. - - Here is where we also allow candidate Nodes to alter the list of - Nodes that should be examined. This is used, for example, when - invoking SCons in a source directory. A source directory Node can - return its corresponding build directory Node, essentially saying, - "Hey, you really need to build this thing over here instead." - """ - try: - return self.candidates.pop() - except IndexError: - pass - try: - node = self.top_targets_left.pop() - except IndexError: - return None - self.current_top = node - alt, message = node.alter_targets() - if alt: - self.message = message - self.candidates.append(node) - self.candidates.extend(self.order(alt)) - node = self.candidates.pop() - return node - - def no_next_candidate(self): - """ - Stops Taskmaster processing by not returning a next candidate. - - Note that we have to clean-up the Taskmaster candidate list - because the cycle detection depends on the fact all nodes have - been processed somehow. - """ - while self.candidates: - candidates = self.candidates - self.candidates = [] - self.will_not_build(candidates) - return None - - def _validate_pending_children(self): - """ - Validate the content of the pending_children set. Assert if an - internal error is found. - - This function is used strictly for debugging the taskmaster by - checking that no invariants are violated. It is not used in - normal operation. - - The pending_children set is used to detect cycles in the - dependency graph. We call a "pending child" a child that is - found in the "pending" state when checking the dependencies of - its parent node. - - A pending child can occur when the Taskmaster completes a loop - through a cycle. For example, let's imagine a graph made of - three nodes (A, B and C) making a cycle. The evaluation starts - at node A. The Taskmaster first considers whether node A's - child B is up-to-date. Then, recursively, node B needs to - check whether node C is up-to-date. This leaves us with a - dependency graph looking like:: - - Next candidate \ - \ - Node A (Pending) --> Node B(Pending) --> Node C (NoState) - ^ | - | | - +-------------------------------------+ - - Now, when the Taskmaster examines the Node C's child Node A, - it finds that Node A is in the "pending" state. Therefore, - Node A is a pending child of node C. - - Pending children indicate that the Taskmaster has potentially - loop back through a cycle. We say potentially because it could - also occur when a DAG is evaluated in parallel. For example, - consider the following graph:: - - Node A (Pending) --> Node B(Pending) --> Node C (Pending) --> ... - | ^ - | | - +----------> Node D (NoState) --------+ - / - Next candidate / - - The Taskmaster first evaluates the nodes A, B, and C and - starts building some children of node C. Assuming, that the - maximum parallel level has not been reached, the Taskmaster - will examine Node D. It will find that Node C is a pending - child of Node D. - - In summary, evaluating a graph with a cycle will always - involve a pending child at one point. A pending child might - indicate either a cycle or a diamond-shaped DAG. Only a - fraction of the nodes ends-up being a "pending child" of - another node. This keeps the pending_children set small in - practice. - - We can differentiate between the two cases if we wait until - the end of the build. At this point, all the pending children - nodes due to a diamond-shaped DAG will have been properly - built (or will have failed to build). But, the pending - children involved in a cycle will still be in the pending - state. - - The taskmaster removes nodes from the pending_children set as - soon as a pending_children node moves out of the pending - state. This also helps to keep the pending_children set small. - """ - - for n in self.pending_children: - assert n.state in (NODE_PENDING, NODE_EXECUTING), \ - (str(n), StateString[n.state]) - assert len(n.waiting_parents) != 0, (str(n), len(n.waiting_parents)) - for p in n.waiting_parents: - assert p.ref_count > 0, (str(n), str(p), p.ref_count) - - - def trace_message(self, message): - return 'Taskmaster: %s\n' % message - - def trace_node(self, node): - return '<%-10s %-3s %s>' % (StateString[node.get_state()], - node.ref_count, - repr(str(node))) - - def _find_next_ready_node(self): - """ - Finds the next node that is ready to be built. - - This is *the* main guts of the DAG walk. We loop through the - list of candidates, looking for something that has no un-built - children (i.e., that is a leaf Node or has dependencies that are - all leaf Nodes or up-to-date). Candidate Nodes are re-scanned - (both the target Node itself and its sources, which are always - scanned in the context of a given target) to discover implicit - dependencies. A Node that must wait for some children to be - built will be put back on the candidates list after the children - have finished building. A Node that has been put back on the - candidates list in this way may have itself (or its sources) - re-scanned, in order to handle generated header files (e.g.) and - the implicit dependencies therein. - - Note that this method does not do any signature calculation or - up-to-date check itself. All of that is handled by the Task - class. This is purely concerned with the dependency graph walk. - """ - - self.ready_exc = None - - T = self.trace - if T: T.write('\n' + self.trace_message('Looking for a node to evaluate')) - - while True: - node = self.next_candidate() - if node is None: - if T: T.write(self.trace_message('No candidate anymore.') + '\n') - return None - - node = node.disambiguate() - state = node.get_state() - - # For debugging only: - # - # try: - # self._validate_pending_children() - # except: - # self.ready_exc = sys.exc_info() - # return node - - if CollectStats: - if not hasattr(node.attributes, 'stats'): - node.attributes.stats = Stats() - StatsNodes.append(node) - S = node.attributes.stats - S.considered = S.considered + 1 - else: - S = None - - if T: T.write(self.trace_message(' Considering node %s and its children:' % self.trace_node(node))) - - if state == NODE_NO_STATE: - # Mark this node as being on the execution stack: - node.set_state(NODE_PENDING) - elif state > NODE_PENDING: - # Skip this node if it has already been evaluated: - if S: S.already_handled = S.already_handled + 1 - if T: T.write(self.trace_message(' already handled (executed)')) - continue - - executor = node.get_executor() - - try: - children = executor.get_all_children() - except SystemExit: - exc_value = sys.exc_info()[1] - e = SCons.Errors.ExplicitExit(node, exc_value.code) - self.ready_exc = (SCons.Errors.ExplicitExit, e) - if T: T.write(self.trace_message(' SystemExit')) - return node - except Exception as e: - # We had a problem just trying to figure out the - # children (like a child couldn't be linked in to a - # VariantDir, or a Scanner threw something). Arrange to - # raise the exception when the Task is "executed." - self.ready_exc = sys.exc_info() - if S: S.problem = S.problem + 1 - if T: T.write(self.trace_message(' exception %s while scanning children.\n' % e)) - return node - - children_not_visited = [] - children_pending = set() - children_not_ready = [] - children_failed = False - - for child in chain(executor.get_all_prerequisites(), children): - childstate = child.get_state() - - if T: T.write(self.trace_message(' ' + self.trace_node(child))) - - if childstate == NODE_NO_STATE: - children_not_visited.append(child) - elif childstate == NODE_PENDING: - children_pending.add(child) - elif childstate == NODE_FAILED: - children_failed = True - - if childstate <= NODE_EXECUTING: - children_not_ready.append(child) - - # These nodes have not even been visited yet. Add - # them to the list so that on some next pass we can - # take a stab at evaluating them (or their children). - if children_not_visited: - if len(children_not_visited) > 1: - children_not_visited.reverse() - self.candidates.extend(self.order(children_not_visited)) - - # if T and children_not_visited: - # T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) - # T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) - - # Skip this node if any of its children have failed. - # - # This catches the case where we're descending a top-level - # target and one of our children failed while trying to be - # built by a *previous* descent of an earlier top-level - # target. - # - # It can also occur if a node is reused in multiple - # targets. One first descends though the one of the - # target, the next time occurs through the other target. - # - # Note that we can only have failed_children if the - # --keep-going flag was used, because without it the build - # will stop before diving in the other branch. - # - # Note that even if one of the children fails, we still - # added the other children to the list of candidate nodes - # to keep on building (--keep-going). - if children_failed: - for n in executor.get_action_targets(): - n.set_state(NODE_FAILED) - - if S: S.child_failed = S.child_failed + 1 - if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) - continue - - if children_not_ready: - for child in children_not_ready: - # We're waiting on one or more derived targets - # that have not yet finished building. - if S: S.not_built = S.not_built + 1 - - # Add this node to the waiting parents lists of - # anything we're waiting on, with a reference - # count so we can be put back on the list for - # re-evaluation when they've all finished. - node.ref_count = node.ref_count + child.add_to_waiting_parents(node) - if T: T.write(self.trace_message(' adjusted ref count: %s, child %s' % - (self.trace_node(node), repr(str(child))))) - - if T: - for pc in children_pending: - T.write(self.trace_message(' adding %s to the pending children set\n' % - self.trace_node(pc))) - self.pending_children = self.pending_children | children_pending - - continue - - # Skip this node if it has side-effects that are - # currently being built: - wait_side_effects = False - for se in executor.get_action_side_effects(): - if se.get_state() == NODE_EXECUTING: - se.add_to_waiting_s_e(node) - wait_side_effects = True - - if wait_side_effects: - if S: S.side_effects = S.side_effects + 1 - continue - - # The default when we've gotten through all of the checks above: - # this node is ready to be built. - if S: S.build = S.build + 1 - if T: T.write(self.trace_message('Evaluating %s\n' % - self.trace_node(node))) - - # For debugging only: - # - # try: - # self._validate_pending_children() - # except: - # self.ready_exc = sys.exc_info() - # return node - - return node - - return None - - def next_task(self): - """ - Returns the next task to be executed. - - This simply asks for the next Node to be evaluated, and then wraps - it in the specific Task subclass with which we were initialized. - """ - node = self._find_next_ready_node() - - if node is None: - return None - - executor = node.get_executor() - if executor is None: - return None - - tlist = executor.get_all_targets() - - task = self.tasker(self, tlist, node in self.original_top, node) - try: - task.make_ready() - except Exception as e : - # We had a problem just trying to get this task ready (like - # a child couldn't be linked to a VariantDir when deciding - # whether this node is current). Arrange to raise the - # exception when the Task is "executed." - self.ready_exc = sys.exc_info() - - if self.ready_exc: - task.exception_set(self.ready_exc) - - self.ready_exc = None - - return task - - def will_not_build(self, nodes, node_func=lambda n: None): - """ - Perform clean-up about nodes that will never be built. Invokes - a user defined function on all of these nodes (including all - of their parents). - """ - - T = self.trace - - pending_children = self.pending_children - - to_visit = set(nodes) - pending_children = pending_children - to_visit - - if T: - for n in nodes: - T.write(self.trace_message(' removing node %s from the pending children set\n' % - self.trace_node(n))) - try: - while len(to_visit): - node = to_visit.pop() - node_func(node) - - # Prune recursion by flushing the waiting children - # list immediately. - parents = node.waiting_parents - node.waiting_parents = set() - - to_visit = to_visit | parents - pending_children = pending_children - parents - - for p in parents: - p.ref_count = p.ref_count - 1 - if T: T.write(self.trace_message(' removing parent %s from the pending children set\n' % - self.trace_node(p))) - except KeyError: - # The container to_visit has been emptied. - pass - - # We have the stick back the pending_children list into the - # taskmaster because the python 1.5.2 compatibility does not - # allow us to use in-place updates - self.pending_children = pending_children - - def stop(self): - """ - Stops the current build completely. - """ - self.next_candidate = self.no_next_candidate - - def cleanup(self): - """ - Check for dependency cycles. - """ - if not self.pending_children: - return - - nclist = [(n, find_cycle([n], set())) for n in self.pending_children] - - genuine_cycles = [ - node for node,cycle in nclist - if cycle or node.get_state() != NODE_EXECUTED - ] - if not genuine_cycles: - # All of the "cycles" found were single nodes in EXECUTED state, - # which is to say, they really weren't cycles. Just return. - return - - desc = 'Found dependency cycle(s):\n' - for node, cycle in nclist: - if cycle: - desc = desc + " " + " -> ".join(map(str, cycle)) + "\n" - else: - desc = desc + \ - " Internal Error: no cycle found for node %s (%s) in state %s\n" % \ - (node, repr(node), StateString[node.get_state()]) - - raise SCons.Errors.UserError(desc) - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Taskmaster/Job.py b/SCons/Taskmaster/Job.py new file mode 100644 index 0000000..b398790 --- /dev/null +++ b/SCons/Taskmaster/Job.py @@ -0,0 +1,439 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Serial and Parallel classes to execute build tasks. + +The Jobs class provides a higher level interface to start, +stop, and wait on jobs. +""" + +import SCons.compat + +import os +import signal + +import SCons.Errors +import SCons.Warnings + +# The default stack size (in kilobytes) of the threads used to execute +# jobs in parallel. +# +# We use a stack size of 256 kilobytes. The default on some platforms +# is too large and prevents us from creating enough threads to fully +# parallelized the build. For example, the default stack size on linux +# is 8 MBytes. + +explicit_stack_size = None +default_stack_size = 256 + +interrupt_msg = 'Build interrupted.' + + +class InterruptState: + def __init__(self): + self.interrupted = False + + def set(self): + self.interrupted = True + + def __call__(self): + return self.interrupted + + +class Jobs: + """An instance of this class initializes N jobs, and provides + methods for starting, stopping, and waiting on all N jobs. + """ + + def __init__(self, num, taskmaster): + """ + Create 'num' jobs using the given taskmaster. + + If 'num' is 1 or less, then a serial job will be used, + otherwise a parallel job with 'num' worker threads will + be used. + + The 'num_jobs' attribute will be set to the actual number of jobs + allocated. If more than one job is requested but the Parallel + class can't do it, it gets reset to 1. Wrapping interfaces that + care should check the value of 'num_jobs' after initialization. + """ + + self.job = None + if num > 1: + stack_size = explicit_stack_size + if stack_size is None: + stack_size = default_stack_size + + try: + self.job = Parallel(taskmaster, num, stack_size) + self.num_jobs = num + except NameError: + pass + if self.job is None: + self.job = Serial(taskmaster) + self.num_jobs = 1 + + def run(self, postfunc=lambda: None): + """Run the jobs. + + postfunc() will be invoked after the jobs has run. It will be + invoked even if the jobs are interrupted by a keyboard + interrupt (well, in fact by a signal such as either SIGINT, + SIGTERM or SIGHUP). The execution of postfunc() is protected + against keyboard interrupts and is guaranteed to run to + completion.""" + self._setup_sig_handler() + try: + self.job.start() + finally: + postfunc() + self._reset_sig_handler() + + def were_interrupted(self): + """Returns whether the jobs were interrupted by a signal.""" + return self.job.interrupted() + + def _setup_sig_handler(self): + """Setup an interrupt handler so that SCons can shutdown cleanly in + various conditions: + + a) SIGINT: Keyboard interrupt + b) SIGTERM: kill or system shutdown + c) SIGHUP: Controlling shell exiting + + We handle all of these cases by stopping the taskmaster. It + turns out that it's very difficult to stop the build process + by throwing asynchronously an exception such as + KeyboardInterrupt. For example, the python Condition + variables (threading.Condition) and queues do not seem to be + asynchronous-exception-safe. It would require adding a whole + bunch of try/finally block and except KeyboardInterrupt all + over the place. + + Note also that we have to be careful to handle the case when + SCons forks before executing another process. In that case, we + want the child to exit immediately. + """ + def handler(signum, stack, self=self, parentpid=os.getpid()): + if os.getpid() == parentpid: + self.job.taskmaster.stop() + self.job.interrupted.set() + else: + os._exit(2) # pylint: disable=protected-access + + self.old_sigint = signal.signal(signal.SIGINT, handler) + self.old_sigterm = signal.signal(signal.SIGTERM, handler) + try: + self.old_sighup = signal.signal(signal.SIGHUP, handler) + except AttributeError: + pass + if (self.old_sigint is None) or (self.old_sigterm is None) or \ + (hasattr(self, "old_sighup") and self.old_sighup is None): + msg = "Overwritting previous signal handler which was not installed from Python. " + \ + "Will not be able to reinstate and so will return to default handler." + SCons.Warnings.warn(SCons.Warnings.SConsWarning, msg) + + def _reset_sig_handler(self): + """Restore the signal handlers to their previous state (before the + call to _setup_sig_handler().""" + sigint_to_use = self.old_sigint if self.old_sigint is not None else signal.SIG_DFL + sigterm_to_use = self.old_sigterm if self.old_sigterm is not None else signal.SIG_DFL + signal.signal(signal.SIGINT, sigint_to_use) + signal.signal(signal.SIGTERM, sigterm_to_use) + try: + sigterm_to_use = self.old_sighup if self.old_sighup is not None else signal.SIG_DFL + signal.signal(signal.SIGHUP, sigterm_to_use) + except AttributeError: + pass + +class Serial: + """This class is used to execute tasks in series, and is more efficient + than Parallel, but is only appropriate for non-parallel builds. Only + one instance of this class should be in existence at a time. + + This class is not thread safe. + """ + + def __init__(self, taskmaster): + """Create a new serial job given a taskmaster. + + The taskmaster's next_task() method should return the next task + that needs to be executed, or None if there are no more tasks. The + taskmaster's executed() method will be called for each task when it + is successfully executed, or failed() will be called if it failed to + execute (e.g. execute() raised an exception).""" + + self.taskmaster = taskmaster + self.interrupted = InterruptState() + + def start(self): + """Start the job. This will begin pulling tasks from the taskmaster + and executing them, and return when there are no more tasks. If a task + fails to execute (i.e. execute() raises an exception), then the job will + stop.""" + + while True: + task = self.taskmaster.next_task() + + if task is None: + break + + try: + task.prepare() + if task.needs_execute(): + task.execute() + except Exception: + if self.interrupted(): + try: + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + except: + task.exception_set() + else: + task.exception_set() + + # Let the failed() callback function arrange for the + # build to stop if that's appropriate. + task.failed() + else: + task.executed() + + task.postprocess() + self.taskmaster.cleanup() + + +# Trap import failure so that everything in the Job module but the +# Parallel class (and its dependent classes) will work if the interpreter +# doesn't support threads. +try: + import queue + import threading +except ImportError: + pass +else: + class Worker(threading.Thread): + """A worker thread waits on a task to be posted to its request queue, + dequeues the task, executes it, and posts a tuple including the task + and a boolean indicating whether the task executed successfully. """ + + def __init__(self, requestQueue, resultsQueue, interrupted): + super().__init__() + self.daemon = True + self.requestQueue = requestQueue + self.resultsQueue = resultsQueue + self.interrupted = interrupted + self.start() + + def run(self): + while True: + task = self.requestQueue.get() + + if task is None: + # The "None" value is used as a sentinel by + # ThreadPool.cleanup(). This indicates that there + # are no more tasks, so we should quit. + break + + try: + if self.interrupted(): + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + task.execute() + except: + task.exception_set() + ok = False + else: + ok = True + + self.resultsQueue.put((task, ok)) + + class ThreadPool: + """This class is responsible for spawning and managing worker threads.""" + + def __init__(self, num, stack_size, interrupted): + """Create the request and reply queues, and 'num' worker threads. + + One must specify the stack size of the worker threads. The + stack size is specified in kilobytes. + """ + self.requestQueue = queue.Queue(0) + self.resultsQueue = queue.Queue(0) + + try: + prev_size = threading.stack_size(stack_size*1024) + except AttributeError as e: + # Only print a warning if the stack size has been + # explicitly set. + if explicit_stack_size is not None: + msg = "Setting stack size is unsupported by this version of Python:\n " + \ + e.args[0] + SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) + except ValueError as e: + msg = "Setting stack size failed:\n " + str(e) + SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) + + # Create worker threads + self.workers = [] + for _ in range(num): + worker = Worker(self.requestQueue, self.resultsQueue, interrupted) + self.workers.append(worker) + + if 'prev_size' in locals(): + threading.stack_size(prev_size) + + def put(self, task): + """Put task into request queue.""" + self.requestQueue.put(task) + + def get(self): + """Remove and return a result tuple from the results queue.""" + return self.resultsQueue.get() + + def preparation_failed(self, task): + self.resultsQueue.put((task, False)) + + def cleanup(self): + """ + Shuts down the thread pool, giving each worker thread a + chance to shut down gracefully. + """ + # For each worker thread, put a sentinel "None" value + # on the requestQueue (indicating that there's no work + # to be done) so that each worker thread will get one and + # terminate gracefully. + for _ in self.workers: + self.requestQueue.put(None) + + # Wait for all of the workers to terminate. + # + # If we don't do this, later Python versions (2.4, 2.5) often + # seem to raise exceptions during shutdown. This happens + # in requestQueue.get(), as an assertion failure that + # requestQueue.not_full is notified while not acquired, + # seemingly because the main thread has shut down (or is + # in the process of doing so) while the workers are still + # trying to pull sentinels off the requestQueue. + # + # Normally these terminations should happen fairly quickly, + # but we'll stick a one-second timeout on here just in case + # someone gets hung. + for worker in self.workers: + worker.join(1.0) + self.workers = [] + + class Parallel: + """This class is used to execute tasks in parallel, and is somewhat + less efficient than Serial, but is appropriate for parallel builds. + + This class is thread safe. + """ + + def __init__(self, taskmaster, num, stack_size): + """Create a new parallel job given a taskmaster. + + The taskmaster's next_task() method should return the next + task that needs to be executed, or None if there are no more + tasks. The taskmaster's executed() method will be called + for each task when it is successfully executed, or failed() + will be called if the task failed to execute (i.e. execute() + raised an exception). + + Note: calls to taskmaster are serialized, but calls to + execute() on distinct tasks are not serialized, because + that is the whole point of parallel jobs: they can execute + multiple tasks simultaneously. """ + + self.taskmaster = taskmaster + self.interrupted = InterruptState() + self.tp = ThreadPool(num, stack_size, self.interrupted) + + self.maxjobs = num + + def start(self): + """Start the job. This will begin pulling tasks from the + taskmaster and executing them, and return when there are no + more tasks. If a task fails to execute (i.e. execute() raises + an exception), then the job will stop.""" + + jobs = 0 + + while True: + # Start up as many available tasks as we're + # allowed to. + while jobs < self.maxjobs: + task = self.taskmaster.next_task() + if task is None: + break + + try: + # prepare task for execution + task.prepare() + except: + task.exception_set() + task.failed() + task.postprocess() + else: + if task.needs_execute(): + # dispatch task + self.tp.put(task) + jobs += 1 + else: + task.executed() + task.postprocess() + + if not task and not jobs: break + + # Let any/all completed tasks finish up before we go + # back and put the next batch of tasks on the queue. + while True: + task, ok = self.tp.get() + jobs -= 1 + + if ok: + task.executed() + else: + if self.interrupted(): + try: + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + except: + task.exception_set() + + # Let the failed() callback function arrange + # for the build to stop if that's appropriate. + task.failed() + + task.postprocess() + + if self.tp.resultsQueue.empty(): + break + + self.tp.cleanup() + self.taskmaster.cleanup() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Taskmaster/JobTests.py b/SCons/Taskmaster/JobTests.py new file mode 100644 index 0000000..374d3f3 --- /dev/null +++ b/SCons/Taskmaster/JobTests.py @@ -0,0 +1,574 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import unittest +import random +import math +import sys +import os + +import TestUnit + +import SCons.Taskmaster.Job + + +def get_cpu_nums(): + # Linux, Unix and MacOS: + if hasattr( os, "sysconf" ): + if "SC_NPROCESSORS_ONLN" in os.sysconf_names: + # Linux & Unix: + ncpus = os.sysconf( "SC_NPROCESSORS_ONLN" ) + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + else: # OSX: + return int(os.popen2("sysctl -n hw.ncpu")[1].read() ) + # Windows: + if "NUMBER_OF_PROCESSORS" in os.environ: + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + return 1 # Default + +# a large number +num_sines = 500 + +# how many parallel jobs to perform for the test +num_jobs = get_cpu_nums()*2 + +# in case we werent able to detect num cpus for this test +# just make a hardcoded suffcient large number, though not future proof +if num_jobs == 2: + num_jobs = 33 + +# how many tasks to perform for the test +num_tasks = num_jobs*5 + +class DummyLock: + """fake lock class to use if threads are not supported""" + def acquire(self): + pass + + def release(self): + pass + +class NoThreadsException(Exception): + """raised by the ParallelTestCase if threads are not supported""" + + def __str__(self): + return "the interpreter doesn't support threads" + +class Task: + """A dummy task class for testing purposes.""" + + def __init__(self, i, taskmaster): + self.i = i + self.taskmaster = taskmaster + self.was_executed = 0 + self.was_prepared = 0 + + def prepare(self): + self.was_prepared = 1 + + def _do_something(self): + pass + + def needs_execute(self): + return True + + def execute(self): + self.taskmaster.test_case.assertTrue(self.was_prepared, + "the task wasn't prepared") + + self.taskmaster.guard.acquire() + self.taskmaster.begin_list.append(self.i) + self.taskmaster.guard.release() + + # while task is executing, represent this in the parallel_list + # and then turn it off + self.taskmaster.parallel_list[self.i] = 1 + self._do_something() + self.taskmaster.parallel_list[self.i] = 0 + + # check if task was executing while another was also executing + for j in range(1, self.taskmaster.num_tasks): + if self.taskmaster.parallel_list[j + 1] == 1: + self.taskmaster.found_parallel = True + break + + self.was_executed = 1 + + self.taskmaster.guard.acquire() + self.taskmaster.end_list.append(self.i) + self.taskmaster.guard.release() + + def executed(self): + self.taskmaster.num_executed = self.taskmaster.num_executed + 1 + + self.taskmaster.test_case.assertTrue(self.was_prepared, + "the task wasn't prepared") + self.taskmaster.test_case.assertTrue(self.was_executed, + "the task wasn't really executed") + self.taskmaster.test_case.assertTrue(isinstance(self, Task), + "the task wasn't really a Task instance") + + def failed(self): + self.taskmaster.num_failed = self.taskmaster.num_failed + 1 + self.taskmaster.stop = 1 + self.taskmaster.test_case.assertTrue(self.was_prepared, + "the task wasn't prepared") + + def postprocess(self): + self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + + def exception_set(self): + pass + +class RandomTask(Task): + def _do_something(self): + # do something that will take some random amount of time: + for i in range(random.randrange(0, 100 + num_sines, 1)): + x = math.sin(i) + time.sleep(0.01) + +class ExceptionTask: + """A dummy task class for testing purposes.""" + + def __init__(self, i, taskmaster): + self.taskmaster = taskmaster + self.was_prepared = 0 + + def prepare(self): + self.was_prepared = 1 + + def needs_execute(self): + return True + + def execute(self): + raise Exception + + def executed(self): + self.taskmaster.num_executed = self.taskmaster.num_executed + 1 + + self.taskmaster.test_case.assertTrue(self.was_prepared, + "the task wasn't prepared") + self.taskmaster.test_case.assertTrue(self.was_executed, + "the task wasn't really executed") + self.taskmaster.test_case.assertTrue(self.__class__ is Task, + "the task wasn't really a Task instance") + + def failed(self): + self.taskmaster.num_failed = self.taskmaster.num_failed + 1 + self.taskmaster.stop = 1 + self.taskmaster.test_case.assertTrue(self.was_prepared, + "the task wasn't prepared") + + def postprocess(self): + self.taskmaster.num_postprocessed = self.taskmaster.num_postprocessed + 1 + + def exception_set(self): + self.taskmaster.exception_set() + +class Taskmaster: + """A dummy taskmaster class for testing the job classes.""" + + def __init__(self, n, test_case, Task): + """n is the number of dummy tasks to perform.""" + + self.test_case = test_case + self.stop = None + self.num_tasks = n + self.num_iterated = 0 + self.num_executed = 0 + self.num_failed = 0 + self.num_postprocessed = 0 + self.parallel_list = [0] * (n+1) + self.found_parallel = False + self.Task = Task + + # 'guard' guards 'task_begin_list' and 'task_end_list' + try: + import threading + self.guard = threading.Lock() + except ImportError: + self.guard = DummyLock() + + # keep track of the order tasks are begun in + self.begin_list = [] + + # keep track of the order tasks are completed in + self.end_list = [] + + def next_task(self): + if self.stop or self.all_tasks_are_iterated(): + return None + else: + self.num_iterated = self.num_iterated + 1 + return self.Task(self.num_iterated, self) + + def all_tasks_are_executed(self): + return self.num_executed == self.num_tasks + + def all_tasks_are_iterated(self): + return self.num_iterated == self.num_tasks + + def all_tasks_are_postprocessed(self): + return self.num_postprocessed == self.num_tasks + + def tasks_were_serial(self): + """analyze the task order to see if they were serial""" + return not self.found_parallel + + def exception_set(self): + pass + + def cleanup(self): + pass + +SaveThreadPool = None +ThreadPoolCallList = [] + +class ParallelTestCase(unittest.TestCase): + def runTest(self): + """test parallel jobs""" + + try: + import threading + except ImportError: + raise NoThreadsException() + + taskmaster = Taskmaster(num_tasks, self, RandomTask) + jobs = SCons.Taskmaster.Job.Jobs(num_jobs, taskmaster) + jobs.run() + + self.assertTrue(not taskmaster.tasks_were_serial(), + "the tasks were not executed in parallel") + self.assertTrue(taskmaster.all_tasks_are_executed(), + "all the tests were not executed") + self.assertTrue(taskmaster.all_tasks_are_iterated(), + "all the tests were not iterated over") + self.assertTrue(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") + self.assertFalse(taskmaster.num_failed, + "some task(s) failed to execute") + + # Verify that parallel jobs will pull all of the completed tasks + # out of the queue at once, instead of one by one. We do this by + # replacing the default ThreadPool class with one that records the + # order in which tasks are put() and get() to/from the pool, and + # which sleeps a little bit before call get() to let the initial + # tasks complete and get their notifications on the resultsQueue. + + class SleepTask(Task): + def _do_something(self): + time.sleep(0.01) + + global SaveThreadPool + SaveThreadPool = SCons.Taskmaster.Job.ThreadPool + + class WaitThreadPool(SaveThreadPool): + def put(self, task): + ThreadPoolCallList.append('put(%s)' % task.i) + return SaveThreadPool.put(self, task) + def get(self): + time.sleep(0.05) + result = SaveThreadPool.get(self) + ThreadPoolCallList.append('get(%s)' % result[0].i) + return result + + SCons.Taskmaster.Job.ThreadPool = WaitThreadPool + + try: + taskmaster = Taskmaster(3, self, SleepTask) + jobs = SCons.Taskmaster.Job.Jobs(2, taskmaster) + jobs.run() + + # The key here is that we get(1) and get(2) from the + # resultsQueue before we put(3), but get(1) and get(2) can + # be in either order depending on how the first two parallel + # tasks get scheduled by the operating system. + expect = [ + ['put(1)', 'put(2)', 'get(1)', 'get(2)', 'put(3)', 'get(3)'], + ['put(1)', 'put(2)', 'get(2)', 'get(1)', 'put(3)', 'get(3)'], + ] + assert ThreadPoolCallList in expect, ThreadPoolCallList + + finally: + SCons.Taskmaster.Job.ThreadPool = SaveThreadPool + +class SerialTestCase(unittest.TestCase): + def runTest(self): + """test a serial job""" + + taskmaster = Taskmaster(num_tasks, self, RandomTask) + jobs = SCons.Taskmaster.Job.Jobs(1, taskmaster) + jobs.run() + + self.assertTrue(taskmaster.tasks_were_serial(), + "the tasks were not executed in series") + self.assertTrue(taskmaster.all_tasks_are_executed(), + "all the tests were not executed") + self.assertTrue(taskmaster.all_tasks_are_iterated(), + "all the tests were not iterated over") + self.assertTrue(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") + self.assertFalse(taskmaster.num_failed, + "some task(s) failed to execute") + +class NoParallelTestCase(unittest.TestCase): + def runTest(self): + """test handling lack of parallel support""" + def NoParallel(tm, num, stack_size): + raise NameError + save_Parallel = SCons.Taskmaster.Job.Parallel + SCons.Taskmaster.Job.Parallel = NoParallel + try: + taskmaster = Taskmaster(num_tasks, self, RandomTask) + jobs = SCons.Taskmaster.Job.Jobs(2, taskmaster) + self.assertTrue(jobs.num_jobs == 1, + "unexpected number of jobs %d" % jobs.num_jobs) + jobs.run() + self.assertTrue(taskmaster.tasks_were_serial(), + "the tasks were not executed in series") + self.assertTrue(taskmaster.all_tasks_are_executed(), + "all the tests were not executed") + self.assertTrue(taskmaster.all_tasks_are_iterated(), + "all the tests were not iterated over") + self.assertTrue(taskmaster.all_tasks_are_postprocessed(), + "all the tests were not postprocessed") + self.assertFalse(taskmaster.num_failed, + "some task(s) failed to execute") + finally: + SCons.Taskmaster.Job.Parallel = save_Parallel + + +class SerialExceptionTestCase(unittest.TestCase): + def runTest(self): + """test a serial job with tasks that raise exceptions""" + + taskmaster = Taskmaster(num_tasks, self, ExceptionTask) + jobs = SCons.Taskmaster.Job.Jobs(1, taskmaster) + jobs.run() + + self.assertFalse(taskmaster.num_executed, + "a task was executed") + self.assertTrue(taskmaster.num_iterated == 1, + "exactly one task should have been iterated") + self.assertTrue(taskmaster.num_failed == 1, + "exactly one task should have failed") + self.assertTrue(taskmaster.num_postprocessed == 1, + "exactly one task should have been postprocessed") + +class ParallelExceptionTestCase(unittest.TestCase): + def runTest(self): + """test parallel jobs with tasks that raise exceptions""" + + taskmaster = Taskmaster(num_tasks, self, ExceptionTask) + jobs = SCons.Taskmaster.Job.Jobs(num_jobs, taskmaster) + jobs.run() + + self.assertFalse(taskmaster.num_executed, + "a task was executed") + self.assertTrue(taskmaster.num_iterated >= 1, + "one or more task should have been iterated") + self.assertTrue(taskmaster.num_failed >= 1, + "one or more tasks should have failed") + self.assertTrue(taskmaster.num_postprocessed >= 1, + "one or more tasks should have been postprocessed") + +#--------------------------------------------------------------------- +# Above tested Job object with contrived Task and Taskmaster objects. +# Now test Job object with actual Task and Taskmaster objects. + +import SCons.Taskmaster +import SCons.Node +import time + +class DummyNodeInfo: + def update(self, obj): + pass + +class testnode (SCons.Node.Node): + def __init__(self): + super().__init__() + self.expect_to_be = SCons.Node.executed + self.ninfo = DummyNodeInfo() + +class goodnode (testnode): + def __init__(self): + super().__init__() + self.expect_to_be = SCons.Node.up_to_date + self.ninfo = DummyNodeInfo() + +class slowgoodnode (goodnode): + def prepare(self): + # Delay to allow scheduled Jobs to run while the dispatcher + # sleeps. Keep this short because it affects the time taken + # by this test. + time.sleep(0.15) + goodnode.prepare(self) + +class badnode (goodnode): + def __init__(self): + super().__init__() + self.expect_to_be = SCons.Node.failed + def build(self, **kw): + raise Exception('badnode exception') + +class slowbadnode (badnode): + def build(self, **kw): + # Appears to take a while to build, allowing faster builds to + # overlap. Time duration is not especially important, but if + # it is faster than slowgoodnode then these could complete + # while the scheduler is sleeping. + time.sleep(0.05) + raise Exception('slowbadnode exception') + +class badpreparenode (badnode): + def prepare(self): + raise Exception('badpreparenode exception') + +class _SConsTaskTest(unittest.TestCase): + + def _test_seq(self, num_jobs): + for node_seq in [ + [goodnode], + [badnode], + [slowbadnode], + [slowgoodnode], + [badpreparenode], + [goodnode, badnode], + [slowgoodnode, badnode], + [goodnode, slowbadnode], + [goodnode, goodnode, goodnode, slowbadnode], + [goodnode, slowbadnode, badpreparenode, slowgoodnode], + [goodnode, slowbadnode, slowgoodnode, badnode] + ]: + + self._do_test(num_jobs, node_seq) + + def _do_test(self, num_jobs, node_seq): + + testnodes = [] + for tnum in range(num_tasks): + testnodes.append(node_seq[tnum % len(node_seq)]()) + + taskmaster = SCons.Taskmaster.Taskmaster(testnodes, + tasker=SCons.Taskmaster.AlwaysTask) + + jobs = SCons.Taskmaster.Job.Jobs(num_jobs, taskmaster) + + # Exceptions thrown by tasks are not actually propagated to + # this level, but are instead stored in the Taskmaster. + + jobs.run() + + # Now figure out if tests proceeded correctly. The first test + # that fails will shutdown the initiation of subsequent tests, + # but any tests currently queued for execution will still be + # processed, and any tests that completed before the failure + # would have resulted in new tests being queued for execution. + + # Apply the following operational heuristics of Job.py: + # 0) An initial jobset of tasks will be queued before any + # good/bad results are obtained (from "execute" of task in + # thread). + # 1) A goodnode will complete immediately on its thread and + # allow another node to be queued for execution. + # 2) A badnode will complete immediately and suppress any + # subsequent execution queuing, but all currently queued + # tasks will still be processed. + # 3) A slowbadnode will fail later. It will block slots in + # the job queue. Nodes that complete immediately will + # allow other nodes to be queued in their place, and this + # will continue until either (#2) above or until all job + # slots are filled with slowbadnode entries. + + # One approach to validating this test would be to try to + # determine exactly how many nodes executed, how many didn't, + # and the results of each, and then to assert failure on any + # mismatch (including the total number of built nodes). + # However, while this is possible to do for a single-processor + # system, it is nearly impossible to predict correctly for a + # multi-processor system and still test the characteristics of + # delayed execution nodes. Stated another way, multithreading + # is inherently non-deterministic unless you can completely + # characterize the entire system, and since that's not + # possible here, we shouldn't try. + + # Therefore, this test will simply scan the set of nodes to + # see if the node was executed or not and if it was executed + # that it obtained the expected value for that node + # (i.e. verifying we don't get failure crossovers or + # mislabelling of results). + + for N in testnodes: + state = N.get_state() + self.assertTrue(state in [SCons.Node.no_state, N.expect_to_be], + "Node %s got unexpected result: %s" % (N, state)) + + self.assertTrue([N for N in testnodes if N.get_state()], + "no nodes ran at all.") + + +class SerialTaskTest(_SConsTaskTest): + def runTest(self): + """test serial jobs with actual Taskmaster and Task""" + self._test_seq(1) + + +class ParallelTaskTest(_SConsTaskTest): + def runTest(self): + """test parallel jobs with actual Taskmaster and Task""" + self._test_seq(num_jobs) + + + +#--------------------------------------------------------------------- + +def suite(): + suite = unittest.TestSuite() + suite.addTest(ParallelTestCase()) + suite.addTest(SerialTestCase()) + suite.addTest(NoParallelTestCase()) + suite.addTest(SerialExceptionTestCase()) + suite.addTest(ParallelExceptionTestCase()) + suite.addTest(SerialTaskTest()) + suite.addTest(ParallelTaskTest()) + return suite + +if __name__ == "__main__": + runner = TestUnit.cli.get_runner() + result = runner().run(suite()) + if (len(result.failures) == 0 + and len(result.errors) == 1 + and isinstance(result.errors[0][0], SerialTestCase) + and isinstance(result.errors[0][1][0], NoThreadsException)): + sys.exit(2) + elif not result.wasSuccessful(): + sys.exit(1) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Taskmaster/TaskmasterTests.py b/SCons/Taskmaster/TaskmasterTests.py new file mode 100644 index 0000000..f20fd71 --- /dev/null +++ b/SCons/Taskmaster/TaskmasterTests.py @@ -0,0 +1,1257 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import SCons.compat + +import sys +import unittest + + +import SCons.Taskmaster +import SCons.Errors + + +built_text = None +cache_text = [] +visited_nodes = [] +executed = None +scan_called = 0 + +class Node: + def __init__(self, name, kids = [], scans = []): + self.name = name + self.kids = kids + self.scans = scans + self.cached = 0 + self.scanned = 0 + self.scanner = None + self.targets = [self] + self.prerequisites = None + class Builder: + def targets(self, node): + return node.targets + self.builder = Builder() + self.bsig = None + self.csig = None + self.state = SCons.Node.no_state + self.prepared = None + self.ref_count = 0 + self.waiting_parents = set() + self.waiting_s_e = set() + self.side_effect = 0 + self.side_effects = [] + self.alttargets = [] + self.postprocessed = None + self._bsig_val = None + self._current_val = 0 + self.always_build = None + + def disambiguate(self): + return self + + def push_to_cache(self): + pass + + def retrieve_from_cache(self): + global cache_text + if self.cached: + cache_text.append(self.name + " retrieved") + return self.cached + + def make_ready(self): + pass + + def prepare(self): + self.prepared = 1 + self.get_binfo() + + def build(self): + global built_text + built_text = self.name + " built" + + def remove(self): + pass + + # The following four methods new_binfo(), del_binfo(), + # get_binfo(), clear() as well as its calls have been added + # to support the cached_execute() test (issue #2720). + # They are full copies (or snippets) of their actual + # counterparts in the Node class... + def new_binfo(self): + binfo = "binfo" + return binfo + + def del_binfo(self): + """Delete the build info from this node.""" + try: + delattr(self, 'binfo') + except AttributeError: + pass + + def get_binfo(self): + """Fetch a node's build information.""" + try: + return self.binfo + except AttributeError: + pass + + binfo = self.new_binfo() + self.binfo = binfo + + return binfo + + def clear(self): + # The del_binfo() call here isn't necessary for normal execution, + # but is for interactive mode, where we might rebuild the same + # target and need to start from scratch. + self.del_binfo() + + def built(self): + global built_text + if not self.cached: + built_text = built_text + " really" + + # Clear the implicit dependency caches of any Nodes + # waiting for this Node to be built. + for parent in self.waiting_parents: + parent.implicit = None + + self.clear() + + def release_target_info(self): + pass + + def has_builder(self): + return self.builder is not None + + def is_derived(self): + return self.has_builder or self.side_effect + + def alter_targets(self): + return self.alttargets, None + + def visited(self): + global visited_nodes + visited_nodes.append(self.name) + + def children(self): + if not self.scanned: + self.scan() + self.scanned = 1 + return self.kids + + def scan(self): + global scan_called + scan_called = scan_called + 1 + self.kids = self.kids + self.scans + self.scans = [] + + def scanner_key(self): + return self.name + + def add_to_waiting_parents(self, node): + wp = self.waiting_parents + if node in wp: + return 0 + wp.add(node) + return 1 + + def get_state(self): + return self.state + + def set_state(self, state): + self.state = state + + def set_bsig(self, bsig): + self.bsig = bsig + + def set_csig(self, csig): + self.csig = csig + + def store_csig(self): + pass + + def store_bsig(self): + pass + + def is_pseudo_derived(self): + pass + + def is_up_to_date(self): + return self._current_val + + def depends_on(self, nodes): + for node in nodes: + if node in self.kids: + return 1 + return 0 + + def __str__(self): + return self.name + + def postprocess(self): + self.postprocessed = 1 + self.waiting_parents = set() + + def get_executor(self): + if not hasattr(self, 'executor'): + class Executor: + def prepare(self): + pass + def get_action_targets(self): + return self.targets + def get_all_targets(self): + return self.targets + def get_all_children(self): + result = [] + for node in self.targets: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] + self.executor = Executor() + self.executor.targets = self.targets + return self.executor + +class OtherError(Exception): + pass + +class MyException(Exception): + pass + + +class TaskmasterTestCase(unittest.TestCase): + + def test_next_task(self): + """Test fetching the next task + """ + global built_text + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1, n1]) + t = tm.next_task() + t.prepare() + t.execute() + t = tm.next_task() + assert t is None + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + + tm = SCons.Taskmaster.Taskmaster([n3]) + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n1 built", built_text + t.executed() + t.postprocess() + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n2 built", built_text + t.executed() + t.postprocess() + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n3 built", built_text + t.executed() + t.postprocess() + + assert tm.next_task() is None + + built_text = "up to date: " + top_node = n3 + + class MyTask(SCons.Taskmaster.AlwaysTask): + def execute(self): + global built_text + if self.targets[0].get_state() == SCons.Node.up_to_date: + if self.top: + built_text = self.targets[0].name + " up-to-date top" + else: + built_text = self.targets[0].name + " up-to-date" + else: + self.targets[0].build() + + n1.set_state(SCons.Node.no_state) + n1._current_val = 1 + n2.set_state(SCons.Node.no_state) + n2._current_val = 1 + n3.set_state(SCons.Node.no_state) + n3._current_val = 1 + tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask) + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n1 up-to-date", built_text + t.executed() + t.postprocess() + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n2 up-to-date", built_text + t.executed() + t.postprocess() + + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n3 up-to-date top", built_text + t.executed() + t.postprocess() + + assert tm.next_task() is None + + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + n4 = Node("n4") + n5 = Node("n5", [n3, n4]) + tm = SCons.Taskmaster.Taskmaster([n5]) + + t1 = tm.next_task() + assert t1.get_target() == n1 + + t2 = tm.next_task() + assert t2.get_target() == n2 + + t4 = tm.next_task() + assert t4.get_target() == n4 + t4.executed() + t4.postprocess() + + t1.executed() + t1.postprocess() + t2.executed() + t2.postprocess() + t3 = tm.next_task() + assert t3.get_target() == n3 + + t3.executed() + t3.postprocess() + t5 = tm.next_task() + assert t5.get_target() == n5, t5.get_target() + t5.executed() + t5.postprocess() + + assert tm.next_task() is None + + + n4 = Node("n4") + n4.set_state(SCons.Node.executed) + tm = SCons.Taskmaster.Taskmaster([n4]) + assert tm.next_task() is None + + n1 = Node("n1") + n2 = Node("n2", [n1]) + tm = SCons.Taskmaster.Taskmaster([n2,n2]) + t = tm.next_task() + t.executed() + t.postprocess() + t = tm.next_task() + assert tm.next_task() is None + + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1], [n2]) + tm = SCons.Taskmaster.Taskmaster([n3]) + t = tm.next_task() + target = t.get_target() + assert target == n1, target + t.executed() + t.postprocess() + t = tm.next_task() + target = t.get_target() + assert target == n2, target + t.executed() + t.postprocess() + t = tm.next_task() + target = t.get_target() + assert target == n3, target + t.executed() + t.postprocess() + assert tm.next_task() is None + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + n4 = Node("n4", [n3]) + n5 = Node("n5", [n3]) + global scan_called + scan_called = 0 + tm = SCons.Taskmaster.Taskmaster([n4]) + t = tm.next_task() + assert t.get_target() == n1 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n2 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n3 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n4 + t.executed() + t.postprocess() + assert tm.next_task() is None + assert scan_called == 4, scan_called + + tm = SCons.Taskmaster.Taskmaster([n5]) + t = tm.next_task() + assert t.get_target() == n5, t.get_target() + t.executed() + assert tm.next_task() is None + assert scan_called == 5, scan_called + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3") + n4 = Node("n4", [n1,n2,n3]) + n5 = Node("n5", [n4]) + n3.side_effect = 1 + n1.side_effects = n2.side_effects = n3.side_effects = [n4] + tm = SCons.Taskmaster.Taskmaster([n1,n2,n3,n4,n5]) + t = tm.next_task() + assert t.get_target() == n1 + assert n4.state == SCons.Node.executing, n4.state + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n2 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n3 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n4 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n5 + assert not tm.next_task() + t.executed() + t.postprocess() + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3") + n4 = Node("n4", [n1,n2,n3]) + def reverse(dependencies): + dependencies.reverse() + return dependencies + tm = SCons.Taskmaster.Taskmaster([n4], order=reverse) + t = tm.next_task() + assert t.get_target() == n3, t.get_target() + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n2, t.get_target() + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n1, t.get_target() + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n4, t.get_target() + t.executed() + t.postprocess() + + n5 = Node("n5") + n6 = Node("n6") + n7 = Node("n7") + n6.alttargets = [n7] + + tm = SCons.Taskmaster.Taskmaster([n5]) + t = tm.next_task() + assert t.get_target() == n5 + t.executed() + t.postprocess() + + tm = SCons.Taskmaster.Taskmaster([n6]) + t = tm.next_task() + assert t.get_target() == n7 + t.executed() + t.postprocess() + t = tm.next_task() + assert t.get_target() == n6 + t.executed() + t.postprocess() + + n1 = Node("n1") + n2 = Node("n2", [n1]) + n1.set_state(SCons.Node.failed) + tm = SCons.Taskmaster.Taskmaster([n2]) + assert tm.next_task() is None + + n1 = Node("n1") + n2 = Node("n2") + n1.targets = [n1, n2] + n1._current_val = 1 + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + t.executed() + t.postprocess() + + s = n1.get_state() + assert s == SCons.Node.executed, s + s = n2.get_state() + assert s == SCons.Node.executed, s + + + def test_make_ready_out_of_date(self): + """Test the Task.make_ready() method's list of out-of-date Nodes + """ + ood = [] + def TaskGen(tm, targets, top, node, ood=ood): + class MyTask(SCons.Taskmaster.AlwaysTask): + def make_ready(self): + SCons.Taskmaster.Task.make_ready(self) + self.ood.extend(self.out_of_date) + + t = MyTask(tm, targets, top, node) + t.ood = ood + return t + + n1 = Node("n1") + c2 = Node("c2") + c2._current_val = 1 + n3 = Node("n3") + c4 = Node("c4") + c4._current_val = 1 + a5 = Node("a5") + a5._current_val = 1 + a5.always_build = 1 + tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4, a5], + tasker = TaskGen) + + del ood[:] + t = tm.next_task() + assert ood == [n1], ood + + del ood[:] + t = tm.next_task() + assert ood == [], ood + + del ood[:] + t = tm.next_task() + assert ood == [n3], ood + + del ood[:] + t = tm.next_task() + assert ood == [], ood + + del ood[:] + t = tm.next_task() + assert ood == [a5], ood + + def test_make_ready_exception(self): + """Test handling exceptions from Task.make_ready() + """ + class MyTask(SCons.Taskmaster.AlwaysTask): + def make_ready(self): + raise MyException("from make_ready()") + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster(targets = [n1], tasker = MyTask) + t = tm.next_task() + exc_type, exc_value, exc_tb = t.exception + assert exc_type == MyException, repr(exc_type) + assert str(exc_value) == "from make_ready()", exc_value + + def test_needs_execute(self): + """Test that we can't instantiate a Task subclass without needs_execute + + We should be getting: + TypeError: Can't instantiate abstract class MyTask with abstract methods needs_execute + """ + class MyTask(SCons.Taskmaster.Task): + pass + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster(targets=[n1], tasker=MyTask) + with self.assertRaises(TypeError): + _ = tm.next_task() + + def test_make_ready_all(self): + """Test the make_ready_all() method""" + class MyTask(SCons.Taskmaster.AlwaysTask): + make_ready = SCons.Taskmaster.Task.make_ready_all + + n1 = Node("n1") + c2 = Node("c2") + c2._current_val = 1 + n3 = Node("n3") + c4 = Node("c4") + c4._current_val = 1 + + tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4]) + + t = tm.next_task() + target = t.get_target() + assert target is n1, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + target = t.get_target() + assert target is c2, target + assert target.state == SCons.Node.up_to_date, target.state + t = tm.next_task() + target = t.get_target() + assert target is n3, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + target = t.get_target() + assert target is c4, target + assert target.state == SCons.Node.up_to_date, target.state + t = tm.next_task() + assert t is None + + n1 = Node("n1") + c2 = Node("c2") + n3 = Node("n3") + c4 = Node("c4") + + tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4], + tasker = MyTask) + + t = tm.next_task() + target = t.get_target() + assert target is n1, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + target = t.get_target() + assert target is c2, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + target = t.get_target() + assert target is n3, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + target = t.get_target() + assert target is c4, target + assert target.state == SCons.Node.executing, target.state + t = tm.next_task() + assert t is None + + + def test_children_errors(self): + """Test errors when fetching the children of a node. + """ + class StopNode(Node): + def children(self): + raise SCons.Errors.StopError("stop!") + class ExitNode(Node): + def children(self): + sys.exit(77) + + n1 = StopNode("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + exc_type, exc_value, exc_tb = t.exception + assert exc_type == SCons.Errors.StopError, repr(exc_type) + assert str(exc_value) == "stop!", exc_value + + n2 = ExitNode("n2") + tm = SCons.Taskmaster.Taskmaster([n2]) + t = tm.next_task() + exc_type, exc_value = t.exception + assert exc_type == SCons.Errors.ExplicitExit, repr(exc_type) + assert exc_value.node == n2, exc_value.node + assert exc_value.status == 77, exc_value.status + + def test_cycle_detection(self): + """Test detecting dependency cycles + """ + n1 = Node("n1") + n2 = Node("n2", [n1]) + n3 = Node("n3", [n2]) + n1.kids = [n3] + + tm = SCons.Taskmaster.Taskmaster([n3]) + try: + t = tm.next_task() + except SCons.Errors.UserError as e: + assert str(e) == "Dependency cycle: n3 -> n1 -> n2 -> n3", str(e) + else: + assert 'Did not catch expected UserError' + + def test_next_top_level_candidate(self): + """Test the next_top_level_candidate() method + """ + n1 = Node("n1") + n2 = Node("n2", [n1]) + n3 = Node("n3", [n2]) + + tm = SCons.Taskmaster.Taskmaster([n3]) + t = tm.next_task() + assert t.targets == [n1], t.targets + t.fail_stop() + assert t.targets == [n3], list(map(str, t.targets)) + assert t.top == 1, t.top + + def test_stop(self): + """Test the stop() method + + Both default and overridden in a subclass. + """ + global built_text + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + + tm = SCons.Taskmaster.Taskmaster([n3]) + t = tm.next_task() + t.prepare() + t.execute() + assert built_text == "n1 built", built_text + t.executed() + t.postprocess() + assert built_text == "n1 built really", built_text + + tm.stop() + assert tm.next_task() is None + + class MyTM(SCons.Taskmaster.Taskmaster): + def stop(self): + global built_text + built_text = "MyTM.stop()" + SCons.Taskmaster.Taskmaster.stop(self) + + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + + built_text = None + tm = MyTM([n3]) + tm.next_task().execute() + assert built_text == "n1 built" + + tm.stop() + assert built_text == "MyTM.stop()" + assert tm.next_task() is None + + def test_executed(self): + """Test when a task has been executed + """ + global built_text + global visited_nodes + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + built_text = "xxx" + visited_nodes = [] + n1.set_state(SCons.Node.executing) + + t.executed() + + s = n1.get_state() + assert s == SCons.Node.executed, s + assert built_text == "xxx really", built_text + assert visited_nodes == ['n1'], visited_nodes + + n2 = Node("n2") + tm = SCons.Taskmaster.Taskmaster([n2]) + t = tm.next_task() + built_text = "should_not_change" + visited_nodes = [] + n2.set_state(None) + + t.executed() + + s = n2.get_state() + assert s is None, s + assert built_text == "should_not_change", built_text + assert visited_nodes == ['n2'], visited_nodes + + n3 = Node("n3") + n4 = Node("n4") + n3.targets = [n3, n4] + tm = SCons.Taskmaster.Taskmaster([n3]) + t = tm.next_task() + visited_nodes = [] + n3.set_state(SCons.Node.up_to_date) + n4.set_state(SCons.Node.executing) + + t.executed() + + s = n3.get_state() + assert s == SCons.Node.up_to_date, s + s = n4.get_state() + assert s == SCons.Node.executed, s + assert visited_nodes == ['n3', 'n4'], visited_nodes + + def test_prepare(self): + """Test preparation of multiple Nodes for a task + """ + n1 = Node("n1") + n2 = Node("n2") + tm = SCons.Taskmaster.Taskmaster([n1, n2]) + t = tm.next_task() + # This next line is moderately bogus. We're just reaching + # in and setting the targets for this task to an array. The + # "right" way to do this would be to have the next_task() call + # set it up by having something that approximates a real Builder + # return this list--but that's more work than is probably + # warranted right now. + n1.get_executor().targets = [n1, n2] + t.prepare() + assert n1.prepared + assert n2.prepared + + n3 = Node("n3") + n4 = Node("n4") + tm = SCons.Taskmaster.Taskmaster([n3, n4]) + t = tm.next_task() + # More bogus reaching in and setting the targets. + n3.set_state(SCons.Node.up_to_date) + n3.get_executor().targets = [n3, n4] + t.prepare() + assert n3.prepared + assert n4.prepared + + # If the Node has had an exception recorded while it was getting + # prepared, then prepare() should raise that exception. + class MyException(Exception): + pass + + built_text = None + n5 = Node("n5") + tm = SCons.Taskmaster.Taskmaster([n5]) + t = tm.next_task() + t.exception_set((MyException, "exception value")) + exc_caught = None + exc_actually_caught = None + exc_value = None + try: + t.prepare() + except MyException as e: + exc_caught = 1 + exc_value = e + except Exception as exc_actually_caught: + pass + assert exc_caught, "did not catch expected MyException: %s" % exc_actually_caught + assert str(exc_value) == "exception value", exc_value + assert built_text is None, built_text + + # Regression test, make sure we prepare not only + # all targets, but their side effects as well. + n6 = Node("n6") + n7 = Node("n7") + n8 = Node("n8") + n9 = Node("n9") + n10 = Node("n10") + + n6.side_effects = [ n8 ] + n7.side_effects = [ n9, n10 ] + + tm = SCons.Taskmaster.Taskmaster([n6, n7]) + t = tm.next_task() + # More bogus reaching in and setting the targets. + n6.get_executor().targets = [n6, n7] + t.prepare() + assert n6.prepared + assert n7.prepared + assert n8.prepared + assert n9.prepared + assert n10.prepared + + # Make sure we call an Executor's prepare() method. + class ExceptionExecutor: + def prepare(self): + raise Exception("Executor.prepare() exception") + def get_all_targets(self): + return self.nodes + def get_all_children(self): + result = [] + for node in self.nodes: + result.extend(node.children()) + return result + def get_all_prerequisites(self): + return [] + def get_action_side_effects(self): + return [] + + n11 = Node("n11") + n11.executor = ExceptionExecutor() + n11.executor.nodes = [n11] + tm = SCons.Taskmaster.Taskmaster([n11]) + t = tm.next_task() + try: + t.prepare() + except Exception as e: + assert str(e) == "Executor.prepare() exception", e + else: + raise AssertionError("did not catch expected exception") + + def test_execute(self): + """Test executing a task + """ + global built_text + global cache_text + + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + t.execute() + assert built_text == "n1 built", built_text + + def raise_UserError(): + raise SCons.Errors.UserError + n2 = Node("n2") + n2.build = raise_UserError + tm = SCons.Taskmaster.Taskmaster([n2]) + t = tm.next_task() + try: + t.execute() + except SCons.Errors.UserError: + pass + else: + self.fail("did not catch expected UserError") + + def raise_BuildError(): + raise SCons.Errors.BuildError + n3 = Node("n3") + n3.build = raise_BuildError + tm = SCons.Taskmaster.Taskmaster([n3]) + t = tm.next_task() + try: + t.execute() + except SCons.Errors.BuildError: + pass + else: + self.fail("did not catch expected BuildError") + + # On a generic (non-BuildError) exception from a Builder, + # the target should throw a BuildError exception with the + # args set to the exception value, instance, and traceback. + def raise_OtherError(): + raise OtherError + n4 = Node("n4") + n4.build = raise_OtherError + tm = SCons.Taskmaster.Taskmaster([n4]) + t = tm.next_task() + try: + t.execute() + except SCons.Errors.BuildError as e: + assert e.node == n4, e.node + assert e.errstr == "OtherError : ", e.errstr + assert len(e.exc_info) == 3, e.exc_info + exc_traceback = sys.exc_info()[2] + assert isinstance(e.exc_info[2], type(exc_traceback)), e.exc_info[2] + else: + self.fail("did not catch expected BuildError") + + built_text = None + cache_text = [] + n5 = Node("n5") + n6 = Node("n6") + n6.cached = 1 + tm = SCons.Taskmaster.Taskmaster([n5]) + t = tm.next_task() + # This next line is moderately bogus. We're just reaching + # in and setting the targets for this task to an array. The + # "right" way to do this would be to have the next_task() call + # set it up by having something that approximates a real Builder + # return this list--but that's more work than is probably + # warranted right now. + t.targets = [n5, n6] + t.execute() + assert built_text == "n5 built", built_text + assert cache_text == [], cache_text + + built_text = None + cache_text = [] + n7 = Node("n7") + n8 = Node("n8") + n7.cached = 1 + n8.cached = 1 + tm = SCons.Taskmaster.Taskmaster([n7]) + t = tm.next_task() + # This next line is moderately bogus. We're just reaching + # in and setting the targets for this task to an array. The + # "right" way to do this would be to have the next_task() call + # set it up by having something that approximates a real Builder + # return this list--but that's more work than is probably + # warranted right now. + t.targets = [n7, n8] + t.execute() + assert built_text is None, built_text + assert cache_text == ["n7 retrieved", "n8 retrieved"], cache_text + + def test_cached_execute(self): + """Test executing a task with cached targets + """ + # In issue #2720 Alexei Klimkin detected that the previous + # workflow for execute() led to problems in a multithreaded build. + # We have: + # task.prepare() + # task.execute() + # task.executed() + # -> node.visited() + # for the Serial flow, but + # - Parallel - - Worker - + # task.prepare() + # requestQueue.put(task) + # task = requestQueue.get() + # task.execute() + # resultQueue.put(task) + # task = resultQueue.get() + # task.executed() + # ->node.visited() + # in parallel. Since execute() used to call built() when a target + # was cached, it could unblock dependent nodes before the binfo got + # restored again in visited(). This resulted in spurious + # "file not found" build errors, because files fetched from cache would + # be seen as not up to date and wouldn't be scanned for implicit + # dependencies. + # + # The following test ensures that execute() only marks targets as cached, + # but the actual call to built() happens in executed() only. + # Like this, the binfo should still be intact after calling execute()... + global cache_text + + n1 = Node("n1") + # Mark the node as being cached + n1.cached = 1 + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + t.prepare() + t.execute() + assert cache_text == ["n1 retrieved"], cache_text + # If no binfo exists anymore, something has gone wrong... + has_binfo = hasattr(n1, 'binfo') + assert has_binfo, has_binfo + + def test_exception(self): + """Test generic Taskmaster exception handling + + """ + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + t = tm.next_task() + + t.exception_set((1, 2)) + exc_type, exc_value = t.exception + assert exc_type == 1, exc_type + assert exc_value == 2, exc_value + + t.exception_set(3) + assert t.exception == 3 + + try: 1//0 + except: + # Moved from below + t.exception_set(None) + #pass + +# import pdb; pdb.set_trace() + + # Having this here works for python 2.x, + # but it is a tuple (None, None, None) when called outside + # an except statement + # t.exception_set(None) + + exc_type, exc_value, exc_tb = t.exception + assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s"%exc_type + exception_values = [ + "integer division or modulo", + "integer division or modulo by zero", + "integer division by zero", # PyPy2 + ] + assert str(exc_value) in exception_values, exc_value + + class Exception1(Exception): + pass + + # Previously value was None, but while PY2 None = "", in Py3 None != "", so set to "" + t.exception_set((Exception1, "")) + try: + t.exception_raise() + except: + exc_type, exc_value = sys.exc_info()[:2] + assert exc_type == Exception1, exc_type + assert str(exc_value) == '', "Expecting empty string got:%s (type %s)"%(exc_value,type(exc_value)) + else: + assert 0, "did not catch expected exception" + + class Exception2(Exception): + pass + + t.exception_set((Exception2, "xyzzy")) + try: + t.exception_raise() + except: + exc_type, exc_value = sys.exc_info()[:2] + assert exc_type == Exception2, exc_type + assert str(exc_value) == "xyzzy", exc_value + else: + assert 0, "did not catch expected exception" + + class Exception3(Exception): + pass + + try: + 1//0 + except: + tb = sys.exc_info()[2] + t.exception_set((Exception3, "arg", tb)) + try: + t.exception_raise() + except: + exc_type, exc_value, exc_tb = sys.exc_info() + assert exc_type == Exception3, exc_type + assert str(exc_value) == "arg", exc_value + import traceback + x = traceback.extract_tb(tb)[-1] + y = traceback.extract_tb(exc_tb)[-1] + assert x == y, "x = %s, y = %s" % (x, y) + else: + assert 0, "did not catch expected exception" + + def test_postprocess(self): + """Test postprocessing targets to give them a chance to clean up + """ + n1 = Node("n1") + tm = SCons.Taskmaster.Taskmaster([n1]) + + t = tm.next_task() + assert not n1.postprocessed + t.postprocess() + assert n1.postprocessed + + n2 = Node("n2") + n3 = Node("n3") + tm = SCons.Taskmaster.Taskmaster([n2, n3]) + + assert not n2.postprocessed + assert not n3.postprocessed + t = tm.next_task() + t.postprocess() + assert n2.postprocessed + assert not n3.postprocessed + t = tm.next_task() + t.postprocess() + assert n2.postprocessed + assert n3.postprocessed + + def test_trace(self): + """Test Taskmaster tracing + """ + import io + + trace = io.StringIO() + n1 = Node("n1") + n2 = Node("n2") + n3 = Node("n3", [n1, n2]) + tm = SCons.Taskmaster.Taskmaster([n1, n1, n3], trace=trace) + t = tm.next_task() + t.prepare() + t.execute() + t.postprocess() + n1.set_state(SCons.Node.executed) + t = tm.next_task() + t.prepare() + t.execute() + t.postprocess() + n2.set_state(SCons.Node.executed) + t = tm.next_task() + t.prepare() + t.execute() + t.postprocess() + t = tm.next_task() + assert t is None + + value = trace.getvalue() + expect = """\ + +Taskmaster: Looking for a node to evaluate +Taskmaster: Considering node and its children: +Taskmaster: Evaluating + +Task.make_ready_current(): node +Task.prepare(): node +Task.execute(): node +Task.postprocess(): node + +Taskmaster: Looking for a node to evaluate +Taskmaster: Considering node and its children: +Taskmaster: already handled (executed) +Taskmaster: Considering node and its children: +Taskmaster: +Taskmaster: +Taskmaster: adjusted ref count: , child 'n2' +Taskmaster: Considering node and its children: +Taskmaster: Evaluating + +Task.make_ready_current(): node +Task.prepare(): node +Task.execute(): node +Task.postprocess(): node +Task.postprocess(): removing +Task.postprocess(): adjusted parent ref count + +Taskmaster: Looking for a node to evaluate +Taskmaster: Considering node and its children: +Taskmaster: +Taskmaster: +Taskmaster: Evaluating + +Task.make_ready_current(): node +Task.prepare(): node +Task.execute(): node +Task.postprocess(): node + +Taskmaster: Looking for a node to evaluate +Taskmaster: No candidate anymore. + +""" + assert value == expect, value + + + +if __name__ == "__main__": + unittest.main() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py new file mode 100644 index 0000000..d571795 --- /dev/null +++ b/SCons/Taskmaster/__init__.py @@ -0,0 +1,1059 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Generic Taskmaster module for the SCons build engine. + +This module contains the primary interface(s) between a wrapping user +interface and the SCons build engine. There are two key classes here: + +Taskmaster + This is the main engine for walking the dependency graph and + calling things to decide what does or doesn't need to be built. + +Task + This is the base class for allowing a wrapping interface to + decide what does or doesn't actually need to be done. The + intention is for a wrapping interface to subclass this as + appropriate for different types of behavior it may need. + + The canonical example is the SCons native Python interface, + which has Task subclasses that handle its specific behavior, + like printing "'foo' is up to date" when a top-level target + doesn't need to be built, and handling the -c option by removing + targets as its "build" action. There is also a separate subclass + for suppressing this output when the -q option is used. + + The Taskmaster instantiates a Task object for each (set of) + target(s) that it decides need to be evaluated and/or built. +""" + +import sys +from abc import ABC, abstractmethod +from itertools import chain + +import SCons.Errors +import SCons.Node +import SCons.Warnings + +StateString = SCons.Node.StateString +NODE_NO_STATE = SCons.Node.no_state +NODE_PENDING = SCons.Node.pending +NODE_EXECUTING = SCons.Node.executing +NODE_UP_TO_DATE = SCons.Node.up_to_date +NODE_EXECUTED = SCons.Node.executed +NODE_FAILED = SCons.Node.failed + +print_prepare = False # set by option --debug=prepare + +# A subsystem for recording stats about how different Nodes are handled by +# the main Taskmaster loop. There's no external control here (no need for +# a --debug= option); enable it by changing the value of CollectStats. + +CollectStats = None + +class Stats: + """ + A simple class for holding statistics about the disposition of a + Node by the Taskmaster. If we're collecting statistics, each Node + processed by the Taskmaster gets one of these attached, in which case + the Taskmaster records its decision each time it processes the Node. + (Ideally, that's just once per Node.) + """ + def __init__(self): + """ + Instantiates a Taskmaster.Stats object, initializing all + appropriate counters to zero. + """ + self.considered = 0 + self.already_handled = 0 + self.problem = 0 + self.child_failed = 0 + self.not_built = 0 + self.side_effects = 0 + self.build = 0 + +StatsNodes = [] + +fmt = "%(considered)3d "\ + "%(already_handled)3d " \ + "%(problem)3d " \ + "%(child_failed)3d " \ + "%(not_built)3d " \ + "%(side_effects)3d " \ + "%(build)3d " + +def dump_stats(): + for n in sorted(StatsNodes, key=lambda a: str(a)): + print((fmt % n.attributes.stats.__dict__) + str(n)) + + +class Task(ABC): + """ SCons build engine abstract task class. + + This controls the interaction of the actual building of node + and the rest of the engine. + + This is expected to handle all of the normally-customizable + aspects of controlling a build, so any given application + *should* be able to do what it wants by sub-classing this + class and overriding methods as appropriate. If an application + needs to customize something by sub-classing Taskmaster (or + some other build engine class), we should first try to migrate + that functionality into this class. + + Note that it's generally a good idea for sub-classes to call + these methods explicitly to update state, etc., rather than + roll their own interaction with Taskmaster from scratch. + """ + def __init__(self, tm, targets, top, node): + self.tm = tm + self.targets = targets + self.top = top + self.node = node + self.exc_clear() + + def trace_message(self, method, node, description='node'): + fmt = '%-20s %s %s\n' + return fmt % (method + ':', description, self.tm.trace_node(node)) + + def display(self, message): + """ + Hook to allow the calling interface to display a message. + + This hook gets called as part of preparing a task for execution + (that is, a Node to be built). As part of figuring out what Node + should be built next, the actual target list may be altered, + along with a message describing the alteration. The calling + interface can subclass Task and provide a concrete implementation + of this method to see those messages. + """ + pass + + def prepare(self): + """ + Called just before the task is executed. + + This is mainly intended to give the target Nodes a chance to + unlink underlying files and make all necessary directories before + the Action is actually called to build the targets. + """ + global print_prepare + T = self.tm.trace + if T: T.write(self.trace_message('Task.prepare()', self.node)) + + # Now that it's the appropriate time, give the TaskMaster a + # chance to raise any exceptions it encountered while preparing + # this task. + self.exception_raise() + + if self.tm.message: + self.display(self.tm.message) + self.tm.message = None + + # Let the targets take care of any necessary preparations. + # This includes verifying that all of the necessary sources + # and dependencies exist, removing the target file(s), etc. + # + # As of April 2008, the get_executor().prepare() method makes + # sure that all of the aggregate sources necessary to build this + # Task's target(s) exist in one up-front check. The individual + # target t.prepare() methods check that each target's explicit + # or implicit dependencies exists, and also initialize the + # .sconsign info. + executor = self.targets[0].get_executor() + if executor is None: + return + executor.prepare() + for t in executor.get_action_targets(): + if print_prepare: + print("Preparing target %s..."%t) + for s in t.side_effects: + print("...with side-effect %s..."%s) + t.prepare() + for s in t.side_effects: + if print_prepare: + print("...Preparing side-effect %s..."%s) + s.prepare() + + def get_target(self): + """Fetch the target being built or updated by this task. + """ + return self.node + + @abstractmethod + def needs_execute(self): + return + + def execute(self): + """ + Called to execute the task. + + This method is called from multiple threads in a parallel build, + so only do thread safe stuff here. Do thread unsafe stuff in + prepare(), executed() or failed(). + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.execute()', self.node)) + + try: + cached_targets = [] + for t in self.targets: + if not t.retrieve_from_cache(): + break + cached_targets.append(t) + if len(cached_targets) < len(self.targets): + # Remove targets before building. It's possible that we + # partially retrieved targets from the cache, leaving + # them in read-only mode. That might cause the command + # to fail. + # + for t in cached_targets: + try: + t.fs.unlink(t.get_internal_path()) + except (IOError, OSError): + pass + self.targets[0].build() + else: + for t in cached_targets: + t.cached = 1 + except SystemExit: + exc_value = sys.exc_info()[1] + raise SCons.Errors.ExplicitExit(self.targets[0], exc_value.code) + except SCons.Errors.UserError: + raise + except SCons.Errors.BuildError: + raise + except Exception as e: + buildError = SCons.Errors.convert_to_BuildError(e) + buildError.node = self.targets[0] + buildError.exc_info = sys.exc_info() + raise buildError + + def executed_without_callbacks(self): + """ + Called when the task has been successfully executed + and the Taskmaster instance doesn't want to call + the Node's callback methods. + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.executed_without_callbacks()', + self.node)) + + for t in self.targets: + if t.get_state() == NODE_EXECUTING: + for side_effect in t.side_effects: + side_effect.set_state(NODE_NO_STATE) + t.set_state(NODE_EXECUTED) + + def executed_with_callbacks(self): + """ + Called when the task has been successfully executed and + the Taskmaster instance wants to call the Node's callback + methods. + + This may have been a do-nothing operation (to preserve build + order), so we must check the node's state before deciding whether + it was "built", in which case we call the appropriate Node method. + In any event, we always call "visited()", which will handle any + post-visit actions that must take place regardless of whether + or not the target was an actual built target or a source Node. + """ + global print_prepare + T = self.tm.trace + if T: T.write(self.trace_message('Task.executed_with_callbacks()', + self.node)) + + for t in self.targets: + if t.get_state() == NODE_EXECUTING: + for side_effect in t.side_effects: + side_effect.set_state(NODE_NO_STATE) + t.set_state(NODE_EXECUTED) + if not t.cached: + t.push_to_cache() + t.built() + t.visited() + if (not print_prepare and + (not hasattr(self, 'options') or not self.options.debug_includes)): + t.release_target_info() + else: + t.visited() + + executed = executed_with_callbacks + + def failed(self): + """ + Default action when a task fails: stop the build. + + Note: Although this function is normally invoked on nodes in + the executing state, it might also be invoked on up-to-date + nodes when using Configure(). + """ + self.fail_stop() + + def fail_stop(self): + """ + Explicit stop-the-build failure. + + This sets failure status on the target nodes and all of + their dependent parent nodes. + + Note: Although this function is normally invoked on nodes in + the executing state, it might also be invoked on up-to-date + nodes when using Configure(). + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.failed_stop()', self.node)) + + # Invoke will_not_build() to clean-up the pending children + # list. + self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) + + # Tell the taskmaster to not start any new tasks + self.tm.stop() + + # We're stopping because of a build failure, but give the + # calling Task class a chance to postprocess() the top-level + # target under which the build failure occurred. + self.targets = [self.tm.current_top] + self.top = 1 + + def fail_continue(self): + """ + Explicit continue-the-build failure. + + This sets failure status on the target nodes and all of + their dependent parent nodes. + + Note: Although this function is normally invoked on nodes in + the executing state, it might also be invoked on up-to-date + nodes when using Configure(). + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.failed_continue()', self.node)) + + self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) + + def make_ready_all(self): + """ + Marks all targets in a task ready for execution. + + This is used when the interface needs every target Node to be + visited--the canonical example being the "scons -c" option. + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.make_ready_all()', self.node)) + + self.out_of_date = self.targets[:] + for t in self.targets: + t.disambiguate().set_state(NODE_EXECUTING) + for s in t.side_effects: + # add disambiguate here to mirror the call on targets above + s.disambiguate().set_state(NODE_EXECUTING) + + def make_ready_current(self): + """ + Marks all targets in a task ready for execution if any target + is not current. + + This is the default behavior for building only what's necessary. + """ + global print_prepare + T = self.tm.trace + if T: T.write(self.trace_message('Task.make_ready_current()', + self.node)) + + self.out_of_date = [] + needs_executing = False + for t in self.targets: + try: + t.disambiguate().make_ready() + is_up_to_date = not t.has_builder() or \ + (not t.always_build and t.is_up_to_date()) + except EnvironmentError as e: + raise SCons.Errors.BuildError(node=t, errstr=e.strerror, filename=e.filename) + + if not is_up_to_date: + self.out_of_date.append(t) + needs_executing = True + + if needs_executing: + for t in self.targets: + t.set_state(NODE_EXECUTING) + for s in t.side_effects: + # add disambiguate here to mirror the call on targets in first loop above + s.disambiguate().set_state(NODE_EXECUTING) + else: + for t in self.targets: + # We must invoke visited() to ensure that the node + # information has been computed before allowing the + # parent nodes to execute. (That could occur in a + # parallel build...) + t.visited() + t.set_state(NODE_UP_TO_DATE) + if (not print_prepare and + (not hasattr(self, 'options') or not self.options.debug_includes)): + t.release_target_info() + + make_ready = make_ready_current + + def postprocess(self): + """ + Post-processes a task after it's been executed. + + This examines all the targets just built (or not, we don't care + if the build was successful, or even if there was no build + because everything was up-to-date) to see if they have any + waiting parent Nodes, or Nodes waiting on a common side effect, + that can be put back on the candidates list. + """ + T = self.tm.trace + if T: T.write(self.trace_message('Task.postprocess()', self.node)) + + # We may have built multiple targets, some of which may have + # common parents waiting for this build. Count up how many + # targets each parent was waiting for so we can subtract the + # values later, and so we *don't* put waiting side-effect Nodes + # back on the candidates list if the Node is also a waiting + # parent. + + targets = set(self.targets) + + pending_children = self.tm.pending_children + parents = {} + for t in targets: + # A node can only be in the pending_children set if it has + # some waiting_parents. + if t.waiting_parents: + if T: T.write(self.trace_message('Task.postprocess()', + t, + 'removing')) + pending_children.discard(t) + for p in t.waiting_parents: + parents[p] = parents.get(p, 0) + 1 + t.waiting_parents = set() + + for t in targets: + if t.side_effects is not None: + for s in t.side_effects: + if s.get_state() == NODE_EXECUTING: + s.set_state(NODE_NO_STATE) + + # The side-effects may have been transferred to + # NODE_NO_STATE by executed_with{,out}_callbacks, but was + # not taken out of the waiting parents/pending children + # data structures. Check for that now. + if s.get_state() == NODE_NO_STATE and s.waiting_parents: + pending_children.discard(s) + for p in s.waiting_parents: + parents[p] = parents.get(p, 0) + 1 + s.waiting_parents = set() + for p in s.waiting_s_e: + if p.ref_count == 0: + self.tm.candidates.append(p) + + for p, subtract in parents.items(): + p.ref_count = p.ref_count - subtract + if T: T.write(self.trace_message('Task.postprocess()', + p, + 'adjusted parent ref count')) + if p.ref_count == 0: + self.tm.candidates.append(p) + + for t in targets: + t.postprocess() + + # Exception handling subsystem. + # + # Exceptions that occur while walking the DAG or examining Nodes + # must be raised, but must be raised at an appropriate time and in + # a controlled manner so we can, if necessary, recover gracefully, + # possibly write out signature information for Nodes we've updated, + # etc. This is done by having the Taskmaster tell us about the + # exception, and letting + + def exc_info(self): + """ + Returns info about a recorded exception. + """ + return self.exception + + def exc_clear(self): + """ + Clears any recorded exception. + + This also changes the "exception_raise" attribute to point + to the appropriate do-nothing method. + """ + self.exception = (None, None, None) + self.exception_raise = self._no_exception_to_raise + + def exception_set(self, exception=None): + """ + Records an exception to be raised at the appropriate time. + + This also changes the "exception_raise" attribute to point + to the method that will, in fact + """ + if not exception: + exception = sys.exc_info() + self.exception = exception + self.exception_raise = self._exception_raise + + def _no_exception_to_raise(self): + pass + + def _exception_raise(self): + """ + Raises a pending exception that was recorded while getting a + Task ready for execution. + """ + exc = self.exc_info()[:] + try: + exc_type, exc_value, exc_traceback = exc + except ValueError: + exc_type, exc_value = exc # pylint: disable=unbalanced-tuple-unpacking + exc_traceback = None + + # raise exc_type(exc_value).with_traceback(exc_traceback) + if isinstance(exc_value, Exception): #hasattr(exc_value, 'with_traceback'): + # If exc_value is an exception, then just reraise + raise exc_value.with_traceback(exc_traceback) + else: + # else we'll create an exception using the value and raise that + raise exc_type(exc_value).with_traceback(exc_traceback) + + + # raise e.__class__, e.__class__(e), sys.exc_info()[2] + # exec("raise exc_type(exc_value).with_traceback(exc_traceback)") + + + +class AlwaysTask(Task): + def needs_execute(self): + """ + Always returns True (indicating this Task should always + be executed). + + Subclasses that need this behavior (as opposed to the default + of only executing Nodes that are out of date w.r.t. their + dependencies) can use this as follows: + + class MyTaskSubclass(SCons.Taskmaster.Task): + needs_execute = SCons.Taskmaster.AlwaysTask.needs_execute + """ + return True + +class OutOfDateTask(Task): + def needs_execute(self): + """ + Returns True (indicating this Task should be executed) if this + Task's target state indicates it needs executing, which has + already been determined by an earlier up-to-date check. + """ + return self.targets[0].get_state() == SCons.Node.executing + + +def find_cycle(stack, visited): + if stack[-1] in visited: + return None + visited.add(stack[-1]) + for n in stack[-1].waiting_parents: + stack.append(n) + if stack[0] == stack[-1]: + return stack + if find_cycle(stack, visited): + return stack + stack.pop() + return None + + +class Taskmaster: + """ + The Taskmaster for walking the dependency DAG. + """ + + def __init__(self, targets=[], tasker=None, order=None, trace=None): + self.original_top = targets + self.top_targets_left = targets[:] + self.top_targets_left.reverse() + self.candidates = [] + if tasker is None: + tasker = OutOfDateTask + self.tasker = tasker + if not order: + order = lambda l: l + self.order = order + self.message = None + self.trace = trace + self.next_candidate = self.find_next_candidate + self.pending_children = set() + + def find_next_candidate(self): + """ + Returns the next candidate Node for (potential) evaluation. + + The candidate list (really a stack) initially consists of all of + the top-level (command line) targets provided when the Taskmaster + was initialized. While we walk the DAG, visiting Nodes, all the + children that haven't finished processing get pushed on to the + candidate list. Each child can then be popped and examined in + turn for whether *their* children are all up-to-date, in which + case a Task will be created for their actual evaluation and + potential building. + + Here is where we also allow candidate Nodes to alter the list of + Nodes that should be examined. This is used, for example, when + invoking SCons in a source directory. A source directory Node can + return its corresponding build directory Node, essentially saying, + "Hey, you really need to build this thing over here instead." + """ + try: + return self.candidates.pop() + except IndexError: + pass + try: + node = self.top_targets_left.pop() + except IndexError: + return None + self.current_top = node + alt, message = node.alter_targets() + if alt: + self.message = message + self.candidates.append(node) + self.candidates.extend(self.order(alt)) + node = self.candidates.pop() + return node + + def no_next_candidate(self): + """ + Stops Taskmaster processing by not returning a next candidate. + + Note that we have to clean-up the Taskmaster candidate list + because the cycle detection depends on the fact all nodes have + been processed somehow. + """ + while self.candidates: + candidates = self.candidates + self.candidates = [] + self.will_not_build(candidates) + return None + + def _validate_pending_children(self): + """ + Validate the content of the pending_children set. Assert if an + internal error is found. + + This function is used strictly for debugging the taskmaster by + checking that no invariants are violated. It is not used in + normal operation. + + The pending_children set is used to detect cycles in the + dependency graph. We call a "pending child" a child that is + found in the "pending" state when checking the dependencies of + its parent node. + + A pending child can occur when the Taskmaster completes a loop + through a cycle. For example, let's imagine a graph made of + three nodes (A, B and C) making a cycle. The evaluation starts + at node A. The Taskmaster first considers whether node A's + child B is up-to-date. Then, recursively, node B needs to + check whether node C is up-to-date. This leaves us with a + dependency graph looking like:: + + Next candidate \ + \ + Node A (Pending) --> Node B(Pending) --> Node C (NoState) + ^ | + | | + +-------------------------------------+ + + Now, when the Taskmaster examines the Node C's child Node A, + it finds that Node A is in the "pending" state. Therefore, + Node A is a pending child of node C. + + Pending children indicate that the Taskmaster has potentially + loop back through a cycle. We say potentially because it could + also occur when a DAG is evaluated in parallel. For example, + consider the following graph:: + + Node A (Pending) --> Node B(Pending) --> Node C (Pending) --> ... + | ^ + | | + +----------> Node D (NoState) --------+ + / + Next candidate / + + The Taskmaster first evaluates the nodes A, B, and C and + starts building some children of node C. Assuming, that the + maximum parallel level has not been reached, the Taskmaster + will examine Node D. It will find that Node C is a pending + child of Node D. + + In summary, evaluating a graph with a cycle will always + involve a pending child at one point. A pending child might + indicate either a cycle or a diamond-shaped DAG. Only a + fraction of the nodes ends-up being a "pending child" of + another node. This keeps the pending_children set small in + practice. + + We can differentiate between the two cases if we wait until + the end of the build. At this point, all the pending children + nodes due to a diamond-shaped DAG will have been properly + built (or will have failed to build). But, the pending + children involved in a cycle will still be in the pending + state. + + The taskmaster removes nodes from the pending_children set as + soon as a pending_children node moves out of the pending + state. This also helps to keep the pending_children set small. + """ + + for n in self.pending_children: + assert n.state in (NODE_PENDING, NODE_EXECUTING), \ + (str(n), StateString[n.state]) + assert len(n.waiting_parents) != 0, (str(n), len(n.waiting_parents)) + for p in n.waiting_parents: + assert p.ref_count > 0, (str(n), str(p), p.ref_count) + + + def trace_message(self, message): + return 'Taskmaster: %s\n' % message + + def trace_node(self, node): + return '<%-10s %-3s %s>' % (StateString[node.get_state()], + node.ref_count, + repr(str(node))) + + def _find_next_ready_node(self): + """ + Finds the next node that is ready to be built. + + This is *the* main guts of the DAG walk. We loop through the + list of candidates, looking for something that has no un-built + children (i.e., that is a leaf Node or has dependencies that are + all leaf Nodes or up-to-date). Candidate Nodes are re-scanned + (both the target Node itself and its sources, which are always + scanned in the context of a given target) to discover implicit + dependencies. A Node that must wait for some children to be + built will be put back on the candidates list after the children + have finished building. A Node that has been put back on the + candidates list in this way may have itself (or its sources) + re-scanned, in order to handle generated header files (e.g.) and + the implicit dependencies therein. + + Note that this method does not do any signature calculation or + up-to-date check itself. All of that is handled by the Task + class. This is purely concerned with the dependency graph walk. + """ + + self.ready_exc = None + + T = self.trace + if T: T.write('\n' + self.trace_message('Looking for a node to evaluate')) + + while True: + node = self.next_candidate() + if node is None: + if T: T.write(self.trace_message('No candidate anymore.') + '\n') + return None + + node = node.disambiguate() + state = node.get_state() + + # For debugging only: + # + # try: + # self._validate_pending_children() + # except: + # self.ready_exc = sys.exc_info() + # return node + + if CollectStats: + if not hasattr(node.attributes, 'stats'): + node.attributes.stats = Stats() + StatsNodes.append(node) + S = node.attributes.stats + S.considered = S.considered + 1 + else: + S = None + + if T: T.write(self.trace_message(' Considering node %s and its children:' % self.trace_node(node))) + + if state == NODE_NO_STATE: + # Mark this node as being on the execution stack: + node.set_state(NODE_PENDING) + elif state > NODE_PENDING: + # Skip this node if it has already been evaluated: + if S: S.already_handled = S.already_handled + 1 + if T: T.write(self.trace_message(' already handled (executed)')) + continue + + executor = node.get_executor() + + try: + children = executor.get_all_children() + except SystemExit: + exc_value = sys.exc_info()[1] + e = SCons.Errors.ExplicitExit(node, exc_value.code) + self.ready_exc = (SCons.Errors.ExplicitExit, e) + if T: T.write(self.trace_message(' SystemExit')) + return node + except Exception as e: + # We had a problem just trying to figure out the + # children (like a child couldn't be linked in to a + # VariantDir, or a Scanner threw something). Arrange to + # raise the exception when the Task is "executed." + self.ready_exc = sys.exc_info() + if S: S.problem = S.problem + 1 + if T: T.write(self.trace_message(' exception %s while scanning children.\n' % e)) + return node + + children_not_visited = [] + children_pending = set() + children_not_ready = [] + children_failed = False + + for child in chain(executor.get_all_prerequisites(), children): + childstate = child.get_state() + + if T: T.write(self.trace_message(' ' + self.trace_node(child))) + + if childstate == NODE_NO_STATE: + children_not_visited.append(child) + elif childstate == NODE_PENDING: + children_pending.add(child) + elif childstate == NODE_FAILED: + children_failed = True + + if childstate <= NODE_EXECUTING: + children_not_ready.append(child) + + # These nodes have not even been visited yet. Add + # them to the list so that on some next pass we can + # take a stab at evaluating them (or their children). + if children_not_visited: + if len(children_not_visited) > 1: + children_not_visited.reverse() + self.candidates.extend(self.order(children_not_visited)) + + # if T and children_not_visited: + # T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) + # T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) + + # Skip this node if any of its children have failed. + # + # This catches the case where we're descending a top-level + # target and one of our children failed while trying to be + # built by a *previous* descent of an earlier top-level + # target. + # + # It can also occur if a node is reused in multiple + # targets. One first descends though the one of the + # target, the next time occurs through the other target. + # + # Note that we can only have failed_children if the + # --keep-going flag was used, because without it the build + # will stop before diving in the other branch. + # + # Note that even if one of the children fails, we still + # added the other children to the list of candidate nodes + # to keep on building (--keep-going). + if children_failed: + for n in executor.get_action_targets(): + n.set_state(NODE_FAILED) + + if S: S.child_failed = S.child_failed + 1 + if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) + continue + + if children_not_ready: + for child in children_not_ready: + # We're waiting on one or more derived targets + # that have not yet finished building. + if S: S.not_built = S.not_built + 1 + + # Add this node to the waiting parents lists of + # anything we're waiting on, with a reference + # count so we can be put back on the list for + # re-evaluation when they've all finished. + node.ref_count = node.ref_count + child.add_to_waiting_parents(node) + if T: T.write(self.trace_message(' adjusted ref count: %s, child %s' % + (self.trace_node(node), repr(str(child))))) + + if T: + for pc in children_pending: + T.write(self.trace_message(' adding %s to the pending children set\n' % + self.trace_node(pc))) + self.pending_children = self.pending_children | children_pending + + continue + + # Skip this node if it has side-effects that are + # currently being built: + wait_side_effects = False + for se in executor.get_action_side_effects(): + if se.get_state() == NODE_EXECUTING: + se.add_to_waiting_s_e(node) + wait_side_effects = True + + if wait_side_effects: + if S: S.side_effects = S.side_effects + 1 + continue + + # The default when we've gotten through all of the checks above: + # this node is ready to be built. + if S: S.build = S.build + 1 + if T: T.write(self.trace_message('Evaluating %s\n' % + self.trace_node(node))) + + # For debugging only: + # + # try: + # self._validate_pending_children() + # except: + # self.ready_exc = sys.exc_info() + # return node + + return node + + return None + + def next_task(self): + """ + Returns the next task to be executed. + + This simply asks for the next Node to be evaluated, and then wraps + it in the specific Task subclass with which we were initialized. + """ + node = self._find_next_ready_node() + + if node is None: + return None + + executor = node.get_executor() + if executor is None: + return None + + tlist = executor.get_all_targets() + + task = self.tasker(self, tlist, node in self.original_top, node) + try: + task.make_ready() + except Exception as e : + # We had a problem just trying to get this task ready (like + # a child couldn't be linked to a VariantDir when deciding + # whether this node is current). Arrange to raise the + # exception when the Task is "executed." + self.ready_exc = sys.exc_info() + + if self.ready_exc: + task.exception_set(self.ready_exc) + + self.ready_exc = None + + return task + + def will_not_build(self, nodes, node_func=lambda n: None): + """ + Perform clean-up about nodes that will never be built. Invokes + a user defined function on all of these nodes (including all + of their parents). + """ + + T = self.trace + + pending_children = self.pending_children + + to_visit = set(nodes) + pending_children = pending_children - to_visit + + if T: + for n in nodes: + T.write(self.trace_message(' removing node %s from the pending children set\n' % + self.trace_node(n))) + try: + while len(to_visit): + node = to_visit.pop() + node_func(node) + + # Prune recursion by flushing the waiting children + # list immediately. + parents = node.waiting_parents + node.waiting_parents = set() + + to_visit = to_visit | parents + pending_children = pending_children - parents + + for p in parents: + p.ref_count = p.ref_count - 1 + if T: T.write(self.trace_message(' removing parent %s from the pending children set\n' % + self.trace_node(p))) + except KeyError: + # The container to_visit has been emptied. + pass + + # We have the stick back the pending_children list into the + # taskmaster because the python 1.5.2 compatibility does not + # allow us to use in-place updates + self.pending_children = pending_children + + def stop(self): + """ + Stops the current build completely. + """ + self.next_candidate = self.no_next_candidate + + def cleanup(self): + """ + Check for dependency cycles. + """ + if not self.pending_children: + return + + nclist = [(n, find_cycle([n], set())) for n in self.pending_children] + + genuine_cycles = [ + node for node,cycle in nclist + if cycle or node.get_state() != NODE_EXECUTED + ] + if not genuine_cycles: + # All of the "cycles" found were single nodes in EXECUTED state, + # which is to say, they really weren't cycles. Just return. + return + + desc = 'Found dependency cycle(s):\n' + for node, cycle in nclist: + if cycle: + desc = desc + " " + " -> ".join(map(str, cycle)) + "\n" + else: + desc = desc + \ + " Internal Error: no cycle found for node %s (%s) in state %s\n" % \ + (node, repr(node), StateString[node.get_state()]) + + raise SCons.Errors.UserError(desc) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/TaskmasterTests.py b/SCons/TaskmasterTests.py deleted file mode 100644 index f20fd71..0000000 --- a/SCons/TaskmasterTests.py +++ /dev/null @@ -1,1257 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import SCons.compat - -import sys -import unittest - - -import SCons.Taskmaster -import SCons.Errors - - -built_text = None -cache_text = [] -visited_nodes = [] -executed = None -scan_called = 0 - -class Node: - def __init__(self, name, kids = [], scans = []): - self.name = name - self.kids = kids - self.scans = scans - self.cached = 0 - self.scanned = 0 - self.scanner = None - self.targets = [self] - self.prerequisites = None - class Builder: - def targets(self, node): - return node.targets - self.builder = Builder() - self.bsig = None - self.csig = None - self.state = SCons.Node.no_state - self.prepared = None - self.ref_count = 0 - self.waiting_parents = set() - self.waiting_s_e = set() - self.side_effect = 0 - self.side_effects = [] - self.alttargets = [] - self.postprocessed = None - self._bsig_val = None - self._current_val = 0 - self.always_build = None - - def disambiguate(self): - return self - - def push_to_cache(self): - pass - - def retrieve_from_cache(self): - global cache_text - if self.cached: - cache_text.append(self.name + " retrieved") - return self.cached - - def make_ready(self): - pass - - def prepare(self): - self.prepared = 1 - self.get_binfo() - - def build(self): - global built_text - built_text = self.name + " built" - - def remove(self): - pass - - # The following four methods new_binfo(), del_binfo(), - # get_binfo(), clear() as well as its calls have been added - # to support the cached_execute() test (issue #2720). - # They are full copies (or snippets) of their actual - # counterparts in the Node class... - def new_binfo(self): - binfo = "binfo" - return binfo - - def del_binfo(self): - """Delete the build info from this node.""" - try: - delattr(self, 'binfo') - except AttributeError: - pass - - def get_binfo(self): - """Fetch a node's build information.""" - try: - return self.binfo - except AttributeError: - pass - - binfo = self.new_binfo() - self.binfo = binfo - - return binfo - - def clear(self): - # The del_binfo() call here isn't necessary for normal execution, - # but is for interactive mode, where we might rebuild the same - # target and need to start from scratch. - self.del_binfo() - - def built(self): - global built_text - if not self.cached: - built_text = built_text + " really" - - # Clear the implicit dependency caches of any Nodes - # waiting for this Node to be built. - for parent in self.waiting_parents: - parent.implicit = None - - self.clear() - - def release_target_info(self): - pass - - def has_builder(self): - return self.builder is not None - - def is_derived(self): - return self.has_builder or self.side_effect - - def alter_targets(self): - return self.alttargets, None - - def visited(self): - global visited_nodes - visited_nodes.append(self.name) - - def children(self): - if not self.scanned: - self.scan() - self.scanned = 1 - return self.kids - - def scan(self): - global scan_called - scan_called = scan_called + 1 - self.kids = self.kids + self.scans - self.scans = [] - - def scanner_key(self): - return self.name - - def add_to_waiting_parents(self, node): - wp = self.waiting_parents - if node in wp: - return 0 - wp.add(node) - return 1 - - def get_state(self): - return self.state - - def set_state(self, state): - self.state = state - - def set_bsig(self, bsig): - self.bsig = bsig - - def set_csig(self, csig): - self.csig = csig - - def store_csig(self): - pass - - def store_bsig(self): - pass - - def is_pseudo_derived(self): - pass - - def is_up_to_date(self): - return self._current_val - - def depends_on(self, nodes): - for node in nodes: - if node in self.kids: - return 1 - return 0 - - def __str__(self): - return self.name - - def postprocess(self): - self.postprocessed = 1 - self.waiting_parents = set() - - def get_executor(self): - if not hasattr(self, 'executor'): - class Executor: - def prepare(self): - pass - def get_action_targets(self): - return self.targets - def get_all_targets(self): - return self.targets - def get_all_children(self): - result = [] - for node in self.targets: - result.extend(node.children()) - return result - def get_all_prerequisites(self): - return [] - def get_action_side_effects(self): - return [] - self.executor = Executor() - self.executor.targets = self.targets - return self.executor - -class OtherError(Exception): - pass - -class MyException(Exception): - pass - - -class TaskmasterTestCase(unittest.TestCase): - - def test_next_task(self): - """Test fetching the next task - """ - global built_text - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1, n1]) - t = tm.next_task() - t.prepare() - t.execute() - t = tm.next_task() - assert t is None - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - - tm = SCons.Taskmaster.Taskmaster([n3]) - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n1 built", built_text - t.executed() - t.postprocess() - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n2 built", built_text - t.executed() - t.postprocess() - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n3 built", built_text - t.executed() - t.postprocess() - - assert tm.next_task() is None - - built_text = "up to date: " - top_node = n3 - - class MyTask(SCons.Taskmaster.AlwaysTask): - def execute(self): - global built_text - if self.targets[0].get_state() == SCons.Node.up_to_date: - if self.top: - built_text = self.targets[0].name + " up-to-date top" - else: - built_text = self.targets[0].name + " up-to-date" - else: - self.targets[0].build() - - n1.set_state(SCons.Node.no_state) - n1._current_val = 1 - n2.set_state(SCons.Node.no_state) - n2._current_val = 1 - n3.set_state(SCons.Node.no_state) - n3._current_val = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask) - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n1 up-to-date", built_text - t.executed() - t.postprocess() - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n2 up-to-date", built_text - t.executed() - t.postprocess() - - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n3 up-to-date top", built_text - t.executed() - t.postprocess() - - assert tm.next_task() is None - - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - n4 = Node("n4") - n5 = Node("n5", [n3, n4]) - tm = SCons.Taskmaster.Taskmaster([n5]) - - t1 = tm.next_task() - assert t1.get_target() == n1 - - t2 = tm.next_task() - assert t2.get_target() == n2 - - t4 = tm.next_task() - assert t4.get_target() == n4 - t4.executed() - t4.postprocess() - - t1.executed() - t1.postprocess() - t2.executed() - t2.postprocess() - t3 = tm.next_task() - assert t3.get_target() == n3 - - t3.executed() - t3.postprocess() - t5 = tm.next_task() - assert t5.get_target() == n5, t5.get_target() - t5.executed() - t5.postprocess() - - assert tm.next_task() is None - - - n4 = Node("n4") - n4.set_state(SCons.Node.executed) - tm = SCons.Taskmaster.Taskmaster([n4]) - assert tm.next_task() is None - - n1 = Node("n1") - n2 = Node("n2", [n1]) - tm = SCons.Taskmaster.Taskmaster([n2,n2]) - t = tm.next_task() - t.executed() - t.postprocess() - t = tm.next_task() - assert tm.next_task() is None - - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1], [n2]) - tm = SCons.Taskmaster.Taskmaster([n3]) - t = tm.next_task() - target = t.get_target() - assert target == n1, target - t.executed() - t.postprocess() - t = tm.next_task() - target = t.get_target() - assert target == n2, target - t.executed() - t.postprocess() - t = tm.next_task() - target = t.get_target() - assert target == n3, target - t.executed() - t.postprocess() - assert tm.next_task() is None - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - n4 = Node("n4", [n3]) - n5 = Node("n5", [n3]) - global scan_called - scan_called = 0 - tm = SCons.Taskmaster.Taskmaster([n4]) - t = tm.next_task() - assert t.get_target() == n1 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n2 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n3 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n4 - t.executed() - t.postprocess() - assert tm.next_task() is None - assert scan_called == 4, scan_called - - tm = SCons.Taskmaster.Taskmaster([n5]) - t = tm.next_task() - assert t.get_target() == n5, t.get_target() - t.executed() - assert tm.next_task() is None - assert scan_called == 5, scan_called - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) - n5 = Node("n5", [n4]) - n3.side_effect = 1 - n1.side_effects = n2.side_effects = n3.side_effects = [n4] - tm = SCons.Taskmaster.Taskmaster([n1,n2,n3,n4,n5]) - t = tm.next_task() - assert t.get_target() == n1 - assert n4.state == SCons.Node.executing, n4.state - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n2 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n3 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n4 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n5 - assert not tm.next_task() - t.executed() - t.postprocess() - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) - def reverse(dependencies): - dependencies.reverse() - return dependencies - tm = SCons.Taskmaster.Taskmaster([n4], order=reverse) - t = tm.next_task() - assert t.get_target() == n3, t.get_target() - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n2, t.get_target() - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n1, t.get_target() - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n4, t.get_target() - t.executed() - t.postprocess() - - n5 = Node("n5") - n6 = Node("n6") - n7 = Node("n7") - n6.alttargets = [n7] - - tm = SCons.Taskmaster.Taskmaster([n5]) - t = tm.next_task() - assert t.get_target() == n5 - t.executed() - t.postprocess() - - tm = SCons.Taskmaster.Taskmaster([n6]) - t = tm.next_task() - assert t.get_target() == n7 - t.executed() - t.postprocess() - t = tm.next_task() - assert t.get_target() == n6 - t.executed() - t.postprocess() - - n1 = Node("n1") - n2 = Node("n2", [n1]) - n1.set_state(SCons.Node.failed) - tm = SCons.Taskmaster.Taskmaster([n2]) - assert tm.next_task() is None - - n1 = Node("n1") - n2 = Node("n2") - n1.targets = [n1, n2] - n1._current_val = 1 - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - t.executed() - t.postprocess() - - s = n1.get_state() - assert s == SCons.Node.executed, s - s = n2.get_state() - assert s == SCons.Node.executed, s - - - def test_make_ready_out_of_date(self): - """Test the Task.make_ready() method's list of out-of-date Nodes - """ - ood = [] - def TaskGen(tm, targets, top, node, ood=ood): - class MyTask(SCons.Taskmaster.AlwaysTask): - def make_ready(self): - SCons.Taskmaster.Task.make_ready(self) - self.ood.extend(self.out_of_date) - - t = MyTask(tm, targets, top, node) - t.ood = ood - return t - - n1 = Node("n1") - c2 = Node("c2") - c2._current_val = 1 - n3 = Node("n3") - c4 = Node("c4") - c4._current_val = 1 - a5 = Node("a5") - a5._current_val = 1 - a5.always_build = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4, a5], - tasker = TaskGen) - - del ood[:] - t = tm.next_task() - assert ood == [n1], ood - - del ood[:] - t = tm.next_task() - assert ood == [], ood - - del ood[:] - t = tm.next_task() - assert ood == [n3], ood - - del ood[:] - t = tm.next_task() - assert ood == [], ood - - del ood[:] - t = tm.next_task() - assert ood == [a5], ood - - def test_make_ready_exception(self): - """Test handling exceptions from Task.make_ready() - """ - class MyTask(SCons.Taskmaster.AlwaysTask): - def make_ready(self): - raise MyException("from make_ready()") - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster(targets = [n1], tasker = MyTask) - t = tm.next_task() - exc_type, exc_value, exc_tb = t.exception - assert exc_type == MyException, repr(exc_type) - assert str(exc_value) == "from make_ready()", exc_value - - def test_needs_execute(self): - """Test that we can't instantiate a Task subclass without needs_execute - - We should be getting: - TypeError: Can't instantiate abstract class MyTask with abstract methods needs_execute - """ - class MyTask(SCons.Taskmaster.Task): - pass - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster(targets=[n1], tasker=MyTask) - with self.assertRaises(TypeError): - _ = tm.next_task() - - def test_make_ready_all(self): - """Test the make_ready_all() method""" - class MyTask(SCons.Taskmaster.AlwaysTask): - make_ready = SCons.Taskmaster.Task.make_ready_all - - n1 = Node("n1") - c2 = Node("c2") - c2._current_val = 1 - n3 = Node("n3") - c4 = Node("c4") - c4._current_val = 1 - - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4]) - - t = tm.next_task() - target = t.get_target() - assert target is n1, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - target = t.get_target() - assert target is c2, target - assert target.state == SCons.Node.up_to_date, target.state - t = tm.next_task() - target = t.get_target() - assert target is n3, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - target = t.get_target() - assert target is c4, target - assert target.state == SCons.Node.up_to_date, target.state - t = tm.next_task() - assert t is None - - n1 = Node("n1") - c2 = Node("c2") - n3 = Node("n3") - c4 = Node("c4") - - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4], - tasker = MyTask) - - t = tm.next_task() - target = t.get_target() - assert target is n1, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - target = t.get_target() - assert target is c2, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - target = t.get_target() - assert target is n3, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - target = t.get_target() - assert target is c4, target - assert target.state == SCons.Node.executing, target.state - t = tm.next_task() - assert t is None - - - def test_children_errors(self): - """Test errors when fetching the children of a node. - """ - class StopNode(Node): - def children(self): - raise SCons.Errors.StopError("stop!") - class ExitNode(Node): - def children(self): - sys.exit(77) - - n1 = StopNode("n1") - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - exc_type, exc_value, exc_tb = t.exception - assert exc_type == SCons.Errors.StopError, repr(exc_type) - assert str(exc_value) == "stop!", exc_value - - n2 = ExitNode("n2") - tm = SCons.Taskmaster.Taskmaster([n2]) - t = tm.next_task() - exc_type, exc_value = t.exception - assert exc_type == SCons.Errors.ExplicitExit, repr(exc_type) - assert exc_value.node == n2, exc_value.node - assert exc_value.status == 77, exc_value.status - - def test_cycle_detection(self): - """Test detecting dependency cycles - """ - n1 = Node("n1") - n2 = Node("n2", [n1]) - n3 = Node("n3", [n2]) - n1.kids = [n3] - - tm = SCons.Taskmaster.Taskmaster([n3]) - try: - t = tm.next_task() - except SCons.Errors.UserError as e: - assert str(e) == "Dependency cycle: n3 -> n1 -> n2 -> n3", str(e) - else: - assert 'Did not catch expected UserError' - - def test_next_top_level_candidate(self): - """Test the next_top_level_candidate() method - """ - n1 = Node("n1") - n2 = Node("n2", [n1]) - n3 = Node("n3", [n2]) - - tm = SCons.Taskmaster.Taskmaster([n3]) - t = tm.next_task() - assert t.targets == [n1], t.targets - t.fail_stop() - assert t.targets == [n3], list(map(str, t.targets)) - assert t.top == 1, t.top - - def test_stop(self): - """Test the stop() method - - Both default and overridden in a subclass. - """ - global built_text - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - - tm = SCons.Taskmaster.Taskmaster([n3]) - t = tm.next_task() - t.prepare() - t.execute() - assert built_text == "n1 built", built_text - t.executed() - t.postprocess() - assert built_text == "n1 built really", built_text - - tm.stop() - assert tm.next_task() is None - - class MyTM(SCons.Taskmaster.Taskmaster): - def stop(self): - global built_text - built_text = "MyTM.stop()" - SCons.Taskmaster.Taskmaster.stop(self) - - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - - built_text = None - tm = MyTM([n3]) - tm.next_task().execute() - assert built_text == "n1 built" - - tm.stop() - assert built_text == "MyTM.stop()" - assert tm.next_task() is None - - def test_executed(self): - """Test when a task has been executed - """ - global built_text - global visited_nodes - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - built_text = "xxx" - visited_nodes = [] - n1.set_state(SCons.Node.executing) - - t.executed() - - s = n1.get_state() - assert s == SCons.Node.executed, s - assert built_text == "xxx really", built_text - assert visited_nodes == ['n1'], visited_nodes - - n2 = Node("n2") - tm = SCons.Taskmaster.Taskmaster([n2]) - t = tm.next_task() - built_text = "should_not_change" - visited_nodes = [] - n2.set_state(None) - - t.executed() - - s = n2.get_state() - assert s is None, s - assert built_text == "should_not_change", built_text - assert visited_nodes == ['n2'], visited_nodes - - n3 = Node("n3") - n4 = Node("n4") - n3.targets = [n3, n4] - tm = SCons.Taskmaster.Taskmaster([n3]) - t = tm.next_task() - visited_nodes = [] - n3.set_state(SCons.Node.up_to_date) - n4.set_state(SCons.Node.executing) - - t.executed() - - s = n3.get_state() - assert s == SCons.Node.up_to_date, s - s = n4.get_state() - assert s == SCons.Node.executed, s - assert visited_nodes == ['n3', 'n4'], visited_nodes - - def test_prepare(self): - """Test preparation of multiple Nodes for a task - """ - n1 = Node("n1") - n2 = Node("n2") - tm = SCons.Taskmaster.Taskmaster([n1, n2]) - t = tm.next_task() - # This next line is moderately bogus. We're just reaching - # in and setting the targets for this task to an array. The - # "right" way to do this would be to have the next_task() call - # set it up by having something that approximates a real Builder - # return this list--but that's more work than is probably - # warranted right now. - n1.get_executor().targets = [n1, n2] - t.prepare() - assert n1.prepared - assert n2.prepared - - n3 = Node("n3") - n4 = Node("n4") - tm = SCons.Taskmaster.Taskmaster([n3, n4]) - t = tm.next_task() - # More bogus reaching in and setting the targets. - n3.set_state(SCons.Node.up_to_date) - n3.get_executor().targets = [n3, n4] - t.prepare() - assert n3.prepared - assert n4.prepared - - # If the Node has had an exception recorded while it was getting - # prepared, then prepare() should raise that exception. - class MyException(Exception): - pass - - built_text = None - n5 = Node("n5") - tm = SCons.Taskmaster.Taskmaster([n5]) - t = tm.next_task() - t.exception_set((MyException, "exception value")) - exc_caught = None - exc_actually_caught = None - exc_value = None - try: - t.prepare() - except MyException as e: - exc_caught = 1 - exc_value = e - except Exception as exc_actually_caught: - pass - assert exc_caught, "did not catch expected MyException: %s" % exc_actually_caught - assert str(exc_value) == "exception value", exc_value - assert built_text is None, built_text - - # Regression test, make sure we prepare not only - # all targets, but their side effects as well. - n6 = Node("n6") - n7 = Node("n7") - n8 = Node("n8") - n9 = Node("n9") - n10 = Node("n10") - - n6.side_effects = [ n8 ] - n7.side_effects = [ n9, n10 ] - - tm = SCons.Taskmaster.Taskmaster([n6, n7]) - t = tm.next_task() - # More bogus reaching in and setting the targets. - n6.get_executor().targets = [n6, n7] - t.prepare() - assert n6.prepared - assert n7.prepared - assert n8.prepared - assert n9.prepared - assert n10.prepared - - # Make sure we call an Executor's prepare() method. - class ExceptionExecutor: - def prepare(self): - raise Exception("Executor.prepare() exception") - def get_all_targets(self): - return self.nodes - def get_all_children(self): - result = [] - for node in self.nodes: - result.extend(node.children()) - return result - def get_all_prerequisites(self): - return [] - def get_action_side_effects(self): - return [] - - n11 = Node("n11") - n11.executor = ExceptionExecutor() - n11.executor.nodes = [n11] - tm = SCons.Taskmaster.Taskmaster([n11]) - t = tm.next_task() - try: - t.prepare() - except Exception as e: - assert str(e) == "Executor.prepare() exception", e - else: - raise AssertionError("did not catch expected exception") - - def test_execute(self): - """Test executing a task - """ - global built_text - global cache_text - - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - t.execute() - assert built_text == "n1 built", built_text - - def raise_UserError(): - raise SCons.Errors.UserError - n2 = Node("n2") - n2.build = raise_UserError - tm = SCons.Taskmaster.Taskmaster([n2]) - t = tm.next_task() - try: - t.execute() - except SCons.Errors.UserError: - pass - else: - self.fail("did not catch expected UserError") - - def raise_BuildError(): - raise SCons.Errors.BuildError - n3 = Node("n3") - n3.build = raise_BuildError - tm = SCons.Taskmaster.Taskmaster([n3]) - t = tm.next_task() - try: - t.execute() - except SCons.Errors.BuildError: - pass - else: - self.fail("did not catch expected BuildError") - - # On a generic (non-BuildError) exception from a Builder, - # the target should throw a BuildError exception with the - # args set to the exception value, instance, and traceback. - def raise_OtherError(): - raise OtherError - n4 = Node("n4") - n4.build = raise_OtherError - tm = SCons.Taskmaster.Taskmaster([n4]) - t = tm.next_task() - try: - t.execute() - except SCons.Errors.BuildError as e: - assert e.node == n4, e.node - assert e.errstr == "OtherError : ", e.errstr - assert len(e.exc_info) == 3, e.exc_info - exc_traceback = sys.exc_info()[2] - assert isinstance(e.exc_info[2], type(exc_traceback)), e.exc_info[2] - else: - self.fail("did not catch expected BuildError") - - built_text = None - cache_text = [] - n5 = Node("n5") - n6 = Node("n6") - n6.cached = 1 - tm = SCons.Taskmaster.Taskmaster([n5]) - t = tm.next_task() - # This next line is moderately bogus. We're just reaching - # in and setting the targets for this task to an array. The - # "right" way to do this would be to have the next_task() call - # set it up by having something that approximates a real Builder - # return this list--but that's more work than is probably - # warranted right now. - t.targets = [n5, n6] - t.execute() - assert built_text == "n5 built", built_text - assert cache_text == [], cache_text - - built_text = None - cache_text = [] - n7 = Node("n7") - n8 = Node("n8") - n7.cached = 1 - n8.cached = 1 - tm = SCons.Taskmaster.Taskmaster([n7]) - t = tm.next_task() - # This next line is moderately bogus. We're just reaching - # in and setting the targets for this task to an array. The - # "right" way to do this would be to have the next_task() call - # set it up by having something that approximates a real Builder - # return this list--but that's more work than is probably - # warranted right now. - t.targets = [n7, n8] - t.execute() - assert built_text is None, built_text - assert cache_text == ["n7 retrieved", "n8 retrieved"], cache_text - - def test_cached_execute(self): - """Test executing a task with cached targets - """ - # In issue #2720 Alexei Klimkin detected that the previous - # workflow for execute() led to problems in a multithreaded build. - # We have: - # task.prepare() - # task.execute() - # task.executed() - # -> node.visited() - # for the Serial flow, but - # - Parallel - - Worker - - # task.prepare() - # requestQueue.put(task) - # task = requestQueue.get() - # task.execute() - # resultQueue.put(task) - # task = resultQueue.get() - # task.executed() - # ->node.visited() - # in parallel. Since execute() used to call built() when a target - # was cached, it could unblock dependent nodes before the binfo got - # restored again in visited(). This resulted in spurious - # "file not found" build errors, because files fetched from cache would - # be seen as not up to date and wouldn't be scanned for implicit - # dependencies. - # - # The following test ensures that execute() only marks targets as cached, - # but the actual call to built() happens in executed() only. - # Like this, the binfo should still be intact after calling execute()... - global cache_text - - n1 = Node("n1") - # Mark the node as being cached - n1.cached = 1 - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - t.prepare() - t.execute() - assert cache_text == ["n1 retrieved"], cache_text - # If no binfo exists anymore, something has gone wrong... - has_binfo = hasattr(n1, 'binfo') - assert has_binfo, has_binfo - - def test_exception(self): - """Test generic Taskmaster exception handling - - """ - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() - - t.exception_set((1, 2)) - exc_type, exc_value = t.exception - assert exc_type == 1, exc_type - assert exc_value == 2, exc_value - - t.exception_set(3) - assert t.exception == 3 - - try: 1//0 - except: - # Moved from below - t.exception_set(None) - #pass - -# import pdb; pdb.set_trace() - - # Having this here works for python 2.x, - # but it is a tuple (None, None, None) when called outside - # an except statement - # t.exception_set(None) - - exc_type, exc_value, exc_tb = t.exception - assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s"%exc_type - exception_values = [ - "integer division or modulo", - "integer division or modulo by zero", - "integer division by zero", # PyPy2 - ] - assert str(exc_value) in exception_values, exc_value - - class Exception1(Exception): - pass - - # Previously value was None, but while PY2 None = "", in Py3 None != "", so set to "" - t.exception_set((Exception1, "")) - try: - t.exception_raise() - except: - exc_type, exc_value = sys.exc_info()[:2] - assert exc_type == Exception1, exc_type - assert str(exc_value) == '', "Expecting empty string got:%s (type %s)"%(exc_value,type(exc_value)) - else: - assert 0, "did not catch expected exception" - - class Exception2(Exception): - pass - - t.exception_set((Exception2, "xyzzy")) - try: - t.exception_raise() - except: - exc_type, exc_value = sys.exc_info()[:2] - assert exc_type == Exception2, exc_type - assert str(exc_value) == "xyzzy", exc_value - else: - assert 0, "did not catch expected exception" - - class Exception3(Exception): - pass - - try: - 1//0 - except: - tb = sys.exc_info()[2] - t.exception_set((Exception3, "arg", tb)) - try: - t.exception_raise() - except: - exc_type, exc_value, exc_tb = sys.exc_info() - assert exc_type == Exception3, exc_type - assert str(exc_value) == "arg", exc_value - import traceback - x = traceback.extract_tb(tb)[-1] - y = traceback.extract_tb(exc_tb)[-1] - assert x == y, "x = %s, y = %s" % (x, y) - else: - assert 0, "did not catch expected exception" - - def test_postprocess(self): - """Test postprocessing targets to give them a chance to clean up - """ - n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster([n1]) - - t = tm.next_task() - assert not n1.postprocessed - t.postprocess() - assert n1.postprocessed - - n2 = Node("n2") - n3 = Node("n3") - tm = SCons.Taskmaster.Taskmaster([n2, n3]) - - assert not n2.postprocessed - assert not n3.postprocessed - t = tm.next_task() - t.postprocess() - assert n2.postprocessed - assert not n3.postprocessed - t = tm.next_task() - t.postprocess() - assert n2.postprocessed - assert n3.postprocessed - - def test_trace(self): - """Test Taskmaster tracing - """ - import io - - trace = io.StringIO() - n1 = Node("n1") - n2 = Node("n2") - n3 = Node("n3", [n1, n2]) - tm = SCons.Taskmaster.Taskmaster([n1, n1, n3], trace=trace) - t = tm.next_task() - t.prepare() - t.execute() - t.postprocess() - n1.set_state(SCons.Node.executed) - t = tm.next_task() - t.prepare() - t.execute() - t.postprocess() - n2.set_state(SCons.Node.executed) - t = tm.next_task() - t.prepare() - t.execute() - t.postprocess() - t = tm.next_task() - assert t is None - - value = trace.getvalue() - expect = """\ - -Taskmaster: Looking for a node to evaluate -Taskmaster: Considering node and its children: -Taskmaster: Evaluating - -Task.make_ready_current(): node -Task.prepare(): node -Task.execute(): node -Task.postprocess(): node - -Taskmaster: Looking for a node to evaluate -Taskmaster: Considering node and its children: -Taskmaster: already handled (executed) -Taskmaster: Considering node and its children: -Taskmaster: -Taskmaster: -Taskmaster: adjusted ref count: , child 'n2' -Taskmaster: Considering node and its children: -Taskmaster: Evaluating - -Task.make_ready_current(): node -Task.prepare(): node -Task.execute(): node -Task.postprocess(): node -Task.postprocess(): removing -Task.postprocess(): adjusted parent ref count - -Taskmaster: Looking for a node to evaluate -Taskmaster: Considering node and its children: -Taskmaster: -Taskmaster: -Taskmaster: Evaluating - -Task.make_ready_current(): node -Task.prepare(): node -Task.execute(): node -Task.postprocess(): node - -Taskmaster: Looking for a node to evaluate -Taskmaster: No candidate anymore. - -""" - assert value == expect, value - - - -if __name__ == "__main__": - unittest.main() - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 6da5ce45b03c4120d90cad3e7ccb1808199611f6 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 10 Oct 2022 15:51:09 -0700 Subject: Fix references to SCons.Jobs -> SCons.Taskmaster.Jobs which weren't caught by pycharm refactoring --- SCons/Tool/ninja/Overrides.py | 2 +- SCons/Tool/ninja/__init__.py | 2 +- doc/sphinx/SCons.rst | 8 -------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/SCons/Tool/ninja/Overrides.py b/SCons/Tool/ninja/Overrides.py index eb8dbb5..e98ec40 100644 --- a/SCons/Tool/ninja/Overrides.py +++ b/SCons/Tool/ninja/Overrides.py @@ -84,7 +84,7 @@ def ninja_always_serial(self, num, taskmaster): # builds. So here we lie so the Main.py will not give a false # warning to users. self.num_jobs = num - self.job = SCons.Job.Serial(taskmaster) + self.job = SCons.Taskmaster.Job.Serial(taskmaster) # pylint: disable=too-few-public-methods diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index f0bdee5..c4023c3 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -416,7 +416,7 @@ def generate(env): # The Serial job class is SIGNIFICANTLY (almost twice as) faster # than the Parallel job class for generating Ninja files. So we # monkey the Jobs constructor to only use the Serial Job class. - SCons.Job.Jobs.__init__ = ninja_always_serial + SCons.Taskmaster.Job.Jobs.__init__ = ninja_always_serial ninja_syntax = importlib.import_module(".ninja_syntax", package='ninja') diff --git a/doc/sphinx/SCons.rst b/doc/sphinx/SCons.rst index c6320e7..49fed4c 100644 --- a/doc/sphinx/SCons.rst +++ b/doc/sphinx/SCons.rst @@ -99,14 +99,6 @@ SCons.Executor module :undoc-members: :show-inheritance: -SCons.Job module ----------------- - -.. automodule:: SCons.Job - :members: - :undoc-members: - :show-inheritance: - SCons.Memoize module -------------------- -- cgit v0.12 From 1f82d0e7f690617c56ec8484a45e191a059e61ef Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 12 Oct 2022 12:38:30 -0700 Subject: Updated RELEASE.txt and CHANGES.txt --- CHANGES.txt | 3 +++ RELEASE.txt | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index bba6e10..e9cd653 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added ValidateOptions() which will check that all command line options are in either those specified by SCons itself, or by AddOption() in SConstruct/SConscript. It should not be called until all AddOption() calls are completed. Resolves Issue #4187 + - Refactored SCons/Taskmaster into a package. Moved SCons/Jobs.py into that package. + NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs + From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). diff --git a/RELEASE.txt b/RELEASE.txt index 78fcc16..8ad105b 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -78,7 +78,8 @@ DOCUMENTATION DEVELOPMENT ----------- -- List visible changes in the way SCons is developed +- Refactored SCons/Taskmaster into a package. Moved SCons/Jobs.py into that package. + NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs Thanks to the following contributors listed below for their contributions to this release. ========================================================================================== -- cgit v0.12 From 3bd3df5a4979bc8fa2fbfb17dd6cd4b284892cc0 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Mon, 16 May 2022 23:19:59 -0400 Subject: Implement new parallel scheduler --- SCons/Taskmaster/Job.py | 266 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 265 insertions(+), 1 deletion(-) diff --git a/SCons/Taskmaster/Job.py b/SCons/Taskmaster/Job.py index b398790..a0bc1d0 100644 --- a/SCons/Taskmaster/Job.py +++ b/SCons/Taskmaster/Job.py @@ -31,6 +31,9 @@ import SCons.compat import os import signal +import threading + +from enum import Enum import SCons.Errors import SCons.Warnings @@ -342,7 +345,7 @@ else: worker.join(1.0) self.workers = [] - class Parallel: + class LegacyParallel: """This class is used to execute tasks in parallel, and is somewhat less efficient than Serial, but is appropriate for parallel builds. @@ -432,6 +435,267 @@ else: self.tp.cleanup() self.taskmaster.cleanup() + + class NewParallel: + + class State(Enum): + READY = 0 + SEARCHING = 1 + STALLED = 2 + COMPLETED = 3 + + class Worker(threading.Thread): + def __init__(self, owner): + super().__init__() + self.daemon = True + self.owner = owner + self.start() + + def run(self): + self.owner._work() + + def __init__(self, taskmaster, num, stack_size): + self.taskmaster = taskmaster + self.num_workers = num + self.stack_size = stack_size + self.interrupted = InterruptState() + self.workers = [] + + # The `tm_lock` is what ensures that we only have one + # thread interacting with the taskmaster at a time. It + # also protects access to our state that gets updated + # concurrently. The `can_search_cv` is associated with + # this mutex. + self.tm_lock = threading.Lock() + + # Guarded under `tm_lock`. + self.jobs = 0 + self.state = NewParallel.State.READY + + # The `can_search_cv` is used to manage a leader / + # follower pattern for access to the taskmaster, and to + # awaken from stalls. + self.can_search_cv = threading.Condition(self.tm_lock) + + # The queue of tasks that have completed execution. The + # next thread to obtain `tm_lock`` will retire them. + self.results_queue_lock = threading.Lock() + self.results_queue = [] + + def start(self): + self._start_workers() + for worker in self.workers: + worker.join() + self.workers = [] + self.taskmaster.cleanup() + + def _start_workers(self): + prev_size = self._adjust_stack_size() + for _ in range(self.num_workers): + self.workers.append(NewParallel.Worker(self)) + self._restore_stack_size(prev_size) + + def _adjust_stack_size(self): + try: + prev_size = threading.stack_size(self.stack_size*1024) + return prev_size + except AttributeError as e: + # Only print a warning if the stack size has been + # explicitly set. + if explicit_stack_size is not None: + msg = "Setting stack size is unsupported by this version of Python:\n " + \ + e.args[0] + SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) + except ValueError as e: + msg = "Setting stack size failed:\n " + str(e) + SCons.Warnings.warn(SCons.Warnings.StackSizeWarning, msg) + + return None + + def _restore_stack_size(self, prev_size): + if prev_size is not None: + threading.stack_size(prev_size) + + def _work(self): + + task = None + + while True: + + # Obtain `tm_lock`, granting exclusive access to the taskmaster. + with self.can_search_cv: + + # print(f"XXX {threading.get_ident()} Gained exclusive access") + + # Capture whether we got here with `task` set, + # then drop our reference to the task as we are no + # longer interested in the actual object. + completed_task = (task is not None) + task = None + + # We will only have `completed_task` set here if + # we have looped back after executing a task. If + # we have completed a task and find that we are + # stalled, we should speculatively indicate that + # we are no longer stalled by transitioning to the + # 'ready' state which will bypass the condition + # wait so that we immediately process the results + # queue and hopefully light up new + # work. Otherwise, stay stalled, and we will wait + # in the condvar. Some other thread will come back + # here with a completed task. + if self.state == NewParallel.State.STALLED and completed_task: + # print(f"XXX {threading.get_ident()} Detected stall with completed task, bypassing wait") + self.state = NewParallel.State.READY + + # Wait until we are neither searching nor stalled. + while self.state == NewParallel.State.SEARCHING or self.state == NewParallel.State.STALLED: + # print(f"XXX {threading.get_ident()} Search already in progress, waiting") + self.can_search_cv.wait() + + # If someone set the completed flag, bail. + if self.state == NewParallel.State.COMPLETED: + # print(f"XXX {threading.get_ident()} Completion detected, breaking from main loop") + break + + # Set the searching flag to indicate that a thread + # is currently in the critical section for + # taskmaster work. + # + # print(f"XXX {threading.get_ident()} Starting search") + self.state = NewParallel.State.SEARCHING + + # Bulk acquire the tasks in the results queue + # under the result queue lock, then process them + # all outside that lock. We need to process the + # tasks in the results queue before looking for + # new work because we might be unable to find new + # work if we don't. + results_queue = [] + with self.results_queue_lock: + results_queue, self.results_queue = self.results_queue, results_queue + + # print(f"XXX {threading.get_ident()} Found {len(results_queue)} completed tasks to process") + for (rtask, rresult) in results_queue: + if rresult: + rtask.executed() + else: + if self.interrupted(): + try: + raise SCons.Errors.BuildError( + rtask.targets[0], errstr=interrupt_msg) + except: + rtask.exception_set() + + # Let the failed() callback function arrange + # for the build to stop if that's appropriate. + rtask.failed() + + rtask.postprocess() + self.jobs -= 1 + + # We are done with any task objects that were in + # the results queue. + results_queue.clear() + + # Now, turn the crank on the taskmaster until we + # either run out of tasks, or find a task that + # needs execution. If we run out of tasks, go idle + # until results arrive if jobs are pending, or + # mark the walk as complete if not. + while self.state == NewParallel.State.SEARCHING: + # print(f"XXX {threading.get_ident()} Searching for new tasks") + task = self.taskmaster.next_task() + + if task: + # We found a task. Walk it through the + # task lifecycle. If it does not need + # execution, just complete the task and + # look for the next one. Otherwise, + # indicate that we are no longer searching + # so we can drop out of this loop, execute + # the task outside the lock, and allow + # another thread in to search. + try: + task.prepare() + except: + task.exception_set() + task.failed() + task.postprocess() + else: + if not task.needs_execute(): + # print(f"XXX {threading.get_ident()} Found internal task") + task.executed() + task.postprocess() + else: + self.jobs += 1 + # print(f"XXX {threading.get_ident()} Found task requiring execution") + self.state = NewParallel.State.READY + self.can_search_cv.notify() + + else: + # We failed to find a task, so this thread + # cannot continue turning the taskmaster + # crank. We must exit the loop. + if self.jobs: + # No task was found, but there are + # outstanding jobs executing that + # might unblock new tasks when they + # complete. Transition to the stalled + # state. We do not need a notify, + # because we know there are threads + # outstanding that will re-enter the + # loop. + # + # print(f"XXX {threading.get_ident()} Found no task requiring execution, but have jobs: marking stalled") + self.state = NewParallel.State.STALLED + else: + # We didn't find a task and there are + # no jobs outstanding, so there is + # nothing that will ever return + # results which might unblock new + # tasks. We can conclude that the walk + # is complete. Update our state to + # note completion and awaken anyone + # sleeping on the condvar. + # + # print(f"XXX {threading.get_ident()} Found no task requiring execution, and have no jobs: marking complete") + self.state = NewParallel.State.COMPLETED + self.can_search_cv.notify_all() + + # We no longer hold `tm_lock` here. If we have a task, + # we can now execute it. If there are threads waiting + # to search, one of them can now begin turning the + # taskmaster crank in parallel. + if task: + # print(f"XXX {threading.get_ident()} Executing task") + ok = True + try: + if self.interrupted(): + raise SCons.Errors.BuildError( + task.targets[0], errstr=interrupt_msg) + task.execute() + except: + ok = False + task.exception_set() + + # Grab the results queue lock and enqueue the + # executed task and state. The next thread into + # the searching loop will complete the + # postprocessing work under the taskmaster lock. + # + # print(f"XXX {threading.get_ident()} Enqueueing executed task results") + with self.results_queue_lock: + self.results_queue.append((task, ok)) + + # Tricky state "fallthrough" here. We are going back + # to the top of the loop, which behaves differently + # depending on whether `task` is set. Do not perturb + # the value of the `task` variable if you add new code + # after this comment. + + Parallel = NewParallel + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 8b4c611eee0fd97f6b60ad7db5d380c1ec4c702e Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Thu, 13 Oct 2022 15:05:39 -0400 Subject: Restore original Parallel as the default and rename new parallel as experimental --- SCons/Taskmaster/Job.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/SCons/Taskmaster/Job.py b/SCons/Taskmaster/Job.py index a0bc1d0..e9624cf 100644 --- a/SCons/Taskmaster/Job.py +++ b/SCons/Taskmaster/Job.py @@ -345,7 +345,7 @@ else: worker.join(1.0) self.workers = [] - class LegacyParallel: + class Parallel: """This class is used to execute tasks in parallel, and is somewhat less efficient than Serial, but is appropriate for parallel builds. @@ -436,7 +436,8 @@ else: self.taskmaster.cleanup() - class NewParallel: + # An experimental new parallel scheduler that uses a leaders/followers pattern. + class ExperimentalParallel: class State(Enum): READY = 0 @@ -470,7 +471,7 @@ else: # Guarded under `tm_lock`. self.jobs = 0 - self.state = NewParallel.State.READY + self.state = ExperimentalParallel.State.READY # The `can_search_cv` is used to manage a leader / # follower pattern for access to the taskmaster, and to @@ -492,7 +493,7 @@ else: def _start_workers(self): prev_size = self._adjust_stack_size() for _ in range(self.num_workers): - self.workers.append(NewParallel.Worker(self)) + self.workers.append(ExperimentalParallel.Worker(self)) self._restore_stack_size(prev_size) def _adjust_stack_size(self): @@ -544,17 +545,17 @@ else: # work. Otherwise, stay stalled, and we will wait # in the condvar. Some other thread will come back # here with a completed task. - if self.state == NewParallel.State.STALLED and completed_task: + if self.state == ExperimentalParallel.State.STALLED and completed_task: # print(f"XXX {threading.get_ident()} Detected stall with completed task, bypassing wait") - self.state = NewParallel.State.READY + self.state = ExperimentalParallel.State.READY # Wait until we are neither searching nor stalled. - while self.state == NewParallel.State.SEARCHING or self.state == NewParallel.State.STALLED: + while self.state == ExperimentalParallel.State.SEARCHING or self.state == ExperimentalParallel.State.STALLED: # print(f"XXX {threading.get_ident()} Search already in progress, waiting") self.can_search_cv.wait() # If someone set the completed flag, bail. - if self.state == NewParallel.State.COMPLETED: + if self.state == ExperimentalParallel.State.COMPLETED: # print(f"XXX {threading.get_ident()} Completion detected, breaking from main loop") break @@ -563,7 +564,7 @@ else: # taskmaster work. # # print(f"XXX {threading.get_ident()} Starting search") - self.state = NewParallel.State.SEARCHING + self.state = ExperimentalParallel.State.SEARCHING # Bulk acquire the tasks in the results queue # under the result queue lock, then process them @@ -603,7 +604,7 @@ else: # needs execution. If we run out of tasks, go idle # until results arrive if jobs are pending, or # mark the walk as complete if not. - while self.state == NewParallel.State.SEARCHING: + while self.state == ExperimentalParallel.State.SEARCHING: # print(f"XXX {threading.get_ident()} Searching for new tasks") task = self.taskmaster.next_task() @@ -630,7 +631,7 @@ else: else: self.jobs += 1 # print(f"XXX {threading.get_ident()} Found task requiring execution") - self.state = NewParallel.State.READY + self.state = ExperimentalParallel.State.READY self.can_search_cv.notify() else: @@ -648,7 +649,7 @@ else: # loop. # # print(f"XXX {threading.get_ident()} Found no task requiring execution, but have jobs: marking stalled") - self.state = NewParallel.State.STALLED + self.state = ExperimentalParallel.State.STALLED else: # We didn't find a task and there are # no jobs outstanding, so there is @@ -660,7 +661,7 @@ else: # sleeping on the condvar. # # print(f"XXX {threading.get_ident()} Found no task requiring execution, and have no jobs: marking complete") - self.state = NewParallel.State.COMPLETED + self.state = ExperimentalParallel.State.COMPLETED self.can_search_cv.notify_all() # We no longer hold `tm_lock` here. If we have a task, @@ -694,8 +695,6 @@ else: # the value of the `task` variable if you add new code # after this comment. - Parallel = NewParallel - # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 32b4cc4932c378845da8891514c497eba86a6697 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 14 Oct 2022 13:49:53 -0700 Subject: Updated URL to SCons users mailing list --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6fc99e9..8702711 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -40,7 +40,7 @@ One of the easiest ways to contribute is by filing bugs. The SCons project welcomes bug reports and feature requests, but we *do* have a preference for having talked about them first - we request you send an email to the -`SCons Users Mailing List `_ +`SCons Users Mailing List `_ or hop on the Discord channel (see link above), and if so instructed, then proceed to an issue report. -- cgit v0.12 From cbe5d046c82d6f3c3e9952ffdbcf457be1dccb72 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 16 Oct 2022 12:43:12 -0600 Subject: Java scanner don't split JAVACLASSPATH on space The scanner now splits JAVACLASSPATH before searching only if it is passed as a scalar (string); it no longer recurses to split elements of a list value - this is more consistent with other SCons usage. No longer splits on space, rather splits on os.pathsep, as considerably more useful for a search-path variable. The latter change avoids breaking paths with an embedded space (usually from Windows). This is a "breaking change" from the new behavior introduced in 4.4 where a JAVACLASSPATH like "aaa bbb ccc" would have been split into ["aaa", "bbb", "ccc"] - now it will be left unchanged in the scanner. However a JAVACLASSPATH like "aaa;bbb;ccc" will now be split into ["aaa", "bbb", "ccc"]. This behavior only affects the scanner - there is no change in how JAVACLASSPATH gets transformed into the "-classpath ARG" argument when calling JDK elements. The former behavior was presumably unintended, and definitely broke on a space-containing path (e.g. "My Classes". The unit test is expanded to test for a path string with spaces, and for a path string containing a search-path separator. Documentation tweaks: explain better how JAVACLASSPATH (and the other two Java path variables) can be specified, and whether or not SCons changes them before calling the JDK commands. Added note that JAVABOOTCLASSPATH is no longer useful, and a note about the side effect that SCons always supplies a "-sourcepath" argument when calling javac. Fixes #4243 Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 +++ RELEASE.txt | 10 ++++ SCons/Scanner/Java.py | 57 +++++++++--------- SCons/Scanner/JavaTests.py | 43 +++++++++++--- SCons/Tool/jar.xml | 4 +- SCons/Tool/javac.xml | 143 +++++++++++++++++++++++++++++++-------------- SCons/Tool/javah.xml | 4 +- SCons/Tool/rmic.xml | 28 +++++---- doc/user/java.xml | 49 +++++----------- 9 files changed, 220 insertions(+), 125 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9cd653..c5d55a9 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -40,6 +40,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - doc: EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and SConscriptChdir are now listed as Global functions only; the Environment versions still work but are not documented. + - The Java scanner processing of JAVACLASSPATH for dependencies was + changed to split on os.pathsep instead of space, to match usage of + passing a path string like "xxx:yyy:zzz". This is not portable - + passing a POSIX-style path string (with ':') won't work on Windows + (';') - which is now documented with a hint to use a list instead + to be portable. Splitting on space broke paths with embedded spaces. + Fixes #4243. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 8ad105b..a8541e6 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,6 +43,10 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. +- The Java scanner processing of JAVACLASSPATH for dependencies (introduced + in 4.4.0) is adjusted to split on the system's search path separator + instead of on space - the latter disallowed passing a completed search + string as a scalar value, and broke on paths with embedded spaces. IMPROVEMENTS ------------ @@ -74,6 +78,12 @@ DOCUMENTATION SConscriptChdir are now listed as Global functions only. - Updated the docs for Glob. - Updated SHELL_ENV_GENERATORS description and added versionadded indicator. +- JAVABOOTCLASSPATH, JAVACLASSPATH and JAVASOURCEPATH better document the + acceptable syntax for values, and how they will be interpreted, + including that JAVACLASSPATH will be scanned for dependencies. + Added note on the possibly surprising feature that SCons always passes + -sourcepath when calling javac, which affects how the class path is + used when finding sources. DEVELOPMENT ----------- diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index ab1f4e6..0c6df37 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -21,56 +21,58 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import os +import os import SCons.Node import SCons.Node.FS import SCons.Scanner -import SCons.Util +from SCons.Util import flatten, is_String -def _subst_libs(env, libs): - """ - Substitute environment variables and split into list. +def _subst_paths(env, paths) -> list: + """Return a list of substituted path elements. + + If *paths* is a string, it is split on the search-path separator + (this makes the interpretation system-specitic - this is warned about + in the manpage). This helps support behavior like pulling in the + external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. + Otherwise, substitution is done on string-valued list elements + but not splitting. """ - if SCons.Util.is_String(libs): - libs = env.subst(libs) - if SCons.Util.is_String(libs): - libs = libs.split() - elif SCons.Util.is_Sequence(libs): - _libs = [] - for lib in libs: - _libs += _subst_libs(env, lib) - libs = _libs + if is_String(paths): + paths = env.subst(paths) + if SCons.Util.is_String(paths): + paths = paths.split(os.pathsep) else: - # libs is an object (Node, for example) - libs = [libs] - return libs + paths = flatten(paths) + paths = [env.subst(path) if is_String(path) else path for path in paths] + return paths -def _collect_classes(list, dirname, files): +def _collect_classes(classlist, dirname, files): for fname in files: - if os.path.splitext(fname)[1] == ".class": - list.append(os.path.join(str(dirname), fname)) + if fname.endswith(".class"): + classlist.append(os.path.join(str(dirname), fname)) -def scan(node, env, libpath=()): +def scan(node, env, libpath=()) -> list: """Scan for files on the JAVACLASSPATH. - The classpath can contain: + JAVACLASSPATH path can contain: - Explicit paths to JAR/Zip files - Wildcards (*) - Directories which contain classes in an unnamed package - Parent directories of the root package for classes in a named package - Class path entries that are neither directories nor archives (.zip or JAR files) nor the asterisk (*) wildcard character are ignored. - """ + Class path entries that are neither directories nor archives (.zip + or JAR files) nor the asterisk (*) wildcard character are ignored. + """ classpath = env.get('JAVACLASSPATH', []) - classpath = _subst_libs(env, classpath) + classpath = _subst_paths(env, classpath) result = [] for path in classpath: - if SCons.Util.is_String(path) and "*" in path: + if is_String(path) and "*" in path: libs = env.Glob(path) else: libs = [path] @@ -89,8 +91,7 @@ def scan(node, env, libpath=()): def JavaScanner(): - return SCons.Scanner.Base(scan, 'JavaScanner', - skeys=['.java']) + return SCons.Scanner.Base(scan, 'JavaScanner', skeys=['.java']) # Local Variables: # tab-width:4 diff --git a/SCons/Scanner/JavaTests.py b/SCons/Scanner/JavaTests.py index 9fb39ce..77cd560 100644 --- a/SCons/Scanner/JavaTests.py +++ b/SCons/Scanner/JavaTests.py @@ -33,18 +33,25 @@ import SCons.Warnings test = TestCmd.TestCmd(workdir = '') -test.subdir('com') files = [ 'bootclasspath.jar', 'classpath.jar', 'Test.class', - 'com/Test.class' ] for fname in files: test.write(fname, "\n") +test.subdir('com') +test.subdir('java space') +subfiles = [ + 'com/Test.class', + 'java space/Test.class' +] + +for fname in subfiles: + test.write(fname.split('/'), "\n") class DummyEnvironment(collections.UserDict): def __init__(self,**kw): @@ -52,7 +59,7 @@ class DummyEnvironment(collections.UserDict): self.data.update(kw) self.fs = SCons.Node.FS.FS(test.workpath('')) self['ENV'] = {} - + def Dictionary(self, *args): return self.data @@ -80,7 +87,7 @@ class DummyEnvironment(collections.UserDict): def File(self, filename): return self.fs.File(filename) - + def Glob(self, path): return self.fs.Glob(path) @@ -111,8 +118,7 @@ def deps_match(self, deps, headers): class JavaScannerEmptyClasspath(unittest.TestCase): def runTest(self): path = [] - env = DummyEnvironment(JAVASUFFIXES=['.java'], - JAVACLASSPATH=path) + env = DummyEnvironment(JAVASUFFIXES=['.java'], JAVACLASSPATH=path) s = SCons.Scanner.Java.JavaScanner() deps = s(DummyNode('dummy'), env) expected = [] @@ -145,10 +151,33 @@ class JavaScannerDirClasspath(unittest.TestCase): JAVACLASSPATH=[test.workpath()]) s = SCons.Scanner.Java.JavaScanner() deps = s(DummyNode('dummy'), env) - expected = ['Test.class', 'com/Test.class'] + expected = ['Test.class', 'com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + + +class JavaScannerNamedDirClasspath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVACLASSPATH=[test.workpath('com'), test.workpath('java space')], + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] deps_match(self, deps, expected) +class JavaScannerSearchPathClasspath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVACLASSPATH=os.pathsep.join([test.workpath('com'), test.workpath('java space')]), + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + if __name__ == "__main__": unittest.main() diff --git a/SCons/Tool/jar.xml b/SCons/Tool/jar.xml index 1713495..403aacb 100644 --- a/SCons/Tool/jar.xml +++ b/SCons/Tool/jar.xml @@ -1,6 +1,8 @@ @@ -206,7 +187,7 @@ public class AdditionalClass3 Will not only tell you reliably that the .class files - in the classes subdirectory + in the classes subdirectory are up-to-date: @@ -250,7 +231,7 @@ public class AdditionalClass3 variable to specify the version in use. With Java 1.6, the one-liner example can then be defined like this: - + Java('classes', 'src', JAVAVERSION='1.6') @@ -280,8 +261,8 @@ Java('classes', 'src', JAVAVERSION='1.6') -Java(target = 'classes', source = 'src') -Jar(target = 'test.jar', source = 'classes') +Java(target='classes', source='src') +Jar(target='test.jar', source='classes') public class Example1 @@ -344,10 +325,10 @@ public class Example3 -prog1_class_files = Java(target = 'classes', source = 'prog1') -prog2_class_files = Java(target = 'classes', source = 'prog2') -Jar(target = 'prog1.jar', source = prog1_class_files) -Jar(target = 'prog2.jar', source = prog2_class_files) +prog1_class_files = Java(target='classes', source='prog1') +prog2_class_files = Java(target='classes', source='prog2') +Jar(target='prog1.jar', source=prog1_class_files) +Jar(target='prog2.jar', source=prog2_class_files) public class Example1 @@ -418,8 +399,8 @@ public class Example4 -classes = Java(target = 'classes', source = 'src/pkg/sub') -JavaH(target = 'native', source = classes) +classes = Java(target='classes', source='src/pkg/sub') +JavaH(target='native', source=classes) package pkg.sub; @@ -642,8 +623,8 @@ public class Example3 -classes = Java(target = 'classes', source = 'src/pkg/sub') -RMIC(target = 'outdir', source = classes) +classes = Java(target='classes', source='src/pkg/sub') +RMIC(target='outdir', source=classes) package pkg.sub; -- cgit v0.12 From 984e3f58b9f2c33cc551e2a3d2962c5f6420b3c8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 18 Oct 2022 11:33:36 -0600 Subject: Java scanner: fixes per review comments Signed-off-by: Mats Wichmann --- RELEASE.txt | 13 +++++++++---- SCons/Scanner/Java.py | 10 +++++++++- SCons/Tool/javac.xml | 27 ++++++++++++++------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index a8541e6..8bdbe43 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,10 +43,15 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. -- The Java scanner processing of JAVACLASSPATH for dependencies (introduced - in 4.4.0) is adjusted to split on the system's search path separator - instead of on space - the latter disallowed passing a completed search - string as a scalar value, and broke on paths with embedded spaces. +- The Java Scanner processing of JAVACLASSPATH for dependencies (behavior + that was introduced in SCons 4.4.0) is adjusted to split on the system's + search path separator instead of on a space. The previous behavior meant + that a path containing spaces (e.g. r"C:\somepath\My Classes") would + lead to unexpected errors. If the split-on-space behavior is desired, + pre-split the value: instead of: env["JAVACLASSPATH"] = "foo bar baz" + use: env["JAVACLASSPATH"] = env.Split("foo bar baz") + There is no change in how JAVACLASSPATH gets turned into the -classpath + argument passed to the JDK tools. IMPROVEMENTS ------------ diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index 0c6df37..f6edf93 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -33,7 +33,7 @@ def _subst_paths(env, paths) -> list: """Return a list of substituted path elements. If *paths* is a string, it is split on the search-path separator - (this makes the interpretation system-specitic - this is warned about + (this makes the interpretation system-specific - this is warned about in the manpage). This helps support behavior like pulling in the external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. Otherwise, substitution is done on string-valued list elements @@ -44,6 +44,7 @@ def _subst_paths(env, paths) -> list: if SCons.Util.is_String(paths): paths = paths.split(os.pathsep) else: + # TODO: may want to revisit splitting list-element strings if requested paths = flatten(paths) paths = [env.subst(path) if is_String(path) else path for path in paths] return paths @@ -73,6 +74,9 @@ def scan(node, env, libpath=()) -> list: result = [] for path in classpath: if is_String(path) and "*" in path: + # This matches more than the Java docs describe: a '*' only + # matches jar files. The filter later should trim this down. + # TODO: should we filter here? use .endswith('*') rather than "in"? libs = env.Glob(path) else: libs = [path] @@ -91,6 +95,10 @@ def scan(node, env, libpath=()) -> list: def JavaScanner(): + """Scanner for .java files. + + .. versionadded:: 4.4 + """ return SCons.Scanner.Base(scan, 'JavaScanner', skeys=['.java']) # Local Variables: diff --git a/SCons/Tool/javac.xml b/SCons/Tool/javac.xml index 922ae2b..014d905 100644 --- a/SCons/Tool/javac.xml +++ b/SCons/Tool/javac.xml @@ -140,10 +140,10 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' search path separator characters (: for POSIX systems or ; for Windows), it will not be modified; + and so is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVABOOTCLASSPATH; as a list of paths instead. - Can only be used when compiling for releases prior to JDK 9. @@ -239,12 +239,13 @@ env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") If &cv-JAVACLASSPATH; is a single string containing search path separator characters (: for POSIX systems or - ; for Windows), it will not be modified; + ; for Windows), + it will be split on the separator into a list of individual + paths for dependency scanning purposes. + It will not be modified for JDK command-line usage, + so such a string is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVACLASSPATH; as a list of paths instead. - Such a string will, however, - be split on the separator into a list of individual paths - for dependency scanning purposes. @@ -289,24 +290,24 @@ env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") The value will be added to the JDK command lines - via the option. - The JDK option requires a - system-specific search path separator, - which will be supplied by &SCons; as needed when it + via the option, + which requires a system-specific search path separator, + This will be supplied by &SCons; as needed when it constructs the command line if &cv-JAVASOURCEPATH; is provided in list form. If &cv-JAVASOURCEPATH; is a single string containing search path separator characters (: for POSIX systems or - ; for Windows), it will not be modified; + ; for Windows), it will not be modified, + and so is inherently system-specific; to supply the path in a system-independent manner, give &cv-JAVASOURCEPATH; as a list of paths instead. - Note that this currently just adds the specified - directories via the option. + Note that the specified directories are only added to + the command line via the option. &SCons; does not currently search the - &cv-JAVASOURCEPATH; directories for dependency + &cv-JAVASOURCEPATH; directories for dependent .java files. -- cgit v0.12 From 897ab9da7a09ec3f4705b84ad43485697761c594 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Wed, 19 Oct 2022 10:20:50 -0400 Subject: Obtain a small perf win by using types that iterate faster for children --- CHANGES.txt | 5 +++++ SCons/Executor.py | 12 ++++++------ SCons/Util.py | 41 ++++------------------------------------- 3 files changed, 15 insertions(+), 43 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9cd653..5039eed 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -41,6 +41,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER SConscriptChdir are now listed as Global functions only; the Environment versions still work but are not documented. + From Andrew Morrow + - Avoid returning UniqueList for `children` and other `Executor` APIs. This type + iterates more slowly than the builtin types. Also simplify uniquer_hashables to + use an faster implementation under the assumption of ordered dictionaries. + RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 From Joseph Brill: diff --git a/SCons/Executor.py b/SCons/Executor.py index 492ebe3..274af6a 100644 --- a/SCons/Executor.py +++ b/SCons/Executor.py @@ -306,30 +306,30 @@ class Executor(object, metaclass=NoSlotsPyPy): over and over), so removing the duplicates once up front should save the Taskmaster a lot of work. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_all_targets(): result.extend(target.children()) - return result + return SCons.Util.uniquer_hashables(result) def get_all_prerequisites(self): """Returns all unique (order-only) prerequisites for all batches of this Executor. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_all_targets(): if target.prerequisites is not None: result.extend(target.prerequisites) - return result + return SCons.Util.uniquer_hashables(result) def get_action_side_effects(self): """Returns all side effects for all batches of this Executor used by the underlying Action. """ - result = SCons.Util.UniqueList([]) + result = [] for target in self.get_action_targets(): result.extend(target.side_effects) - return result + return SCons.Util.uniquer_hashables(result) @SCons.Memoize.CountMethodCall def get_build_env(self): diff --git a/SCons/Util.py b/SCons/Util.py index 49a3a0f..4b06ca4 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1363,44 +1363,11 @@ def unique(seq): u.append(x) return u - -# From Alex Martelli, -# https://code.activestate.com/recipes/52560 -# ASPN: Python Cookbook: Remove duplicates from a sequence -# First comment, dated 2001/10/13. -# (Also in the printed Python Cookbook.) -# This not currently used, in favor of the next function... - -def uniquer(seq, idfun=None): - def default_idfun(x): - return x - if not idfun: - idfun = default_idfun - seen = {} - result = [] - result_append = result.append # perf: avoid repeated method lookups - for item in seq: - marker = idfun(item) - if marker in seen: - continue - seen[marker] = 1 - result_append(item) - return result - -# A more efficient implementation of Alex's uniquer(), this avoids the -# idfun() argument and function-call overhead by assuming that all -# items in the sequence are hashable. Order-preserving. - +# Best way (assuming Python 3.7, but effectively 3.6) to remove +# duplicates from a list in while preserving order, according to +# https://discord.com/channels/@me/627187920079421471/1032001231952027648 def uniquer_hashables(seq): - seen = {} - result = [] - result_append = result.append # perf: avoid repeated method lookups - for item in seq: - if item not in seen: - seen[item] = 1 - result_append(item) - return result - + return list(dict.fromkeys(seq)) # Recipe 19.11 "Reading Lines with Continuation Characters", # by Alex Martelli, straight from the Python CookBook (2nd edition). -- cgit v0.12 From c0f80cbbeec176f303d3ef1b7ab94582c018fcd8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 20 Oct 2022 09:10:31 -0600 Subject: Add Java scanner to generated API docs Signed-off-by: Mats Wichmann --- SCons/Scanner/Java.py | 14 ++++++++------ doc/sphinx/SCons.Scanner.rst | 8 ++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index f6edf93..8c31bc1 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -32,12 +32,14 @@ from SCons.Util import flatten, is_String def _subst_paths(env, paths) -> list: """Return a list of substituted path elements. - If *paths* is a string, it is split on the search-path separator - (this makes the interpretation system-specific - this is warned about - in the manpage). This helps support behavior like pulling in the - external ``CLASSPATH`` and setting it directly into ``JAVACLASSPATH``. - Otherwise, substitution is done on string-valued list elements - but not splitting. + If *paths* is a string, it is split on the search-path separator. + Otherwise, substitution is done on string-valued list elements but + they are not split. + + Note helps support behavior like pulling in the external ``CLASSPATH`` + and setting it directly into ``JAVACLASSPATH``, however splitting on + ``os.pathsep`` makes the interpretation system-specific (this is + warned about in the manpage entry for ``JAVACLASSPATH``). """ if is_String(paths): paths = env.subst(paths) diff --git a/doc/sphinx/SCons.Scanner.rst b/doc/sphinx/SCons.Scanner.rst index 181dbde..484bd43 100644 --- a/doc/sphinx/SCons.Scanner.rst +++ b/doc/sphinx/SCons.Scanner.rst @@ -44,6 +44,14 @@ SCons.Scanner.IDL module :undoc-members: :show-inheritance: +SCons.Scanner.Java module +------------------------- + +.. automodule:: SCons.Scanner.Java + :members: + :undoc-members: + :show-inheritance: + SCons.Scanner.LaTeX module -------------------------- -- cgit v0.12 From edff9134a860534df8b00f1321ccc79d27d5df17 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 20 Oct 2022 10:40:35 -0600 Subject: Adjust Sphinx doc build for moved Taskmaster [skip appveyor] Signed-off-by: Mats Wichmann --- doc/sphinx/SCons.Taskmaster.rst | 21 +++++++++++++++++++++ doc/sphinx/SCons.rst | 9 +-------- doc/sphinx/index.rst | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 doc/sphinx/SCons.Taskmaster.rst diff --git a/doc/sphinx/SCons.Taskmaster.rst b/doc/sphinx/SCons.Taskmaster.rst new file mode 100644 index 0000000..adf9013 --- /dev/null +++ b/doc/sphinx/SCons.Taskmaster.rst @@ -0,0 +1,21 @@ +SCons.Taskmaster package +======================== + +Submodules +---------- + +SCons.Taskmaster.Job module +--------------------------- + +.. automodule:: SCons.Taskmaster.Job + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: SCons.Taskmaster + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/sphinx/SCons.rst b/doc/sphinx/SCons.rst index 49fed4c..45e20ee 100644 --- a/doc/sphinx/SCons.rst +++ b/doc/sphinx/SCons.rst @@ -18,6 +18,7 @@ Subpackages SCons.Platform SCons.Scanner SCons.Script + SCons.Taskmaster SCons.Tool SCons.Variables SCons.compat @@ -139,14 +140,6 @@ SCons.Subst module :undoc-members: :show-inheritance: -SCons.Taskmaster module ------------------------ - -.. automodule:: SCons.Taskmaster - :members: - :undoc-members: - :show-inheritance: - SCons.Util module ----------------- diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index a6bbec1..f8d5f47 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -31,6 +31,7 @@ SCons API Documentation SCons.Platform SCons.Scanner SCons.Script + SCons.Taskmaster SCons.Tool SCons.Variables -- cgit v0.12 From 92f12d78cb46100115812f83b2d1e6f2a67163ae Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 21 Oct 2022 09:17:37 -0600 Subject: Minor doc update for Value nodes [skip appveyor] Add a "changed in version 4.0" for a past change. Fiddled some docstrings. Tweaked docstring for env.Tool as well - proximity in the files was the motivation (it's just above in Environment.xml) - trying to get to a consistent added/changed notation format, following the Sphinx docstring directive style. Signed-off-by: Mats Wichmann --- SCons/Environment.py | 15 +++++++++++---- SCons/Environment.xml | 36 ++++++++++++++++++++++-------------- SCons/Node/Python.py | 15 +++++++++++---- SCons/Node/__init__.py | 10 +++++----- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/SCons/Environment.py b/SCons/Environment.py index e458f25..293447f 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -1897,6 +1897,11 @@ class Base(SubstitutionEnvironment): return self.fs.Dir(self.subst(tp)).srcnode().get_abspath() def Tool(self, tool, toolpath=None, **kwargs) -> SCons.Tool.Tool: + """Find and run tool module *tool*. + + .. versionchanged:: 4.2 + returns the tool module rather than ``None``. + """ if is_String(tool): tool = self.subst(tool) if toolpath is None: @@ -2335,7 +2340,10 @@ class Base(SubstitutionEnvironment): return [self.subst(arg)] def Value(self, value, built_value=None, name=None): - """ + """Return a Value (Python expression) node. + + .. versionchanged:: 4.0 + the *name* parameter was added. """ return SCons.Node.Python.ValueWithMemo(value, built_value, name) @@ -2344,9 +2352,8 @@ class Base(SubstitutionEnvironment): src_dir = self.arg2nodes(src_dir, self.fs.Dir)[0] self.fs.VariantDir(variant_dir, src_dir, duplicate) - def FindSourceFiles(self, node='.'): - """ returns a list of all source files. - """ + def FindSourceFiles(self, node='.') -> list: + """Return a list of all source files.""" node = self.arg2nodes(node, self.fs.Entry)[0] sources = [] diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 2c5d563..2e06b1e 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -3311,6 +3311,12 @@ appended to the &cv-link-TOOLS; +Changed in version 4.2: +&f-env-Tool; now returns the tool object, +previously it did not return (i.e. returned None). + + + Examples: @@ -3347,12 +3353,6 @@ msvctool(env) # adds 'msvc' to the TOOLS variable gltool = Tool('opengl', toolpath = ['tools']) gltool(env) # adds 'opengl' to the TOOLS variable - - -Changed in &SCons; 4.2: &f-env-Tool; now returns -the tool object, previously it did not return -(i.e. returned None). - @@ -3373,10 +3373,6 @@ will be rebuilt. files are up-to-date.) When using timestamp source signatures, Value Nodes' timestamps are equal to the system time when the Node is created. -name can be provided as an alternative name -for the resulting Value node; this is advised -if the value parameter can't be converted to -a string. @@ -3396,6 +3392,18 @@ method that will return the built value of the Node. +The optional name parameter can be provided as an +alternative name for the resulting Value node; +this is advised if the value parameter +cannot be converted to a string. + + + +Changed in version 4.0: +the name parameter was added. + + + Examples: @@ -3415,8 +3423,8 @@ prefix = ARGUMENTS.get('prefix', '/usr/local') # Attach a .Config() builder for the above function action # to the construction environment. -env['BUILDERS']['Config'] = Builder(action = create) -env.Config(target = 'package-config', source = Value(prefix)) +env['BUILDERS']['Config'] = Builder(action=create) +env.Config(target='package-config', source=Value(prefix)) def build_value(target, source, env): # A function that "builds" a Python Value by updating @@ -3429,8 +3437,8 @@ input = env.Value('after') # Attach a .UpdateValue() builder for the above function # action to the construction environment. -env['BUILDERS']['UpdateValue'] = Builder(action = build_value) -env.UpdateValue(target = Value(output), source = Value(input)) +env['BUILDERS']['UpdateValue'] = Builder(action=build_value) +env.UpdateValue(target=Value(output), source=Value(input)) diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index 80d2762..57416ef 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -75,8 +75,13 @@ class ValueBuildInfo(SCons.Node.BuildInfoBase): class Value(SCons.Node.Node): - """A class for Python variables, typically passed on the command line - or generated by a script, but not from a file or some other source. + """A Node class for values represented by Python expressions. + + Values are typically passed on the command line or generated + by a script, but not from a file or some other source. + + .. versionchanged:: 4.0 + the *name* parameter was added. """ NodeInfo = ValueNodeInfo @@ -165,8 +170,10 @@ class Value(SCons.Node.Node): def ValueWithMemo(value, built_value=None, name=None): - """ - Memoized Value() node factory. + """Memoized :class:`Value` node factory. + + .. versionchanged:: 4.0 + the *name* parameter was added. """ global _memo_lookup_map diff --git a/SCons/Node/__init__.py b/SCons/Node/__init__.py index ec742a6..bb09868 100644 --- a/SCons/Node/__init__.py +++ b/SCons/Node/__init__.py @@ -685,8 +685,8 @@ class Node(object, metaclass=NoSlotsPyPy): """Try to retrieve the node's content from a cache This method is called from multiple threads in a parallel build, - so only do thread safe stuff here. Do thread unsafe stuff in - built(). + so only do thread safe stuff here. Do thread unsafe stuff + in :meth:`built`. Returns true if the node was successfully retrieved. """ @@ -743,12 +743,12 @@ class Node(object, metaclass=NoSlotsPyPy): """Actually build the node. This is called by the Taskmaster after it's decided that the - Node is out-of-date and must be rebuilt, and after the prepare() - method has gotten everything, uh, prepared. + Node is out-of-date and must be rebuilt, and after the + :meth:`prepare` method has gotten everything, uh, prepared. This method is called from multiple threads in a parallel build, so only do thread safe stuff here. Do thread unsafe stuff - in built(). + in :meth:`built`. """ try: -- cgit v0.12 From 742ee9a4ed63c88158a12be92b7218d4eed31207 Mon Sep 17 00:00:00 2001 From: Andrew Morrow Date: Fri, 21 Oct 2022 12:24:10 -0400 Subject: fix link about unique --- SCons/Util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Util.py b/SCons/Util.py index 4b06ca4..33e02f4 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1365,7 +1365,7 @@ def unique(seq): # Best way (assuming Python 3.7, but effectively 3.6) to remove # duplicates from a list in while preserving order, according to -# https://discord.com/channels/@me/627187920079421471/1032001231952027648 +# https://stackoverflow.com/questions/480214/how-do-i-remove-duplicates-from-a-list-while-preserving-order/17016257#17016257 def uniquer_hashables(seq): return list(dict.fromkeys(seq)) -- cgit v0.12 From da4b75ded0a5ba97af5bb7124d4436df66464c99 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 21 Oct 2022 13:35:36 -0700 Subject: [ci skip] Resolve outstanding pep8 errors (fix sider complaints and more) --- SCons/Taskmaster/Job.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/SCons/Taskmaster/Job.py b/SCons/Taskmaster/Job.py index e9624cf..d9c98e8 100644 --- a/SCons/Taskmaster/Job.py +++ b/SCons/Taskmaster/Job.py @@ -145,7 +145,7 @@ class Jobs: else: os._exit(2) # pylint: disable=protected-access - self.old_sigint = signal.signal(signal.SIGINT, handler) + self.old_sigint = signal.signal(signal.SIGINT, handler) self.old_sigterm = signal.signal(signal.SIGTERM, handler) try: self.old_sighup = signal.signal(signal.SIGHUP, handler) @@ -170,6 +170,7 @@ class Jobs: except AttributeError: pass + class Serial: """This class is used to execute tasks in series, and is more efficient than Parallel, but is only appropriate for non-parallel builds. Only @@ -211,7 +212,7 @@ class Serial: try: raise SCons.Errors.BuildError( task.targets[0], errstr=interrupt_msg) - except: + except Exception: task.exception_set() else: task.exception_set() @@ -263,7 +264,7 @@ else: raise SCons.Errors.BuildError( task.targets[0], errstr=interrupt_msg) task.execute() - except: + except Exception: task.exception_set() ok = False else: @@ -284,7 +285,7 @@ else: self.resultsQueue = queue.Queue(0) try: - prev_size = threading.stack_size(stack_size*1024) + prev_size = threading.stack_size(stack_size * 1024) except AttributeError as e: # Only print a warning if the stack size has been # explicitly set. @@ -392,7 +393,7 @@ else: try: # prepare task for execution task.prepare() - except: + except Exception: task.exception_set() task.failed() task.postprocess() @@ -405,7 +406,8 @@ else: task.executed() task.postprocess() - if not task and not jobs: break + if not task and not jobs: + break # Let any/all completed tasks finish up before we go # back and put the next batch of tasks on the queue. @@ -420,7 +422,7 @@ else: try: raise SCons.Errors.BuildError( task.targets[0], errstr=interrupt_msg) - except: + except Exception: task.exception_set() # Let the failed() callback function arrange @@ -435,7 +437,6 @@ else: self.tp.cleanup() self.taskmaster.cleanup() - # An experimental new parallel scheduler that uses a leaders/followers pattern. class ExperimentalParallel: @@ -498,7 +499,7 @@ else: def _adjust_stack_size(self): try: - prev_size = threading.stack_size(self.stack_size*1024) + prev_size = threading.stack_size(self.stack_size * 1024) return prev_size except AttributeError as e: # Only print a warning if the stack size has been @@ -585,7 +586,7 @@ else: try: raise SCons.Errors.BuildError( rtask.targets[0], errstr=interrupt_msg) - except: + except Exception: rtask.exception_set() # Let the failed() callback function arrange @@ -619,7 +620,7 @@ else: # another thread in to search. try: task.prepare() - except: + except Exception: task.exception_set() task.failed() task.postprocess() @@ -676,7 +677,7 @@ else: raise SCons.Errors.BuildError( task.targets[0], errstr=interrupt_msg) task.execute() - except: + except Exception: ok = False task.exception_set() -- cgit v0.12 From 20ffb3e37f8b46ef232c28e32006294e4d513e3e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 21 Oct 2022 17:16:13 -0700 Subject: Migrate Taskmaster tracing to use python logging --- SCons/Script/Main.py | 13 +-- SCons/Taskmaster/TaskmasterTests.py | 15 +++- SCons/Taskmaster/__init__.py | 152 ++++++++++++++++++++++++------------ SCons/Util.py | 12 +++ 4 files changed, 128 insertions(+), 64 deletions(-) diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index 22042f5..b902842 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -1291,23 +1291,12 @@ def _build_targets(fs, options, targets, target_top): """Leave the order of dependencies alone.""" return dependencies - def tmtrace_cleanup(tfile): - tfile.close() - - if options.taskmastertrace_file == '-': - tmtrace = sys.stdout - elif options.taskmastertrace_file: - tmtrace = open(options.taskmastertrace_file, 'w') - atexit.register(tmtrace_cleanup, tmtrace) - else: - tmtrace = None - taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, tmtrace) + taskmaster = SCons.Taskmaster.Taskmaster(nodes, task_class, order, options.taskmastertrace_file) # Let the BuildTask objects get at the options to respond to the # various print_* settings, tree_printer list, etc. BuildTask.options = options - is_pypy = platform.python_implementation() == 'PyPy' # As of 3.7, python removed support for threadless platforms. # See https://www.python.org/dev/peps/pep-0011/ diff --git a/SCons/Taskmaster/TaskmasterTests.py b/SCons/Taskmaster/TaskmasterTests.py index f20fd71..2ff15f0 100644 --- a/SCons/Taskmaster/TaskmasterTests.py +++ b/SCons/Taskmaster/TaskmasterTests.py @@ -1241,10 +1241,19 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - """ - assert value == expect, value - + v_split=value.split('\n') + e_split=expect.split('\n') + if len(v_split) != len(e_split): + print("different number of lines:%d %d"%(len(v_split), len(e_split))) + + # breakpoint() + for v, e in zip(v_split, e_split): + # print("%s:%s"%(v,e)) + if v != e: + print("\n[%s]\n[%s]" % (v, e)) + + assert value == expect, "Expected:\n%s\nGot:\n%s"%(expect, value) if __name__ == "__main__": diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index d571795..59b5886 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -46,14 +46,16 @@ Task The Taskmaster instantiates a Task object for each (set of) target(s) that it decides need to be evaluated and/or built. """ - +import io import sys from abc import ABC, abstractmethod from itertools import chain +import logging import SCons.Errors import SCons.Node import SCons.Warnings +from SCons.Util import DispatchingFormatter StateString = SCons.Node.StateString NODE_NO_STATE = SCons.Node.no_state @@ -71,6 +73,7 @@ print_prepare = False # set by option --debug=prepare CollectStats = None + class Stats: """ A simple class for holding statistics about the disposition of a @@ -92,6 +95,7 @@ class Stats: self.side_effects = 0 self.build = 0 + StatsNodes = [] fmt = "%(considered)3d "\ @@ -102,6 +106,7 @@ fmt = "%(considered)3d "\ "%(side_effects)3d " \ "%(build)3d " + def dump_stats(): for n in sorted(StatsNodes, key=lambda a: str(a)): print((fmt % n.attributes.stats.__dict__) + str(n)) @@ -125,6 +130,9 @@ class Task(ABC): these methods explicitly to update state, etc., rather than roll their own interaction with Taskmaster from scratch. """ + + LOGGER = None + def __init__(self, tm, targets, top, node): self.tm = tm self.targets = targets @@ -132,9 +140,10 @@ class Task(ABC): self.node = node self.exc_clear() - def trace_message(self, method, node, description='node'): - fmt = '%-20s %s %s\n' - return fmt % (method + ':', description, self.tm.trace_node(node)) + def trace_message(self, node, description='node'): + # This grabs the name of the function which calls trace_message() + method_name=sys._getframe(1).f_code.co_name+"():" + Task.LOGGER.debug('%-15s %s %s' % (method_name, description, self.tm.tm_trace_node(node))) def display(self, message): """ @@ -159,7 +168,7 @@ class Task(ABC): """ global print_prepare T = self.tm.trace - if T: T.write(self.trace_message('Task.prepare()', self.node)) + if T: self.trace_message(self.node) # Now that it's the appropriate time, give the TaskMaster a # chance to raise any exceptions it encountered while preparing @@ -213,7 +222,7 @@ class Task(ABC): prepare(), executed() or failed(). """ T = self.tm.trace - if T: T.write(self.trace_message('Task.execute()', self.node)) + if T: self.trace_message(self.node) try: cached_targets = [] @@ -256,8 +265,7 @@ class Task(ABC): the Node's callback methods. """ T = self.tm.trace - if T: T.write(self.trace_message('Task.executed_without_callbacks()', - self.node)) + if T: self.trace_message(self.node) for t in self.targets: if t.get_state() == NODE_EXECUTING: @@ -280,8 +288,7 @@ class Task(ABC): """ global print_prepare T = self.tm.trace - if T: T.write(self.trace_message('Task.executed_with_callbacks()', - self.node)) + if T: self.trace_message(self.node) for t in self.targets: if t.get_state() == NODE_EXECUTING: @@ -322,7 +329,7 @@ class Task(ABC): nodes when using Configure(). """ T = self.tm.trace - if T: T.write(self.trace_message('Task.failed_stop()', self.node)) + if T: self.trace_message(self.node) # Invoke will_not_build() to clean-up the pending children # list. @@ -349,7 +356,7 @@ class Task(ABC): nodes when using Configure(). """ T = self.tm.trace - if T: T.write(self.trace_message('Task.failed_continue()', self.node)) + if T: self.trace_message(self.node) self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) @@ -361,7 +368,7 @@ class Task(ABC): visited--the canonical example being the "scons -c" option. """ T = self.tm.trace - if T: T.write(self.trace_message('Task.make_ready_all()', self.node)) + if T: self.trace_message(self.node) self.out_of_date = self.targets[:] for t in self.targets: @@ -379,8 +386,9 @@ class Task(ABC): """ global print_prepare T = self.tm.trace - if T: T.write(self.trace_message('Task.make_ready_current()', - self.node)) + if T: + T.log_handler.stream.write('\n') # Prefix message with new line. This is a hack + self.trace_message(self.node) self.out_of_date = [] needs_executing = False @@ -427,7 +435,7 @@ class Task(ABC): that can be put back on the candidates list. """ T = self.tm.trace - if T: T.write(self.trace_message('Task.postprocess()', self.node)) + if T: self.trace_message(self.node) # We may have built multiple targets, some of which may have # common parents waiting for this build. Count up how many @@ -444,9 +452,7 @@ class Task(ABC): # A node can only be in the pending_children set if it has # some waiting_parents. if t.waiting_parents: - if T: T.write(self.trace_message('Task.postprocess()', - t, - 'removing')) + if T: self.trace_message(t, 'removing') pending_children.discard(t) for p in t.waiting_parents: parents[p] = parents.get(p, 0) + 1 @@ -473,9 +479,7 @@ class Task(ABC): for p, subtract in parents.items(): p.ref_count = p.ref_count - subtract - if T: T.write(self.trace_message('Task.postprocess()', - p, - 'adjusted parent ref count')) + if T: self.trace_message(p, 'adjusted parent ref count') if p.ref_count == 0: self.tm.candidates.append(p) @@ -604,9 +608,59 @@ class Taskmaster: order = lambda l: l self.order = order self.message = None - self.trace = trace self.next_candidate = self.find_next_candidate self.pending_children = set() + self.trace = False + self.configure_trace(trace) + + def configure_trace(self, trace=None): + """ + This handles the command line option --taskmastertrace= + It can be: + - : output to stdout + : output to a file + False/None : Do not trace + """ + if not trace: + self.trace = False + return + + # TODO: May want to switch format to something like this. + # log_format = ( + # '%(relativeCreated)05dms' + # ':%(relfilename)s' + # ':%(funcName)s' + # '#%(lineno)s' + # ': %(message)s' + # ) + tm_formatter = logging.Formatter('Taskmaster: %(message)s') + if isinstance(trace, io.StringIO): + log_handler = logging.StreamHandler(trace) + elif trace == '-': + log_handler = logging.StreamHandler(sys.stdout) + elif trace: + log_handler = logging.FileHandler(filename=trace) + + logger = logging.getLogger('Taskmaster') + logger.setLevel(level=logging.DEBUG) + logger.addHandler(log_handler) + self.trace = logger + + logger.log_handler = log_handler + + # Now setup Task's logger. + tl = logging.getLogger("Task") + tl.setLevel(level=logging.DEBUG) + tl.addHandler(log_handler) + task_formatter = logging.Formatter('%(name)s.%(message)s') + Task.LOGGER = tl + + log_handler.setFormatter(DispatchingFormatter({ + 'Taskmaster': tm_formatter, + 'Task': task_formatter, + }, + logging.Formatter('%(message)s') + )) def find_next_candidate(self): """ @@ -735,14 +789,13 @@ class Taskmaster: for p in n.waiting_parents: assert p.ref_count > 0, (str(n), str(p), p.ref_count) + def tm_trace_message(self, message): + return 'Taskmaster: %s' % message - def trace_message(self, message): - return 'Taskmaster: %s\n' % message - - def trace_node(self, node): - return '<%-10s %-3s %s>' % (StateString[node.get_state()], + def tm_trace_node(self, node): + return('<%-10s %-3s %s>' % (StateString[node.get_state()], node.ref_count, - repr(str(node))) + repr(str(node)))) def _find_next_ready_node(self): """ @@ -769,12 +822,14 @@ class Taskmaster: self.ready_exc = None T = self.trace - if T: T.write('\n' + self.trace_message('Looking for a node to evaluate')) + if T: + T.log_handler.stream.write('\n') # Prefix message with new line. This is a hack + self.trace.debug('Looking for a node to evaluate') while True: node = self.next_candidate() if node is None: - if T: T.write(self.trace_message('No candidate anymore.') + '\n') + if T: self.trace.debug('No candidate anymore.') return None node = node.disambiguate() @@ -797,7 +852,7 @@ class Taskmaster: else: S = None - if T: T.write(self.trace_message(' Considering node %s and its children:' % self.trace_node(node))) + if T: self.trace.debug(' Considering node %s and its children:' % self.tm_trace_node(node)) if state == NODE_NO_STATE: # Mark this node as being on the execution stack: @@ -805,7 +860,7 @@ class Taskmaster: elif state > NODE_PENDING: # Skip this node if it has already been evaluated: if S: S.already_handled = S.already_handled + 1 - if T: T.write(self.trace_message(' already handled (executed)')) + if T: self.trace.debug(' already handled (executed)') continue executor = node.get_executor() @@ -816,7 +871,7 @@ class Taskmaster: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) self.ready_exc = (SCons.Errors.ExplicitExit, e) - if T: T.write(self.trace_message(' SystemExit')) + if T: self.trace.debug(' SystemExit') return node except Exception as e: # We had a problem just trying to figure out the @@ -825,7 +880,7 @@ class Taskmaster: # raise the exception when the Task is "executed." self.ready_exc = sys.exc_info() if S: S.problem = S.problem + 1 - if T: T.write(self.trace_message(' exception %s while scanning children.\n' % e)) + if T: self.trace.debug(' exception %s while scanning children.' % e) return node children_not_visited = [] @@ -836,7 +891,7 @@ class Taskmaster: for child in chain(executor.get_all_prerequisites(), children): childstate = child.get_state() - if T: T.write(self.trace_message(' ' + self.trace_node(child))) + if T: self.trace.debug(' ' + self.tm_trace_node(child)) if childstate == NODE_NO_STATE: children_not_visited.append(child) @@ -857,8 +912,8 @@ class Taskmaster: self.candidates.extend(self.order(children_not_visited)) # if T and children_not_visited: - # T.write(self.trace_message(' adding to candidates: %s' % map(str, children_not_visited))) - # T.write(self.trace_message(' candidates now: %s\n' % map(str, self.candidates))) + # self.trace.debug(' adding to candidates: %s' % map(str, children_not_visited)) + # self.trace.debug(' candidates now: %s\n' % map(str, self.candidates)) # Skip this node if any of its children have failed. # @@ -883,7 +938,7 @@ class Taskmaster: n.set_state(NODE_FAILED) if S: S.child_failed = S.child_failed + 1 - if T: T.write(self.trace_message('****** %s\n' % self.trace_node(node))) + if T: self.trace.debug('****** %s' % self.tm_trace_node(node)) continue if children_not_ready: @@ -897,13 +952,13 @@ class Taskmaster: # count so we can be put back on the list for # re-evaluation when they've all finished. node.ref_count = node.ref_count + child.add_to_waiting_parents(node) - if T: T.write(self.trace_message(' adjusted ref count: %s, child %s' % - (self.trace_node(node), repr(str(child))))) + if T: self.trace.debug(' adjusted ref count: %s, child %s' % + (self.tm_trace_node(node), repr(str(child)))) if T: for pc in children_pending: - T.write(self.trace_message(' adding %s to the pending children set\n' % - self.trace_node(pc))) + self.trace.debug(' adding %s to the pending children set' % + self.tm_trace_node(pc)) self.pending_children = self.pending_children | children_pending continue @@ -923,8 +978,7 @@ class Taskmaster: # The default when we've gotten through all of the checks above: # this node is ready to be built. if S: S.build = S.build + 1 - if T: T.write(self.trace_message('Evaluating %s\n' % - self.trace_node(node))) + if T: self.trace.debug('Evaluating %s' % self.tm_trace_node(node)) # For debugging only: # @@ -989,8 +1043,8 @@ class Taskmaster: if T: for n in nodes: - T.write(self.trace_message(' removing node %s from the pending children set\n' % - self.trace_node(n))) + self.trace.debug(' removing node %s from the pending children set\n' % + self.tm_trace_node(n)) try: while len(to_visit): node = to_visit.pop() @@ -1006,8 +1060,8 @@ class Taskmaster: for p in parents: p.ref_count = p.ref_count - 1 - if T: T.write(self.trace_message(' removing parent %s from the pending children set\n' % - self.trace_node(p))) + if T: self.trace.debug(' removing parent %s from the pending children set\n' % + self.tm_trace_node(p)) except KeyError: # The container to_visit has been emptied. pass diff --git a/SCons/Util.py b/SCons/Util.py index 49a3a0f..476d4d6 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -35,6 +35,7 @@ from collections.abc import MappingView from contextlib import suppress from types import MethodType, FunctionType from typing import Optional, Union +from logging import Formatter # Note: Util module cannot import other bits of SCons globally without getting # into import loops. Both the below modules import SCons.Util early on. @@ -2159,6 +2160,17 @@ def wait_for_process_to_die(pid): else: time.sleep(0.1) +# From: https://stackoverflow.com/questions/1741972/how-to-use-different-formatters-with-the-same-logging-handler-in-python +class DispatchingFormatter(Formatter): + + def __init__(self, formatters, default_formatter): + self._formatters = formatters + self._default_formatter = default_formatter + + def format(self, record): + formatter = self._formatters.get(record.name, self._default_formatter) + return formatter.format(record) + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 31dd7e91b4b5a0b303e975c0380fdbc313c2429c Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 22 Oct 2022 10:49:23 -0600 Subject: Small cleanup in BoolVariable [skip appveyor] For consistency, have BoolVariable usage use True and False instead of 0 and 1 - mainly this is tests and doc examples. Reformatted a few embedded test files. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + SCons/Variables/BoolVariable.py | 24 ++++---- SCons/Variables/BoolVariableTests.py | 27 ++++---- doc/man/scons.xml | 8 +-- doc/user/command-line.xml | 4 +- test/Configure/issue-3469/fixture/SConstruct | 6 +- test/Copy-Option.py | 25 ++++---- test/Parallel/multiple-parents.py | 92 +++++++++++++++------------- test/Variables/BoolVariable.py | 21 +++---- test/Variables/help.py | 74 +++++++++++++--------- test/no-global-dependencies.py | 7 +-- 11 files changed, 158 insertions(+), 132 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f2c7fd5..1f588c7 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -47,6 +47,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER (';') - which is now documented with a hint to use a list instead to be portable. Splitting on space broke paths with embedded spaces. Fixes #4243. + - Cleanup: make sure BoolVariable usage in tests and examples uses Python + boolean values instead of 0/1. From Andrew Morrow - Avoid returning UniqueList for `children` and other `Executor` APIs. This type diff --git a/SCons/Variables/BoolVariable.py b/SCons/Variables/BoolVariable.py index d4a7de7..71e44c9 100644 --- a/SCons/Variables/BoolVariable.py +++ b/SCons/Variables/BoolVariable.py @@ -26,10 +26,10 @@ Usage example:: opts = Variables() - opts.Add(BoolVariable('embedded', 'build for an embedded system', 0)) - ... - if env['embedded'] == 1: + opts.Add(BoolVariable('embedded', 'build for an embedded system', False)) ... + if env['embedded']: + ... """ from typing import Tuple, Callable @@ -42,17 +42,17 @@ TRUE_STRINGS = ('y', 'yes', 'true', 't', '1', 'on' , 'all') FALSE_STRINGS = ('n', 'no', 'false', 'f', '0', 'off', 'none') -def _text2bool(val) -> bool: - """Converts strings to True/False. +def _text2bool(val: str) -> bool: + """Convert boolean-like string to boolean. If *val* looks like it expresses a bool-like value, based on - the :data:`TRUE_STRINGS` and :data:`FALSE_STRINGS` tuples, + the :const:`TRUE_STRINGS` and :const:`FALSE_STRINGS` tuples, return the appropriate value. This is usable as a converter function for SCons Variables. Raises: - ValueError: if the string cannot be converted. + ValueError: if *val* cannot be converted to boolean. """ lval = val.lower() @@ -64,13 +64,15 @@ def _text2bool(val) -> bool: def _validator(key, val, env) -> None: - """Validates the given value to be either true or false. + """Validate that the value of *key* in *env* is a boolean. + + Parmaeter *val* is not used in the check. - This is usable as a validator function for SCons Variables. + Usable as a validator function for SCons Variables. Raises: - KeyError: if key is not set in env - UserError: if key does not validate. + KeyError: if *key* is not set in *env* + UserError: if the value of *key* is not ``True`` or ``False``. """ if not env[key] in (True, False): raise SCons.Errors.UserError( diff --git a/SCons/Variables/BoolVariableTests.py b/SCons/Variables/BoolVariableTests.py index e486e4b..868e7e0 100644 --- a/SCons/Variables/BoolVariableTests.py +++ b/SCons/Variables/BoolVariableTests.py @@ -30,7 +30,7 @@ class BoolVariableTestCase(unittest.TestCase): def test_BoolVariable(self): """Test BoolVariable creation""" opts = SCons.Variables.Variables() - opts.Add(SCons.Variables.BoolVariable('test', 'test option help', 0)) + opts.Add(SCons.Variables.BoolVariable('test', 'test option help', False)) o = opts.options[0] assert o.key == 'test', o.key @@ -42,7 +42,7 @@ class BoolVariableTestCase(unittest.TestCase): def test_converter(self): """Test the BoolVariable converter""" opts = SCons.Variables.Variables() - opts.Add(SCons.Variables.BoolVariable('test', 'test option help', 0)) + opts.Add(SCons.Variables.BoolVariable('test', 'test option help', False)) o = opts.options[0] @@ -73,17 +73,17 @@ class BoolVariableTestCase(unittest.TestCase): x = o.converter(f) assert not x, "converter returned true for '%s'" % f - caught = None + caught = False try: o.converter('x') except ValueError: - caught = 1 - assert caught, "did not catch expected ValueError" + caught = True + assert caught, "did not catch expected ValueError for 'x'" def test_validator(self): """Test the BoolVariable validator""" opts = SCons.Variables.Variables() - opts.Add(SCons.Variables.BoolVariable('test', 'test option help', 0)) + opts.Add(SCons.Variables.BoolVariable('test', 'test option help', False)) o = opts.options[0] @@ -93,23 +93,24 @@ class BoolVariableTestCase(unittest.TestCase): 'N' : 'xyzzy', } + # positive checks o.validator('T', 0, env) - o.validator('F', 0, env) - caught = None + # negative checks + caught = False try: o.validator('N', 0, env) except SCons.Errors.UserError: - caught = 1 - assert caught, "did not catch expected UserError for N" + caught = True + assert caught, "did not catch expected UserError for value %s" % env['N'] - caught = None + caught = False try: o.validator('NOSUCHKEY', 0, env) except KeyError: - caught = 1 - assert caught, "did not catch expected KeyError for NOSUCHKEY" + caught = True + assert caught, "did not catch expected KeyError for 'NOSUCHKEY'" if __name__ == "__main__": diff --git a/doc/man/scons.xml b/doc/man/scons.xml index e00db3d..0278d87 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -4737,7 +4737,7 @@ the &Add; or &AddVariables; method: BoolVariable(key, help, default) -Returns a tuple of arguments +Return a tuple of arguments to set up a Boolean option. The option will use the specified name @@ -4746,7 +4746,7 @@ have a default value of default, and help will form the descriptive part of the help text. -The option will interpret the values +The option will interpret the command-line values y, yes, t, @@ -4756,7 +4756,7 @@ The option will interpret the values and all as true, -and the values +and the command-line values n, no, f, @@ -4990,7 +4990,7 @@ vars.AddVariables( BoolVariable( "warnings", help="compilation with -Wall and similar", - default=1, + default=True, ), EnumVariable( "debug", diff --git a/doc/user/command-line.xml b/doc/user/command-line.xml index a14800d..500b2ef 100644 --- a/doc/user/command-line.xml +++ b/doc/user/command-line.xml @@ -1281,7 +1281,7 @@ vars = Variables('custom.py', ARGUMENTS) vars = Variables('custom.py') -vars.Add(BoolVariable('RELEASE', help='Set to build for release', default=0)) +vars.Add(BoolVariable('RELEASE', help='Set to build for release', default=False)) env = Environment(variables=vars, CPPDEFINES={'RELEASE_BUILD': '${RELEASE}'}) env.Program('foo.c') @@ -1964,7 +1964,7 @@ vars = Variables() vars.AddVariables( ('RELEASE', 'Set to 1 to build for release', 0), ('CONFIG', 'Configuration file', '/etc/my_config'), - BoolVariable('warnings', help='compilation with -Wall and similiar', default=1), + BoolVariable('warnings', help='compilation with -Wall and similiar', default=True), EnumVariable( 'debug', help='debug output and symbols', diff --git a/test/Configure/issue-3469/fixture/SConstruct b/test/Configure/issue-3469/fixture/SConstruct index 4b5bedc..ae1fb30 100644 --- a/test/Configure/issue-3469/fixture/SConstruct +++ b/test/Configure/issue-3469/fixture/SConstruct @@ -1,3 +1,7 @@ +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + """ This tests if we add/remove a test in between other tests if a rerun will properly cache the results. Github issue #3469 @@ -6,7 +10,7 @@ Github issue #3469 DefaultEnvironment(tools=[]) vars = Variables() -vars.Add(BoolVariable('SKIP', 'Skip Middle Conf test', 0)) +vars.Add(BoolVariable('SKIP', 'Skip Middle Conf test', False)) env = Environment(variables=vars) conf = Configure(env) diff --git a/test/Copy-Option.py b/test/Copy-Option.py index f06c3b0..170dbbe 100644 --- a/test/Copy-Option.py +++ b/test/Copy-Option.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test that setting Variables in an Environment doesn't prevent the @@ -37,20 +36,22 @@ test.write('SConstruct', """ gpib_options = ['NI_GPIB', 'NI_ENET'] gpib_include = '/' -#0.96 broke copying ListVariables ??? +# 0.96 broke copying ListVariables ??? opts = Variables('config.py', ARGUMENTS) opts.AddVariables( - BoolVariable('gpib', 'enable gpib support', 1), - ListVariable('gpib_options', + BoolVariable('gpib', 'enable gpib support', True), + ListVariable( + 'gpib_options', 'whether and what kind of gpib support shall be enabled', 'all', - gpib_options), - ) -env = Environment(options = opts, CPPPATH = ['#/']) -new_env=env.Clone() + gpib_options, + ), +) +env = Environment(options=opts, CPPPATH=['#/']) +new_env = env.Clone() """) -test.run(arguments = '.') +test.run(arguments='.') test.pass_test() diff --git a/test/Parallel/multiple-parents.py b/test/Parallel/multiple-parents.py index 5a52f28..609e493 100644 --- a/test/Parallel/multiple-parents.py +++ b/test/Parallel/multiple-parents.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,14 +22,11 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# """ Verify that a failed build action with -j works as expected. """ -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import sys import TestSCons @@ -58,67 +57,76 @@ test = TestSCons.TestSCons() test.write('SConstruct', """ vars = Variables() -vars.Add( BoolVariable('interrupt', 'Interrupt the build.', 0 ) ) +vars.Add(BoolVariable('interrupt', 'Interrupt the build.', False)) varEnv = Environment(variables=vars) -def fail_action(target = None, source = None, env = None): +def fail_action(target=None, source=None, env=None): return 2 -def simulate_keyboard_interrupt(target = None, source = None, env = None): +def simulate_keyboard_interrupt(target=None, source=None, env=None): # Directly invoked the SIGINT handler to simulate a # KeyboardInterrupt. This hack is necessary because there is no # easy way to get access to the current Job/Taskmaster object. import signal + handler = signal.getsignal(signal.SIGINT) handler(signal.SIGINT, None) return 0 -interrupt = Command(target='interrupt', source='', action=simulate_keyboard_interrupt) +interrupt = Command(target='interrupt', source='', action=simulate_keyboard_interrupt) touch0 = Touch('${TARGETS[0]}') touch1 = Touch('${TARGETS[1]}') touch2 = Touch('${TARGETS[2]}') -failed0 = Command(target='failed00', source='', action=fail_action) -ok0 = Command(target=['ok00a', 'ok00b', 'ok00c'], - source='', - action=[touch0, touch1, touch2]) -prereq0 = Command(target='prereq00', source='', action=touch0) -ignore0 = Command(target='ignore00', source='', action=touch0) -igreq0 = Command(target='igreq00', source='', action=touch0) +failed0 = Command(target='failed00', source='', action=fail_action) +ok0 = Command( + target=['ok00a', 'ok00b', 'ok00c'], + source='', + action=[touch0, touch1, touch2], +) +prereq0 = Command(target='prereq00', source='', action=touch0) +ignore0 = Command(target='ignore00', source='', action=touch0) +igreq0 = Command(target='igreq00', source='', action=touch0) missing0 = Command(target='missing00', source='MissingSrc', action=touch0) -withSE0 = Command(target=['withSE00a', 'withSE00b', 'withSE00c'], - source='', - action=[touch0, touch1, touch2, Touch('side_effect')]) -SideEffect('side_effect', withSE0) - -prev_level = failed0 + ok0 + ignore0 + missing0 + withSE0 +withSE0 = Command( + target=['withSE00a', 'withSE00b', 'withSE00c'], + source='', + action=[touch0, touch1, touch2, Touch('side_effect')], +) +SideEffect('side_effect', withSE0) + +prev_level = failed0 + ok0 + ignore0 + missing0 + withSE0 prev_prereq = prereq0 prev_ignore = ignore0 -prev_igreq = igreq0 +prev_igreq = igreq0 if varEnv['interrupt']: prev_level = prev_level + interrupt -for i in range(1,20): - - failed = Command(target='failed%02d' % i, source='', action=fail_action) - ok = Command(target=['ok%02da' % i, 'ok%02db' % i, 'ok%02dc' % i], - source='', - action=[touch0, touch1, touch2]) - prereq = Command(target='prereq%02d' % i, source='', action=touch0) - ignore = Command(target='ignore%02d' % i, source='', action=touch0) - igreq = Command(target='igreq%02d' % i, source='', action=touch0) - missing = Command(target='missing%02d' %i, source='MissingSrc', action=touch0) - withSE = Command(target=['withSE%02da' % i, 'withSE%02db' % i, 'withSE%02dc' % i], - source='', - action=[touch0, touch1, touch2, Touch('side_effect')]) - SideEffect('side_effect', withSE) +for i in range(1, 20): + + failed = Command(target='failed%02d' % i, source='', action=fail_action) + ok = Command( + target=['ok%02da' % i, 'ok%02db' % i, 'ok%02dc' % i], + source='', + action=[touch0, touch1, touch2], + ) + prereq = Command(target='prereq%02d' % i, source='', action=touch0) + ignore = Command(target='ignore%02d' % i, source='', action=touch0) + igreq = Command(target='igreq%02d' % i, source='', action=touch0) + missing = Command(target='missing%02d' % i, source='MissingSrc', action=touch0) + withSE = Command( + target=['withSE%02da' % i, 'withSE%02db' % i, 'withSE%02dc' % i], + source='', + action=[touch0, touch1, touch2, Touch('side_effect')], + ) + SideEffect('side_effect', withSE) next_level = failed + ok + ignore + igreq + missing + withSE - for j in range(1,10): - a = Alias('a%02d%02d' % (i,j), prev_level) + for j in range(1, 10): + a = Alias('a%02d%02d' % (i, j), prev_level) Requires(a, prev_prereq) Ignore(a, prev_ignore) @@ -128,18 +136,18 @@ for i in range(1,20): next_level = next_level + a - prev_level = next_level + prev_level = next_level prev_prereq = prereq prev_ignore = ignore - prev_igreq = igreq + prev_igreq = igreq all = Alias('all', prev_level) Requires(all, prev_prereq) -Ignore(all, prev_ignore) +Ignore(all, prev_ignore) Requires(all, prev_igreq) -Ignore(all, prev_igreq) +Ignore(all, prev_igreq) Default(all) """) diff --git a/test/Variables/BoolVariable.py b/test/Variables/BoolVariable.py index eaf496a..9a95d85 100644 --- a/test/Variables/BoolVariable.py +++ b/test/Variables/BoolVariable.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test the BoolVariable canned Variable type. @@ -40,18 +39,18 @@ def check(expect): assert result[1:len(expect)+1] == expect, (result[1:len(expect)+1], expect) - test.write(SConstruct_path, """\ from SCons.Variables.BoolVariable import BoolVariable + BV = BoolVariable from SCons.Variables import BoolVariable opts = Variables(args=ARGUMENTS) opts.AddVariables( - BoolVariable('warnings', 'compilation with -Wall and similiar', 1), - BV('profile', 'create profiling informations', 0), - ) + BoolVariable('warnings', 'compilation with -Wall and similiar', True), + BV('profile', 'create profiling informations', False), +) env = Environment(variables=opts) Help(opts.GenerateHelpText(env)) @@ -62,8 +61,6 @@ print(env['profile']) Default(env.Alias('dummy', None)) """) - - test.run() check([str(True), str(False)]) @@ -73,12 +70,10 @@ check([str(False), str(True)]) expect_stderr = """ scons: *** Error converting option: warnings Invalid value for boolean option: irgendwas -""" + test.python_file_line(SConstruct_path, 12) +""" + test.python_file_line(SConstruct_path, 13) test.run(arguments='warnings=irgendwas', stderr = expect_stderr, status=2) - - test.pass_test() # Local Variables: diff --git a/test/Variables/help.py b/test/Variables/help.py index bee6011..cab7a67 100644 --- a/test/Variables/help.py +++ b/test/Variables/help.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test the Variables help messages. @@ -46,37 +45,52 @@ test.subdir(qtpath) test.subdir(libpath) test.write('SConstruct', """ -from SCons.Variables import BoolVariable, EnumVariable, ListVariable, \ - PackageVariable, PathVariable +from SCons.Variables import ( + BoolVariable, + EnumVariable, + ListVariable, + PackageVariable, + PathVariable, +) list_of_libs = Split('x11 gl qt ical') qtdir = r'%(qtpath)s' opts = Variables(args=ARGUMENTS) opts.AddVariables( - BoolVariable('warnings', 'compilation with -Wall and similiar', 1), - BoolVariable('profile', 'create profiling informations', 0), - EnumVariable('debug', 'debug output and symbols', 'no', - allowed_values=('yes', 'no', 'full'), - map={}, ignorecase=0), # case sensitive - EnumVariable('guilib', 'gui lib to use', 'gtk', - allowed_values=('motif', 'gtk', 'kde'), - map={}, ignorecase=1), # case insensitive - EnumVariable('some', 'some option', 'xaver', - allowed_values=('xaver', 'eins'), - map={}, ignorecase=2), # make lowercase - ListVariable('shared', - 'libraries to build as shared libraries', - 'all', - names = list_of_libs), - PackageVariable('x11', - 'use X11 installed here (yes = search some places)', - 'yes'), + BoolVariable('warnings', 'compilation with -Wall and similiar', True), + BoolVariable('profile', 'create profiling informations', False), + EnumVariable( + 'debug', + 'debug output and symbols', + 'no', + allowed_values=('yes', 'no', 'full'), + map={}, + ignorecase=0, + ), # case sensitive + EnumVariable( + 'guilib', + 'gui lib to use', + 'gtk', + allowed_values=('motif', 'gtk', 'kde'), + map={}, + ignorecase=1, + ), # case insensitive + EnumVariable( + 'some', + 'some option', + 'xaver', + allowed_values=('xaver', 'eins'), + map={}, + ignorecase=2, + ), # make lowercase + ListVariable( + 'shared', 'libraries to build as shared libraries', 'all', names=list_of_libs + ), + PackageVariable('x11', 'use X11 installed here (yes = search some places)', 'yes'), PathVariable('qtdir', 'where the root of Qt is installed', qtdir), - PathVariable('qt_libraries', - 'where the Qt library is installed', - r'%(libdirvar)s'), - ) + PathVariable('qt_libraries', 'where the Qt library is installed', r'%(libdirvar)s'), +) env = Environment(variables=opts) Help(opts.GenerateHelpText(env)) @@ -96,11 +110,11 @@ scons: Reading SConscript files ... scons: done reading SConscript files. warnings: compilation with -Wall and similiar (yes|no) - default: 1 + default: True actual: %(str_True)s profile: create profiling informations (yes|no) - default: 0 + default: False actual: %(str_False)s debug: debug output and symbols (yes|no|full) diff --git a/test/no-global-dependencies.py b/test/no-global-dependencies.py index 18b674c..95761e7 100644 --- a/test/no-global-dependencies.py +++ b/test/no-global-dependencies.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test that files are correctly located in the variant directory even when -- cgit v0.12 From e31090743567389160633af207e41e1efe8a56a0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Oct 2022 11:09:50 -0700 Subject: [ci skip] Fix sider/PEP8 complaints --- SCons/Taskmaster/TaskmasterTests.py | 4 +-- SCons/Taskmaster/__init__.py | 63 +++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/SCons/Taskmaster/TaskmasterTests.py b/SCons/Taskmaster/TaskmasterTests.py index 2ff15f0..fad028e 100644 --- a/SCons/Taskmaster/TaskmasterTests.py +++ b/SCons/Taskmaster/TaskmasterTests.py @@ -1245,7 +1245,7 @@ Taskmaster: No candidate anymore. v_split=value.split('\n') e_split=expect.split('\n') if len(v_split) != len(e_split): - print("different number of lines:%d %d"%(len(v_split), len(e_split))) + print("different number of lines:%d %d" % (len(v_split), len(e_split))) # breakpoint() for v, e in zip(v_split, e_split): @@ -1253,7 +1253,7 @@ Taskmaster: No candidate anymore. if v != e: print("\n[%s]\n[%s]" % (v, e)) - assert value == expect, "Expected:\n%s\nGot:\n%s"%(expect, value) + assert value == expect, "Expected:\n%s\nGot:\n%s" % (expect, value) if __name__ == "__main__": diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index 59b5886..d94c892 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -168,7 +168,8 @@ class Task(ABC): """ global print_prepare T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) # Now that it's the appropriate time, give the TaskMaster a # chance to raise any exceptions it encountered while preparing @@ -222,7 +223,8 @@ class Task(ABC): prepare(), executed() or failed(). """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) try: cached_targets = [] @@ -265,7 +267,8 @@ class Task(ABC): the Node's callback methods. """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) for t in self.targets: if t.get_state() == NODE_EXECUTING: @@ -288,7 +291,8 @@ class Task(ABC): """ global print_prepare T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) for t in self.targets: if t.get_state() == NODE_EXECUTING: @@ -329,7 +333,8 @@ class Task(ABC): nodes when using Configure(). """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) # Invoke will_not_build() to clean-up the pending children # list. @@ -356,7 +361,8 @@ class Task(ABC): nodes when using Configure(). """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) self.tm.will_not_build(self.targets, lambda n: n.set_state(NODE_FAILED)) @@ -368,7 +374,8 @@ class Task(ABC): visited--the canonical example being the "scons -c" option. """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) self.out_of_date = self.targets[:] for t in self.targets: @@ -435,7 +442,8 @@ class Task(ABC): that can be put back on the candidates list. """ T = self.tm.trace - if T: self.trace_message(self.node) + if T: + self.trace_message(self.node) # We may have built multiple targets, some of which may have # common parents waiting for this build. Count up how many @@ -452,7 +460,8 @@ class Task(ABC): # A node can only be in the pending_children set if it has # some waiting_parents. if t.waiting_parents: - if T: self.trace_message(t, 'removing') + if T: + self.trace_message(t, 'removing') pending_children.discard(t) for p in t.waiting_parents: parents[p] = parents.get(p, 0) + 1 @@ -479,7 +488,8 @@ class Task(ABC): for p, subtract in parents.items(): p.ref_count = p.ref_count - subtract - if T: self.trace_message(p, 'adjusted parent ref count') + if T: + self.trace_message(p, 'adjusted parent ref count') if p.ref_count == 0: self.tm.candidates.append(p) @@ -789,9 +799,6 @@ class Taskmaster: for p in n.waiting_parents: assert p.ref_count > 0, (str(n), str(p), p.ref_count) - def tm_trace_message(self, message): - return 'Taskmaster: %s' % message - def tm_trace_node(self, node): return('<%-10s %-3s %s>' % (StateString[node.get_state()], node.ref_count, @@ -829,7 +836,8 @@ class Taskmaster: while True: node = self.next_candidate() if node is None: - if T: self.trace.debug('No candidate anymore.') + if T: + self.trace.debug('No candidate anymore.') return None node = node.disambiguate() @@ -852,7 +860,8 @@ class Taskmaster: else: S = None - if T: self.trace.debug(' Considering node %s and its children:' % self.tm_trace_node(node)) + if T: + self.trace.debug(' Considering node %s and its children:' % self.tm_trace_node(node)) if state == NODE_NO_STATE: # Mark this node as being on the execution stack: @@ -860,7 +869,8 @@ class Taskmaster: elif state > NODE_PENDING: # Skip this node if it has already been evaluated: if S: S.already_handled = S.already_handled + 1 - if T: self.trace.debug(' already handled (executed)') + if T: + self.trace.debug(' already handled (executed)') continue executor = node.get_executor() @@ -871,7 +881,8 @@ class Taskmaster: exc_value = sys.exc_info()[1] e = SCons.Errors.ExplicitExit(node, exc_value.code) self.ready_exc = (SCons.Errors.ExplicitExit, e) - if T: self.trace.debug(' SystemExit') + if T: + self.trace.debug(' SystemExit') return node except Exception as e: # We had a problem just trying to figure out the @@ -880,7 +891,8 @@ class Taskmaster: # raise the exception when the Task is "executed." self.ready_exc = sys.exc_info() if S: S.problem = S.problem + 1 - if T: self.trace.debug(' exception %s while scanning children.' % e) + if T: + elf.trace.debug(' exception %s while scanning children.' % e) return node children_not_visited = [] @@ -891,7 +903,8 @@ class Taskmaster: for child in chain(executor.get_all_prerequisites(), children): childstate = child.get_state() - if T: self.trace.debug(' ' + self.tm_trace_node(child)) + if T: + self.trace.debug(' ' + self.tm_trace_node(child)) if childstate == NODE_NO_STATE: children_not_visited.append(child) @@ -938,7 +951,8 @@ class Taskmaster: n.set_state(NODE_FAILED) if S: S.child_failed = S.child_failed + 1 - if T: self.trace.debug('****** %s' % self.tm_trace_node(node)) + if T: + self.trace.debug('****** %s' % self.tm_trace_node(node)) continue if children_not_ready: @@ -952,7 +966,8 @@ class Taskmaster: # count so we can be put back on the list for # re-evaluation when they've all finished. node.ref_count = node.ref_count + child.add_to_waiting_parents(node) - if T: self.trace.debug(' adjusted ref count: %s, child %s' % + if T: + self.trace.debug(' adjusted ref count: %s, child %s' % (self.tm_trace_node(node), repr(str(child)))) if T: @@ -978,7 +993,8 @@ class Taskmaster: # The default when we've gotten through all of the checks above: # this node is ready to be built. if S: S.build = S.build + 1 - if T: self.trace.debug('Evaluating %s' % self.tm_trace_node(node)) + if T: + self.trace.debug('Evaluating %s' % self.tm_trace_node(node)) # For debugging only: # @@ -1060,7 +1076,8 @@ class Taskmaster: for p in parents: p.ref_count = p.ref_count - 1 - if T: self.trace.debug(' removing parent %s from the pending children set\n' % + if T: + self.trace.debug(' removing parent %s from the pending children set\n' % self.tm_trace_node(p)) except KeyError: # The container to_visit has been emptied. -- cgit v0.12 From 0401fe27103967a34a7ce6dc5b446b697d9f0ac9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Oct 2022 11:13:09 -0700 Subject: Address style comment on cal to DispatchingFormatter() from mwichmann. Good call --- SCons/Taskmaster/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index d94c892..ef5b7d4 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -665,11 +665,12 @@ class Taskmaster: task_formatter = logging.Formatter('%(name)s.%(message)s') Task.LOGGER = tl - log_handler.setFormatter(DispatchingFormatter({ + log_handler.setFormatter(DispatchingFormatter( + formatters={ 'Taskmaster': tm_formatter, 'Task': task_formatter, }, - logging.Formatter('%(message)s') + default_formatter=logging.Formatter('%(message)s') )) def find_next_candidate(self): -- cgit v0.12 From 7f93a59d162292f03a74c984c7d6aa141e372eaa Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Oct 2022 11:18:30 -0700 Subject: [ci skip] address sider complain on call to DispatchingFormatter() --- SCons/Taskmaster/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index ef5b7d4..fe81f99 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -667,9 +667,9 @@ class Taskmaster: log_handler.setFormatter(DispatchingFormatter( formatters={ - 'Taskmaster': tm_formatter, - 'Task': task_formatter, - }, + 'Taskmaster': tm_formatter, + 'Task': task_formatter, + }, default_formatter=logging.Formatter('%(message)s') )) -- cgit v0.12 From d9c359a081d9f0a6e923b2ae15d530ebc0d6fc04 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Oct 2022 11:21:59 -0700 Subject: [ci skip] Fix Typo --- SCons/Taskmaster/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index fe81f99..3f0e700 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -893,7 +893,7 @@ class Taskmaster: self.ready_exc = sys.exc_info() if S: S.problem = S.problem + 1 if T: - elf.trace.debug(' exception %s while scanning children.' % e) + self.trace.debug(' exception %s while scanning children.' % e) return node children_not_visited = [] -- cgit v0.12 From eded27097384dfa54172f955e0a3ab7ccdb187bf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Oct 2022 11:29:39 -0700 Subject: [ci skip] Updated CHANGES.txt and RELEASE.txt --- CHANGES.txt | 3 +++ RELEASE.txt | 12 +----------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index e9cd653..9c9344a 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -20,6 +20,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER not be called until all AddOption() calls are completed. Resolves Issue #4187 - Refactored SCons/Taskmaster into a package. Moved SCons/Jobs.py into that package. NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs + - Changed the Taskmaster trace logic to use python's logging module. The output formatting + should remain (mostly) the same. Minor update to unittest for this to adjust for 1 less newline. + From Dan Mezhiborsky: diff --git a/RELEASE.txt b/RELEASE.txt index 8ad105b..63717a0 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -37,8 +37,6 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY FIXES ----- -- List fixes of outright bugs - - Added missing newline to generated compilation database (compile_commands.json) - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress @@ -47,15 +45,11 @@ FIXES IMPROVEMENTS ------------ -- List improvements that wouldn't be visible to the user in the - documentation: performance improvements (describe the circumstances - under which they would be observed), or major code cleanups +- Changed the Taskmaster trace logic to use python's logging module. PACKAGING --------- -- List changes in the way SCons is packaged and/or released - - SCons now has three requirements files: requirements.txt describes requirements to run scons; requirements-dev.txt requirements to develop it - mainly things needed to run the testsuite; @@ -65,10 +59,6 @@ PACKAGING DOCUMENTATION ------------- -- List any significant changes to the documentation (not individual - typo fixes, even if they're mentioned in src/CHANGES.txt to give - the contributor credit) - - Updated the --hash-format manpage entry. - EnsureSConsVersion, EnsurePythonVersion, Exit, GetLaunchDir and SConscriptChdir are now listed as Global functions only. -- cgit v0.12 From de04bf85a087d2c914b30aaa4e3d563e53b06454 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 26 Oct 2022 10:34:34 -0700 Subject: Fixed taskmaster trace tests. Previously there was an extra line at the end of the file. It's no longer there. Added TestCommon.detailed_diff() function which can be used to diff large text blobs expected vs actual --- SCons/Taskmaster/TaskmasterTests.py | 107 ++++++++++++++++++++---------------- test/Interactive/taskmastertrace.py | 11 ++-- test/option/taskmastertrace.py | 9 +-- testing/framework/TestCommon.py | 16 ++++++ 4 files changed, 84 insertions(+), 59 deletions(-) diff --git a/SCons/Taskmaster/TaskmasterTests.py b/SCons/Taskmaster/TaskmasterTests.py index fad028e..9d7f959 100644 --- a/SCons/Taskmaster/TaskmasterTests.py +++ b/SCons/Taskmaster/TaskmasterTests.py @@ -26,10 +26,10 @@ import SCons.compat import sys import unittest - import SCons.Taskmaster import SCons.Errors +import TestCommon built_text = None cache_text = [] @@ -37,8 +37,9 @@ visited_nodes = [] executed = None scan_called = 0 + class Node: - def __init__(self, name, kids = [], scans = []): + def __init__(self, name, kids=[], scans=[]): self.name = name self.kids = kids self.scans = scans @@ -47,9 +48,11 @@ class Node: self.scanner = None self.targets = [self] self.prerequisites = None + class Builder: def targets(self, node): return node.targets + self.builder = Builder() self.bsig = None self.csig = None @@ -83,7 +86,7 @@ class Node: def prepare(self): self.prepared = 1 - self.get_binfo() + self.get_binfo() def build(self): global built_text @@ -119,7 +122,7 @@ class Node: self.binfo = binfo return binfo - + def clear(self): # The del_binfo() call here isn't necessary for normal execution, # but is for interactive mode, where we might rebuild the same @@ -130,14 +133,14 @@ class Node: global built_text if not self.cached: built_text = built_text + " really" - + # Clear the implicit dependency caches of any Nodes # waiting for this Node to be built. for parent in self.waiting_parents: parent.implicit = None self.clear() - + def release_target_info(self): pass @@ -199,7 +202,7 @@ class Node: def is_up_to_date(self): return self._current_val - + def depends_on(self, nodes): for node in nodes: if node in self.kids: @@ -218,26 +221,34 @@ class Node: class Executor: def prepare(self): pass + def get_action_targets(self): return self.targets + def get_all_targets(self): return self.targets + def get_all_children(self): result = [] for node in self.targets: result.extend(node.children()) return result + def get_all_prerequisites(self): return [] + def get_action_side_effects(self): return [] + self.executor = Executor() self.executor.targets = self.targets return self.executor + class OtherError(Exception): pass + class MyException(Exception): pass @@ -306,7 +317,7 @@ class TaskmasterTestCase(unittest.TestCase): n2._current_val = 1 n3.set_state(SCons.Node.no_state) n3._current_val = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n3], tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n3], tasker=MyTask) t = tm.next_task() t.prepare() @@ -331,7 +342,6 @@ class TaskmasterTestCase(unittest.TestCase): assert tm.next_task() is None - n1 = Node("n1") n2 = Node("n2") n3 = Node("n3", [n1, n2]) @@ -366,7 +376,6 @@ class TaskmasterTestCase(unittest.TestCase): assert tm.next_task() is None - n4 = Node("n4") n4.set_state(SCons.Node.executed) tm = SCons.Taskmaster.Taskmaster([n4]) @@ -374,14 +383,13 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2", [n1]) - tm = SCons.Taskmaster.Taskmaster([n2,n2]) + tm = SCons.Taskmaster.Taskmaster([n2, n2]) t = tm.next_task() t.executed() t.postprocess() t = tm.next_task() assert tm.next_task() is None - n1 = Node("n1") n2 = Node("n2") n3 = Node("n3", [n1], [n2]) @@ -440,11 +448,11 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2") n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) + n4 = Node("n4", [n1, n2, n3]) n5 = Node("n5", [n4]) n3.side_effect = 1 n1.side_effects = n2.side_effects = n3.side_effects = [n4] - tm = SCons.Taskmaster.Taskmaster([n1,n2,n3,n4,n5]) + tm = SCons.Taskmaster.Taskmaster([n1, n2, n3, n4, n5]) t = tm.next_task() assert t.get_target() == n1 assert n4.state == SCons.Node.executing, n4.state @@ -471,10 +479,12 @@ class TaskmasterTestCase(unittest.TestCase): n1 = Node("n1") n2 = Node("n2") n3 = Node("n3") - n4 = Node("n4", [n1,n2,n3]) + n4 = Node("n4", [n1, n2, n3]) + def reverse(dependencies): dependencies.reverse() return dependencies + tm = SCons.Taskmaster.Taskmaster([n4], order=reverse) t = tm.next_task() assert t.get_target() == n3, t.get_target() @@ -534,11 +544,11 @@ class TaskmasterTestCase(unittest.TestCase): s = n2.get_state() assert s == SCons.Node.executed, s - def test_make_ready_out_of_date(self): """Test the Task.make_ready() method's list of out-of-date Nodes """ ood = [] + def TaskGen(tm, targets, top, node, ood=ood): class MyTask(SCons.Taskmaster.AlwaysTask): def make_ready(self): @@ -558,8 +568,8 @@ class TaskmasterTestCase(unittest.TestCase): a5 = Node("a5") a5._current_val = 1 a5.always_build = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4, a5], - tasker = TaskGen) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4, a5], + tasker=TaskGen) del ood[:] t = tm.next_task() @@ -584,12 +594,13 @@ class TaskmasterTestCase(unittest.TestCase): def test_make_ready_exception(self): """Test handling exceptions from Task.make_ready() """ + class MyTask(SCons.Taskmaster.AlwaysTask): def make_ready(self): raise MyException("from make_ready()") n1 = Node("n1") - tm = SCons.Taskmaster.Taskmaster(targets = [n1], tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n1], tasker=MyTask) t = tm.next_task() exc_type, exc_value, exc_tb = t.exception assert exc_type == MyException, repr(exc_type) @@ -601,6 +612,7 @@ class TaskmasterTestCase(unittest.TestCase): We should be getting: TypeError: Can't instantiate abstract class MyTask with abstract methods needs_execute """ + class MyTask(SCons.Taskmaster.Task): pass @@ -611,6 +623,7 @@ class TaskmasterTestCase(unittest.TestCase): def test_make_ready_all(self): """Test the make_ready_all() method""" + class MyTask(SCons.Taskmaster.AlwaysTask): make_ready = SCons.Taskmaster.Task.make_ready_all @@ -621,7 +634,7 @@ class TaskmasterTestCase(unittest.TestCase): c4 = Node("c4") c4._current_val = 1 - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4]) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4]) t = tm.next_task() target = t.get_target() @@ -647,8 +660,8 @@ class TaskmasterTestCase(unittest.TestCase): n3 = Node("n3") c4 = Node("c4") - tm = SCons.Taskmaster.Taskmaster(targets = [n1, c2, n3, c4], - tasker = MyTask) + tm = SCons.Taskmaster.Taskmaster(targets=[n1, c2, n3, c4], + tasker=MyTask) t = tm.next_task() target = t.get_target() @@ -669,13 +682,14 @@ class TaskmasterTestCase(unittest.TestCase): t = tm.next_task() assert t is None - def test_children_errors(self): """Test errors when fetching the children of a node. """ + class StopNode(Node): def children(self): raise SCons.Errors.StopError("stop!") + class ExitNode(Node): def children(self): sys.exit(77) @@ -879,8 +893,8 @@ class TaskmasterTestCase(unittest.TestCase): n9 = Node("n9") n10 = Node("n10") - n6.side_effects = [ n8 ] - n7.side_effects = [ n9, n10 ] + n6.side_effects = [n8] + n7.side_effects = [n9, n10] tm = SCons.Taskmaster.Taskmaster([n6, n7]) t = tm.next_task() @@ -897,15 +911,19 @@ class TaskmasterTestCase(unittest.TestCase): class ExceptionExecutor: def prepare(self): raise Exception("Executor.prepare() exception") + def get_all_targets(self): return self.nodes + def get_all_children(self): result = [] for node in self.nodes: result.extend(node.children()) return result + def get_all_prerequisites(self): return [] + def get_action_side_effects(self): return [] @@ -935,6 +953,7 @@ class TaskmasterTestCase(unittest.TestCase): def raise_UserError(): raise SCons.Errors.UserError + n2 = Node("n2") n2.build = raise_UserError tm = SCons.Taskmaster.Taskmaster([n2]) @@ -948,6 +967,7 @@ class TaskmasterTestCase(unittest.TestCase): def raise_BuildError(): raise SCons.Errors.BuildError + n3 = Node("n3") n3.build = raise_BuildError tm = SCons.Taskmaster.Taskmaster([n3]) @@ -964,6 +984,7 @@ class TaskmasterTestCase(unittest.TestCase): # args set to the exception value, instance, and traceback. def raise_OtherError(): raise OtherError + n4 = Node("n4") n4.build = raise_OtherError tm = SCons.Taskmaster.Taskmaster([n4]) @@ -1066,7 +1087,7 @@ class TaskmasterTestCase(unittest.TestCase): """ n1 = Node("n1") tm = SCons.Taskmaster.Taskmaster([n1]) - t = tm.next_task() + t = tm.next_task() t.exception_set((1, 2)) exc_type, exc_value = t.exception @@ -1076,25 +1097,26 @@ class TaskmasterTestCase(unittest.TestCase): t.exception_set(3) assert t.exception == 3 - try: 1//0 + try: + 1 // 0 except: # Moved from below t.exception_set(None) - #pass + # pass -# import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() # Having this here works for python 2.x, # but it is a tuple (None, None, None) when called outside # an except statement # t.exception_set(None) - + exc_type, exc_value, exc_tb = t.exception - assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s"%exc_type + assert exc_type is ZeroDivisionError, "Expecting ZeroDevisionError got:%s" % exc_type exception_values = [ "integer division or modulo", "integer division or modulo by zero", - "integer division by zero", # PyPy2 + "integer division by zero", # PyPy2 ] assert str(exc_value) in exception_values, exc_value @@ -1108,7 +1130,7 @@ class TaskmasterTestCase(unittest.TestCase): except: exc_type, exc_value = sys.exc_info()[:2] assert exc_type == Exception1, exc_type - assert str(exc_value) == '', "Expecting empty string got:%s (type %s)"%(exc_value,type(exc_value)) + assert str(exc_value) == '', "Expecting empty string got:%s (type %s)" % (exc_value, type(exc_value)) else: assert 0, "did not catch expected exception" @@ -1129,7 +1151,7 @@ class TaskmasterTestCase(unittest.TestCase): pass try: - 1//0 + 1 // 0 except: tb = sys.exc_info()[2] t.exception_set((Exception3, "arg", tb)) @@ -1242,18 +1264,11 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. """ - v_split=value.split('\n') - e_split=expect.split('\n') - if len(v_split) != len(e_split): - print("different number of lines:%d %d" % (len(v_split), len(e_split))) - - # breakpoint() - for v, e in zip(v_split, e_split): - # print("%s:%s"%(v,e)) - if v != e: - print("\n[%s]\n[%s]" % (v, e)) - - assert value == expect, "Expected:\n%s\nGot:\n%s" % (expect, value) + + if value != expect: + TestCommon.TestCommon.detailed_diff(value, expect) + + assert value == expect, "Expected taskmaster trace contents didn't match. See above" if __name__ == "__main__": diff --git a/test/Interactive/taskmastertrace.py b/test/Interactive/taskmastertrace.py index 93ee068..04e95fd 100644 --- a/test/Interactive/taskmastertrace.py +++ b/test/Interactive/taskmastertrace.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,8 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + """ Verify use of the --taskmastertrace= option to the "build" command @@ -42,7 +43,6 @@ Command('2', [], Touch('$TARGET')) test.write('foo.in', "foo.in 1\n") - scons = test.start(arguments = '-Q --interactive') scons.send("build foo.out 1\n") @@ -101,7 +101,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - scons>>> Touch("2") scons>>> scons: `foo.out' is up to date. scons>>> @@ -109,8 +108,6 @@ scons>>> test.finish(scons, stdout = expect_stdout) - - test.pass_test() # Local Variables: diff --git a/test/option/taskmastertrace.py b/test/option/taskmastertrace.py index fc8522c..9d0a606 100644 --- a/test/option/taskmastertrace.py +++ b/test/option/taskmastertrace.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Simple tests of the --taskmastertrace= option. @@ -123,7 +122,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - """) test.run(arguments='--taskmastertrace=- .', stdout=expect_stdout) @@ -212,7 +210,6 @@ Task.postprocess(): node Taskmaster: Looking for a node to evaluate Taskmaster: No candidate anymore. - """ test.must_match('trace.out', expect_trace, mode='r') diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index f45b856..df005b3 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -796,6 +796,22 @@ class TestCommon(TestCmd): # so this is an Aegis invocation; pass the test (exit 0). self.pass_test() + @staticmethod + def detailed_diff(value, expect): + v_split = value.split('\n') + e_split = expect.split('\n') + if len(v_split) != len(e_split): + print("different number of lines:%d %d" % (len(v_split), len(e_split))) + + # breakpoint() + for v, e in zip(v_split, e_split): + # print("%s:%s"%(v,e)) + if v != e: + print("\n[%s]\n[%s]" % (v, e)) + + return "Expected:\n%s\nGot:\n%s" % (expect, value) + + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 4d440cef23ea297ecb73457a807db385ff779f44 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 26 Oct 2022 20:04:26 -0700 Subject: Initial logic to add logging to ExperimentalParallel job class. Not quite working --- SCons/Taskmaster/Job.py | 61 ++++++++++++++++++++++++++++++++++---------- SCons/Taskmaster/__init__.py | 3 +++ 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/SCons/Taskmaster/Job.py b/SCons/Taskmaster/Job.py index d9c98e8..667d46b 100644 --- a/SCons/Taskmaster/Job.py +++ b/SCons/Taskmaster/Job.py @@ -29,8 +29,10 @@ stop, and wait on jobs. import SCons.compat +import logging import os import signal +import sys import threading from enum import Enum @@ -484,6 +486,23 @@ else: self.results_queue_lock = threading.Lock() self.results_queue = [] + if self.taskmaster.trace: + self.trace = self._setup_logging() + else: + self.trace = False + + def _setup_logging(self): + jl = logging.getLogger("Job") + jl.setLevel(level=logging.DEBUG) + jl.addHandler(self.taskmaster.trace.log_handler) + return jl + + def trace_message(self, message): + # This grabs the name of the function which calls trace_message() + method_name = sys._getframe(1).f_code.co_name + "():" + self.trace.debug('%-15s %s' % (method_name, message)) + print('%-15s %s' % (method_name, message)) + def start(self): self._start_workers() for worker in self.workers: @@ -527,7 +546,8 @@ else: # Obtain `tm_lock`, granting exclusive access to the taskmaster. with self.can_search_cv: - # print(f"XXX {threading.get_ident()} Gained exclusive access") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Gained exclusive access") # Capture whether we got here with `task` set, # then drop our reference to the task as we are no @@ -547,24 +567,28 @@ else: # in the condvar. Some other thread will come back # here with a completed task. if self.state == ExperimentalParallel.State.STALLED and completed_task: - # print(f"XXX {threading.get_ident()} Detected stall with completed task, bypassing wait") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Detected stall with completed task, bypassing wait") self.state = ExperimentalParallel.State.READY # Wait until we are neither searching nor stalled. while self.state == ExperimentalParallel.State.SEARCHING or self.state == ExperimentalParallel.State.STALLED: - # print(f"XXX {threading.get_ident()} Search already in progress, waiting") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Search already in progress, waiting") self.can_search_cv.wait() # If someone set the completed flag, bail. if self.state == ExperimentalParallel.State.COMPLETED: - # print(f"XXX {threading.get_ident()} Completion detected, breaking from main loop") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Completion detected, breaking from main loop") break # Set the searching flag to indicate that a thread # is currently in the critical section for # taskmaster work. # - # print(f"XXX {threading.get_ident()} Starting search") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Starting search") self.state = ExperimentalParallel.State.SEARCHING # Bulk acquire the tasks in the results queue @@ -577,7 +601,8 @@ else: with self.results_queue_lock: results_queue, self.results_queue = self.results_queue, results_queue - # print(f"XXX {threading.get_ident()} Found {len(results_queue)} completed tasks to process") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Found {len(results_queue)} completed tasks to process") for (rtask, rresult) in results_queue: if rresult: rtask.executed() @@ -606,7 +631,8 @@ else: # until results arrive if jobs are pending, or # mark the walk as complete if not. while self.state == ExperimentalParallel.State.SEARCHING: - # print(f"XXX {threading.get_ident()} Searching for new tasks") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Searching for new tasks") task = self.taskmaster.next_task() if task: @@ -626,12 +652,14 @@ else: task.postprocess() else: if not task.needs_execute(): - # print(f"XXX {threading.get_ident()} Found internal task") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Found internal task") task.executed() task.postprocess() else: self.jobs += 1 - # print(f"XXX {threading.get_ident()} Found task requiring execution") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Found task requiring execution") self.state = ExperimentalParallel.State.READY self.can_search_cv.notify() @@ -649,7 +677,8 @@ else: # outstanding that will re-enter the # loop. # - # print(f"XXX {threading.get_ident()} Found no task requiring execution, but have jobs: marking stalled") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Found no task requiring execution, but have jobs: marking stalled") self.state = ExperimentalParallel.State.STALLED else: # We didn't find a task and there are @@ -661,7 +690,8 @@ else: # note completion and awaken anyone # sleeping on the condvar. # - # print(f"XXX {threading.get_ident()} Found no task requiring execution, and have no jobs: marking complete") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Found no task requiring execution, and have no jobs: marking complete") self.state = ExperimentalParallel.State.COMPLETED self.can_search_cv.notify_all() @@ -670,7 +700,8 @@ else: # to search, one of them can now begin turning the # taskmaster crank in parallel. if task: - # print(f"XXX {threading.get_ident()} Executing task") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Executing task") ok = True try: if self.interrupted(): @@ -686,7 +717,8 @@ else: # the searching loop will complete the # postprocessing work under the taskmaster lock. # - # print(f"XXX {threading.get_ident()} Enqueueing executed task results") + if self.trace: + self.trace_message(f"XXX {threading.get_ident()} Enqueueing executed task results") with self.results_queue_lock: self.results_queue.append((task, ok)) @@ -696,6 +728,9 @@ else: # the value of the `task` variable if you add new code # after this comment. +# TODO: Remove this is just for testing. +Parallel = ExperimentalParallel + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/SCons/Taskmaster/__init__.py b/SCons/Taskmaster/__init__.py index 3f0e700..2e08a8b 100644 --- a/SCons/Taskmaster/__init__.py +++ b/SCons/Taskmaster/__init__.py @@ -665,10 +665,13 @@ class Taskmaster: task_formatter = logging.Formatter('%(name)s.%(message)s') Task.LOGGER = tl + self.trace.log_handler = log_handler + log_handler.setFormatter(DispatchingFormatter( formatters={ 'Taskmaster': tm_formatter, 'Task': task_formatter, + 'Job': task_formatter, }, default_formatter=logging.Formatter('%(message)s') )) -- cgit v0.12 From 2c791e4d7f4ab304fda9a03eabc3916056b0e36f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 2 Nov 2022 23:21:57 -0400 Subject: Move execution environment sanitation from Action._suproc to SCons.Util.sanitize_shell_env(ENV). Have ninja's spawning logic use this same function to clean it's execution environment --- CHANGES.txt | 5 +++++ RELEASE.txt | 3 +++ SCons/Action.py | 19 +------------------ SCons/Platform/darwin.py | 1 + SCons/Tool/ninja/Utils.py | 16 ++-------------- SCons/Tool/ninja/__init__.py | 8 +++++++- SCons/Util.py | 27 +++++++++++++++++++++++++++ test/ninja/shell_command.py | 6 ++++++ 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0a9f697..146a861 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -22,6 +22,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs - Changed the Taskmaster trace logic to use python's logging module. The output formatting should remain (mostly) the same. Minor update to unittest for this to adjust for 1 less newline. + - Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an + execution environment variable set to a python list it would crash. Now it + will create a string joining the list with os.pathsep + - Move execution environment sanitation from Action._subproc() to + SCons.Util.sanitize_shell_env(ENV) diff --git a/RELEASE.txt b/RELEASE.txt index 94ba72e..b93c449 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -50,6 +50,9 @@ FIXES use: env["JAVACLASSPATH"] = env.Split("foo bar baz") There is no change in how JAVACLASSPATH gets turned into the -classpath argument passed to the JDK tools. +- Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an + execution environment variable set to a python list it would crash. Now it + will create a string joining the list with os.pathsep IMPROVEMENTS ------------ diff --git a/SCons/Action.py b/SCons/Action.py index ccb3a2c..8151b2a 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -806,24 +806,7 @@ def _subproc(scons_env, cmd, error='ignore', **kw): ENV = kw.get('env', None) if ENV is None: ENV = get_default_ENV(scons_env) - # Ensure that the ENV values are all strings: - new_env = {} - for key, value in ENV.items(): - if is_List(value): - # If the value is a list, then we assume it is a path list, - # because that's a pretty common list-like value to stick - # in an environment variable: - value = SCons.Util.flatten_sequence(value) - new_env[key] = os.pathsep.join(map(str, value)) - else: - # It's either a string or something else. If it's a string, - # we still want to call str() because it might be a *Unicode* - # string, which makes subprocess.Popen() gag. If it isn't a - # string or a list, then we just coerce it to a string, which - # is the proper way to handle Dir and File instances and will - # produce something reasonable for just about everything else: - new_env[key] = str(value) - kw['env'] = new_env + kw['env'] = SCons.Util.sanitize_shell_env(ENV) try: pobj = subprocess.Popen(cmd, **kw) diff --git a/SCons/Platform/darwin.py b/SCons/Platform/darwin.py index f997a7d..d57f810 100644 --- a/SCons/Platform/darwin.py +++ b/SCons/Platform/darwin.py @@ -40,6 +40,7 @@ def generate(env): # env['ENV']['PATH'] = '/opt/local/bin:/opt/local/sbin:' + env['ENV']['PATH'] + ':/sw/bin' # Store extra system paths in env['ENV']['PATHOSX'] + env['ENV']['PATHOSX'] = '/opt/local/bin' filelist = ['/etc/paths',] # make sure this works on Macs with Tiger or earlier diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 583301e..7269fb2 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -285,6 +285,7 @@ def ninja_sorted_build(ninja, **build): sorted_dict = ninja_recursive_sorted_dict(build) ninja.build(**sorted_dict) + def get_command_env(env, target, source): """ Return a string that sets the environment for any environment variables that @@ -311,21 +312,8 @@ def get_command_env(env, target, source): windows = env["PLATFORM"] == "win32" command_env = "" + scons_specified_env = SCons.Util.sanitize_shell_env(scons_specified_env) for key, value in scons_specified_env.items(): - # Ensure that the ENV values are all strings: - if is_List(value): - # If the value is a list, then we assume it is a - # path list, because that's a pretty common list-like - # value to stick in an environment variable: - value = flatten_sequence(value) - value = joinpath(map(str, value)) - else: - # If it isn't a string or a list, then we just coerce - # it to a string, which is the proper way to handle - # Dir and File instances and will produce something - # reasonable for just about everything else: - value = str(value) - if windows: command_env += "set '{}={}' && ".format(key, value) else: diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index c4023c3..2622641 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -34,6 +34,7 @@ import SCons import SCons.Script import SCons.Tool.ninja.Globals from SCons.Script import GetOption +from SCons.Util import sanitize_shell_env from .Globals import NINJA_RULES, NINJA_POOLS, NINJA_CUSTOM_HANDLERS, NINJA_DEFAULT_TARGETS, NINJA_CMDLINE_TARGETS from .Methods import register_custom_handler, register_custom_rule_mapping, register_custom_rule, register_custom_pool, \ @@ -100,11 +101,16 @@ def ninja_builder(env, target, source): # reproduce the output like a ninja build would def execute_ninja(): + if env['PLATFORM'] == 'win32': + spawn_env = os.environ + else: + spawn_env = sanitize_shell_env(env['ENV']) + proc = subprocess.Popen(cmd, stderr=sys.stderr, stdout=subprocess.PIPE, universal_newlines=True, - env=os.environ if env["PLATFORM"] == "win32" else env['ENV'] + env=spawn_env ) for stdout_line in iter(proc.stdout.readline, ""): yield stdout_line diff --git a/SCons/Util.py b/SCons/Util.py index 7eb2005..c914af0 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2138,6 +2138,33 @@ class DispatchingFormatter(Formatter): formatter = self._formatters.get(record.name, self._default_formatter) return formatter.format(record) + +def sanitize_shell_env(env): + """ + Sanitize all values from env['ENV'] which will be propagated to the shell + :param env: The shell environment variables to be propagated to spawned shell + :return: sanitize dictionary of env variables (similar to what you'd get from os.environ) + """ + + # Ensure that the ENV values are all strings: + new_env = {} + for key, value in env.items(): + if is_List(value): + # If the value is a list, then we assume it is a path list, + # because that's a pretty common list-like value to stick + # in an environment variable: + value = flatten_sequence(value) + new_env[key] = os.pathsep.join(map(str, value)) + else: + # It's either a string or something else. If it's a string, + # we still want to call str() because it might be a *Unicode* + # string, which makes subprocess.Popen() gag. If it isn't a + # string or a list, then we just coerce it to a string, which + # is the proper way to handle Dir and File instances and will + # produce something reasonable for just about everything else: + new_env[key] = str(value) + return new_env + # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/test/ninja/shell_command.py b/test/ninja/shell_command.py index a6926c7..a6c48c4 100644 --- a/test/ninja/shell_command.py +++ b/test/ninja/shell_command.py @@ -53,6 +53,12 @@ SetOption('experimental','ninja') DefaultEnvironment(tools=[]) env = Environment() + +# Added to verify that SCons Ninja tool is sanitizing the shell environment +# before it spawns a new shell +env['ENV']['ZPATH']=['/a/b/c','/c/d/e'] + + env.Tool('ninja') prog = env.Program(target = 'foo', source = 'foo.c') env.Command('foo.out', prog, '%(shell)sfoo%(_exe)s > foo.out') -- cgit v0.12 From ef9879e9d8de55631e07a7e05510ca33aaf61185 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 3 Nov 2022 10:35:17 -0400 Subject: Changed argument name for sanitize_shell_env from env -> execution_env per feedback from mwichmann --- SCons/Util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/SCons/Util.py b/SCons/Util.py index c914af0..a7a6307 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -2139,16 +2139,16 @@ class DispatchingFormatter(Formatter): return formatter.format(record) -def sanitize_shell_env(env): +def sanitize_shell_env(execution_env): """ - Sanitize all values from env['ENV'] which will be propagated to the shell - :param env: The shell environment variables to be propagated to spawned shell + Sanitize all values in execution_env (typically this is env['ENV']) which will be propagated to the shell + :param execution_env: The shell environment variables to be propagated to spawned shell :return: sanitize dictionary of env variables (similar to what you'd get from os.environ) """ # Ensure that the ENV values are all strings: new_env = {} - for key, value in env.items(): + for key, value in execution_env.items(): if is_List(value): # If the value is a list, then we assume it is a path list, # because that's a pretty common list-like value to stick -- cgit v0.12 From c309375eb9972bab48cc8e84187a40c33a59e713 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sat, 5 Nov 2022 07:36:08 -0600 Subject: Maintenance: fix some fiddly checker errors A rawstring to avoid an invalid escape warning. A couple of instances where code is comparing type () calls, advice is to is isinstance() instead. Missing backslashes on escapes in framewok test (rarely run) Updating expected traceback msg in framework test after the code which forces the exception changed in a previous revision. Signed-off-by: Mats Wichmann --- .../docbook/docbook-xsl-1.76.1/extensions/docbook.py | 6 +++--- testing/framework/TestCommonTests.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py b/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py index d79ece3..17241bb 100644 --- a/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py +++ b/SCons/Tool/docbook/docbook-xsl-1.76.1/extensions/docbook.py @@ -157,7 +157,7 @@ def convertLength(length): global pixelsPerInch global unitHash - m = re.search('([+-]?[\d.]+)(\S+)', length) + m = re.search(r'([+-]?[\d.]+)(\S+)', length) if m is not None and m.lastindex > 1: unit = pixelsPerInch if m.group(2) in unitHash: @@ -204,11 +204,11 @@ def lookupVariable(tctxt, varName, default): return default # If it's a list, get the first element - if type(varString) == type([]): + if isinstance(varString, list): varString = varString[0] # If it's not a string, it must be a node, get its content - if type(varString) != type(""): + if not isinstance(varString, str): varString = varString.content return varString diff --git a/testing/framework/TestCommonTests.py b/testing/framework/TestCommonTests.py index 525f263..68f52cf 100644 --- a/testing/framework/TestCommonTests.py +++ b/testing/framework/TestCommonTests.py @@ -1880,7 +1880,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*fail \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -1940,13 +1940,13 @@ class run_TestCase(TestCommonTestCase): Traceback \\(most recent call last\\): File "", line \\d+, in (\\?|) File "[^"]+TestCommon.py", line \\d+, in run - TestCmd.run\\(self, \\*\\*kw\\) + super().run\\(\\*\\*kw\\) File "[^"]+TestCmd.py", line \\d+, in run - .* + p = self.start(program=program, File "[^"]+TestCommon.py", line \\d+, in start raise e File "[^"]+TestCommon.py", line \\d+, in start - return TestCmd.start\\(self, program, interpreter, arguments, + return super().start\\(program, interpreter, arguments, File "", line \\d+, in raise_exception TypeError: forced TypeError """ % re.escape(repr(sys.executable))) @@ -2066,7 +2066,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*pass \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2096,7 +2096,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*fail \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2128,7 +2128,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*pass \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) @@ -2162,7 +2162,7 @@ class run_TestCase(TestCommonTestCase): FAILED test of .*stderr \\tat line \\d+ of .*TestCommon\\.py \\(_complete\\) \\tfrom line \\d+ of .*TestCommon\\.py \\(run\\) - \\tfrom line \\d+ of ( \(\))? + \\tfrom line \\d+ of ( \\(\\))? """) expect_stderr = re.compile(expect_stderr, re.M) -- cgit v0.12 From e9fe1258201d598ce1feec35345d08bf524238fc Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 11 Nov 2022 11:05:19 -0700 Subject: Update setup/installation chapter in User Guide [skip appveyor] No longer tell people to install via "python setup.py install", possibly with arguments on library directories and paths. Simplify, and mention use of virtualenvs as the current way to have multiple SCons versions installed. Fixes #4259 Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 + RELEASE.txt | 3 + doc/user/build-install.xml | 428 ++++++++++++++++++--------------------------- 3 files changed, 180 insertions(+), 255 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0a9f697..e7ba3f3 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -52,6 +52,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER Fixes #4243. - Cleanup: make sure BoolVariable usage in tests and examples uses Python boolean values instead of 0/1. + - Stop telling people to run "python setup.py install" in the User Guide. + Adds new content on using virtualenvs to be able to have multiple + different SCons versions available on one system. + From Andrew Morrow - Avoid returning UniqueList for `children` and other `Executor` APIs. This type diff --git a/RELEASE.txt b/RELEASE.txt index 94ba72e..a796668 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -79,6 +79,9 @@ DOCUMENTATION Added note on the possibly surprising feature that SCons always passes -sourcepath when calling javac, which affects how the class path is used when finding sources. +- Updated the User Guide chapter on installation: modernized the notes + on Python installs, SCons installs, and having multiple SCons versions + present on a single system. DEVELOPMENT ----------- diff --git a/doc/user/build-install.xml b/doc/user/build-install.xml index c106dc5..697f5a2 100644 --- a/doc/user/build-install.xml +++ b/doc/user/build-install.xml @@ -1,4 +1,10 @@ + + @@ -22,46 +28,19 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Building and Installing &SCons; - - +Building and Installing &SCons; This chapter will take you through the basic steps - of installing &SCons; on your system, - and building &SCons; if you don't have a - pre-built package available - (or simply prefer the flexibility of building it yourself). + of installing &SCons; so you can use it for your your projects. Before that, however, this chapter will also describe the basic steps - involved in installing Python on your system, + involved in installing &Python; on your system, in case that is necessary. - Fortunately, both &SCons; and Python - are very easy to install on almost any system, - and Python already comes installed on many systems. + Fortunately, both &SCons; and &Python; + are easy to install on almost any system, + and &Python; already comes installed on many systems. @@ -70,31 +49,31 @@ Lastly, this chapter also contains a section that - provides a brief overview of the Python programming language, + provides a brief overview of the &Python; programming language, which is the language used to implement &SCons;, and which forms the basis of the &SCons; configuration files. - Becoming familiar with some Python concepts will make it easier + Becoming familiar with some &Python; concepts will make it easier to understand many of the examples in this User's Guide. Nevertheless, it is possible - to configure simple &SCons; builds without knowing Python, + to configure simple &SCons; builds without knowing &Python;, so you can skip this section if you want to dive in and pick up things by example- -or, of course, if you are - already familiar with Python. + already familiar with &Python;. --> -
                    +
                    Installing Python - Because &SCons; is written in Python, - you need to have Python installed on your system + Because &SCons; is written in the &Python; programming language, + you need to have a &Python; interpreter available on your system to use &SCons;. - Before you try to install Python, - you should check to see if Python is already + Before you try to install &Python;, + check to see if &Python; is already available on your system by typing python -V (capital 'V') @@ -106,28 +85,34 @@ $ python -V -Python 3.7.1 +Python 3.9.15 - Note to Windows users: there are a number of different ways Python + If you get a version like 2.7.x, you may need to try using the + name python3 - current &SCons; no longer + works with &Python; 2. + + + + Note to Windows users: there are a number of different ways &Python; can be installed or invoked on Windows, it is beyond the scope - of this guide to unravel all of them. Many will have an additional + of this guide to unravel all of them. Some have an additional program called the Python launcher (described, somewhat technically, in PEP 397): try using the command name py instead of python, if that is not available drop - back to trying python. + back to trying python C:\>py -V -Python 3.7.1 +Python 3.9.15 - If Python is not installed on your system, + If &Python; is not installed on your system, or is not findable in the current search path, you will see an error message stating something like "command not found" @@ -135,14 +120,14 @@ Python 3.7.1 or "'python' is not recognized as an internal or external command, operable progam or batch file" (on Windows cmd). - In that case, you need to either install Python + In that case, you need to either install &Python; or fix the search path before you can install &SCons;. - The canonical location for downloading Python - from Python's own website is: + The link for downloading &Python; installers (Windows and Mac) + from the project's own website is: https://www.python.org/download. There are useful system-specific entries on setup and usage to be found at: @@ -150,34 +135,56 @@ Python 3.7.1 - For Linux systems, Python is - almost certainly available as a supported package, possibly + For Linux systems, &Python; is + almost certainly available as a supported package, probably installed by default; this is often preferred over installing - by other means, and is easier than installing from source code. + by other means as the system package will be built with + carefully chosen optimizations, and will be kept up to date + with bug fixes and security patches. In fact, the &Python; + project itself does not build installers for Linux for this reason. Many such systems have separate packages for - Python 2 and Python 3 - make sure the Python 3 package is + &Python; 2 and &Python; 3 - make sure the &Python; 3 package is installed, as the latest &SCons; requires it. Building from source may still be a - useful option if you need a version that is not offered by + useful option if you need a specific version that is not offered by the distribution you are using. - &SCons; will work with Python 3.5 or later. - If you need to install Python and have a choice, - we recommend using the most recent Python version available. - Newer Pythons have significant improvements + Recent versions of the Mac no longer come with &Python; + pre-installed; older versions came with a rather out of date + version (based on &Python; 2.7) which is insufficient to run + current &SCons;. + The python.org installer can be used on the Mac, but there are + also other sources such as MacPorts and Homebrew. + The Anaconda installation also comes with a bundled &Python;. + + + + Windows has even more choices. The Python.org installer is + a traditional .exe style; + the same software is also released as a Windows application through + the Microsoft Store. Several alternative builds also exist + such as Chocolatey and ActiveState, and, again, + a version of Python comes with Anaconda. + + + + &SCons; will work with &Python; 3.6 or later. + If you need to install &Python; and have a choice, + we recommend using the most recent &Python; version available. + Newer &Python; versions have significant improvements that help speed up the performance of &SCons;.
                    -
                    +
                    Installing &SCons; - The canonical way to install &SCons; is from the Python Package - Index (PyPi): + The recommended way to install &SCons; is from the &Python; Package + Index (PyPI): @@ -185,9 +192,9 @@ Python 3.7.1 - If you prefer not to install to the Python system location, - or do not have privileges to do so, you can add a flag to - install to a location specific to your own account: + If you prefer not to install to the &Python; system location, + or do not have privileges to do so, you can add a flag to install + to a location specific to your own account and &Python; version: @@ -197,7 +204,7 @@ Python 3.7.1 For those users using Anaconda or Miniconda, use the conda installer instead, so the &scons; - install location will match the version of Python that + install location will match the version of &Python; that system will be using. For example: @@ -206,43 +213,61 @@ Python 3.7.1 - &SCons; comes pre-packaged for installation on many Linux systems. - Check your package installation system - to see if there is an &SCons; package available. - Many people prefer to install distribution-native packages if available, - as they provide a central point for management and updating. - During the still-ongoing Python 2 to 3 transition, - some distributions may still have two &SCons; packages available, - one which uses Python 2 and one which uses Python 3. Since - the latest &scons; only runs on Python 3, to get the current version - you should choose the Python 3 package. + If you need a specific + version of &SCons; that is different from the current version, + pip has a version option + (e.g. python -m pip install scons==3.1.2), + or you can follow the instructions in the following sections. - If you need a specific - version of &SCons; that is different from the package available, - pip has a version option or you can follow - the instructions in the next section. + &SCons; does comes pre-packaged for installation on many Linux systems. + Check your package installation system + to see if there is an up-to-date &SCons; package available. + Many people prefer to install distribution-native packages if available, + as they provide a central point for management and updating; + however not all distributions update in a timely fashion. + During the still-ongoing &Python; 2 to 3 transition, + some distributions may still have two &SCons; packages available, + one which uses &Python; 2 and one which uses &Python; 3. Since + the latest &scons; only runs on &Python; 3, to get the current version + you should choose the &Python; 3 package.
                    -
                    - Building and Installing &SCons; on Any System +
                    + Using &SCons; Without Installing - If a pre-built &SCons; package is not available for your system, - and installing using pip is not suitable, - then you can still easily build and install &SCons; using the native - Python setuptools package. + You don't actually need to "install" &SCons; to use it. + Nor do you need to "build" it, unless you are interested in + producing the &SCons; documentation, which does use several + tools to produce HTML, PDF and other output formats from + files in the source tree. + All you need to do is + call the scons.py driver script in a + location that contains an &SCons; tree, and it will figure out + the rest. You can test that like this: + +$ python /path/to/unpacked/scripts/scons.py --version + + - The first step is to download either the + To make use of an uninstalled &SCons;, + the first step is to download either the scons-&buildversion;.tar.gz or scons-&buildversion;.zip, which are available from the SCons download page at https://scons.org/pages/download.html. + There is also a scons-local bundle you can make + use of. It is arranged a little bit differently, with the idea + that you can include it with your own project if you want people + to be able to do builds without having to download or install &SCons;. + Finally, you can also use a checkout of the git tree from GitHub + at a location to point to. @@ -252,195 +277,88 @@ Python 3.7.1 or WinZip on Windows. This will create a directory called scons-&buildversion;, - usually in your local directory. - Then change your working directory to that directory - and install &SCons; by executing the following commands: + usually in your local directory. The driver script + will be in a subdirectory named scripts, + unless you are using scons-local, + in which case it will be in the top directory. + Now you only need to call scons.py by + giving a full or relative path to it in order to use that + &SCons; version. - -# cd scons-&buildversion; -# python setup.py install - - - - This will build &SCons;, - install the &scons; script - in the python which is used to run the setup.py's scripts directory - (/usr/local/bin or - C:\Python37\Scripts), - and will install the &SCons; build engine - in the corresponding library directory for the python used - (/usr/local/lib/scons or - C:\Python37\scons). - Because these are system directories, - you may need root (on Linux or UNIX) or Administrator (on Windows) - privileges to install &SCons; like this. - + Note that instructions for older versions may have suggested + running python setup.py install to + "build and install" &SCons;. This is no longer recommended + (in fact, it is not recommended by the wider &Python; packaging + community for any end-user installations + of &Python; software). There is a setup.py file, + but it is only tested and used for the automated procedure which + prepares an &SCons; bundle for making a release on PyPI, + and even that is not guaranteed to work in future. - - -
                    - Building and Installing Multiple Versions of &SCons; Side-by-Side - - +
                    - The &SCons; setup.py script - has some extensions that support - easy installation of multiple versions of &SCons; - in side-by-side locations. - This makes it easier to download and - experiment with different versions of &SCons; - before moving your official build process to a new version, - for example. - - +
                    + Running Multiple Versions of &SCons; Side-by-Side - - To install &SCons; in a version-specific location, - add the option - when you call setup.py: - + In some cases you may need several versions of &SCons; + present on a system at the same time - perhaps you have + an older project to build that has not yet been "ported" + to a newer &SCons; version, or maybe you want to test a + new &SCons; release side-by-side with a previous one + before switching over. + The use of an "uninstalled" package as described in the + previous section can be of use for this purpose. - -# python setup.py install --version-lib - - - - This will install the &SCons; build engine - in the - /usr/lib/scons-&buildversion; - or - C:\Python27\scons-&buildversion; - directory, for example. - + Another approach to multiple versions is to create + &Python; virtualenvs, and install different &SCons; versions in each. + A Python virtual environment + is a directory with an isolated set of Python packages, + where packages you install/upgrade/remove inside the + environment do not affect anything outside it, + and those you install/upgrade/remove outside of it + do not affect anything inside it. + In other words, anything you do with pip + in the environment stays in that environment. + The &Python; standard library provides a module called + venv for creating these + (), + although there are also other tools which provide more precise + control of the setup. - - If you use the option - the first time you install &SCons;, - you do not need to specify it each time you install - a new version. - The &SCons; setup.py script - will detect the version-specific directory name(s) - and assume you want to install all versions - in version-specific directories. - You can override that assumption in the future - by explicitly specifying the option. - + Using a virtualenv can be useful even for a single version of + &SCons;, to gain the advantages of having an isolated environment. + It also gets around the problem of not having administrative + privileges on a particular system to install a distribution + package or use pip to install to a + system location, as the virtualenv is completely under your control. -
                    - -
                    - Installing &SCons; in Other Locations - - - You can install &SCons; in locations other than - the default by specifying the option: - + The following outline shows how this could be set up + on a Linux/POSIX system (the syntax will be a bit different + on Windows): -# python setup.py install --prefix=/opt/scons +$ create virtualenv named scons3 +$ create virtualenv named scons4 +$ source scons3/bin/activate +$ pip install scons==3.1.2 +$ deactivate +$ source scons4/bin/activate +$ pip install scons +$ deactivate +$ activate a virtualenv and run 'scons' to use that version - - - This would - install the scons script in - /opt/scons/bin - and the build engine in - /opt/scons/lib/scons, - - - - - - Note that you can specify both the - and the options - at the same type, - in which case setup.py - will install the build engine - in a version-specific directory - relative to the specified prefix. - Adding to the - above example would install the build engine in - /opt/scons/lib/scons-&buildversion;. - - - -
                    - -
                    - Building and Installing &SCons; Without Administrative Privileges - - - - If you don't have the right privileges to install &SCons; - in a system location, - simply use the --prefix= option - to install it in a location of your choosing. - For example, - to install &SCons; in appropriate locations - relative to the user's $HOME directory, - the &scons; script in - $HOME/bin - and the build engine in - $HOME/lib/scons, - simply type: - - - - -$ python setup.py install --prefix=$HOME - - - - - You may, of course, specify any other location you prefer, - and may use the option - if you would like to install version-specific directories - relative to the specified prefix. - - - - - - This can also be used to experiment with a newer - version of &SCons; than the one installed - in your system locations. - Of course, the location in which you install the - newer version of the &scons; script - ($HOME/bin in the above example) - must be configured in your &PATH; variable - before the directory containing - the system-installed version - of the &scons; script. - - - -
                    -
                    - + , , @@ -2314,7 +2315,7 @@ repositories are searched in the order specified. SConscript Files The build configuration is described by one or more files, -known as SConscript files. +known as &SConscript; files. There must be at least one file for a valid build (&scons; will quit if it does not find one). &scons; by default looks for this file by the name @@ -2334,20 +2335,20 @@ included or excluded at run-time depending on how &scons; is invoked. -Each SConscript file in a build configuration is invoked +Each &SConscript; file in a build configuration is invoked independently in a separate context. This provides necessary isolation so that different parts of the build don't accidentally step on each other. You have to be explicit about sharing information, by using the &f-link-Export; function or the &exports; argument -to the &SConscript; function, as well as the &f-link-Return; function -in a called SConscript file, and comsume shared information by using the +to the &f-link-SConscript; function, as well as the &f-link-Return; function +in a called &SConscript; file, and comsume shared information by using the &f-link-Import; function. The following sections describe the various &SCons; facilities -that can be used in SConscript files. Quick links: +that can be used in &SConscript; files. Quick links: @@ -2546,7 +2547,7 @@ See for details. (more properly, tool specification modules) which are used to help initialize the &consenv;. An &SCons; tool is only responsible for setup. -For example, if an SConscript file declares +For example, if an &SConscript; file declares the need to construct an object file from a C-language source file by calling the &b-link-Object; builder, then a tool representing @@ -2734,7 +2735,7 @@ contains separator characters, the search follows down from the starting point, which is the top of the directory tree for an absolute path and the current directory for a relative path. The "current directory" in this context is the directory -of the SConscript file currently being processed. +of the &SConscript; file currently being processed. @@ -2868,7 +2869,7 @@ target file's directory. -Python only keeps one current directory +&Python; only keeps one current directory location even if there are multiple threads. This means that use of the chdir @@ -2879,7 +2880,7 @@ work with the SCons option, because individual worker threads spawned -by SCons interfere with each other +by &SCons; interfere with each other when they start changing directory. @@ -2973,7 +2974,7 @@ has determined are appropriate for the local system. environment (indicated in the listing of builders below without a leading env.) may be called from custom &Python; modules that you -import into an SConscript file by adding the following +import into an &SConscript; file by adding the following to the &Python; module: @@ -3114,13 +3115,13 @@ the object file. When trying to handle errors that may occur in a builder method, consider that the corresponding Action is executed at a different -time than the SConscript file statement calling the builder. +time than the &SConscript; file statement calling the builder. It is not useful to wrap a builder call in a try block, since success in the builder call is not the same as the builder itself succeeding. If necessary, a Builder's Action should be coded to exit with -a useful exception message indicating the problem in the SConscript files - +a useful exception message indicating the problem in the &SConscript; files - programmatically recovering from build errors is rarely useful. @@ -3276,7 +3277,7 @@ see the next section on &consvars;. Global functions may be called from custom Python modules that you -import into an SConscript file by adding the following import +import into an &SConscript; file by adding the following import to the Python module: @@ -3316,7 +3317,7 @@ include: In addition to the global functions and methods, &scons; supports a number of variables -that can be used in SConscript files +that can be used in &SConscript; files to affect how you want the build to be performed. @@ -3489,7 +3490,7 @@ print([str(t) for t in DEFAULT_TARGETS]) # back to [] only after you've made all of your &Default;() calls, or else simply be careful of the order -of these statements in your SConscript files +of these statements in your &SConscript; files so that you don't look for a specific default target before it's actually been added to the list. @@ -3498,7 +3499,7 @@ default target before it's actually been added to the list. These variables may be accessed from custom Python modules that you -import into an SConscript file by adding the following +import into an &SConscript; file by adding the following to the Python module: @@ -3654,14 +3655,16 @@ discovered while running tests. The context includes a local &consenv; which is used when running the tests and which can be updated with the check results. Only one context may be active -at a time (since 4.0, &scons; will raise an exception -on an attempt to create a new context when there is -an active context), but a new context can be created +at a time, but a new context can be created after the active one is completed. For the global function form, the required env describes the initial values for the context's local &consenv;; for the &consenv; method form the instance provides the values. + +Changed in version 4.0: raises an exception +on an attempt to create a new context when there is an active context. + custom_tests specifies a dictionary containing custom tests (see the section on custom tests below). @@ -4426,7 +4429,8 @@ from various sources, often from the command line: scons VARIABLE=foo -The variable values can also be specified in a configuration file or an SConscript file. +The variable values can also be specified in a configuration file +or an &SConscript; file. To obtain the object for manipulating values, call the &Variables; function: @@ -5062,7 +5066,7 @@ Builders return the target Node(s) in the form of a list, which you can then make use of. However, since filesystem Nodes have some useful public attributes and methods -that you can use in SConscript files, +that you can use in &SConscript; files, it is sometimes appropriate to create them manually, outside the regular context of a Builder call. @@ -6394,7 +6398,7 @@ are intended to be used. First, if you need to perform the action -at the time the SConscript +at the time the &SConscript; file is being read, you can use the &f-link-Execute; @@ -6763,8 +6767,8 @@ builder action. The value of env['BAR'] will be exactly as it was set: "$FOO baz". This can make debugging tricky, as the substituted result is not available at the time -the SConscript files are being interpreted and -thus not available to print(). +the &SConscript; files are being interpreted and +thus not available to the print function. However, you can perform the substitution on demand by calling the &f-link-env-subst; method for this purpose. @@ -7041,9 +7045,9 @@ echo BAR > foo.out according to the current value of env['COND'] when the command is executed. The evaluation takes place when the target is being -built, not when the SConscript is being read. So if +built, not when the &SConscript; is being read. So if env['COND'] is changed -later in the SConscript, the final value will be used. +later in the &SConscript;, the final value will be used. Here's a more complete example. Note that all of COND, @@ -7078,7 +7082,7 @@ which &SCons; passes to eval which returns the value. - + Use of the Python eval function is considered to have security implications, since, depending on input sources, @@ -7086,7 +7090,7 @@ arbitrary unchecked strings of code can be executed by the Python interpreter. Although &SCons; makes use of it in a somewhat restricted context, you should be aware of this issue when using the ${python-expression-for-subst} form. - + @@ -7229,7 +7233,7 @@ it will be expanded into a list by the current environment. A Python function that takes four or five arguments: a &consenv;, a Node for the directory containing -the SConscript file in which +the &SConscript; file in which the first target was defined, a list of target nodes, a list of source nodes, @@ -7713,7 +7717,7 @@ may yield unpredictable results. "Mixing and matching" in this way can be made to work, but it requires careful attention to the use of path names -in your SConscript files. +in your &SConscript; files. In practice, users can sidestep the issue by adopting the following guidelines: @@ -8092,7 +8096,7 @@ scanner = Scanner( Creating a Hierarchical Build Notice that the file names specified in a subdirectory's -SConscript file are relative to that subdirectory. +&SConscript; file are relative to that subdirectory. SConstruct: @@ -8128,7 +8132,7 @@ env.Program(target='foo', source='foo.c') You must explicitly call &f-link-Export; and &f-link-Import; for variables that -you want to share between SConscript files. +you want to share between &SConscript; files. SConstruct: @@ -8179,7 +8183,7 @@ env.Program(target='src', source='src.c') Note the use of the &f-link-Export; method to set the cppdefines variable to a different -value each time we call the &SConscriptFunc; function. +value each time we call the &f-link-SConscript; function. @@ -8362,7 +8366,7 @@ env.Program('MyApp', ['Foo.cpp', 'Bar.cpp']) In general, &scons; is not controlled by environment variables set in the shell used to invoke it, leaving it -up to the SConscript file author to import those if desired. +up to the &SConscript; file author to import those if desired. However the following variables are imported by &scons; itself if set: -- cgit v0.12 From b6acf5ee664d1d4440bc02be33d4cf0f155e349f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 8 Dec 2022 09:10:06 -0700 Subject: Tweak no-exec manpage wording [skip appveyor] Minor change - this was one of the options where the xml id was changed, and the previous wording just didn't seem to be that smooth. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 4d72180..8dda049 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -1611,24 +1611,29 @@ no matter how old the file is. -Set no execute mode. +Set no-exec mode. Print the commands that would be executed to build -any out-of-date target files, but do not execute the commands. - -The output is a best effort, as &SCons; cannot always precisely -determine what would be built. For example, if a file is generated -by a builder action that is later used in the build, -that file is not available to scan for dependencies on an unbuilt tree, -or may contain out of date information in a built tree. +any out-of-date targets, but do not execute those commands. + -Work which is done directly in &Python; code in &SConscript; files, -as opposed to work done by builder actions during the build phase, -will still be carried out. If it is important to avoid some -such work from taking place in no execute mode, it should be protected. -An &SConscript; file can determine which mode -is active by querying &f-link-GetOption;, as in the call -if GetOption("no_exec"): +Only target building is suppressed - any work in the build +system that is done directly (in regular &Python; code) +will still be carried out. You can add guards around +code which should not be executed in no-exec mode by +checking the value of the option at run time with &f-link-GetOption;: + + +if not GetOption("no_exec"): + # run regular instructions + + +The output is a best effort, as &SCons; cannot always precisely +determine what would be built. For example, if a file generated +by a builder action is also used as a source in the build, +that file is not available to scan for dependencies at all +in an unbuilt tree, and may contain out of date information in a +previously built tree. @@ -1682,7 +1687,7 @@ directories to the toolpath. -The type +The type of package to create when using the &b-link-Package; builder. Multiple types can be specified by using a comma-separated string, in which case &SCons; will try to build for all of those package types. -- cgit v0.12 From d04f2b3a804ee5dca41608c7ff44404b681b8fc8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 10 Dec 2022 17:08:12 -1000 Subject: Solution for Issue #4275 (Compilation db get's tmp file command line), also extending serveral APIs to allow specifying an overrides dictionary which would override (at the last minute) any envvar or potentially env TARGET/SOURCE when subst is being called a subst() or subst_list(). Tests created. Docs still need to be updated. --- SCons/Action.py | 14 +++++++------- SCons/ActionTests.py | 8 ++++---- SCons/Environment.py | 8 ++++---- SCons/Subst.py | 13 +++++++++++-- SCons/SubstTests.py | 15 +++++++++++++++ SCons/Tool/compilation_db.py | 9 +++++++++ 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/SCons/Action.py b/SCons/Action.py index 8151b2a..1f5e548 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -875,11 +875,11 @@ class CommandAction(_ActionAction): return ' '.join(map(str, self.cmd_list)) return str(self.cmd_list) - def process(self, target, source, env, executor=None): + def process(self, target, source, env, executor=None, overrides=False): if executor: - result = env.subst_list(self.cmd_list, 0, executor=executor) + result = env.subst_list(self.cmd_list, 0, executor=executor, overrides=overrides) else: - result = env.subst_list(self.cmd_list, 0, target, source) + result = env.subst_list(self.cmd_list, 0, target, source, overrides=overrides) silent = None ignore = None while True: @@ -896,18 +896,18 @@ class CommandAction(_ActionAction): pass return result, ignore, silent - def strfunction(self, target, source, env, executor=None): + def strfunction(self, target, source, env, executor=None, overrides=False): if self.cmdstr is None: return None if self.cmdstr is not _null: from SCons.Subst import SUBST_RAW if executor: - c = env.subst(self.cmdstr, SUBST_RAW, executor=executor) + c = env.subst(self.cmdstr, SUBST_RAW, executor=executor, overrides=overrides) else: - c = env.subst(self.cmdstr, SUBST_RAW, target, source) + c = env.subst(self.cmdstr, SUBST_RAW, target, source, overrides=overrides) if c: return c - cmd_list, ignore, silent = self.process(target, source, env, executor) + cmd_list, ignore, silent = self.process(target, source, env, executor, overrides=overrides) if silent: return '' return _string_from_cmd_list(cmd_list[0]) diff --git a/SCons/ActionTests.py b/SCons/ActionTests.py index 0a7f25b..101953b 100644 --- a/SCons/ActionTests.py +++ b/SCons/ActionTests.py @@ -142,15 +142,15 @@ class Environment: self.d[k] = v # Just use the underlying scons_subst*() utility methods. - def subst(self, strSubst, raw=0, target=[], source=[], conv=None): + def subst(self, strSubst, raw=0, target=[], source=[], conv=None, overrides=False): return SCons.Subst.scons_subst(strSubst, self, raw, - target, source, self.d, conv=conv) + target, source, self.d, conv=conv, overrides=overrides) subst_target_source = subst - def subst_list(self, strSubst, raw=0, target=[], source=[], conv=None): + def subst_list(self, strSubst, raw=0, target=[], source=[], conv=None, overrides=False): return SCons.Subst.scons_subst_list(strSubst, self, raw, - target, source, self.d, conv=conv) + target, source, self.d, conv=conv, overrides=overrides) def __getitem__(self, item): return self.d[item] diff --git a/SCons/Environment.py b/SCons/Environment.py index 293447f..7212c89 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -481,7 +481,7 @@ class SubstitutionEnvironment: def lvars(self): return {} - def subst(self, string, raw=0, target=None, source=None, conv=None, executor=None): + def subst(self, string, raw=0, target=None, source=None, conv=None, executor=None, overrides=False): """Recursively interpolates construction variables from the Environment into the specified string, returning the expanded result. Construction variables are specified by a $ prefix @@ -496,7 +496,7 @@ class SubstitutionEnvironment: lvars['__env__'] = self if executor: lvars.update(executor.get_lvars()) - return SCons.Subst.scons_subst(string, self, raw, target, source, gvars, lvars, conv) + return SCons.Subst.scons_subst(string, self, raw, target, source, gvars, lvars, conv, overrides=overrides) def subst_kw(self, kw, raw=0, target=None, source=None): nkw = {} @@ -507,7 +507,7 @@ class SubstitutionEnvironment: nkw[k] = v return nkw - def subst_list(self, string, raw=0, target=None, source=None, conv=None, executor=None): + def subst_list(self, string, raw=0, target=None, source=None, conv=None, executor=None, overrides=False): """Calls through to SCons.Subst.scons_subst_list(). See the documentation for that function.""" gvars = self.gvars() @@ -515,7 +515,7 @@ class SubstitutionEnvironment: lvars['__env__'] = self if executor: lvars.update(executor.get_lvars()) - return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv) + return SCons.Subst.scons_subst_list(string, self, raw, target, source, gvars, lvars, conv, overrides=overrides) def subst_path(self, path, target=None, source=None): """Substitute a path list, turning EntryProxies into Nodes diff --git a/SCons/Subst.py b/SCons/Subst.py index 7535772..645639b 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -804,7 +804,8 @@ _separate_args = re.compile(r'(%s|\s+|[^\s$]+|\$)' % _dollar_exps_str) # space characters in the string result from the scons_subst() function. _space_sep = re.compile(r'[\t ]+(?![^{]*})') -def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None): + +def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None, overrides=False): """Expand a string or list containing construction variable substitutions. @@ -834,6 +835,10 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ lvars = lvars.copy() lvars.update(d) + # Allow last ditch chance to override lvars + if overrides: + lvars.update(overrides) + # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), # it copies the current global values for you. Avoid this by @@ -882,7 +887,7 @@ def scons_subst(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={ return result -def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None): +def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gvars={}, lvars={}, conv=None,overrides=False): """Substitute construction variables in a string (or list or other object) and separate the arguments into a command list. @@ -908,6 +913,10 @@ def scons_subst_list(strSubst, env, mode=SUBST_RAW, target=None, source=None, gv lvars = lvars.copy() lvars.update(d) + # Allow caller to specify last ditch override of lvars + if overrides: + lvars.update(overrides) + # We're (most likely) going to eval() things. If Python doesn't # find a __builtins__ value in the global dictionary used for eval(), # it copies the current global values for you. Avoid this by diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index 1a203a0..16bd3c7 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -713,6 +713,14 @@ class CLVar_TestCase(unittest.TestCase): assert cmd_list[0][3] == "call", cmd_list[0][3] assert cmd_list[0][4] == "test", cmd_list[0][4] + + def test_subst_overriding_lvars_overrides(self): + """Test that optional passed arg overrides overrides gvars, and existing lvars.""" + env=DummyEnv({'XXX' : 'xxx'}) + result = scons_subst('$XXX', env, gvars=env.Dictionary(), overrides={'XXX': 'yyz'}) + assert result == 'yyz', result + + class scons_subst_list_TestCase(SubstTestCase): basic_cases = [ @@ -1102,6 +1110,13 @@ class scons_subst_list_TestCase(SubstTestCase): result = scons_subst_list('$XXX', env, gvars={'XXX' : 'yyy'}) assert result == [['yyy']], result + def test_subst_list_overriding_lvars_overrides(self): + """Test that optional passed arg overrides overrides gvars, and existing lvars.""" + env = DummyEnv({'XXX':'xxx'}) + result = scons_subst_list('$XXX', env, gvars=env.Dictionary(), overrides={'XXX': 'yyy'}) + assert result == [['yyy']], result + + class scons_subst_once_TestCase(unittest.TestCase): loc = { diff --git a/SCons/Tool/compilation_db.py b/SCons/Tool/compilation_db.py index 9e72fe9..a4954c1 100644 --- a/SCons/Tool/compilation_db.py +++ b/SCons/Tool/compilation_db.py @@ -34,6 +34,8 @@ import itertools import fnmatch import SCons +from SCons.Platform import TempFileMunge + from .cxx import CXXSuffixes from .cc import CSuffixes from .asm import ASSuffixes, ASPPSuffixes @@ -53,6 +55,7 @@ class __CompilationDbNode(SCons.Node.Python.Value): SCons.Node.Python.Value.__init__(self, value) self.Decider(changed_since_last_build_node) + def changed_since_last_build_node(child, target, prev_ni, node): """ Dummy decider to force always building""" return True @@ -103,6 +106,11 @@ def make_emit_compilation_DB_entry(comstr): return emit_compilation_db_entry +class CompDBTEMPFILE(TempFileMunge): + def __call__(self, target, source, env, for_signature): + return self.cmd + + def compilation_db_entry_action(target, source, env, **kw): """ Create a dictionary with evaluated command line, target, source @@ -119,6 +127,7 @@ def compilation_db_entry_action(target, source, env, **kw): target=env["__COMPILATIONDB_UOUTPUT"], source=env["__COMPILATIONDB_USOURCE"], env=env["__COMPILATIONDB_ENV"], + overrides={'TEMPFILE': CompDBTEMPFILE} ) entry = { -- cgit v0.12 From e3403f4ac807e22a5cadb716f76b4d9f48f44211 Mon Sep 17 00:00:00 2001 From: TZe Date: Mon, 12 Dec 2022 09:50:18 +0100 Subject: Added specific test for Compilation Database to reproduce issue #4275 with TempFileMunge --- .../fixture/SConstruct_tempfile | 45 ++++++++++ test/CompilationDatabase/tmpfile.py | 96 ++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 test/CompilationDatabase/fixture/SConstruct_tempfile create mode 100644 test/CompilationDatabase/tmpfile.py diff --git a/test/CompilationDatabase/fixture/SConstruct_tempfile b/test/CompilationDatabase/fixture/SConstruct_tempfile new file mode 100644 index 0000000..d7c2efd --- /dev/null +++ b/test/CompilationDatabase/fixture/SConstruct_tempfile @@ -0,0 +1,45 @@ +import sys + +DefaultEnvironment(tools=[]) +env = Environment( + PYTHON=sys.executable, + LINK='$PYTHON mylink.py', + LINKFLAGS=[], + CC='$PYTHON mygcc.py cc', + CXX='$PYTHON mygcc.py c++', + tools=['gcc','g++','gnulink'], + MAXLINELENGTH=10, +) +# make sure TempFileMunge is used +if 'TEMPFILE' not in env['CCCOM']: + env['CCCOM'] = '${TEMPFILE("%s")}'%(env['CCCOM']) + +env.Tool('compilation_db') + +outputs = [] +env_abs = env.Clone(COMPILATIONDB_USE_ABSPATH=True) +outputs+= env_abs.CompilationDatabase('compile_commands_clone_abs.json') + +# Should be relative paths +outputs+= env.CompilationDatabase('compile_commands_only_arg.json') +outputs+= env.CompilationDatabase(target='compile_commands_target.json') + +# Should default name compile_commands.json +outputs+= env.CompilationDatabase() + +# Should be absolute paths +outputs+= env.CompilationDatabase('compile_commands_over_abs.json', COMPILATIONDB_USE_ABSPATH=True) +outputs+= env.CompilationDatabase(target='compile_commands_target_over_abs.json', COMPILATIONDB_USE_ABSPATH=True) + +# Should be relative paths +outputs+= env.CompilationDatabase('compile_commands_over_rel.json', COMPILATIONDB_USE_ABSPATH=False) + +# Try 1/0 for COMPILATIONDB_USE_ABSPATH +outputs+= env.CompilationDatabase('compile_commands_over_abs_1.json', COMPILATIONDB_USE_ABSPATH=1) +outputs+= env.CompilationDatabase('compile_commands_over_abs_0.json', COMPILATIONDB_USE_ABSPATH=0) + +env.Program('main', 'test_main.c') + +# Prevent actual call of $PYTHON @tempfile since "mygcc.py cc ..." is not a proper python statement +# Interesting outputs are json databases +env.Default(outputs) diff --git a/test/CompilationDatabase/tmpfile.py b/test/CompilationDatabase/tmpfile.py new file mode 100644 index 0000000..ea28a07 --- /dev/null +++ b/test/CompilationDatabase/tmpfile.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +""" +Test CompilationDatabase and several variations of ways to call it +and values of COMPILATIONDB_USE_ABSPATH +""" + +import sys +import os +import os.path +import TestSCons + +test = TestSCons.TestSCons() + +test.file_fixture('mylink.py') +test.file_fixture('mygcc.py') + +test.verbose_set(1) +test.file_fixture('fixture/SConstruct_tempfile', 'SConstruct') +test.file_fixture('test_main.c') +test.run() + +rel_files = [ + 'compile_commands_only_arg.json', + 'compile_commands_target.json', + 'compile_commands.json', + 'compile_commands_over_rel.json', + 'compile_commands_over_abs_0.json' +] + +abs_files = [ + 'compile_commands_clone_abs.json', + 'compile_commands_over_abs.json', + 'compile_commands_target_over_abs.json', + 'compile_commands_over_abs_1.json', +] + +example_rel_file = """[ + { + "command": "%s mygcc.py cc -o test_main.o -c test_main.c", + "directory": "%s", + "file": "test_main.c", + "output": "test_main.o" + } +] +""" % (sys.executable, test.workdir) + +if sys.platform == 'win32': + example_rel_file = example_rel_file.replace('\\', '\\\\') + +for f in rel_files: + # print("Checking:%s" % f) + test.must_exist(f) + test.must_match(f, example_rel_file, mode='r') + +example_abs_file = """[ + { + "command": "%s mygcc.py cc -o test_main.o -c test_main.c", + "directory": "%s", + "file": "%s", + "output": "%s" + } +] +""" % (sys.executable, test.workdir, os.path.join(test.workdir, 'test_main.c'), os.path.join(test.workdir, 'test_main.o')) + +if sys.platform == 'win32': + example_abs_file = example_abs_file.replace('\\', '\\\\') + + +for f in abs_files: + test.must_exist(f) + test.must_match(f, example_abs_file, mode='r') + + +test.pass_test() -- cgit v0.12 From d83d7cf50d95bf1959c984d2467f96dfdb56472b Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 14 Dec 2022 14:54:55 -0700 Subject: Turn Util module into a package Change Util.py into a package. Some routines are split out into separate files in this package. Signed-off-by: Mats Wichmann --- CHANGES.txt | 3 + SCons/Util.py | 2170 ------------------------------------------------ SCons/Util/__init__.py | 1325 +++++++++++++++++++++++++++++ SCons/Util/envs.py | 325 ++++++++ SCons/Util/hashes.py | 401 +++++++++ SCons/Util/types.py | 309 +++++++ SCons/UtilTests.py | 10 +- 7 files changed, 2369 insertions(+), 2174 deletions(-) delete mode 100644 SCons/Util.py create mode 100644 SCons/Util/__init__.py create mode 100644 SCons/Util/envs.py create mode 100644 SCons/Util/hashes.py create mode 100644 SCons/Util/types.py diff --git a/CHANGES.txt b/CHANGES.txt index f2a42e2..04ce9e7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -86,6 +86,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER still run into problems on some configurations), and automated tests are now run on changes in this area so future problems can be spotted. + - The single-file Util module was split into a package with a few + functional areas getting their own files - Util.py had grown to + over 2100 lines. diff --git a/SCons/Util.py b/SCons/Util.py deleted file mode 100644 index 33dc0d3..0000000 --- a/SCons/Util.py +++ /dev/null @@ -1,2170 +0,0 @@ -# MIT License -# -# Copyright The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""Various SCons utility functions.""" - -import copy -import hashlib -import os -import pprint -import re -import sys -import time -from collections import UserDict, UserList, UserString, OrderedDict -from collections.abc import MappingView -from contextlib import suppress -from types import MethodType, FunctionType -from typing import Optional, Union -from logging import Formatter - -# Note: Util module cannot import other bits of SCons globally without getting -# into import loops. Both the below modules import SCons.Util early on. -# --> SCons.Warnings -# --> SCons.Errors -# Thus the local imports, which are annotated for pylint to show we mean it. - - -PYPY = hasattr(sys, 'pypy_translation_info') - -# this string will be hashed if a Node refers to a file that doesn't exist -# in order to distinguish from a file that exists but is empty. -NOFILE = "SCONS_MAGIC_MISSING_FILE_STRING" - -# unused? -def dictify(keys, values, result=None) -> dict: - if result is None: - result = {} - result.update(zip(keys, values)) - return result - -_ALTSEP = os.altsep -if _ALTSEP is None and sys.platform == 'win32': - # My ActivePython 2.0.1 doesn't set os.altsep! What gives? - _ALTSEP = '/' -if _ALTSEP: - def rightmost_separator(path, sep): - return max(path.rfind(sep), path.rfind(_ALTSEP)) -else: - def rightmost_separator(path, sep): - return path.rfind(sep) - -# First two from the Python Cookbook, just for completeness. -# (Yeah, yeah, YAGNI...) -def containsAny(s, pat) -> bool: - """Check whether string `s` contains ANY of the items in `pat`.""" - return any(c in s for c in pat) - -def containsAll(s, pat) -> bool: - """Check whether string `s` contains ALL of the items in `pat`.""" - return all(c in s for c in pat) - -def containsOnly(s, pat) -> bool: - """Check whether string `s` contains ONLY items in `pat`.""" - for c in s: - if c not in pat: - return False - return True - - -# TODO: Verify this method is STILL faster than os.path.splitext -def splitext(path) -> tuple: - """Split `path` into a (root, ext) pair. - - Same as :mod:`os.path.splitext` but faster. - """ - sep = rightmost_separator(path, os.sep) - dot = path.rfind('.') - # An ext is only real if it has at least one non-digit char - if dot > sep and not path[dot + 1:].isdigit(): - return path[:dot], path[dot:] - - return path, "" - -def updrive(path) -> str: - """Make the drive letter (if any) upper case. - - This is useful because Windows is inconsistent on the case - of the drive letter, which can cause inconsistencies when - calculating command signatures. - """ - drive, rest = os.path.splitdrive(path) - if drive: - path = drive.upper() + rest - return path - -class NodeList(UserList): - """A list of Nodes with special attribute retrieval. - - Unlike an ordinary list, access to a member's attribute returns a - `NodeList` containing the same attribute for each member. Although - this can hold any object, it is intended for use when processing - Nodes, where fetching an attribute of each member is very commone, - for example getting the content signature of each node. The term - "attribute" here includes the string representation. - - Example: - - >>> someList = NodeList([' foo ', ' bar ']) - >>> someList.strip() - ['foo', 'bar'] - """ - - def __bool__(self): - return bool(self.data) - - def __str__(self): - return ' '.join(map(str, self.data)) - - def __iter__(self): - return iter(self.data) - - def __call__(self, *args, **kwargs) -> 'NodeList': - result = [x(*args, **kwargs) for x in self.data] - return self.__class__(result) - - def __getattr__(self, name) -> 'NodeList': - """Returns a NodeList of `name` from each member.""" - result = [getattr(x, name) for x in self.data] - return self.__class__(result) - - def __getitem__(self, index): - """Returns one item, forces a `NodeList` if `index` is a slice.""" - # TODO: annotate return how? Union[] - don't know type of single item - if isinstance(index, slice): - return self.__class__(self.data[index]) - return self.data[index] - - -_get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') - -def get_environment_var(varstr) -> Optional[str]: - """Return undecorated construction variable string. - - Determine if `varstr` looks like a reference - to a single environment variable, like `"$FOO"` or `"${FOO}"`. - If so, return that variable with no decorations, like `"FOO"`. - If not, return `None`. - """ - - mo = _get_env_var.match(to_String(varstr)) - if mo: - var = mo.group(1) - if var[0] == '{': - return var[1:-1] - return var - - return None - - -class DisplayEngine: - """A callable class used to display SCons messages.""" - - print_it = True - - def __call__(self, text, append_newline=1): - if not self.print_it: - return - - if append_newline: - text = text + '\n' - - try: - sys.stdout.write(str(text)) - except IOError: - # Stdout might be connected to a pipe that has been closed - # by now. The most likely reason for the pipe being closed - # is that the user has press ctrl-c. It this is the case, - # then SCons is currently shutdown. We therefore ignore - # IOError's here so that SCons can continue and shutdown - # properly so that the .sconsign is correctly written - # before SCons exits. - pass - - def set_mode(self, mode): - self.print_it = mode - - -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) -def render_tree(root, child_func, prune=0, margin=[0], visited=None): - """Render a tree of nodes into an ASCII tree view. - - Args: - root: the root node of the tree - child_func: the function called to get the children of a node - prune: don't visit the same node twice - margin: the format of the left margin to use for children of `root`. - 1 results in a pipe, and 0 results in no pipe. - visited: a dictionary of visited nodes in the current branch if - `prune` is 0, or in the whole tree if `prune` is 1. - """ - - rname = str(root) - - # Initialize 'visited' dict, if required - if visited is None: - visited = {} - - children = child_func(root) - retval = "" - for pipe in margin[:-1]: - if pipe: - retval = retval + "| " - else: - retval = retval + " " - - if rname in visited: - return retval + "+-[" + rname + "]\n" - - retval = retval + "+-" + rname + "\n" - if not prune: - visited = copy.copy(visited) - visited[rname] = True - - for i, child in enumerate(children): - margin.append(i < len(children)-1) - retval = retval + render_tree(child, child_func, prune, margin, visited) - margin.pop() - - return retval - -def IDX(n) -> bool: - """Generate in index into strings from the tree legends. - - These are always a choice between two, so bool works fine. - """ - return bool(n) - -# unicode line drawing chars: -BOX_HORIZ = chr(0x2500) # '─' -BOX_VERT = chr(0x2502) # '│' -BOX_UP_RIGHT = chr(0x2514) # 'â””' -BOX_DOWN_RIGHT = chr(0x250c) # '┌' -BOX_DOWN_LEFT = chr(0x2510) # 'â”' -BOX_UP_LEFT = chr(0x2518) # '┘' -BOX_VERT_RIGHT = chr(0x251c) # '├' -BOX_HORIZ_DOWN = chr(0x252c) # '┬' - - -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) -def print_tree( - root, - child_func, - prune=0, - showtags=False, - margin=[0], - visited=None, - lastChild=False, - singleLineDraw=False, -): - """Print a tree of nodes. - - This is like func:`render_tree`, except it prints lines directly instead - of creating a string representation in memory, so that huge trees can - be handled. - - Args: - root: the root node of the tree - child_func: the function called to get the children of a node - prune: don't visit the same node twice - showtags: print status information to the left of each node line - margin: the format of the left margin to use for children of `root`. - 1 results in a pipe, and 0 results in no pipe. - visited: a dictionary of visited nodes in the current branch if - prune` is 0, or in the whole tree if `prune` is 1. - singleLineDraw: use line-drawing characters rather than ASCII. - """ - - rname = str(root) - - # Initialize 'visited' dict, if required - if visited is None: - visited = {} - - if showtags: - - if showtags == 2: - legend = (' E = exists\n' + - ' R = exists in repository only\n' + - ' b = implicit builder\n' + - ' B = explicit builder\n' + - ' S = side effect\n' + - ' P = precious\n' + - ' A = always build\n' + - ' C = current\n' + - ' N = no clean\n' + - ' H = no cache\n' + - '\n') - sys.stdout.write(legend) - - tags = [ - '[', - ' E'[IDX(root.exists())], - ' R'[IDX(root.rexists() and not root.exists())], - ' BbB'[ - [0, 1][IDX(root.has_explicit_builder())] + - [0, 2][IDX(root.has_builder())] - ], - ' S'[IDX(root.side_effect)], - ' P'[IDX(root.precious)], - ' A'[IDX(root.always_build)], - ' C'[IDX(root.is_up_to_date())], - ' N'[IDX(root.noclean)], - ' H'[IDX(root.nocache)], - ']' - ] - - else: - tags = [] - - def MMM(m): - if singleLineDraw: - return [" ", BOX_VERT + " "][m] - - return [" ", "| "][m] - - margins = list(map(MMM, margin[:-1])) - children = child_func(root) - cross = "+-" - if singleLineDraw: - cross = BOX_VERT_RIGHT + BOX_HORIZ # sign used to point to the leaf. - # check if this is the last leaf of the branch - if lastChild: - #if this if the last leaf, then terminate: - cross = BOX_UP_RIGHT + BOX_HORIZ # sign for the last leaf - - # if this branch has children then split it - if children: - # if it's a leaf: - if prune and rname in visited and children: - cross += BOX_HORIZ - else: - cross += BOX_HORIZ_DOWN - - if prune and rname in visited and children: - sys.stdout.write(''.join(tags + margins + [cross,'[', rname, ']']) + '\n') - return - - sys.stdout.write(''.join(tags + margins + [cross, rname]) + '\n') - - visited[rname] = 1 - - # if this item has children: - if children: - margin.append(1) # Initialize margin with 1 for vertical bar. - idx = IDX(showtags) - _child = 0 # Initialize this for the first child. - for C in children[:-1]: - _child = _child + 1 # number the children - print_tree( - C, - child_func, - prune, - idx, - margin, - visited, - (len(children) - _child) <= 0, - singleLineDraw, - ) - # margins are with space (index 0) because we arrived to the last child. - margin[-1] = 0 - # for this call child and nr of children needs to be set 0, to signal the second phase. - print_tree(children[-1], child_func, prune, idx, margin, visited, True, singleLineDraw) - margin.pop() # destroy the last margin added - - -# Functions for deciding if things are like various types, mainly to -# handle UserDict, UserList and UserString like their underlying types. -# -# Yes, all of this manual testing breaks polymorphism, and the real -# Pythonic way to do all of this would be to just try it and handle the -# exception, but handling the exception when it's not the right type is -# often too slow. - -# We are using the following trick to speed up these -# functions. Default arguments are used to take a snapshot of -# the global functions and constants used by these functions. This -# transforms accesses to global variable into local variables -# accesses (i.e. LOAD_FAST instead of LOAD_GLOBAL). -# Since checkers dislike this, it's now annotated for pylint to flag -# (mostly for other readers of this code) we're doing this intentionally. -# TODO: PY3 check these are still valid choices for all of these funcs. - -DictTypes = (dict, UserDict) -ListTypes = (list, UserList) - -# Handle getting dictionary views. -SequenceTypes = (list, tuple, UserList, MappingView) - -# Note that profiling data shows a speed-up when comparing -# explicitly with str instead of simply comparing -# with basestring. (at least on Python 2.5.1) -# TODO: PY3 check this benchmarking is still correct. -StringTypes = (str, UserString) - -# Empirically, it is faster to check explicitly for str than for basestring. -BaseStringTypes = str - -def is_Dict( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, DictTypes=DictTypes -) -> bool: - return isinstance(obj, DictTypes) - - -def is_List( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, ListTypes=ListTypes -) -> bool: - return isinstance(obj, ListTypes) - - -def is_Sequence( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, SequenceTypes=SequenceTypes -) -> bool: - return isinstance(obj, SequenceTypes) - - -def is_Tuple( # pylint: disable=redefined-builtin - obj, isinstance=isinstance, tuple=tuple -) -> bool: - return isinstance(obj, tuple) - - -def is_String( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, StringTypes=StringTypes -) -> bool: - return isinstance(obj, StringTypes) - - -def is_Scalar( # pylint: disable=redefined-outer-name,redefined-builtin - obj, isinstance=isinstance, StringTypes=StringTypes, SequenceTypes=SequenceTypes -) -> bool: - - # Profiling shows that there is an impressive speed-up of 2x - # when explicitly checking for strings instead of just not - # sequence when the argument (i.e. obj) is already a string. - # But, if obj is a not string then it is twice as fast to - # check only for 'not sequence'. The following code therefore - # assumes that the obj argument is a string most of the time. - return isinstance(obj, StringTypes) or not isinstance(obj, SequenceTypes) - - -def do_flatten( - sequence, - result, - isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, -): # pylint: disable=redefined-outer-name,redefined-builtin - for item in sequence: - if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): - result.append(item) - else: - do_flatten(item, result) - - -def flatten( # pylint: disable=redefined-outer-name,redefined-builtin - obj, - isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, - do_flatten=do_flatten, -) -> list: - """Flatten a sequence to a non-nested list. - - Converts either a single scalar or a nested sequence to a non-nested list. - Note that :func:`flatten` considers strings - to be scalars instead of sequences like pure Python would. - """ - if isinstance(obj, StringTypes) or not isinstance(obj, SequenceTypes): - return [obj] - result = [] - for item in obj: - if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): - result.append(item) - else: - do_flatten(item, result) - return result - - -def flatten_sequence( # pylint: disable=redefined-outer-name,redefined-builtin - sequence, - isinstance=isinstance, - StringTypes=StringTypes, - SequenceTypes=SequenceTypes, - do_flatten=do_flatten, -) -> list: - """Flatten a sequence to a non-nested list. - - Same as :func:`flatten`, but it does not handle the single scalar case. - This is slightly more efficient when one knows that the sequence - to flatten can not be a scalar. - """ - result = [] - for item in sequence: - if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): - result.append(item) - else: - do_flatten(item, result) - return result - -# Generic convert-to-string functions. The wrapper -# to_String_for_signature() will use a for_signature() method if the -# specified object has one. - -def to_String( # pylint: disable=redefined-outer-name,redefined-builtin - obj, - isinstance=isinstance, - str=str, - UserString=UserString, - BaseStringTypes=BaseStringTypes, -) -> str: - """Return a string version of obj.""" - - if isinstance(obj, BaseStringTypes): - # Early out when already a string! - return obj - - if isinstance(obj, UserString): - # obj.data can only be a regular string. Please see the UserString initializer. - return obj.data - - return str(obj) - -def to_String_for_subst( # pylint: disable=redefined-outer-name,redefined-builtin - obj, - isinstance=isinstance, - str=str, - BaseStringTypes=BaseStringTypes, - SequenceTypes=SequenceTypes, - UserString=UserString, -) -> str: - """Return a string version of obj for subst usage.""" - - # Note that the test cases are sorted by order of probability. - if isinstance(obj, BaseStringTypes): - return obj - - if isinstance(obj, SequenceTypes): - return ' '.join([to_String_for_subst(e) for e in obj]) - - if isinstance(obj, UserString): - # obj.data can only a regular string. Please see the UserString initializer. - return obj.data - - return str(obj) - -def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-builtin - obj, to_String_for_subst=to_String_for_subst, AttributeError=AttributeError -) -> str: - """Return a string version of obj for signature usage. - - Like :func:`to_String_for_subst` but has special handling for - scons objects that have a :meth:`for_signature` method, and for dicts. - """ - - try: - f = obj.for_signature - except AttributeError: - if isinstance(obj, dict): - # pprint will output dictionary in key sorted order - # with py3.5 the order was randomized. In general depending on dictionary order - # which was undefined until py3.6 (where it's by insertion order) was not wise. - # TODO: Change code when floor is raised to PY36 - return pprint.pformat(obj, width=1000000) - return to_String_for_subst(obj) - else: - return f() - - -# The SCons "semi-deep" copy. -# -# This makes separate copies of lists (including UserList objects) -# dictionaries (including UserDict objects) and tuples, but just copies -# references to anything else it finds. -# -# A special case is any object that has a __semi_deepcopy__() method, -# which we invoke to create the copy. Currently only used by -# BuilderDict to actually prevent the copy operation (as invalid on that object). -# -# The dispatch table approach used here is a direct rip-off from the -# normal Python copy module. - -def semi_deepcopy_dict(obj, exclude=None) -> dict: - if exclude is None: - exclude = [] - return {k: semi_deepcopy(v) for k, v in obj.items() if k not in exclude} - -def _semi_deepcopy_list(obj) -> list: - return [semi_deepcopy(item) for item in obj] - -def _semi_deepcopy_tuple(obj) -> tuple: - return tuple(map(semi_deepcopy, obj)) - -_semi_deepcopy_dispatch = { - dict: semi_deepcopy_dict, - list: _semi_deepcopy_list, - tuple: _semi_deepcopy_tuple, -} - -def semi_deepcopy(obj): - copier = _semi_deepcopy_dispatch.get(type(obj)) - if copier: - return copier(obj) - - if hasattr(obj, '__semi_deepcopy__') and callable(obj.__semi_deepcopy__): - return obj.__semi_deepcopy__() - - if isinstance(obj, UserDict): - return obj.__class__(semi_deepcopy_dict(obj)) - - if isinstance(obj, UserList): - return obj.__class__(_semi_deepcopy_list(obj)) - - return obj - - -class Proxy: - """A simple generic Proxy class, forwarding all calls to subject. - - This means you can take an object, let's call it `'obj_a`, - and wrap it in this Proxy class, with a statement like this:: - - proxy_obj = Proxy(obj_a) - - Then, if in the future, you do something like this:: - - x = proxy_obj.var1 - - since the :class:`Proxy` class does not have a :attr:`var1` attribute - (but presumably `objA` does), the request actually is equivalent to saying:: - - x = obj_a.var1 - - Inherit from this class to create a Proxy. - - With Python 3.5+ this does *not* work transparently - for :class:`Proxy` subclasses that use special .__*__() method names, - because those names are now bound to the class, not the individual - instances. You now need to know in advance which special method names you - want to pass on to the underlying Proxy object, and specifically delegate - their calls like this:: - - class Foo(Proxy): - __str__ = Delegate('__str__') - """ - - def __init__(self, subject): - """Wrap an object as a Proxy object""" - self._subject = subject - - def __getattr__(self, name): - """Retrieve an attribute from the wrapped object. - - Raises: - AttributeError: if attribute `name` doesn't exist. - """ - return getattr(self._subject, name) - - def get(self): - """Retrieve the entire wrapped object""" - return self._subject - - def __eq__(self, other): - if issubclass(other.__class__, self._subject.__class__): - return self._subject == other - return self.__dict__ == other.__dict__ - - -class Delegate: - """A Python Descriptor class that delegates attribute fetches - to an underlying wrapped subject of a Proxy. Typical use:: - - class Foo(Proxy): - __str__ = Delegate('__str__') - """ - def __init__(self, attribute): - self.attribute = attribute - - def __get__(self, obj, cls): - if isinstance(obj, cls): - return getattr(obj._subject, self.attribute) - - return self - - -class MethodWrapper: - """A generic Wrapper class that associates a method with an object. - - As part of creating this MethodWrapper object an attribute with the - specified name (by default, the name of the supplied method) is added - to the underlying object. When that new "method" is called, our - :meth:`__call__` method adds the object as the first argument, simulating - the Python behavior of supplying "self" on method calls. - - We hang on to the name by which the method was added to the underlying - base class so that we can provide a method to "clone" ourselves onto - a new underlying object being copied (without which we wouldn't need - to save that info). - """ - def __init__(self, obj, method, name=None): - if name is None: - name = method.__name__ - self.object = obj - self.method = method - self.name = name - setattr(self.object, name, self) - - def __call__(self, *args, **kwargs): - nargs = (self.object,) + args - return self.method(*nargs, **kwargs) - - def clone(self, new_object): - """ - Returns an object that re-binds the underlying "method" to - the specified new object. - """ - return self.__class__(new_object, self.method, self.name) - - -# attempt to load the windows registry module: -can_read_reg = False -try: - import winreg - - can_read_reg = True - hkey_mod = winreg - -except ImportError: - class _NoError(Exception): - pass - RegError = _NoError - -if can_read_reg: - HKEY_CLASSES_ROOT = hkey_mod.HKEY_CLASSES_ROOT - HKEY_LOCAL_MACHINE = hkey_mod.HKEY_LOCAL_MACHINE - HKEY_CURRENT_USER = hkey_mod.HKEY_CURRENT_USER - HKEY_USERS = hkey_mod.HKEY_USERS - - RegOpenKeyEx = winreg.OpenKeyEx - RegEnumKey = winreg.EnumKey - RegEnumValue = winreg.EnumValue - RegQueryValueEx = winreg.QueryValueEx - RegError = winreg.error - - def RegGetValue(root, key): - r"""Returns a registry value without having to open the key first. - - Only available on Windows platforms with a version of Python that - can read the registry. - - Returns the same thing as :func:`RegQueryValueEx`, except you just - specify the entire path to the value, and don't have to bother - opening the key first. So, instead of:: - - k = SCons.Util.RegOpenKeyEx(SCons.Util.HKEY_LOCAL_MACHINE, - r'SOFTWARE\Microsoft\Windows\CurrentVersion') - out = SCons.Util.RegQueryValueEx(k, 'ProgramFilesDir') - - You can write:: - - out = SCons.Util.RegGetValue(SCons.Util.HKEY_LOCAL_MACHINE, - r'SOFTWARE\Microsoft\Windows\CurrentVersion\ProgramFilesDir') - """ - # I would use os.path.split here, but it's not a filesystem - # path... - p = key.rfind('\\') + 1 - keyp = key[: p - 1] # -1 to omit trailing slash - val = key[p:] - k = RegOpenKeyEx(root, keyp) - return RegQueryValueEx(k, val) - - -else: - HKEY_CLASSES_ROOT = None - HKEY_LOCAL_MACHINE = None - HKEY_CURRENT_USER = None - HKEY_USERS = None - - def RegGetValue(root, key): - raise OSError - - def RegOpenKeyEx(root, key): - raise OSError - - -if sys.platform == 'win32': - - def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: - if path is None: - try: - path = os.environ['PATH'] - except KeyError: - return None - if is_String(path): - path = path.split(os.pathsep) - if pathext is None: - try: - pathext = os.environ['PATHEXT'] - except KeyError: - pathext = '.COM;.EXE;.BAT;.CMD' - if is_String(pathext): - pathext = pathext.split(os.pathsep) - for ext in pathext: - if ext.lower() == file[-len(ext):].lower(): - pathext = [''] - break - if reject is None: - reject = [] - if not is_List(reject) and not is_Tuple(reject): - reject = [reject] - for p in path: - f = os.path.join(p, file) - for ext in pathext: - fext = f + ext - if os.path.isfile(fext): - try: - reject.index(fext) - except ValueError: - return os.path.normpath(fext) - continue - return None - -elif os.name == 'os2': - - def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: - if path is None: - try: - path = os.environ['PATH'] - except KeyError: - return None - if is_String(path): - path = path.split(os.pathsep) - if pathext is None: - pathext = ['.exe', '.cmd'] - for ext in pathext: - if ext.lower() == file[-len(ext):].lower(): - pathext = [''] - break - if reject is None: - reject = [] - if not is_List(reject) and not is_Tuple(reject): - reject = [reject] - for p in path: - f = os.path.join(p, file) - for ext in pathext: - fext = f + ext - if os.path.isfile(fext): - try: - reject.index(fext) - except ValueError: - return os.path.normpath(fext) - continue - return None - -else: - - def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: - import stat # pylint: disable=import-outside-toplevel - - if path is None: - try: - path = os.environ['PATH'] - except KeyError: - return None - if is_String(path): - path = path.split(os.pathsep) - if reject is None: - reject = [] - if not is_List(reject) and not is_Tuple(reject): - reject = [reject] - for p in path: - f = os.path.join(p, file) - if os.path.isfile(f): - try: - st = os.stat(f) - except OSError: - # os.stat() raises OSError, not IOError if the file - # doesn't exist, so in this case we let IOError get - # raised so as to not mask possibly serious disk or - # network issues. - continue - if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: - try: - reject.index(f) - except ValueError: - return os.path.normpath(f) - continue - return None - -WhereIs.__doc__ = """\ -Return the path to an executable that matches `file`. - -Searches the given `path` for `file`, respecting any filename -extensions `pathext` (on the Windows platform only), and -returns the full path to the matching command. If no -command is found, return ``None``. - -If `path` is not specified, :attr:`os.environ[PATH]` is used. -If `pathext` is not specified, :attr:`os.environ[PATHEXT]` -is used. Will not select any path name or names in the optional -`reject` list. -""" - -def PrependPath( - oldpath, newpath, sep=os.pathsep, delete_existing=True, canonicalize=None -) -> Union[list, str]: - """Prepends `newpath` path elements to `oldpath`. - - Will only add any particular path once (leaving the first one it - encounters and ignoring the rest, to preserve path order), and will - :mod:`os.path.normpath` and :mod:`os.path.normcase` all paths to help - assure this. This can also handle the case where `oldpath` - is a list instead of a string, in which case a list will be returned - instead of a string. For example: - - >>> p = PrependPath("/foo/bar:/foo", "/biz/boom:/foo") - >>> print(p) - /biz/boom:/foo:/foo/bar - - If `delete_existing` is ``False``, then adding a path that exists will - not move it to the beginning; it will stay where it is in the list. - - >>> p = PrependPath("/foo/bar:/foo", "/biz/boom:/foo", delete_existing=False) - >>> print(p) - /biz/boom:/foo/bar:/foo - - If `canonicalize` is not ``None``, it is applied to each element of - `newpath` before use. - """ - - orig = oldpath - is_list = True - paths = orig - if not is_List(orig) and not is_Tuple(orig): - paths = paths.split(sep) - is_list = False - - if is_String(newpath): - newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [ newpath ] # might be a Dir - else: - newpaths = newpath - - if canonicalize: - newpaths=list(map(canonicalize, newpaths)) - - if not delete_existing: - # First uniquify the old paths, making sure to - # preserve the first instance (in Unix/Linux, - # the first one wins), and remembering them in normpaths. - # Then insert the new paths at the head of the list - # if they're not already in the normpaths list. - result = [] - normpaths = [] - for path in paths: - if not path: - continue - normpath = os.path.normpath(os.path.normcase(path)) - if normpath not in normpaths: - result.append(path) - normpaths.append(normpath) - newpaths.reverse() # since we're inserting at the head - for path in newpaths: - if not path: - continue - normpath = os.path.normpath(os.path.normcase(path)) - if normpath not in normpaths: - result.insert(0, path) - normpaths.append(normpath) - paths = result - - else: - newpaths = newpaths + paths # prepend new paths - - normpaths = [] - paths = [] - # now we add them only if they are unique - for path in newpaths: - normpath = os.path.normpath(os.path.normcase(path)) - if path and normpath not in normpaths: - paths.append(path) - normpaths.append(normpath) - - if is_list: - return paths - - return sep.join(paths) - -def AppendPath( - oldpath, newpath, sep=os.pathsep, delete_existing=True, canonicalize=None -) -> Union[list, str]: - """Appends `newpath` path elements to `oldpath`. - - Will only add any particular path once (leaving the last one it - encounters and ignoring the rest, to preserve path order), and will - :mod:`os.path.normpath` and :mod:`os.path.normcase` all paths to help - assure this. This can also handle the case where `oldpath` - is a list instead of a string, in which case a list will be returned - instead of a string. For example: - - >>> p = AppendPath("/foo/bar:/foo", "/biz/boom:/foo") - >>> print(p) - /foo/bar:/biz/boom:/foo - - If `delete_existing` is ``False``, then adding a path that exists - will not move it to the end; it will stay where it is in the list. - - >>> p = AppendPath("/foo/bar:/foo", "/biz/boom:/foo", delete_existing=False) - >>> print(p) - /foo/bar:/foo:/biz/boom - - If `canonicalize` is not ``None``, it is applied to each element of - `newpath` before use. - """ - - orig = oldpath - is_list = True - paths = orig - if not is_List(orig) and not is_Tuple(orig): - paths = paths.split(sep) - is_list = False - - if is_String(newpath): - newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [newpath] # might be a Dir - else: - newpaths = newpath - - if canonicalize: - newpaths=list(map(canonicalize, newpaths)) - - if not delete_existing: - # add old paths to result, then - # add new paths if not already present - # (I thought about using a dict for normpaths for speed, - # but it's not clear hashing the strings would be faster - # than linear searching these typically short lists.) - result = [] - normpaths = [] - for path in paths: - if not path: - continue - result.append(path) - normpaths.append(os.path.normpath(os.path.normcase(path))) - for path in newpaths: - if not path: - continue - normpath = os.path.normpath(os.path.normcase(path)) - if normpath not in normpaths: - result.append(path) - normpaths.append(normpath) - paths = result - else: - # start w/ new paths, add old ones if not present, - # then reverse. - newpaths = paths + newpaths # append new paths - newpaths.reverse() - - normpaths = [] - paths = [] - # now we add them only if they are unique - for path in newpaths: - normpath = os.path.normpath(os.path.normcase(path)) - if path and normpath not in normpaths: - paths.append(path) - normpaths.append(normpath) - paths.reverse() - - if is_list: - return paths - - return sep.join(paths) - -def AddPathIfNotExists(env_dict, key, path, sep=os.pathsep): - """Add a path element to a construction variable. - - `key` is looked up in `env_dict`, and `path` is added to it if it - is not already present. `env_dict[key]` is assumed to be in the - format of a PATH variable: a list of paths separated by `sep` tokens. - Example: - - >>> env = {'PATH': '/bin:/usr/bin:/usr/local/bin'} - >>> AddPathIfNotExists(env, 'PATH', '/opt/bin') - >>> print(env['PATH']) - /opt/bin:/bin:/usr/bin:/usr/local/bin - """ - - try: - is_list = True - paths = env_dict[key] - if not is_List(env_dict[key]): - paths = paths.split(sep) - is_list = False - if os.path.normcase(path) not in list(map(os.path.normcase, paths)): - paths = [ path ] + paths - if is_list: - env_dict[key] = paths - else: - env_dict[key] = sep.join(paths) - except KeyError: - env_dict[key] = path - -if sys.platform == 'cygwin': - import subprocess # pylint: disable=import-outside-toplevel - - def get_native_path(path) -> str: - cp = subprocess.run(('cygpath', '-w', path), check=False, stdout=subprocess.PIPE) - return cp.stdout.decode().replace('\n', '') -else: - def get_native_path(path) -> str: - return path - -get_native_path.__doc__ = """\ -Transform an absolute path into a native path for the system. - -In Cygwin, this converts from a Cygwin path to a Windows path, -without regard to whether `path` refers to an existing file -system object. For other platforms, `path` is unchanged. -""" - - -display = DisplayEngine() - -def Split(arg) -> list: - """Returns a list of file names or other objects. - - If `arg` is a string, it will be split on strings of white-space - characters within the string. If `arg` is already a list, the list - will be returned untouched. If `arg` is any other type of object, - it will be returned as a list containing just the object. - - >>> print(Split(" this is a string ")) - ['this', 'is', 'a', 'string'] - >>> print(Split(["stringlist", " preserving ", " spaces "])) - ['stringlist', ' preserving ', ' spaces '] - """ - if is_List(arg) or is_Tuple(arg): - return arg - - if is_String(arg): - return arg.split() - - return [arg] - - -class CLVar(UserList): - """A container for command-line construction variables. - - Forces the use of a list of strings intended as command-line - arguments. Like :class:`collections.UserList`, but the argument - passed to the initializter will be processed by the :func:`Split` - function, which includes special handling for string types: they - will be split into a list of words, not coereced directly to a list. - The same happens if a string is added to a :class:`CLVar`, - which allows doing the right thing with both - :func:`Append`/:func:`Prepend` methods, - as well as with pure Python addition, regardless of whether adding - a list or a string to a construction variable. - - Side effect: spaces will be stripped from individual string - arguments. If you need spaces preserved, pass strings containing - spaces inside a list argument. - - >>> u = UserList("--some --opts and args") - >>> print(len(u), repr(u)) - 22 ['-', '-', 's', 'o', 'm', 'e', ' ', '-', '-', 'o', 'p', 't', 's', ' ', 'a', 'n', 'd', ' ', 'a', 'r', 'g', 's'] - >>> c = CLVar("--some --opts and args") - >>> print(len(c), repr(c)) - 4 ['--some', '--opts', 'and', 'args'] - >>> c += " strips spaces " - >>> print(len(c), repr(c)) - 6 ['--some', '--opts', 'and', 'args', 'strips', 'spaces'] - """ - - def __init__(self, initlist=None): - super().__init__(Split(initlist if initlist is not None else [])) - - def __add__(self, other): - return super().__add__(CLVar(other)) - - def __radd__(self, other): - return super().__radd__(CLVar(other)) - - def __iadd__(self, other): - return super().__iadd__(CLVar(other)) - - def __str__(self): - # Some cases the data can contain Nodes, so make sure they - # processed to string before handing them over to join. - return ' '.join([str(d) for d in self.data]) - - -class Selector(OrderedDict): - """A callable ordered dictionary that maps file suffixes to - dictionary values. We preserve the order in which items are added - so that :func:`get_suffix` calls always return the first suffix added. - """ - def __call__(self, env, source, ext=None): - if ext is None: - try: - ext = source[0].get_suffix() - except IndexError: - ext = "" - try: - return self[ext] - except KeyError: - # Try to perform Environment substitution on the keys of - # the dictionary before giving up. - s_dict = {} - for (k,v) in self.items(): - if k is not None: - s_k = env.subst(k) - if s_k in s_dict: - # We only raise an error when variables point - # to the same suffix. If one suffix is literal - # and a variable suffix contains this literal, - # the literal wins and we don't raise an error. - raise KeyError(s_dict[s_k][0], k, s_k) - s_dict[s_k] = (k,v) - try: - return s_dict[ext][1] - except KeyError: - try: - return self[None] - except KeyError: - return None - - -if sys.platform == 'cygwin': - # On Cygwin, os.path.normcase() lies, so just report back the - # fact that the underlying Windows OS is case-insensitive. - def case_sensitive_suffixes(s1, s2) -> bool: # pylint: disable=unused-argument - return False - -else: - def case_sensitive_suffixes(s1, s2) -> bool: - return os.path.normcase(s1) != os.path.normcase(s2) - - -def adjustixes(fname, pre, suf, ensure_suffix=False) -> str: - """Adjust filename prefixes and suffixes as needed. - - Add `prefix` to `fname` if specified. - Add `suffix` to `fname` if specified and if `ensure_suffix` is ``True`` - """ - - if pre: - path, fn = os.path.split(os.path.normpath(fname)) - - # Handle the odd case where the filename = the prefix. - # In that case, we still want to add the prefix to the file - if not fn.startswith(pre) or fn == pre: - fname = os.path.join(path, pre + fn) - # Only append a suffix if the suffix we're going to add isn't already - # there, and if either we've been asked to ensure the specific suffix - # is present or there's no suffix on it at all. - # Also handle the odd case where the filename = the suffix. - # in that case we still want to append the suffix - if suf and not fname.endswith(suf) and \ - (ensure_suffix or not splitext(fname)[1]): - fname = fname + suf - return fname - - - -# From Tim Peters, -# https://code.activestate.com/recipes/52560 -# ASPN: Python Cookbook: Remove duplicates from a sequence -# (Also in the printed Python Cookbook.) -# Updated. This algorithm is used by some scanners and tools. - -def unique(seq): - """Return a list of the elements in seq without duplicates, ignoring order. - - >>> mylist = unique([1, 2, 3, 1, 2, 3]) - >>> print(sorted(mylist)) - [1, 2, 3] - >>> mylist = unique("abcabc") - >>> print(sorted(mylist)) - ['a', 'b', 'c'] - >>> mylist = unique(([1, 2], [2, 3], [1, 2])) - >>> print(sorted(mylist)) - [[1, 2], [2, 3]] - - For best speed, all sequence elements should be hashable. Then - unique() will usually work in linear time. - - If not possible, the sequence elements should enjoy a total - ordering, and if list(s).sort() doesn't raise TypeError it's - assumed that they do enjoy a total ordering. Then unique() will - usually work in O(N*log2(N)) time. - - If that's not possible either, the sequence elements must support - equality-testing. Then unique() will usually work in quadratic time. - """ - - if not seq: - return [] - - # Try using a dict first, as that's the fastest and will usually - # work. If it doesn't work, it will usually fail quickly, so it - # usually doesn't cost much to *try* it. It requires that all the - # sequence elements be hashable, and support equality comparison. - # TODO: should be even faster: return(list(set(seq))) - with suppress(TypeError): - return list(dict.fromkeys(seq)) - - # We couldn't hash all the elements (got a TypeError). - # Next fastest is to sort, which brings the equal elements together; - # then duplicates are easy to weed out in a single pass. - # NOTE: Python's list.sort() was designed to be efficient in the - # presence of many duplicate elements. This isn't true of all - # sort functions in all languages or libraries, so this approach - # is more effective in Python than it may be elsewhere. - n = len(seq) - try: - t = sorted(seq) - except TypeError: - pass # move on to the next method - else: - last = t[0] - lasti = i = 1 - while i < n: - if t[i] != last: - t[lasti] = last = t[i] - lasti = lasti + 1 - i = i + 1 - return t[:lasti] - - # Brute force is all that's left. - u = [] - for x in seq: - if x not in u: - u.append(x) - return u - -# Best way (assuming Python 3.7, but effectively 3.6) to remove -# duplicates from a list in while preserving order, according to -# https://stackoverflow.com/questions/480214/how-do-i-remove-duplicates-from-a-list-while-preserving-order/17016257#17016257 -def uniquer_hashables(seq): - return list(dict.fromkeys(seq)) - -# Recipe 19.11 "Reading Lines with Continuation Characters", -# by Alex Martelli, straight from the Python CookBook (2nd edition). -def logical_lines(physical_lines, joiner=''.join): - logical_line = [] - for line in physical_lines: - stripped = line.rstrip() - if stripped.endswith('\\'): - # a line which continues w/the next physical line - logical_line.append(stripped[:-1]) - else: - # a line which does not continue, end of logical line - logical_line.append(line) - yield joiner(logical_line) - logical_line = [] - if logical_line: - # end of sequence implies end of last logical line - yield joiner(logical_line) - - -class LogicalLines: - """ Wrapper class for the logical_lines method. - - Allows us to read all "logical" lines at once from a given file object. - """ - - def __init__(self, fileobj): - self.fileobj = fileobj - - def readlines(self): - return list(logical_lines(self.fileobj)) - - -class UniqueList(UserList): - """A list which maintains uniqueness. - - Uniquing is lazy: rather than being assured on list changes, it is fixed - up on access by those methods which need to act on a uniqe list to be - correct. That means things like "in" don't have to eat the uniquing time. - """ - def __init__(self, initlist=None): - super().__init__(initlist) - self.unique = True - - def __make_unique(self): - if not self.unique: - self.data = uniquer_hashables(self.data) - self.unique = True - - def __repr__(self): - self.__make_unique() - return super().__repr__() - - def __lt__(self, other): - self.__make_unique() - return super().__lt__(other) - - def __le__(self, other): - self.__make_unique() - return super().__le__(other) - - def __eq__(self, other): - self.__make_unique() - return super().__eq__(other) - - def __ne__(self, other): - self.__make_unique() - return super().__ne__(other) - - def __gt__(self, other): - self.__make_unique() - return super().__gt__(other) - - def __ge__(self, other): - self.__make_unique() - return super().__ge__(other) - - # __contains__ doesn't need to worry about uniquing, inherit - - def __len__(self): - self.__make_unique() - return super().__len__() - - def __getitem__(self, i): - self.__make_unique() - return super().__getitem__(i) - - def __setitem__(self, i, item): - super().__setitem__(i, item) - self.unique = False - - # __delitem__ doesn't need to worry about uniquing, inherit - - def __add__(self, other): - result = super().__add__(other) - result.unique = False - return result - - def __radd__(self, other): - result = super().__radd__(other) - result.unique = False - return result - - def __iadd__(self, other): - result = super().__iadd__(other) - result.unique = False - return result - - def __mul__(self, other): - result = super().__mul__(other) - result.unique = False - return result - - def __rmul__(self, other): - result = super().__rmul__(other) - result.unique = False - return result - - def __imul__(self, other): - result = super().__imul__(other) - result.unique = False - return result - - def append(self, item): - super().append(item) - self.unique = False - - def insert(self, i, item): - super().insert(i, item) - self.unique = False - - def count(self, item): - self.__make_unique() - return super().count(item) - - def index(self, item, *args): - self.__make_unique() - return super().index(item, *args) - - def reverse(self): - self.__make_unique() - super().reverse() - - # TODO: Py3.8: def sort(self, /, *args, **kwds): - def sort(self, *args, **kwds): - self.__make_unique() - return super().sort(*args, **kwds) - - def extend(self, other): - super().extend(other) - self.unique = False - - -class Unbuffered: - """A proxy that wraps a file object, flushing after every write. - - Delegates everything else to the wrapped object. - """ - def __init__(self, file): - self.file = file - - def write(self, arg): - # Stdout might be connected to a pipe that has been closed - # by now. The most likely reason for the pipe being closed - # is that the user has press ctrl-c. It this is the case, - # then SCons is currently shutdown. We therefore ignore - # IOError's here so that SCons can continue and shutdown - # properly so that the .sconsign is correctly written - # before SCons exits. - with suppress(IOError): - self.file.write(arg) - self.file.flush() - - def writelines(self, arg): - with suppress(IOError): - self.file.writelines(arg) - self.file.flush() - - def __getattr__(self, attr): - return getattr(self.file, attr) - -def make_path_relative(path) -> str: - """Converts an absolute path name to a relative pathname.""" - - if os.path.isabs(path): - drive_s, path = os.path.splitdrive(path) - - if not drive_s: - path=re.compile(r"/*(.*)").findall(path)[0] - else: - path=path[1:] - - assert not os.path.isabs(path), path - return path - - -# The original idea for AddMethod() came from the -# following post to the ActiveState Python Cookbook: -# -# ASPN: Python Cookbook : Install bound methods in an instance -# https://code.activestate.com/recipes/223613 -# -# Changed as follows: -# * Switched the installmethod() "object" and "function" arguments, -# so the order reflects that the left-hand side is the thing being -# "assigned to" and the right-hand side is the value being assigned. -# * The instance/class detection is changed a bit, as it's all -# new-style classes now with Py3. -# * The by-hand construction of the function object from renamefunction() -# is not needed, the remaining bit is now used inline in AddMethod. - -def AddMethod(obj, function, name=None): - """Adds a method to an object. - - Adds `function` to `obj` if `obj` is a class object. - Adds `function` as a bound method if `obj` is an instance object. - If `obj` looks like an environment instance, use `MethodWrapper` - to add it. If `name` is supplied it is used as the name of `function`. - - Although this works for any class object, the intent as a public - API is to be used on Environment, to be able to add a method to all - construction environments; it is preferred to use env.AddMethod - to add to an individual environment. - - >>> class A: - ... ... - - >>> a = A() - - >>> def f(self, x, y): - ... self.z = x + y - - >>> AddMethod(A, f, "add") - >>> a.add(2, 4) - >>> print(a.z) - 6 - >>> a.data = ['a', 'b', 'c', 'd', 'e', 'f'] - >>> AddMethod(a, lambda self, i: self.data[i], "listIndex") - >>> print(a.listIndex(3)) - d - - """ - if name is None: - name = function.__name__ - else: - # "rename" - function = FunctionType( - function.__code__, function.__globals__, name, function.__defaults__ - ) - - if hasattr(obj, '__class__') and obj.__class__ is not type: - # obj is an instance, so it gets a bound method. - if hasattr(obj, "added_methods"): - method = MethodWrapper(obj, function, name) - obj.added_methods.append(method) - else: - method = MethodType(function, obj) - else: - # obj is a class - method = function - - setattr(obj, name, method) - - -# Default hash function and format. SCons-internal. -DEFAULT_HASH_FORMATS = ['md5', 'sha1', 'sha256'] -ALLOWED_HASH_FORMATS = [] -_HASH_FUNCTION = None -_HASH_FORMAT = None - -def _attempt_init_of_python_3_9_hash_object(hash_function_object, sys_used=sys): - """Python 3.9 and onwards lets us initialize the hash function object with the - key "usedforsecurity"=false. This lets us continue to use algorithms that have - been deprecated either by FIPS or by Python itself, as the MD5 algorithm SCons - prefers is not being used for security purposes as much as a short, 32 char - hash that is resistant to accidental collisions. - - In prior versions of python, hashlib returns a native function wrapper, which - errors out when it's queried for the optional parameter, so this function - wraps that call. - - It can still throw a ValueError if the initialization fails due to FIPS - compliance issues, but that is assumed to be the responsibility of the caller. - """ - if hash_function_object is None: - return None - - # https://stackoverflow.com/a/11887885 details how to check versions with the "packaging" library. - # however, for our purposes checking the version is greater than or equal to 3.9 is good enough, as - # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See - # https://docs.python.org/3/library/hashlib.html#:~:text=usedforsecurity for the version support notes. - if (sys_used.version_info.major > 3) or (sys_used.version_info.major == 3 and sys_used.version_info.minor >= 9): - return hash_function_object(usedforsecurity=False) - - # note that this can throw a ValueError in FIPS-enabled versions of Linux prior to 3.9 - # the OpenSSL hashlib will throw on first init here, but that is assumed to be responsibility of - # the caller to diagnose the ValueError & potentially display the error to screen. - return hash_function_object() - -def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): - """Checks if SCons has ability to call the default algorithms normally supported. - - This util class is sometimes called prior to setting the user-selected hash algorithm, - meaning that on FIPS-compliant systems the library would default-initialize MD5 - and throw an exception in set_hash_format. A common case is using the SConf options, - which can run prior to main, and thus ignore the options.hash_format variable. - - This function checks the DEFAULT_HASH_FORMATS and sets the ALLOWED_HASH_FORMATS - to only the ones that can be called. In Python >= 3.9 this will always default to - MD5 as in Python 3.9 there is an optional attribute "usedforsecurity" set for the method. - - Throws if no allowed hash formats are detected. - """ - global ALLOWED_HASH_FORMATS - _last_error = None - # note: if you call this method repeatedly, example using timeout, this is needed. - # otherwise it keeps appending valid formats to the string - ALLOWED_HASH_FORMATS = [] - - for test_algorithm in DEFAULT_HASH_FORMATS: - _test_hash = getattr(hashlib_used, test_algorithm, None) - # we know hashlib claims to support it... check to see if we can call it. - if _test_hash is not None: - # the hashing library will throw an exception on initialization in FIPS mode, - # meaning if we call the default algorithm returned with no parameters, it'll - # throw if it's a bad algorithm, otherwise it will append it to the known - # good formats. - try: - _attempt_init_of_python_3_9_hash_object(_test_hash, sys_used) - ALLOWED_HASH_FORMATS.append(test_algorithm) - except ValueError as e: - _last_error = e - continue - - if len(ALLOWED_HASH_FORMATS) == 0: - from SCons.Errors import SConsEnvironmentError # pylint: disable=import-outside-toplevel - # chain the exception thrown with the most recent error from hashlib. - raise SConsEnvironmentError( - 'No usable hash algorithms found.' - 'Most recent error from hashlib attached in trace.' - ) from _last_error - return - -_set_allowed_viable_default_hashes(hashlib) - - -def get_hash_format(): - """Retrieves the hash format or ``None`` if not overridden. - - A return value of ``None`` - does not guarantee that MD5 is being used; instead, it means that the - default precedence order documented in :func:`SCons.Util.set_hash_format` - is respected. - """ - return _HASH_FORMAT - -def _attempt_get_hash_function(hash_name, hashlib_used=hashlib, sys_used=sys): - """Wrapper used to try to initialize a hash function given. - - If successful, returns the name of the hash function back to the user. - - Otherwise returns None. - """ - try: - _fetch_hash = getattr(hashlib_used, hash_name, None) - if _fetch_hash is None: - return None - _attempt_init_of_python_3_9_hash_object(_fetch_hash, sys_used) - return hash_name - except ValueError: - # if attempt_init_of_python_3_9 throws, this is typically due to FIPS being enabled - # however, if we get to this point, the viable hash function check has either been - # bypassed or otherwise failed to properly restrict the user to only the supported - # functions. As such throw the UserError as an internal assertion-like error. - return None - -def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): - """Sets the default hash format used by SCons. - - If `hash_format` is ``None`` or - an empty string, the default is determined by this function. - - Currently the default behavior is to use the first available format of - the following options: MD5, SHA1, SHA256. - """ - global _HASH_FORMAT, _HASH_FUNCTION - - _HASH_FORMAT = hash_format - if hash_format: - hash_format_lower = hash_format.lower() - if hash_format_lower not in ALLOWED_HASH_FORMATS: - from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - - # user can select something not supported by their OS but normally supported by - # SCons, example, selecting MD5 in an OS with FIPS-mode turned on. Therefore we first - # check if SCons supports it, and then if their local OS supports it. - if hash_format_lower in DEFAULT_HASH_FORMATS: - raise UserError('While hash format "%s" is supported by SCons, the ' - 'local system indicates only the following hash ' - 'formats are supported by the hashlib library: %s' % - (hash_format_lower, - ', '.join(ALLOWED_HASH_FORMATS)) - ) - else: - # the hash format isn't supported by SCons in any case. Warn the user, and - # if we detect that SCons supports more algorithms than their local system - # supports, warn the user about that too. - if ALLOWED_HASH_FORMATS == DEFAULT_HASH_FORMATS: - raise UserError('Hash format "%s" is not supported by SCons. Only ' - 'the following hash formats are supported: %s' % - (hash_format_lower, - ', '.join(ALLOWED_HASH_FORMATS)) - ) - else: - raise UserError('Hash format "%s" is not supported by SCons. ' - 'SCons supports more hash formats than your local system ' - 'is reporting; SCons supports: %s. Your local system only ' - 'supports: %s' % - (hash_format_lower, - ', '.join(DEFAULT_HASH_FORMATS), - ', '.join(ALLOWED_HASH_FORMATS)) - ) - - # this is not expected to fail. If this fails it means the set_allowed_viable_default_hashes - # function did not throw, or when it threw, the exception was caught and ignored, or - # the global ALLOWED_HASH_FORMATS was changed by an external user. - _HASH_FUNCTION = _attempt_get_hash_function(hash_format_lower, hashlib_used, sys_used) - - if _HASH_FUNCTION is None: - from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - - raise UserError( - 'Hash format "%s" is not available in your Python interpreter. ' - 'Expected to be supported algorithm by set_allowed_viable_default_hashes, ' - 'Assertion error in SCons.' - % hash_format_lower - ) - else: - # Set the default hash format based on what is available, defaulting - # to the first supported hash algorithm (usually md5) for backwards compatibility. - # in FIPS-compliant systems this usually defaults to SHA1, unless that too has been - # disabled. - for choice in ALLOWED_HASH_FORMATS: - _HASH_FUNCTION = _attempt_get_hash_function(choice, hashlib_used, sys_used) - - if _HASH_FUNCTION is not None: - break - else: - # This is not expected to happen in practice. - from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - - raise UserError( - 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' - 'SCons requires at least one. Expected to support one or more ' - 'during set_allowed_viable_default_hashes.' - ) - -# Ensure that this is initialized in case either: -# 1. This code is running in a unit test. -# 2. This code is running in a consumer that does hash operations while -# SConscript files are being loaded. -set_hash_format(None) - - -def get_current_hash_algorithm_used(): - """Returns the current hash algorithm name used. - - Where the python version >= 3.9, this is expected to return md5. - If python's version is <= 3.8, this returns md5 on non-FIPS-mode platforms, and - sha1 or sha256 on FIPS-mode Linux platforms. - - This function is primarily useful for testing, where one expects a value to be - one of N distinct hashes, and therefore the test needs to know which hash to select. - """ - return _HASH_FUNCTION - -def _get_hash_object(hash_format, hashlib_used=hashlib, sys_used=sys): - """Allocates a hash object using the requested hash format. - - Args: - hash_format: Hash format to use. - - Returns: - hashlib object. - """ - if hash_format is None: - if _HASH_FUNCTION is None: - from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - - raise UserError('There is no default hash function. Did you call ' - 'a hashing function before SCons was initialized?') - return _attempt_init_of_python_3_9_hash_object(getattr(hashlib_used, _HASH_FUNCTION, None), sys_used) - - if not hasattr(hashlib, hash_format): - from SCons.Errors import UserError # pylint: disable=import-outside-toplevel - - raise UserError( - 'Hash format "%s" is not available in your Python interpreter.' % - hash_format) - - return _attempt_init_of_python_3_9_hash_object(getattr(hashlib, hash_format), sys_used) - - -def hash_signature(s, hash_format=None): - """ - Generate hash signature of a string - - Args: - s: either string or bytes. Normally should be bytes - hash_format: Specify to override default hash format - - Returns: - String of hex digits representing the signature - """ - m = _get_hash_object(hash_format) - try: - m.update(to_bytes(s)) - except TypeError: - m.update(to_bytes(str(s))) - - return m.hexdigest() - - -def hash_file_signature(fname, chunksize=65536, hash_format=None): - """ - Generate the md5 signature of a file - - Args: - fname: file to hash - chunksize: chunk size to read - hash_format: Specify to override default hash format - - Returns: - String of Hex digits representing the signature - """ - - m = _get_hash_object(hash_format) - with open(fname, "rb") as f: - while True: - blck = f.read(chunksize) - if not blck: - break - m.update(to_bytes(blck)) - return m.hexdigest() - - -def hash_collect(signatures, hash_format=None): - """ - Collects a list of signatures into an aggregate signature. - - Args: - signatures: a list of signatures - hash_format: Specify to override default hash format - - Returns: - the aggregate signature - """ - - if len(signatures) == 1: - return signatures[0] - - return hash_signature(', '.join(signatures), hash_format) - - -_MD5_WARNING_SHOWN = False - -def _show_md5_warning(function_name): - """Shows a deprecation warning for various MD5 functions.""" - - global _MD5_WARNING_SHOWN - - if not _MD5_WARNING_SHOWN: - import SCons.Warnings # pylint: disable=import-outside-toplevel - - SCons.Warnings.warn(SCons.Warnings.DeprecatedWarning, - "Function %s is deprecated" % function_name) - _MD5_WARNING_SHOWN = True - - -def MD5signature(s): - """Deprecated. Use :func:`hash_signature` instead.""" - - _show_md5_warning("MD5signature") - return hash_signature(s) - - -def MD5filesignature(fname, chunksize=65536): - """Deprecated. Use :func:`hash_file_signature` instead.""" - - _show_md5_warning("MD5filesignature") - return hash_file_signature(fname, chunksize) - - -def MD5collect(signatures): - """Deprecated. Use :func:`hash_collect` instead.""" - - _show_md5_warning("MD5collect") - return hash_collect(signatures) - - -def silent_intern(x): - """ - Perform :mod:`sys.intern` on the passed argument and return the result. - If the input is ineligible for interning the original argument is - returned and no exception is thrown. - """ - try: - return sys.intern(x) - except TypeError: - return x - - -# From Dinu C. Gherman, -# Python Cookbook, second edition, recipe 6.17, p. 277. -# Also: https://code.activestate.com/recipes/68205 -# ASPN: Python Cookbook: Null Object Design Pattern - -class Null: - """ Null objects always and reliably "do nothing." """ - def __new__(cls, *args, **kwargs): - if '_instance' not in vars(cls): - cls._instance = super(Null, cls).__new__(cls, *args, **kwargs) - return cls._instance - def __init__(self, *args, **kwargs): - pass - def __call__(self, *args, **kwargs): - return self - def __repr__(self): - return "Null(0x%08X)" % id(self) - def __bool__(self): - return False - def __getattr__(self, name): - return self - def __setattr__(self, name, value): - return self - def __delattr__(self, name): - return self - - -class NullSeq(Null): - """ A Null object that can also be iterated over. """ - def __len__(self): - return 0 - def __iter__(self): - return iter(()) - def __getitem__(self, i): - return self - def __delitem__(self, i): - return self - def __setitem__(self, i, v): - return self - - -def to_bytes(s) -> bytes: - if s is None: - return b'None' - if isinstance(s, (bytes, bytearray)): - # if already bytes return. - return s - return bytes(s, 'utf-8') - - -def to_str(s) -> str: - if s is None: - return 'None' - if is_String(s): - return s - return str(s, 'utf-8') - - -def cmp(a, b) -> bool: - """A cmp function because one is no longer available in python3.""" - return (a > b) - (a < b) - - -def get_env_bool(env, name, default=False) -> bool: - """Convert a construction variable to bool. - - If the value of `name` in `env` is 'true', 'yes', 'y', 'on' (case - insensitive) or anything convertible to int that yields non-zero then - return ``True``; if 'false', 'no', 'n', 'off' (case insensitive) - or a number that converts to integer zero return ``False``. - Otherwise, return `default`. - - Args: - env: construction environment, or any dict-like object - name: name of the variable - default: value to return if `name` not in `env` or cannot - be converted (default: False) - - Returns: - the "truthiness" of `name` - """ - try: - var = env[name] - except KeyError: - return default - try: - return bool(int(var)) - except ValueError: - if str(var).lower() in ('true', 'yes', 'y', 'on'): - return True - - if str(var).lower() in ('false', 'no', 'n', 'off'): - return False - - return default - - -def get_os_env_bool(name, default=False) -> bool: - """Convert an environment variable to bool. - - Conversion is the same as for :func:`get_env_bool`. - """ - return get_env_bool(os.environ, name, default) - - -def print_time(): - """Hack to return a value from Main if can't import Main.""" - # pylint: disable=redefined-outer-name,import-outside-toplevel - from SCons.Script.Main import print_time - return print_time - - -def wait_for_process_to_die(pid): - """ - Wait for specified process to die, or alternatively kill it - NOTE: This function operates best with psutil pypi package - TODO: Add timeout which raises exception - """ - # wait for the process to fully killed - try: - import psutil - while True: - if pid not in [proc.pid for proc in psutil.process_iter()]: - break - else: - time.sleep(0.1) - except ImportError: - # if psutil is not installed we can do this the hard way - while True: - if sys.platform == 'win32': - import ctypes - PROCESS_QUERY_INFORMATION = 0x1000 - processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) - if processHandle == 0: - break - else: - ctypes.windll.kernel32.CloseHandle(processHandle) - time.sleep(0.1) - else: - try: - os.kill(pid, 0) - except OSError: - break - else: - time.sleep(0.1) - -# From: https://stackoverflow.com/questions/1741972/how-to-use-different-formatters-with-the-same-logging-handler-in-python -class DispatchingFormatter(Formatter): - - def __init__(self, formatters, default_formatter): - self._formatters = formatters - self._default_formatter = default_formatter - - def format(self, record): - formatter = self._formatters.get(record.name, self._default_formatter) - return formatter.format(record) - - -def sanitize_shell_env(execution_env): - """ - Sanitize all values in execution_env (typically this is env['ENV']) which will be propagated to the shell - :param execution_env: The shell environment variables to be propagated to spawned shell - :return: sanitize dictionary of env variables (similar to what you'd get from os.environ) - """ - - # Ensure that the ENV values are all strings: - new_env = {} - for key, value in execution_env.items(): - if is_List(value): - # If the value is a list, then we assume it is a path list, - # because that's a pretty common list-like value to stick - # in an environment variable: - value = flatten_sequence(value) - new_env[key] = os.pathsep.join(map(str, value)) - else: - # It's either a string or something else. If it isn't a - # string or a list, then we just coerce it to a string, which - # is the proper way to handle Dir and File instances and will - # produce something reasonable for just about everything else: - new_env[key] = str(value) - return new_env - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/__init__.py b/SCons/Util/__init__.py new file mode 100644 index 0000000..6b48d05 --- /dev/null +++ b/SCons/Util/__init__.py @@ -0,0 +1,1325 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""SCons utility functions + +This package contains routines for use by other parts of SCons. +""" + +import copy +import hashlib +import os +import re +import sys +import time +from collections import UserDict, UserList, OrderedDict +from contextlib import suppress +from types import MethodType, FunctionType +from typing import Optional, Union +from logging import Formatter + +from .types import ( + DictTypes, + ListTypes, + SequenceTypes, + StringTypes, + BaseStringTypes, + Null, + NullSeq, + is_Dict, + is_List, + is_Sequence, + is_Tuple, + is_String, + is_Scalar, + to_String, + to_String_for_subst, + to_String_for_signature, + to_bytes, + to_str, + get_env_bool, + get_os_env_bool, + get_environment_var, +) +from .hashes import ( + ALLOWED_HASH_FORMATS, + DEFAULT_HASH_FORMATS, + get_hash_format, + set_hash_format, + get_current_hash_algorithm_used, + hash_signature, + hash_file_signature, + hash_collect, + MD5signature, + MD5filesignature, + MD5collect, +) +from .envs import ( + MethodWrapper, + PrependPath, + AppendPath, + AddPathIfNotExists, + AddMethod, +) + + +# Note: the Util package cannot import other parts of SCons globally without +# hitting import loops. Both of these modules import SCons.Util early on, +# and are imported in many other modules: +# --> SCons.Warnings +# --> SCons.Errors +# If you run into places that have to do local imports for this reason, +# annotate them for pylint and for human readers to know why: +# pylint: disable=import-outside-toplevel +# Be aware that Black can break this if the annotated line is too +# long and it wants to split: +# from SCons.Errors import ( +# SConsEnvironmentError, +# ) # pylint: disable=import-outside-toplevel +# That's syntactically valid, but pylint won't recorgnize it with the +# annotation at the end, it would have to be on the first line +# (issues filed upstream, for now just be aware) + +PYPY = hasattr(sys, 'pypy_translation_info') + +# this string will be hashed if a Node refers to a file that doesn't exist +# in order to distinguish from a file that exists but is empty. +NOFILE = "SCONS_MAGIC_MISSING_FILE_STRING" + +# unused? +def dictify(keys, values, result=None) -> dict: + if result is None: + result = {} + result.update(zip(keys, values)) + return result + +_ALTSEP = os.altsep +if _ALTSEP is None and sys.platform == 'win32': + # My ActivePython 2.0.1 doesn't set os.altsep! What gives? + _ALTSEP = '/' +if _ALTSEP: + def rightmost_separator(path, sep): + return max(path.rfind(sep), path.rfind(_ALTSEP)) +else: + def rightmost_separator(path, sep): + return path.rfind(sep) + +# First two from the Python Cookbook, just for completeness. +# (Yeah, yeah, YAGNI...) +def containsAny(s, pat) -> bool: + """Check whether string `s` contains ANY of the items in `pat`.""" + return any(c in s for c in pat) + +def containsAll(s, pat) -> bool: + """Check whether string `s` contains ALL of the items in `pat`.""" + return all(c in s for c in pat) + +def containsOnly(s, pat) -> bool: + """Check whether string `s` contains ONLY items in `pat`.""" + for c in s: + if c not in pat: + return False + return True + + +# TODO: Verify this method is STILL faster than os.path.splitext +def splitext(path) -> tuple: + """Split `path` into a (root, ext) pair. + + Same as :mod:`os.path.splitext` but faster. + """ + sep = rightmost_separator(path, os.sep) + dot = path.rfind('.') + # An ext is only real if it has at least one non-digit char + if dot > sep and not path[dot + 1:].isdigit(): + return path[:dot], path[dot:] + + return path, "" + +def updrive(path) -> str: + """Make the drive letter (if any) upper case. + + This is useful because Windows is inconsistent on the case + of the drive letter, which can cause inconsistencies when + calculating command signatures. + """ + drive, rest = os.path.splitdrive(path) + if drive: + path = drive.upper() + rest + return path + +class NodeList(UserList): + """A list of Nodes with special attribute retrieval. + + Unlike an ordinary list, access to a member's attribute returns a + `NodeList` containing the same attribute for each member. Although + this can hold any object, it is intended for use when processing + Nodes, where fetching an attribute of each member is very commone, + for example getting the content signature of each node. The term + "attribute" here includes the string representation. + + >>> someList = NodeList([' foo ', ' bar ']) + >>> someList.strip() + ['foo', 'bar'] + """ + + def __bool__(self): + return bool(self.data) + + def __str__(self): + return ' '.join(map(str, self.data)) + + def __iter__(self): + return iter(self.data) + + def __call__(self, *args, **kwargs) -> 'NodeList': + result = [x(*args, **kwargs) for x in self.data] + return self.__class__(result) + + def __getattr__(self, name) -> 'NodeList': + """Returns a NodeList of `name` from each member.""" + result = [getattr(x, name) for x in self.data] + return self.__class__(result) + + def __getitem__(self, index): + """Returns one item, forces a `NodeList` if `index` is a slice.""" + # TODO: annotate return how? Union[] - don't know type of single item + if isinstance(index, slice): + return self.__class__(self.data[index]) + return self.data[index] + + +class DisplayEngine: + """A callable class used to display SCons messages.""" + + print_it = True + + def __call__(self, text, append_newline=1): + if not self.print_it: + return + + if append_newline: + text = text + '\n' + + # Stdout might be connected to a pipe that has been closed + # by now. The most likely reason for the pipe being closed + # is that the user has press ctrl-c. It this is the case, + # then SCons is currently shutdown. We therefore ignore + # IOError's here so that SCons can continue and shutdown + # properly so that the .sconsign is correctly written + # before SCons exits. + with suppress(IOError): + sys.stdout.write(str(text)) + + def set_mode(self, mode): + self.print_it = mode + +display = DisplayEngine() + + +# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) +def render_tree(root, child_func, prune=0, margin=[0], visited=None) -> str: + """Render a tree of nodes into an ASCII tree view. + + Args: + root: the root node of the tree + child_func: the function called to get the children of a node + prune: don't visit the same node twice + margin: the format of the left margin to use for children of `root`. + 1 results in a pipe, and 0 results in no pipe. + visited: a dictionary of visited nodes in the current branch if + `prune` is 0, or in the whole tree if `prune` is 1. + """ + + rname = str(root) + + # Initialize 'visited' dict, if required + if visited is None: + visited = {} + + children = child_func(root) + retval = "" + for pipe in margin[:-1]: + if pipe: + retval = retval + "| " + else: + retval = retval + " " + + if rname in visited: + return retval + "+-[" + rname + "]\n" + + retval = retval + "+-" + rname + "\n" + if not prune: + visited = copy.copy(visited) + visited[rname] = True + + for i, child in enumerate(children): + margin.append(i < len(children)-1) + retval = retval + render_tree(child, child_func, prune, margin, visited) + margin.pop() + + return retval + +def IDX(n) -> bool: + """Generate in index into strings from the tree legends. + + These are always a choice between two, so bool works fine. + """ + return bool(n) + +# unicode line drawing chars: +BOX_HORIZ = chr(0x2500) # '─' +BOX_VERT = chr(0x2502) # '│' +BOX_UP_RIGHT = chr(0x2514) # 'â””' +BOX_DOWN_RIGHT = chr(0x250c) # '┌' +BOX_DOWN_LEFT = chr(0x2510) # 'â”' +BOX_UP_LEFT = chr(0x2518) # '┘' +BOX_VERT_RIGHT = chr(0x251c) # '├' +BOX_HORIZ_DOWN = chr(0x252c) # '┬' + + +# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) +def print_tree( + root, + child_func, + prune=0, + showtags=False, + margin=[0], + visited=None, + lastChild: bool = False, + singleLineDraw: bool = False, +) -> None: + """Print a tree of nodes. + + This is like func:`render_tree`, except it prints lines directly instead + of creating a string representation in memory, so that huge trees can + be handled. + + Args: + root: the root node of the tree + child_func: the function called to get the children of a node + prune: don't visit the same node twice + showtags: print status information to the left of each node line + margin: the format of the left margin to use for children of *root*. + 1 results in a pipe, and 0 results in no pipe. + visited: a dictionary of visited nodes in the current branch if + *prune* is 0, or in the whole tree if *prune* is 1. + lastChild: this is the last leaf of a branch + singleLineDraw: use line-drawing characters rather than ASCII. + """ + + rname = str(root) + + # Initialize 'visited' dict, if required + if visited is None: + visited = {} + + if showtags: + + if showtags == 2: + legend = (' E = exists\n' + + ' R = exists in repository only\n' + + ' b = implicit builder\n' + + ' B = explicit builder\n' + + ' S = side effect\n' + + ' P = precious\n' + + ' A = always build\n' + + ' C = current\n' + + ' N = no clean\n' + + ' H = no cache\n' + + '\n') + sys.stdout.write(legend) + + tags = [ + '[', + ' E'[IDX(root.exists())], + ' R'[IDX(root.rexists() and not root.exists())], + ' BbB'[ + [0, 1][IDX(root.has_explicit_builder())] + + [0, 2][IDX(root.has_builder())] + ], + ' S'[IDX(root.side_effect)], + ' P'[IDX(root.precious)], + ' A'[IDX(root.always_build)], + ' C'[IDX(root.is_up_to_date())], + ' N'[IDX(root.noclean)], + ' H'[IDX(root.nocache)], + ']' + ] + + else: + tags = [] + + def MMM(m): + if singleLineDraw: + return [" ", BOX_VERT + " "][m] + + return [" ", "| "][m] + + margins = list(map(MMM, margin[:-1])) + children = child_func(root) + cross = "+-" + if singleLineDraw: + cross = BOX_VERT_RIGHT + BOX_HORIZ # sign used to point to the leaf. + # check if this is the last leaf of the branch + if lastChild: + #if this if the last leaf, then terminate: + cross = BOX_UP_RIGHT + BOX_HORIZ # sign for the last leaf + + # if this branch has children then split it + if children: + # if it's a leaf: + if prune and rname in visited and children: + cross += BOX_HORIZ + else: + cross += BOX_HORIZ_DOWN + + if prune and rname in visited and children: + sys.stdout.write(''.join(tags + margins + [cross,'[', rname, ']']) + '\n') + return + + sys.stdout.write(''.join(tags + margins + [cross, rname]) + '\n') + + visited[rname] = 1 + + # if this item has children: + if children: + margin.append(1) # Initialize margin with 1 for vertical bar. + idx = IDX(showtags) + _child = 0 # Initialize this for the first child. + for C in children[:-1]: + _child = _child + 1 # number the children + print_tree( + C, + child_func, + prune, + idx, + margin, + visited, + (len(children) - _child) <= 0, + singleLineDraw, + ) + # margins are with space (index 0) because we arrived to the last child. + margin[-1] = 0 + # for this call child and nr of children needs to be set 0, to signal the second phase. + print_tree(children[-1], child_func, prune, idx, margin, visited, True, singleLineDraw) + margin.pop() # destroy the last margin added + + +def do_flatten( + sequence, + result, + isinstance=isinstance, + StringTypes=StringTypes, + SequenceTypes=SequenceTypes, +): # pylint: disable=redefined-outer-name,redefined-builtin + for item in sequence: + if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): + result.append(item) + else: + do_flatten(item, result) + + +def flatten( # pylint: disable=redefined-outer-name,redefined-builtin + obj, + isinstance=isinstance, + StringTypes=StringTypes, + SequenceTypes=SequenceTypes, + do_flatten=do_flatten, +) -> list: + """Flatten a sequence to a non-nested list. + + Converts either a single scalar or a nested sequence to a non-nested list. + Note that :func:`flatten` considers strings + to be scalars instead of sequences like pure Python would. + """ + if isinstance(obj, StringTypes) or not isinstance(obj, SequenceTypes): + return [obj] + result = [] + for item in obj: + if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): + result.append(item) + else: + do_flatten(item, result) + return result + + +def flatten_sequence( # pylint: disable=redefined-outer-name,redefined-builtin + sequence, + isinstance=isinstance, + StringTypes=StringTypes, + SequenceTypes=SequenceTypes, + do_flatten=do_flatten, +) -> list: + """Flatten a sequence to a non-nested list. + + Same as :func:`flatten`, but it does not handle the single scalar case. + This is slightly more efficient when one knows that the sequence + to flatten can not be a scalar. + """ + result = [] + for item in sequence: + if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): + result.append(item) + else: + do_flatten(item, result) + return result + + +# The SCons "semi-deep" copy. +# +# This makes separate copies of lists (including UserList objects) +# dictionaries (including UserDict objects) and tuples, but just copies +# references to anything else it finds. +# +# A special case is any object that has a __semi_deepcopy__() method, +# which we invoke to create the copy. Currently only used by +# BuilderDict to actually prevent the copy operation (as invalid on that object). +# +# The dispatch table approach used here is a direct rip-off from the +# normal Python copy module. + +def semi_deepcopy_dict(obj, exclude=None) -> dict: + if exclude is None: + exclude = [] + return {k: semi_deepcopy(v) for k, v in obj.items() if k not in exclude} + +def _semi_deepcopy_list(obj) -> list: + return [semi_deepcopy(item) for item in obj] + +def _semi_deepcopy_tuple(obj) -> tuple: + return tuple(map(semi_deepcopy, obj)) + +_semi_deepcopy_dispatch = { + dict: semi_deepcopy_dict, + list: _semi_deepcopy_list, + tuple: _semi_deepcopy_tuple, +} + +def semi_deepcopy(obj): + copier = _semi_deepcopy_dispatch.get(type(obj)) + if copier: + return copier(obj) + + if hasattr(obj, '__semi_deepcopy__') and callable(obj.__semi_deepcopy__): + return obj.__semi_deepcopy__() + + if isinstance(obj, UserDict): + return obj.__class__(semi_deepcopy_dict(obj)) + + if isinstance(obj, UserList): + return obj.__class__(_semi_deepcopy_list(obj)) + + return obj + + +class Proxy: + """A simple generic Proxy class, forwarding all calls to subject. + + This means you can take an object, let's call it `'obj_a`, + and wrap it in this Proxy class, with a statement like this:: + + proxy_obj = Proxy(obj_a) + + Then, if in the future, you do something like this:: + + x = proxy_obj.var1 + + since the :class:`Proxy` class does not have a :attr:`var1` attribute + (but presumably `objA` does), the request actually is equivalent to saying:: + + x = obj_a.var1 + + Inherit from this class to create a Proxy. + + With Python 3.5+ this does *not* work transparently + for :class:`Proxy` subclasses that use special .__*__() method names, + because those names are now bound to the class, not the individual + instances. You now need to know in advance which special method names you + want to pass on to the underlying Proxy object, and specifically delegate + their calls like this:: + + class Foo(Proxy): + __str__ = Delegate('__str__') + """ + + def __init__(self, subject): + """Wrap an object as a Proxy object""" + self._subject = subject + + def __getattr__(self, name): + """Retrieve an attribute from the wrapped object. + + Raises: + AttributeError: if attribute `name` doesn't exist. + """ + return getattr(self._subject, name) + + def get(self): + """Retrieve the entire wrapped object""" + return self._subject + + def __eq__(self, other): + if issubclass(other.__class__, self._subject.__class__): + return self._subject == other + return self.__dict__ == other.__dict__ + + +class Delegate: + """A Python Descriptor class that delegates attribute fetches + to an underlying wrapped subject of a Proxy. Typical use:: + + class Foo(Proxy): + __str__ = Delegate('__str__') + """ + def __init__(self, attribute): + self.attribute = attribute + + def __get__(self, obj, cls): + if isinstance(obj, cls): + return getattr(obj._subject, self.attribute) + + return self + + +# attempt to load the windows registry module: +can_read_reg = False +try: + import winreg + + can_read_reg = True + hkey_mod = winreg + +except ImportError: + class _NoError(Exception): + pass + RegError = _NoError + +if can_read_reg: + HKEY_CLASSES_ROOT = hkey_mod.HKEY_CLASSES_ROOT + HKEY_LOCAL_MACHINE = hkey_mod.HKEY_LOCAL_MACHINE + HKEY_CURRENT_USER = hkey_mod.HKEY_CURRENT_USER + HKEY_USERS = hkey_mod.HKEY_USERS + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegQueryValueEx = winreg.QueryValueEx + RegError = winreg.error + + def RegGetValue(root, key): + r"""Returns a registry value without having to open the key first. + + Only available on Windows platforms with a version of Python that + can read the registry. + + Returns the same thing as :func:`RegQueryValueEx`, except you just + specify the entire path to the value, and don't have to bother + opening the key first. So, instead of:: + + k = SCons.Util.RegOpenKeyEx(SCons.Util.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Microsoft\Windows\CurrentVersion') + out = SCons.Util.RegQueryValueEx(k, 'ProgramFilesDir') + + You can write:: + + out = SCons.Util.RegGetValue(SCons.Util.HKEY_LOCAL_MACHINE, + r'SOFTWARE\Microsoft\Windows\CurrentVersion\ProgramFilesDir') + """ + # I would use os.path.split here, but it's not a filesystem + # path... + p = key.rfind('\\') + 1 + keyp = key[: p - 1] # -1 to omit trailing slash + val = key[p:] + k = RegOpenKeyEx(root, keyp) + return RegQueryValueEx(k, val) + + +else: + HKEY_CLASSES_ROOT = None + HKEY_LOCAL_MACHINE = None + HKEY_CURRENT_USER = None + HKEY_USERS = None + + def RegGetValue(root, key): + raise OSError + + def RegOpenKeyEx(root, key): + raise OSError + + +if sys.platform == 'win32': + + def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: + if path is None: + try: + path = os.environ['PATH'] + except KeyError: + return None + if is_String(path): + path = path.split(os.pathsep) + if pathext is None: + try: + pathext = os.environ['PATHEXT'] + except KeyError: + pathext = '.COM;.EXE;.BAT;.CMD' + if is_String(pathext): + pathext = pathext.split(os.pathsep) + for ext in pathext: + if ext.lower() == file[-len(ext):].lower(): + pathext = [''] + break + if reject is None: + reject = [] + if not is_List(reject) and not is_Tuple(reject): + reject = [reject] + for p in path: + f = os.path.join(p, file) + for ext in pathext: + fext = f + ext + if os.path.isfile(fext): + try: + reject.index(fext) + except ValueError: + return os.path.normpath(fext) + continue + return None + +elif os.name == 'os2': + + def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: + if path is None: + try: + path = os.environ['PATH'] + except KeyError: + return None + if is_String(path): + path = path.split(os.pathsep) + if pathext is None: + pathext = ['.exe', '.cmd'] + for ext in pathext: + if ext.lower() == file[-len(ext):].lower(): + pathext = [''] + break + if reject is None: + reject = [] + if not is_List(reject) and not is_Tuple(reject): + reject = [reject] + for p in path: + f = os.path.join(p, file) + for ext in pathext: + fext = f + ext + if os.path.isfile(fext): + try: + reject.index(fext) + except ValueError: + return os.path.normpath(fext) + continue + return None + +else: + + def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: + import stat # pylint: disable=import-outside-toplevel + + if path is None: + try: + path = os.environ['PATH'] + except KeyError: + return None + if is_String(path): + path = path.split(os.pathsep) + if reject is None: + reject = [] + if not is_List(reject) and not is_Tuple(reject): + reject = [reject] + for p in path: + f = os.path.join(p, file) + if os.path.isfile(f): + try: + st = os.stat(f) + except OSError: + # os.stat() raises OSError, not IOError if the file + # doesn't exist, so in this case we let IOError get + # raised so as to not mask possibly serious disk or + # network issues. + continue + if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: + try: + reject.index(f) + except ValueError: + return os.path.normpath(f) + continue + return None + +WhereIs.__doc__ = """\ +Return the path to an executable that matches `file`. + +Searches the given `path` for `file`, respecting any filename +extensions `pathext` (on the Windows platform only), and +returns the full path to the matching command. If no +command is found, return ``None``. + +If `path` is not specified, :attr:`os.environ[PATH]` is used. +If `pathext` is not specified, :attr:`os.environ[PATHEXT]` +is used. Will not select any path name or names in the optional +`reject` list. +""" + + +if sys.platform == 'cygwin': + import subprocess # pylint: disable=import-outside-toplevel + + def get_native_path(path) -> str: + cp = subprocess.run(('cygpath', '-w', path), check=False, stdout=subprocess.PIPE) + return cp.stdout.decode().replace('\n', '') +else: + def get_native_path(path) -> str: + return path + +get_native_path.__doc__ = """\ +Transform an absolute path into a native path for the system. + +In Cygwin, this converts from a Cygwin path to a Windows path, +without regard to whether `path` refers to an existing file +system object. For other platforms, `path` is unchanged. +""" + + +def Split(arg) -> list: + """Returns a list of file names or other objects. + + If `arg` is a string, it will be split on strings of white-space + characters within the string. If `arg` is already a list, the list + will be returned untouched. If `arg` is any other type of object, + it will be returned as a list containing just the object. + + >>> print(Split(" this is a string ")) + ['this', 'is', 'a', 'string'] + >>> print(Split(["stringlist", " preserving ", " spaces "])) + ['stringlist', ' preserving ', ' spaces '] + """ + if is_List(arg) or is_Tuple(arg): + return arg + + if is_String(arg): + return arg.split() + + return [arg] + + +class CLVar(UserList): + """A container for command-line construction variables. + + Forces the use of a list of strings intended as command-line + arguments. Like :class:`collections.UserList`, but the argument + passed to the initializter will be processed by the :func:`Split` + function, which includes special handling for string types: they + will be split into a list of words, not coereced directly to a list. + The same happens if a string is added to a :class:`CLVar`, + which allows doing the right thing with both + :func:`Append`/:func:`Prepend` methods, + as well as with pure Python addition, regardless of whether adding + a list or a string to a construction variable. + + Side effect: spaces will be stripped from individual string + arguments. If you need spaces preserved, pass strings containing + spaces inside a list argument. + + >>> u = UserList("--some --opts and args") + >>> print(len(u), repr(u)) + 22 ['-', '-', 's', 'o', 'm', 'e', ' ', '-', '-', 'o', 'p', 't', 's', ' ', 'a', 'n', 'd', ' ', 'a', 'r', 'g', 's'] + >>> c = CLVar("--some --opts and args") + >>> print(len(c), repr(c)) + 4 ['--some', '--opts', 'and', 'args'] + >>> c += " strips spaces " + >>> print(len(c), repr(c)) + 6 ['--some', '--opts', 'and', 'args', 'strips', 'spaces'] + """ + + def __init__(self, initlist=None): + super().__init__(Split(initlist if initlist is not None else [])) + + def __add__(self, other): + return super().__add__(CLVar(other)) + + def __radd__(self, other): + return super().__radd__(CLVar(other)) + + def __iadd__(self, other): + return super().__iadd__(CLVar(other)) + + def __str__(self): + # Some cases the data can contain Nodes, so make sure they + # processed to string before handing them over to join. + return ' '.join([str(d) for d in self.data]) + + +class Selector(OrderedDict): + """A callable ordered dictionary that maps file suffixes to + dictionary values. We preserve the order in which items are added + so that :func:`get_suffix` calls always return the first suffix added. + """ + def __call__(self, env, source, ext=None): + if ext is None: + try: + ext = source[0].get_suffix() + except IndexError: + ext = "" + try: + return self[ext] + except KeyError: + # Try to perform Environment substitution on the keys of + # the dictionary before giving up. + s_dict = {} + for (k,v) in self.items(): + if k is not None: + s_k = env.subst(k) + if s_k in s_dict: + # We only raise an error when variables point + # to the same suffix. If one suffix is literal + # and a variable suffix contains this literal, + # the literal wins and we don't raise an error. + raise KeyError(s_dict[s_k][0], k, s_k) + s_dict[s_k] = (k,v) + try: + return s_dict[ext][1] + except KeyError: + try: + return self[None] + except KeyError: + return None + + +if sys.platform == 'cygwin': + # On Cygwin, os.path.normcase() lies, so just report back the + # fact that the underlying Windows OS is case-insensitive. + def case_sensitive_suffixes(s1, s2) -> bool: # pylint: disable=unused-argument + return False + +else: + def case_sensitive_suffixes(s1, s2) -> bool: + return os.path.normcase(s1) != os.path.normcase(s2) + + +def adjustixes(fname, pre, suf, ensure_suffix=False) -> str: + """Adjust filename prefixes and suffixes as needed. + + Add `prefix` to `fname` if specified. + Add `suffix` to `fname` if specified and if `ensure_suffix` is ``True`` + """ + + if pre: + path, fn = os.path.split(os.path.normpath(fname)) + + # Handle the odd case where the filename = the prefix. + # In that case, we still want to add the prefix to the file + if not fn.startswith(pre) or fn == pre: + fname = os.path.join(path, pre + fn) + # Only append a suffix if the suffix we're going to add isn't already + # there, and if either we've been asked to ensure the specific suffix + # is present or there's no suffix on it at all. + # Also handle the odd case where the filename = the suffix. + # in that case we still want to append the suffix + if suf and not fname.endswith(suf) and \ + (ensure_suffix or not splitext(fname)[1]): + fname = fname + suf + return fname + + + +# From Tim Peters, +# https://code.activestate.com/recipes/52560 +# ASPN: Python Cookbook: Remove duplicates from a sequence +# (Also in the printed Python Cookbook.) +# Updated. This algorithm is used by some scanners and tools. + +def unique(seq): + """Return a list of the elements in seq without duplicates, ignoring order. + + >>> mylist = unique([1, 2, 3, 1, 2, 3]) + >>> print(sorted(mylist)) + [1, 2, 3] + >>> mylist = unique("abcabc") + >>> print(sorted(mylist)) + ['a', 'b', 'c'] + >>> mylist = unique(([1, 2], [2, 3], [1, 2])) + >>> print(sorted(mylist)) + [[1, 2], [2, 3]] + + For best speed, all sequence elements should be hashable. Then + unique() will usually work in linear time. + + If not possible, the sequence elements should enjoy a total + ordering, and if list(s).sort() doesn't raise TypeError it's + assumed that they do enjoy a total ordering. Then unique() will + usually work in O(N*log2(N)) time. + + If that's not possible either, the sequence elements must support + equality-testing. Then unique() will usually work in quadratic time. + """ + + if not seq: + return [] + + # Try using a dict first, as that's the fastest and will usually + # work. If it doesn't work, it will usually fail quickly, so it + # usually doesn't cost much to *try* it. It requires that all the + # sequence elements be hashable, and support equality comparison. + # TODO: should be even faster: return(list(set(seq))) + with suppress(TypeError): + return list(dict.fromkeys(seq)) + + # We couldn't hash all the elements (got a TypeError). + # Next fastest is to sort, which brings the equal elements together; + # then duplicates are easy to weed out in a single pass. + # NOTE: Python's list.sort() was designed to be efficient in the + # presence of many duplicate elements. This isn't true of all + # sort functions in all languages or libraries, so this approach + # is more effective in Python than it may be elsewhere. + n = len(seq) + try: + t = sorted(seq) + except TypeError: + pass # move on to the next method + else: + last = t[0] + lasti = i = 1 + while i < n: + if t[i] != last: + t[lasti] = last = t[i] + lasti = lasti + 1 + i = i + 1 + return t[:lasti] + + # Brute force is all that's left. + u = [] + for x in seq: + if x not in u: + u.append(x) + return u + +# Best way (assuming Python 3.7, but effectively 3.6) to remove +# duplicates from a list in while preserving order, according to +# https://stackoverflow.com/questions/480214/how-do-i-remove-duplicates-from-a-list-while-preserving-order/17016257#17016257 +def uniquer_hashables(seq): + return list(dict.fromkeys(seq)) + +# Recipe 19.11 "Reading Lines with Continuation Characters", +# by Alex Martelli, straight from the Python CookBook (2nd edition). +def logical_lines(physical_lines, joiner=''.join): + logical_line = [] + for line in physical_lines: + stripped = line.rstrip() + if stripped.endswith('\\'): + # a line which continues w/the next physical line + logical_line.append(stripped[:-1]) + else: + # a line which does not continue, end of logical line + logical_line.append(line) + yield joiner(logical_line) + logical_line = [] + if logical_line: + # end of sequence implies end of last logical line + yield joiner(logical_line) + + +class LogicalLines: + """ Wrapper class for the logical_lines method. + + Allows us to read all "logical" lines at once from a given file object. + """ + + def __init__(self, fileobj): + self.fileobj = fileobj + + def readlines(self): + return list(logical_lines(self.fileobj)) + + +class UniqueList(UserList): + """A list which maintains uniqueness. + + Uniquing is lazy: rather than being assured on list changes, it is fixed + up on access by those methods which need to act on a unique list to be + correct. That means things like "in" don't have to eat the uniquing time. + """ + def __init__(self, initlist=None): + super().__init__(initlist) + self.unique = True + + def __make_unique(self): + if not self.unique: + self.data = uniquer_hashables(self.data) + self.unique = True + + def __repr__(self): + self.__make_unique() + return super().__repr__() + + def __lt__(self, other): + self.__make_unique() + return super().__lt__(other) + + def __le__(self, other): + self.__make_unique() + return super().__le__(other) + + def __eq__(self, other): + self.__make_unique() + return super().__eq__(other) + + def __ne__(self, other): + self.__make_unique() + return super().__ne__(other) + + def __gt__(self, other): + self.__make_unique() + return super().__gt__(other) + + def __ge__(self, other): + self.__make_unique() + return super().__ge__(other) + + # __contains__ doesn't need to worry about uniquing, inherit + + def __len__(self): + self.__make_unique() + return super().__len__() + + def __getitem__(self, i): + self.__make_unique() + return super().__getitem__(i) + + def __setitem__(self, i, item): + super().__setitem__(i, item) + self.unique = False + + # __delitem__ doesn't need to worry about uniquing, inherit + + def __add__(self, other): + result = super().__add__(other) + result.unique = False + return result + + def __radd__(self, other): + result = super().__radd__(other) + result.unique = False + return result + + def __iadd__(self, other): + result = super().__iadd__(other) + result.unique = False + return result + + def __mul__(self, other): + result = super().__mul__(other) + result.unique = False + return result + + def __rmul__(self, other): + result = super().__rmul__(other) + result.unique = False + return result + + def __imul__(self, other): + result = super().__imul__(other) + result.unique = False + return result + + def append(self, item): + super().append(item) + self.unique = False + + def insert(self, i, item): + super().insert(i, item) + self.unique = False + + def count(self, item): + self.__make_unique() + return super().count(item) + + def index(self, item, *args): + self.__make_unique() + return super().index(item, *args) + + def reverse(self): + self.__make_unique() + super().reverse() + + # TODO: Py3.8: def sort(self, /, *args, **kwds): + def sort(self, *args, **kwds): + self.__make_unique() + return super().sort(*args, **kwds) + + def extend(self, other): + super().extend(other) + self.unique = False + + +class Unbuffered: + """A proxy that wraps a file object, flushing after every write. + + Delegates everything else to the wrapped object. + """ + def __init__(self, file): + self.file = file + + def write(self, arg): + # Stdout might be connected to a pipe that has been closed + # by now. The most likely reason for the pipe being closed + # is that the user has press ctrl-c. It this is the case, + # then SCons is currently shutdown. We therefore ignore + # IOError's here so that SCons can continue and shutdown + # properly so that the .sconsign is correctly written + # before SCons exits. + with suppress(IOError): + self.file.write(arg) + self.file.flush() + + def writelines(self, arg): + with suppress(IOError): + self.file.writelines(arg) + self.file.flush() + + def __getattr__(self, attr): + return getattr(self.file, attr) + +def make_path_relative(path) -> str: + """Converts an absolute path name to a relative pathname.""" + + if os.path.isabs(path): + drive_s, path = os.path.splitdrive(path) + + if not drive_s: + path=re.compile(r"/*(.*)").findall(path)[0] + else: + path=path[1:] + + assert not os.path.isabs(path), path + return path + + +def silent_intern(x): + """ + Perform :mod:`sys.intern` on the passed argument and return the result. + If the input is ineligible for interning the original argument is + returned and no exception is thrown. + """ + try: + return sys.intern(x) + except TypeError: + return x + + +def cmp(a, b) -> bool: + """A cmp function because one is no longer available in python3.""" + return (a > b) - (a < b) + + +def print_time(): + """Hack to return a value from Main if can't import Main.""" + # pylint: disable=redefined-outer-name,import-outside-toplevel + from SCons.Script.Main import print_time + return print_time + + +def wait_for_process_to_die(pid): + """ + Wait for specified process to die, or alternatively kill it + NOTE: This function operates best with psutil pypi package + TODO: Add timeout which raises exception + """ + # wait for the process to fully killed + try: + import psutil # pylint: disable=import-outside-toplevel + while True: + if pid not in [proc.pid for proc in psutil.process_iter()]: + break + time.sleep(0.1) + except ImportError: + # if psutil is not installed we can do this the hard way + while True: + if sys.platform == 'win32': + import ctypes # pylint: disable=import-outside-toplevel + PROCESS_QUERY_INFORMATION = 0x1000 + processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) + if processHandle == 0: + break + ctypes.windll.kernel32.CloseHandle(processHandle) + time.sleep(0.1) + else: + try: + os.kill(pid, 0) + except OSError: + break + time.sleep(0.1) + +# From: https://stackoverflow.com/questions/1741972/how-to-use-different-formatters-with-the-same-logging-handler-in-python +class DispatchingFormatter(Formatter): + + def __init__(self, formatters, default_formatter): + self._formatters = formatters + self._default_formatter = default_formatter + + def format(self, record): + formatter = self._formatters.get(record.name, self._default_formatter) + return formatter.format(record) + + +def sanitize_shell_env(execution_env: dict) -> dict: + """Sanitize all values in *execution_env* + + The execution environment (typically comes from (env['ENV']) is + propagated to the shell, and may need to be cleaned first. + + Args: + execution_env: The shell environment variables to be propagated + to the spawned shell. + + Returns: + sanitized dictionary of env variables (similar to what you'd get + from :data:`os.environ`) + """ + # Ensure that the ENV values are all strings: + new_env = {} + for key, value in execution_env.items(): + if is_List(value): + # If the value is a list, then we assume it is a path list, + # because that's a pretty common list-like value to stick + # in an environment variable: + value = flatten_sequence(value) + new_env[key] = os.pathsep.join(map(str, value)) + else: + # It's either a string or something else. If it isn't a + # string or a list, then we just coerce it to a string, which + # is the proper way to handle Dir and File instances and will + # produce something reasonable for just about everything else: + new_env[key] = str(value) + return new_env + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/envs.py b/SCons/Util/envs.py new file mode 100644 index 0000000..9acfe4a --- /dev/null +++ b/SCons/Util/envs.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + +"""Various SCons utility functions + +Routines for working with environemnts and construction variables +that don't need the specifics of Environment. +""" + +import os +from types import MethodType, FunctionType +from typing import Union + +from .types import is_List, is_Tuple, is_String + + +def PrependPath( + oldpath, newpath, sep=os.pathsep, delete_existing=True, canonicalize=None +) -> Union[list, str]: + """Prepend *newpath* path elements to *oldpath*. + + Will only add any particular path once (leaving the first one it + encounters and ignoring the rest, to preserve path order), and will + :mod:`os.path.normpath` and :mod:`os.path.normcase` all paths to help + assure this. This can also handle the case where *oldpath* + is a list instead of a string, in which case a list will be returned + instead of a string. For example: + + >>> p = PrependPath("/foo/bar:/foo", "/biz/boom:/foo") + >>> print(p) + /biz/boom:/foo:/foo/bar + + If *delete_existing* is ``False``, then adding a path that exists will + not move it to the beginning; it will stay where it is in the list. + + >>> p = PrependPath("/foo/bar:/foo", "/biz/boom:/foo", delete_existing=False) + >>> print(p) + /biz/boom:/foo/bar:/foo + + If *canonicalize* is not ``None``, it is applied to each element of + *newpath* before use. + """ + orig = oldpath + is_list = True + paths = orig + if not is_List(orig) and not is_Tuple(orig): + paths = paths.split(sep) + is_list = False + + if is_String(newpath): + newpaths = newpath.split(sep) + elif not is_List(newpath) and not is_Tuple(newpath): + newpaths = [newpath] # might be a Dir + else: + newpaths = newpath + + if canonicalize: + newpaths = list(map(canonicalize, newpaths)) + + if not delete_existing: + # First uniquify the old paths, making sure to + # preserve the first instance (in Unix/Linux, + # the first one wins), and remembering them in normpaths. + # Then insert the new paths at the head of the list + # if they're not already in the normpaths list. + result = [] + normpaths = [] + for path in paths: + if not path: + continue + normpath = os.path.normpath(os.path.normcase(path)) + if normpath not in normpaths: + result.append(path) + normpaths.append(normpath) + newpaths.reverse() # since we're inserting at the head + for path in newpaths: + if not path: + continue + normpath = os.path.normpath(os.path.normcase(path)) + if normpath not in normpaths: + result.insert(0, path) + normpaths.append(normpath) + paths = result + + else: + newpaths = newpaths + paths # prepend new paths + + normpaths = [] + paths = [] + # now we add them only if they are unique + for path in newpaths: + normpath = os.path.normpath(os.path.normcase(path)) + if path and normpath not in normpaths: + paths.append(path) + normpaths.append(normpath) + + if is_list: + return paths + + return sep.join(paths) + + +def AppendPath( + oldpath, newpath, sep=os.pathsep, delete_existing=True, canonicalize=None +) -> Union[list, str]: + """Append *newpath* path elements to *oldpath*. + + Will only add any particular path once (leaving the last one it + encounters and ignoring the rest, to preserve path order), and will + :mod:`os.path.normpath` and :mod:`os.path.normcase` all paths to help + assure this. This can also handle the case where *oldpath* + is a list instead of a string, in which case a list will be returned + instead of a string. For example: + + >>> p = AppendPath("/foo/bar:/foo", "/biz/boom:/foo") + >>> print(p) + /foo/bar:/biz/boom:/foo + + If *delete_existing* is ``False``, then adding a path that exists + will not move it to the end; it will stay where it is in the list. + + >>> p = AppendPath("/foo/bar:/foo", "/biz/boom:/foo", delete_existing=False) + >>> print(p) + /foo/bar:/foo:/biz/boom + + If *canonicalize* is not ``None``, it is applied to each element of + *newpath* before use. + """ + orig = oldpath + is_list = True + paths = orig + if not is_List(orig) and not is_Tuple(orig): + paths = paths.split(sep) + is_list = False + + if is_String(newpath): + newpaths = newpath.split(sep) + elif not is_List(newpath) and not is_Tuple(newpath): + newpaths = [newpath] # might be a Dir + else: + newpaths = newpath + + if canonicalize: + newpaths = list(map(canonicalize, newpaths)) + + if not delete_existing: + # add old paths to result, then + # add new paths if not already present + # (I thought about using a dict for normpaths for speed, + # but it's not clear hashing the strings would be faster + # than linear searching these typically short lists.) + result = [] + normpaths = [] + for path in paths: + if not path: + continue + result.append(path) + normpaths.append(os.path.normpath(os.path.normcase(path))) + for path in newpaths: + if not path: + continue + normpath = os.path.normpath(os.path.normcase(path)) + if normpath not in normpaths: + result.append(path) + normpaths.append(normpath) + paths = result + else: + # start w/ new paths, add old ones if not present, + # then reverse. + newpaths = paths + newpaths # append new paths + newpaths.reverse() + + normpaths = [] + paths = [] + # now we add them only if they are unique + for path in newpaths: + normpath = os.path.normpath(os.path.normcase(path)) + if path and normpath not in normpaths: + paths.append(path) + normpaths.append(normpath) + paths.reverse() + + if is_list: + return paths + + return sep.join(paths) + + +def AddPathIfNotExists(env_dict, key, path, sep=os.pathsep): + """Add a path element to a construction variable. + + `key` is looked up in `env_dict`, and `path` is added to it if it + is not already present. `env_dict[key]` is assumed to be in the + format of a PATH variable: a list of paths separated by `sep` tokens. + + >>> env = {'PATH': '/bin:/usr/bin:/usr/local/bin'} + >>> AddPathIfNotExists(env, 'PATH', '/opt/bin') + >>> print(env['PATH']) + /opt/bin:/bin:/usr/bin:/usr/local/bin + """ + try: + is_list = True + paths = env_dict[key] + if not is_List(env_dict[key]): + paths = paths.split(sep) + is_list = False + if os.path.normcase(path) not in list(map(os.path.normcase, paths)): + paths = [path] + paths + if is_list: + env_dict[key] = paths + else: + env_dict[key] = sep.join(paths) + except KeyError: + env_dict[key] = path + + +class MethodWrapper: + """A generic Wrapper class that associates a method with an object. + + As part of creating this MethodWrapper object an attribute with the + specified name (by default, the name of the supplied method) is added + to the underlying object. When that new "method" is called, our + :meth:`__call__` method adds the object as the first argument, simulating + the Python behavior of supplying "self" on method calls. + + We hang on to the name by which the method was added to the underlying + base class so that we can provide a method to "clone" ourselves onto + a new underlying object being copied (without which we wouldn't need + to save that info). + """ + def __init__(self, obj, method, name=None): + if name is None: + name = method.__name__ + self.object = obj + self.method = method + self.name = name + setattr(self.object, name, self) + + def __call__(self, *args, **kwargs): + nargs = (self.object,) + args + return self.method(*nargs, **kwargs) + + def clone(self, new_object): + """ + Returns an object that re-binds the underlying "method" to + the specified new object. + """ + return self.__class__(new_object, self.method, self.name) + + +# The original idea for AddMethod() came from the +# following post to the ActiveState Python Cookbook: +# +# ASPN: Python Cookbook : Install bound methods in an instance +# https://code.activestate.com/recipes/223613 +# +# Changed as follows: +# * Switched the installmethod() "object" and "function" arguments, +# so the order reflects that the left-hand side is the thing being +# "assigned to" and the right-hand side is the value being assigned. +# * The instance/class detection is changed a bit, as it's all +# new-style classes now with Py3. +# * The by-hand construction of the function object from renamefunction() +# is not needed, the remaining bit is now used inline in AddMethod. + + +def AddMethod(obj, function, name=None): + """Add a method to an object. + + Adds *function* to *obj* if *obj* is a class object. + Adds *function* as a bound method if *obj* is an instance object. + If *obj* looks like an environment instance, use :class:`~SCons.Util.MethodWrapper` + to add it. If *name* is supplied it is used as the name of *function*. + + Although this works for any class object, the intent as a public + API is to be used on Environment, to be able to add a method to all + construction environments; it is preferred to use ``env.AddMethod`` + to add to an individual environment. + + >>> class A: + ... ... + + >>> a = A() + + >>> def f(self, x, y): + ... self.z = x + y + + >>> AddMethod(A, f, "add") + >>> a.add(2, 4) + >>> print(a.z) + 6 + >>> a.data = ['a', 'b', 'c', 'd', 'e', 'f'] + >>> AddMethod(a, lambda self, i: self.data[i], "listIndex") + >>> print(a.listIndex(3)) + d + + """ + if name is None: + name = function.__name__ + else: + # "rename" + function = FunctionType( + function.__code__, function.__globals__, name, function.__defaults__ + ) + + if hasattr(obj, '__class__') and obj.__class__ is not type: + # obj is an instance, so it gets a bound method. + if hasattr(obj, "added_methods"): + method = MethodWrapper(obj, function, name) + obj.added_methods.append(method) + else: + method = MethodType(function, obj) + else: + # obj is a class + method = function + + setattr(obj, name, method) + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/hashes.py b/SCons/Util/hashes.py new file mode 100644 index 0000000..4771c44 --- /dev/null +++ b/SCons/Util/hashes.py @@ -0,0 +1,401 @@ +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + +"""SCons utility functions + +Routines for working with hash formats. +""" + +import hashlib +import sys + +from .types import to_bytes + + +# Default hash function and format. SCons-internal. +DEFAULT_HASH_FORMATS = ['md5', 'sha1', 'sha256'] +ALLOWED_HASH_FORMATS = [] +_HASH_FUNCTION = None +_HASH_FORMAT = None + + +def _attempt_init_of_python_3_9_hash_object(hash_function_object, sys_used=sys): + """Initialize hash function with non-security indicator. + + In Python 3.9 and onwards, :mod:`hashlib` constructors accept a + keyword argument *usedforsecurity*, which, if set to ``False``, + lets us continue to use algorithms that have been deprecated either + by FIPS or by Python itself, as the MD5 algorithm SCons prefers is + not being used for security purposes as much as a short, 32 char + hash that is resistant to accidental collisions. + + In prior versions of python, :mod:`hashlib` returns a native function + wrapper, which errors out when it's queried for the optional + parameter, so this function wraps that call. + + It can still throw a ValueError if the initialization fails due to + FIPS compliance issues, but that is assumed to be the responsibility + of the caller. + """ + if hash_function_object is None: + return None + + # https://stackoverflow.com/a/11887885 details how to check versions + # with the "packaging" library. However, for our purposes, checking + # the version is greater than or equal to 3.9 is good enough, as the API + # is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See + # https://docs.python.org/3/library/hashlib.html#:~:text=usedforsecurity + # for the version support notes. + if (sys_used.version_info.major > 3) or ( + sys_used.version_info.major == 3 and sys_used.version_info.minor >= 9 + ): + return hash_function_object(usedforsecurity=False) + + # Note that this can throw a ValueError in FIPS-enabled versions of + # Linux prior to 3.9. The OpenSSL hashlib will throw on first init here, + # but it is assumed to be responsibility of the caller to diagnose the + # ValueError & potentially display the error to screen. + return hash_function_object() + + +def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys) -> None: + """Check if the default hash algorithms can be called. + + This util class is sometimes called prior to setting the + user-selected hash algorithm, meaning that on FIPS-compliant systems + the library would default-initialize MD5 and throw an exception in + set_hash_format. A common case is using the SConf options, which can + run prior to main, and thus ignore the options.hash_format variable. + + This function checks the DEFAULT_HASH_FORMATS and sets the + ALLOWED_HASH_FORMATS to only the ones that can be called. In Python + >= 3.9 this will always default to MD5 as in Python 3.9 there is an + optional attribute "usedforsecurity" set for the method. + + Throws if no allowed hash formats are detected. + """ + global ALLOWED_HASH_FORMATS + _last_error = None + # note: if you call this method repeatedly, example using timeout, + # this is needed. Otherwise it keeps appending valid formats to the string. + ALLOWED_HASH_FORMATS = [] + + for test_algorithm in DEFAULT_HASH_FORMATS: + _test_hash = getattr(hashlib_used, test_algorithm, None) + # we know hashlib claims to support it... check to see if we can call it. + if _test_hash is not None: + # The hashing library will throw an exception on initialization + # in FIPS mode, meaning if we call the default algorithm returned + # with no parameters, it'll throw if it's a bad algorithm, + # otherwise it will append it to the known good formats. + try: + _attempt_init_of_python_3_9_hash_object(_test_hash, sys_used) + ALLOWED_HASH_FORMATS.append(test_algorithm) + except ValueError as e: + _last_error = e + continue + + if len(ALLOWED_HASH_FORMATS) == 0: + from SCons.Errors import ( + SConsEnvironmentError, + ) # pylint: disable=import-outside-toplevel + + # chain the exception thrown with the most recent error from hashlib. + raise SConsEnvironmentError( + 'No usable hash algorithms found.' + 'Most recent error from hashlib attached in trace.' + ) from _last_error + + +_set_allowed_viable_default_hashes(hashlib) + + +def get_hash_format(): + """Retrieves the hash format or ``None`` if not overridden. + + A return value of ``None`` + does not guarantee that MD5 is being used; instead, it means that the + default precedence order documented in :func:`SCons.Util.set_hash_format` + is respected. + """ + return _HASH_FORMAT + + +def _attempt_get_hash_function(hash_name, hashlib_used=hashlib, sys_used=sys): + """Wrapper used to try to initialize a hash function given. + + If successful, returns the name of the hash function back to the user. + + Otherwise returns None. + """ + try: + _fetch_hash = getattr(hashlib_used, hash_name, None) + if _fetch_hash is None: + return None + _attempt_init_of_python_3_9_hash_object(_fetch_hash, sys_used) + return hash_name + except ValueError: + # If attempt_init_of_python_3_9 throws, this is typically due to FIPS + # being enabled. However, if we get to this point, the viable hash + # function check has either been bypassed or otherwise failed to + # properly restrict the user to only the supported functions. + # As such throw the UserError as an internal assertion-like error. + return None + + +def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): + """Sets the default hash format used by SCons. + + If `hash_format` is ``None`` or + an empty string, the default is determined by this function. + + Currently the default behavior is to use the first available format of + the following options: MD5, SHA1, SHA256. + """ + global _HASH_FORMAT, _HASH_FUNCTION + + _HASH_FORMAT = hash_format + if hash_format: + hash_format_lower = hash_format.lower() + if hash_format_lower not in ALLOWED_HASH_FORMATS: + from SCons.Errors import ( + UserError, + ) # pylint: disable=import-outside-toplevel + + # User can select something not supported by their OS but + # normally supported by SCons, example, selecting MD5 in an + # OS with FIPS-mode turned on. Therefore we first check if + # SCons supports it, and then if their local OS supports it. + if hash_format_lower in DEFAULT_HASH_FORMATS: + raise UserError( + 'While hash format "%s" is supported by SCons, the ' + 'local system indicates only the following hash ' + 'formats are supported by the hashlib library: %s' + % (hash_format_lower, ', '.join(ALLOWED_HASH_FORMATS)) + ) + + # The hash format isn't supported by SCons in any case. + # Warn the user, and if we detect that SCons supports more + # algorithms than their local system supports, + # warn the user about that too. + if ALLOWED_HASH_FORMATS == DEFAULT_HASH_FORMATS: + raise UserError( + 'Hash format "%s" is not supported by SCons. Only ' + 'the following hash formats are supported: %s' + % (hash_format_lower, ', '.join(ALLOWED_HASH_FORMATS)) + ) + + raise UserError( + 'Hash format "%s" is not supported by SCons. ' + 'SCons supports more hash formats than your local system ' + 'is reporting; SCons supports: %s. Your local system only ' + 'supports: %s' + % ( + hash_format_lower, + ', '.join(DEFAULT_HASH_FORMATS), + ', '.join(ALLOWED_HASH_FORMATS), + ) + ) + + # This is not expected to fail. If this fails it means the + # set_allowed_viable_default_hashes function did not throw, + # or when it threw, the exception was caught and ignored, or + # the global ALLOWED_HASH_FORMATS was changed by an external user. + _HASH_FUNCTION = _attempt_get_hash_function( + hash_format_lower, hashlib_used, sys_used + ) + + if _HASH_FUNCTION is None: + from SCons.Errors import ( + UserError, + ) # pylint: disable=import-outside-toplevel + + raise UserError( + 'Hash format "%s" is not available in your Python interpreter. ' + 'Expected to be supported algorithm by set_allowed_viable_default_hashes, ' + 'Assertion error in SCons.' % hash_format_lower + ) + else: + # Set the default hash format based on what is available, defaulting + # to the first supported hash algorithm (usually md5) for backwards + # compatibility. In FIPS-compliant systems this usually defaults to + # SHA1, unless that too has been disabled. + for choice in ALLOWED_HASH_FORMATS: + _HASH_FUNCTION = _attempt_get_hash_function(choice, hashlib_used, sys_used) + + if _HASH_FUNCTION is not None: + break + else: + # This is not expected to happen in practice. + from SCons.Errors import ( + UserError, + ) # pylint: disable=import-outside-toplevel + + raise UserError( + 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' + 'SCons requires at least one. Expected to support one or more ' + 'during set_allowed_viable_default_hashes.' + ) + + +# Ensure that this is initialized in case either: +# 1. This code is running in a unit test. +# 2. This code is running in a consumer that does hash operations while +# SConscript files are being loaded. +set_hash_format(None) + + +def get_current_hash_algorithm_used(): + """Returns the current hash algorithm name used. + + Where the python version >= 3.9, this is expected to return md5. + If python's version is <= 3.8, this returns md5 on non-FIPS-mode platforms, and + sha1 or sha256 on FIPS-mode Linux platforms. + + This function is primarily useful for testing, where one expects a value to be + one of N distinct hashes, and therefore the test needs to know which hash to select. + """ + return _HASH_FUNCTION + + +def _get_hash_object(hash_format, hashlib_used=hashlib, sys_used=sys): + """Allocates a hash object using the requested hash format. + + Args: + hash_format: Hash format to use. + + Returns: + hashlib object. + """ + if hash_format is None: + if _HASH_FUNCTION is None: + from SCons.Errors import ( + UserError, + ) # pylint: disable=import-outside-toplevel + + raise UserError( + 'There is no default hash function. Did you call ' + 'a hashing function before SCons was initialized?' + ) + return _attempt_init_of_python_3_9_hash_object( + getattr(hashlib_used, _HASH_FUNCTION, None), sys_used + ) + + if not hasattr(hashlib, hash_format): + from SCons.Errors import UserError # pylint: disable=import-outside-toplevel + + raise UserError( + f'Hash format "{hash_format}" is not available in your Python interpreter.' + ) + + return _attempt_init_of_python_3_9_hash_object( + getattr(hashlib, hash_format), sys_used + ) + + +def hash_signature(s, hash_format=None): + """ + Generate hash signature of a string + + Args: + s: either string or bytes. Normally should be bytes + hash_format: Specify to override default hash format + + Returns: + String of hex digits representing the signature + """ + m = _get_hash_object(hash_format) + try: + m.update(to_bytes(s)) + except TypeError: + m.update(to_bytes(str(s))) + + return m.hexdigest() + + +def hash_file_signature(fname, chunksize=65536, hash_format=None): + """ + Generate the md5 signature of a file + + Args: + fname: file to hash + chunksize: chunk size to read + hash_format: Specify to override default hash format + + Returns: + String of Hex digits representing the signature + """ + + m = _get_hash_object(hash_format) + with open(fname, "rb") as f: + while True: + blck = f.read(chunksize) + if not blck: + break + m.update(to_bytes(blck)) + return m.hexdigest() + + +def hash_collect(signatures, hash_format=None): + """ + Collects a list of signatures into an aggregate signature. + + Args: + signatures: a list of signatures + hash_format: Specify to override default hash format + + Returns: + the aggregate signature + """ + + if len(signatures) == 1: + return signatures[0] + + return hash_signature(', '.join(signatures), hash_format) + + +_MD5_WARNING_SHOWN = False + + +def _show_md5_warning(function_name): + """Shows a deprecation warning for various MD5 functions.""" + + global _MD5_WARNING_SHOWN + + if not _MD5_WARNING_SHOWN: + import SCons.Warnings # pylint: disable=import-outside-toplevel + + SCons.Warnings.warn( + SCons.Warnings.DeprecatedWarning, + f"Function {function_name} is deprecated", + ) + _MD5_WARNING_SHOWN = True + + +def MD5signature(s): + """Deprecated. Use :func:`hash_signature` instead.""" + + _show_md5_warning("MD5signature") + return hash_signature(s) + + +def MD5filesignature(fname, chunksize=65536): + """Deprecated. Use :func:`hash_file_signature` instead.""" + + _show_md5_warning("MD5filesignature") + return hash_file_signature(fname, chunksize) + + +def MD5collect(signatures): + """Deprecated. Use :func:`hash_collect` instead.""" + + _show_md5_warning("MD5collect") + return hash_collect(signatures) + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util/types.py b/SCons/Util/types.py new file mode 100644 index 0000000..53dacd1 --- /dev/null +++ b/SCons/Util/types.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: MIT +# +# Copyright The SCons Foundation + +"""Various SCons utility functions + +Routines which check types and do type conversions. +""" + +import os +import pprint +import re +from typing import Optional + +from collections import UserDict, UserList, UserString +from collections.abc import MappingView + +# Functions for deciding if things are like various types, mainly to +# handle UserDict, UserList and UserString like their underlying types. +# +# Yes, all of this manual testing breaks polymorphism, and the real +# Pythonic way to do all of this would be to just try it and handle the +# exception, but handling the exception when it's not the right type is +# often too slow. + +# We are using the following trick to speed up these +# functions. Default arguments are used to take a snapshot of +# the global functions and constants used by these functions. This +# transforms accesses to global variable into local variables +# accesses (i.e. LOAD_FAST instead of LOAD_GLOBAL). +# Since checkers dislike this, it's now annotated for pylint to flag +# (mostly for other readers of this code) we're doing this intentionally. +# TODO: PY3 check these are still valid choices for all of these funcs. + +DictTypes = (dict, UserDict) +ListTypes = (list, UserList) + +# Handle getting dictionary views. +SequenceTypes = (list, tuple, UserList, MappingView) + +# Note that profiling data shows a speed-up when comparing +# explicitly with str instead of simply comparing +# with basestring. (at least on Python 2.5.1) +# TODO: PY3 check this benchmarking is still correct. +StringTypes = (str, UserString) + +# Empirically, it is faster to check explicitly for str than for basestring. +BaseStringTypes = str + + +def is_Dict( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, DictTypes=DictTypes +) -> bool: + """Check if object is a dict.""" + return isinstance(obj, DictTypes) + + +def is_List( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, ListTypes=ListTypes +) -> bool: + """Check if object is a list.""" + return isinstance(obj, ListTypes) + + +def is_Sequence( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, SequenceTypes=SequenceTypes +) -> bool: + """Check if object is a sequence.""" + return isinstance(obj, SequenceTypes) + + +def is_Tuple( # pylint: disable=redefined-builtin + obj, isinstance=isinstance, tuple=tuple +) -> bool: + """Check if object is a tuple.""" + return isinstance(obj, tuple) + + +def is_String( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, StringTypes=StringTypes +) -> bool: + """Check if object is a string.""" + return isinstance(obj, StringTypes) + + +def is_Scalar( # pylint: disable=redefined-outer-name,redefined-builtin + obj, isinstance=isinstance, StringTypes=StringTypes, SequenceTypes=SequenceTypes +) -> bool: + """Check if object is a scalar.""" + # Profiling shows that there is an impressive speed-up of 2x + # when explicitly checking for strings instead of just not + # sequence when the argument (i.e. obj) is already a string. + # But, if obj is a not string then it is twice as fast to + # check only for 'not sequence'. The following code therefore + # assumes that the obj argument is a string most of the time. + return isinstance(obj, StringTypes) or not isinstance(obj, SequenceTypes) + + +# From Dinu C. Gherman, +# Python Cookbook, second edition, recipe 6.17, p. 277. +# Also: https://code.activestate.com/recipes/68205 +# ASPN: Python Cookbook: Null Object Design Pattern + + +class Null: + """Null objects always and reliably 'do nothing'.""" + + def __new__(cls, *args, **kwargs): + if '_instance' not in vars(cls): + cls._instance = super(Null, cls).__new__(cls, *args, **kwargs) + return cls._instance + + def __init__(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return self + + def __repr__(self): + return f"Null(0x{id(self):08X})" + + def __bool__(self): + return False + + def __getattr__(self, name): + return self + + def __setattr__(self, name, value): + return self + + def __delattr__(self, name): + return self + + +class NullSeq(Null): + """A Null object that can also be iterated over.""" + + def __len__(self): + return 0 + + def __iter__(self): + return iter(()) + + def __getitem__(self, i): + return self + + def __delitem__(self, i): + return self + + def __setitem__(self, i, v): + return self + + +def to_bytes(s) -> bytes: + """Convert object to bytes.""" + if s is None: + return b'None' + if isinstance(s, (bytes, bytearray)): + # if already bytes return. + return s + return bytes(s, 'utf-8') + + +def to_str(s) -> str: + """Convert object to string.""" + if s is None: + return 'None' + if is_String(s): + return s + return str(s, 'utf-8') + + +# Generic convert-to-string functions. The wrapper +# to_String_for_signature() will use a for_signature() method if the +# specified object has one. + + +def to_String( # pylint: disable=redefined-outer-name,redefined-builtin + obj, + isinstance=isinstance, + str=str, + UserString=UserString, + BaseStringTypes=BaseStringTypes, +) -> str: + """Return a string version of obj.""" + if isinstance(obj, BaseStringTypes): + # Early out when already a string! + return obj + + if isinstance(obj, UserString): + # obj.data can only be a regular string. Please see the UserString initializer. + return obj.data + + return str(obj) + + +def to_String_for_subst( # pylint: disable=redefined-outer-name,redefined-builtin + obj, + isinstance=isinstance, + str=str, + BaseStringTypes=BaseStringTypes, + SequenceTypes=SequenceTypes, + UserString=UserString, +) -> str: + """Return a string version of obj for subst usage.""" + # Note that the test cases are sorted by order of probability. + if isinstance(obj, BaseStringTypes): + return obj + + if isinstance(obj, SequenceTypes): + return ' '.join([to_String_for_subst(e) for e in obj]) + + if isinstance(obj, UserString): + # obj.data can only a regular string. Please see the UserString initializer. + return obj.data + + return str(obj) + + +def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-builtin + obj, to_String_for_subst=to_String_for_subst, AttributeError=AttributeError, +) -> str: + """Return a string version of obj for signature usage. + + Like :func:`to_String_for_subst` but has special handling for + scons objects that have a :meth:`for_signature` method, and for dicts. + """ + try: + f = obj.for_signature + except AttributeError: + if isinstance(obj, dict): + # pprint will output dictionary in key sorted order + # with py3.5 the order was randomized. In general depending on dictionary order + # which was undefined until py3.6 (where it's by insertion order) was not wise. + # TODO: Change code when floor is raised to PY36 + return pprint.pformat(obj, width=1000000) + return to_String_for_subst(obj) + else: + return f() + + +def get_env_bool(env, name, default=False) -> bool: + """Convert a construction variable to bool. + + If the value of *name* in *env* is 'true', 'yes', 'y', 'on' (case + insensitive) or anything convertible to int that yields non-zero then + return ``True``; if 'false', 'no', 'n', 'off' (case insensitive) + or a number that converts to integer zero return ``False``. + Otherwise, return `default`. + + Args: + env: construction environment, or any dict-like object + name: name of the variable + default: value to return if *name* not in *env* or cannot + be converted (default: False) + + Returns: + the "truthiness" of `name` + """ + try: + var = env[name] + except KeyError: + return default + try: + return bool(int(var)) + except ValueError: + if str(var).lower() in ('true', 'yes', 'y', 'on'): + return True + + if str(var).lower() in ('false', 'no', 'n', 'off'): + return False + + return default + + +def get_os_env_bool(name, default=False) -> bool: + """Convert an environment variable to bool. + + Conversion is the same as for :func:`get_env_bool`. + """ + return get_env_bool(os.environ, name, default) + + +_get_env_var = re.compile(r'^\$([_a-zA-Z]\w*|{[_a-zA-Z]\w*})$') + + +def get_environment_var(varstr) -> Optional[str]: + """Return undecorated construction variable string. + + Determine if `varstr` looks like a reference + to a single environment variable, like `"$FOO"` or `"${FOO}"`. + If so, return that variable with no decorations, like `"FOO"`. + If not, return `None`. + """ + mo = _get_env_var.match(to_String(varstr)) + if mo: + var = mo.group(1) + if var[0] == '{': + return var[1:-1] + return var + + return None + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 0cfb70f..785caf7 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -46,10 +46,6 @@ from SCons.Util import ( Proxy, Selector, WhereIs, - _attempt_init_of_python_3_9_hash_object, - _attempt_get_hash_function, - _get_hash_object, - _set_allowed_viable_default_hashes, adjustixes, containsAll, containsAny, @@ -76,6 +72,12 @@ from SCons.Util import ( to_bytes, to_str, ) +from SCons.Util.hashes import ( + _attempt_init_of_python_3_9_hash_object, + _attempt_get_hash_function, + _get_hash_object, + _set_allowed_viable_default_hashes, +) # These Util classes have no unit tests. Some don't make sense to test? # DisplayEngine, Delegate, MethodWrapper, UniqueList, Unbuffered, Null, NullSeq -- cgit v0.12 From a8d1ef73713371da16300fd3bbd53cd993aa9e5d Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 8 Dec 2022 10:18:47 -0700 Subject: uguide: variant dir tweaking [skip appveyor] Try to clarify some more about variant dirs (people are still getting confused), without getting too technical about it, in the spirit of the User Guide. Also some stuff about hierarchical builds - both manpage and user guide. User guide chunks that were modified anyway had section IDs added. This is part of a plan to gradually get rid of horrible-looking autogenerated links which are not constant (problem when Google archives something and then maybe it doesn't work). May look like this in a browser: https://scons.org/doc/production/HTML/scons-user.html#idp105548893220944 With assigned IDs, looks more like: .../HTML/scons-user.html#sect-variant-sconscript So far only three userguide files changed. Signed-off-by: Mats Wichmann --- doc/man/scons.xml | 51 +++++++--- doc/user/hierarchy.xml | 90 ++++++++--------- doc/user/separate.xml | 267 ++++++++++++++++++++++++++++++------------------- doc/user/variants.xml | 34 ++----- 4 files changed, 249 insertions(+), 193 deletions(-) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 8dda049..824cd41 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -145,33 +145,44 @@ to support additional input file types. Information about files involved in the build, -including a cryptographic hash of the contents, +including a cryptographic hash of the contents of source files, is cached for later reuse. By default this hash (the &contentsig;) is used to determine if a file has changed since the last build, -but this can be controlled by selecting an appropriate +although this can be controlled by selecting an appropriate &f-link-Decider; function. Implicit dependency files are also part of out-of-date computation. The scanned implicit dependency information can optionally be cached and used to speed up future builds. -A hash of each executed build action (the &buildsig; +A hash of each executed build action (the &buildsig;) is cached, so that changes to build instructions (changing flags, etc.) or to the build tools themselves (new version) can also trigger a rebuild. +&SCons; supports the concept of separated source and build +directories through the definition of +variant directories +(see the &f-link-VariantDir; function). + + + When invoked, &scons; looks for a file named &SConstruct; in the current directory and reads the build configuration from that file (other names are allowed, -see for more information). -The &SConstruct; +see +and the option +for more information). +The build may be structured in a hierarchical manner: +the &SConstruct; file may specify subsidiary configuration files by calling the -&f-link-SConscript; function. +&f-link-SConscript; function, +and these may, in turn, do the same. By convention, these subsidiary files are named &SConscript;, @@ -182,7 +193,14 @@ is used to refer generically to the complete set of configuration files for a project (including the &SConstruct; file), -regardless of the actual file names or number of such files. +regardless of the actual file names or number of such files. +A hierarchical build is not recursive - all of +the SConscript files are processed in a single pass, +although each is processed in a separate context so +as not to interfere with one another. &SCons; provides +mechanisms for information to be shared between +SConscript files when needed. + Before reading the &SConscript; files, &scons; @@ -870,9 +888,10 @@ files). duplicate -Print a line for each unlink/relink (or copy) of a variant file from -its source file. Includes debugging info for unlinking stale variant -files, as well as unlinking old targets before building them. +Print a line for each unlink/relink (or copy) of a file in +a variant directory from its source file. +Includes debugging info for unlinking stale variant directory files, +as well as unlinking old targets before building them. @@ -3686,9 +3705,9 @@ The default value is specifies a file which collects the output from commands that are executed to check for the existence of header files, libraries, etc. The default is #/config.log. -If you are using the -&VariantDir; function, -you may want to specify a subdirectory under your variant directory. +If you are using variant directories, +you may want to place the log file for a given build +under that build's variant directory. config_h specifies a C header file where the results of tests @@ -6720,7 +6739,7 @@ For example, the command line string: echo Last build occurred $TODAY. > $TARGET -but the build signature added to any target files would be computed from: +but the &buildsig; added to any target files would be computed from: echo Last build occurred . > $TARGET @@ -6741,8 +6760,8 @@ The callable must accept four arguments: env is the &consenv; to use for context, and for_signature is a boolean value that tells the callable -if it is being called for the purpose of generating a build signature. -Since the build signature is used for rebuild determination, +if it is being called for the purpose of generating a &buildsig;. +Since the &buildsig; is used for rebuild determination, variable elements that do not affect whether a rebuild should be triggered should be omitted from the returned string diff --git a/doc/user/hierarchy.xml b/doc/user/hierarchy.xml index e63d230..17874d0 100644 --- a/doc/user/hierarchy.xml +++ b/doc/user/hierarchy.xml @@ -1,4 +1,10 @@ + + %scons; @@ -18,32 +24,8 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Hierarchical Builds - - +Hierarchical Builds + %scons; @@ -17,98 +23,111 @@ xmlns="http://www.scons.org/dbxsd/v1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.scons.org/dbxsd/v1.0 http://www.scons.org/dbxsd/v1.0/scons.xsd"> -Separating Source and Build Trees: Variant Directories - + - It's often useful to keep any built files completely - separate from the source files. Consider if you have a - project to build software for a variety of different - controller hardware. The boards are able to share a - lot of code, so it makes sense to keep them in the same - source tree, but certain build options in the source code - and header files differ. If you build "Controller A" first, - then "Controller B", on the "Controller B" build everything would - have to be rebuilt, because &SCons; recognizes that the build - instructions are different from those used in the "Controller A" - build for each target - the build instructions are part of - &SCons;'s out-of-date calculation. - Now when you go back and build for "Controller A", - things have to be rebuilt from scratch again for the same reason. - However, if you can separate the locations of the output files, - this problem can be avoided. - You can even set up to do both builds in one invocation of &SCons;. + Having a separated build also helps you keep your source tree clean - + there is less chance of accidentally checking in build products + to version control that were not intended to be checked in. + You can add a separated build directory to your + version control system's list of items not to track. + You can even remove the whole build tree with a single command without + risking removing any of the source code. - You can enable this separation by establishing one or more - variant directory trees - that are used to perform the build in, and thus provide a unique - home for object files, libraries, and executable programs, etc. - for a specific flavor, or variant, of build. &SCons; tracks - targets by their path, so when the variant directory is included, - objects belonging to "Controller A" can have different - build instructions than those belonging to "Controller B" without - triggering ping-ponging rebuilds. + The key to making this separation work is the ability to + do out-of-tree builds: building under a separate root + than the sources being built. + You set up out of tree builds by establishing what &SCons; + calls a variant directory, + a place where you can build a single variant of your software + (of course you can define more than one of these if you need to). + Since &SCons; tracks targets by their path, it is able to distinguish + build products like build/A/network.obj + of the Controller A build + from build/B/network.obj + of the Controller B build, + thus avoiding conflicts. - &SCons; provides two ways to do this, - one through the &f-link-SConscript; function that we've already seen, + &SCons; provides two ways to establish variant directories, + one through the &f-link-SConscript; function that we have already seen, and the second through a more flexible &f-link-VariantDir; function. + + The variant directory mechanism does support doing multiple builds + in one invocation of &SCons;, but the remainder of this chapter + will focus on setting up a single build. You can combine these + techniques with ones from the previous chapter and elsewhere + in this Guide to set up more complex scenarios. + + + + - Historical note: the &VariantDir; function - used to be called &BuildDir;, a name which was - removed because the &SCons; functionality + The &VariantDir; function used to be called &BuildDir;, + a name which was changed because it turned out to be confusing: + the &SCons; functionality differs from a familiar model of a "build directory" - implemented by other build systems like GNU Autotools. + implemented by certain other build systems like GNU Autotools. You might still find references to the old name on the Internet in postings about &SCons;, but it no longer works. - + -
                    +
                    Specifying a Variant Directory Tree as Part of an &SConscript; Call The most straightforward way to establish a variant directory tree - relies the fact that the usual way to + relies on the fact that the usual way to set up a build hierarchy is to have an - SConscript file in the source subdirectory. + &SConscript; file in the source directory. If you pass a &variant_dir; argument to the &f-link-SConscript; function call: @@ -130,7 +149,7 @@ int main() { printf("Hello, world!\n"); } &SCons; will then build all of the files in - the &build; subdirectory: + the &build; directory: @@ -143,46 +162,73 @@ int main() { printf("Hello, world!\n"); } - No files were built in &src;, they went to &build;. - The build output might show a bit of a surprise: + No files were built in &src;: the object file build/hello.o and the executable file build/hello - were built in the &build; subdirectory, - as expected. - But even though our &hello_c; file lives in the &src; subdirectory, - &SCons; has actually compiled a + were built in the &build; directory, as expected. + But notice that even though our &hello_c; file actually + lives in the &src; directory, &SCons; has compiled a build/hello.c file to create the object file, and that file is now seen in &build;. + + + + + + You can ask &SCons; to show the dependency tree to illustrate + a bit more: + + scons -Q --tree=prune + + What's happened is that &SCons; has duplicated - the &hello_c; file from the &src; subdirectory - to the &build; subdirectory, + the &hello_c; file from the &src; directory + to the &build; directory, and built the program from there (it also duplicated &SConscript;). The next section explains why &SCons; does this. + + + The nice thing about the &SConscript; approach is it is almost + invisible to you: + this build looks just like an ordinary in-place build + except for the extra &variant_dir; argument in the + &f-link-SConscript; call. + &SCons; handles all the path adjustments for the + out of tree &build; directory while it processes that SConscript file. + + +
                    -
                    +
                    Why &SCons; Duplicates Source Files in a Variant Directory Tree - The important thing to understand is that when you set up a variant directory, - &SCons; performs the build in that directory. - It turns out it's easiest to ensure where build products end up - by just building in place. - Since the build is happening in a place different from where the - sources are, the most straightforward way to guarantee a correct build - is for &SCons; to copy them there. + When you set up a variant directory &SCons; conceptually behaves as + if you requested a build in that directory. + As noted in the previous chapter, + all builds actually happen from the top level directory, + but as an aid to understanding how &SCons; operates, think + of it as build in place in the variant directory, + not build in source but send build artifacts + to the variant directory. + It turns out in place builds are easier to get right than out + of tree builds - so by default &SCons; simulates an in place build + by making the variant directory look just like the source directory. + The most straightforward way to do that is by making copies + of the files needed for the build. @@ -192,7 +238,11 @@ int main() { printf("Hello, world!\n"); } in variant directories is simply that some tools (mostly older versions) are written to only build their output files - in the same directory as the source files. + in the same directory as the source files - such tools often don't + have any option to specify the output file, and the tool just + uses a predefined output file name, + or uses a derived variant of the source file name, + dropping the result in the same directory. In this case, the choices are either to build the output file in the source directory and move it to the variant directory, @@ -204,9 +254,9 @@ int main() { printf("Hello, world!\n"); } Additionally, relative references between files - can cause problems if we don't - just duplicate the hierarchy of source files - in the variant directory. + can cause problems which are resolved by + just duplicating the hierarchy of source files + into the variant directory. You can see this at work in use of the C preprocessor #include mechanism with double quotes, not angle brackets: @@ -240,8 +290,8 @@ int main() { printf("Hello, world!\n"); } Although source-file duplication guarantees a correct build - even in these end-cases, - it can usually be safely disabled. + even in these edge cases, + it can usually be safely disabled. The next section describes how you can disable the duplication of source files in the variant directory. @@ -250,19 +300,19 @@ int main() { printf("Hello, world!\n"); }
                    -
                    +
                    Telling &SCons; to Not Duplicate Source Files in the Variant Directory Tree In most cases and with most tool sets, - &SCons; can place its target files in a build subdirectory + &SCons; can use sources directly from the source directory without - duplicating the source files + duplicating them into the variant directory before building, and everything will work just fine. - You can disable the default &SCons; behavior + You can disable the default &SCons; duplication behavior by specifying duplicate=False - when you call the &SConscript; function: + when you call the &f-link-SConscript; function: @@ -272,11 +322,11 @@ SConscript('src/SConscript', variant_dir='build', duplicate=False) - When this flag is specified, - &SCons; uses the variant directory - like most people expect--that is, - the output files are placed in the variant directory - while the source files stay in the source directory: + When this flag is specified, the results of a build + look more like the mental model people may have from other + build systems - that is, + the output files end up in the variant directory + while the source files do not. @@ -292,14 +342,22 @@ hello hello.o + + + If disabling duplication causes any problems, + just return to the more cautious approach by letting + &SCons; go back to duplicating files. + + +
                    -
                    +
                    The &VariantDir; Function - Use the &VariantDir; function to establish that target + Use the &f-link-VariantDir; function to establish that target files should be built in a separate directory from the source files: @@ -318,8 +376,8 @@ int main() { printf("Hello, world!\n"); } - Note that when you're not using - an &SConscript; file in the &src; subdirectory, + Note that when you are not using + an &SConscript; file in the &src; directory, you must actually specify that the program must be built from the build/hello.c @@ -345,7 +403,7 @@ int main() { printf("Hello, world!\n"); } You can specify the same duplicate=False argument - that you can specify for an &SConscript; call: + that you can specify for an &f-link-SConscript; call: @@ -375,12 +433,12 @@ int main() { printf("Hello, world!\n"); }
                    -
                    +
                    Using &VariantDir; With an &SConscript; File - Even when using the &VariantDir; function, + Even when using the &f-link-VariantDir; function, it's more natural to use it with a subsidiary &SConscript; file, because then you don't have to adjust your individual @@ -428,21 +486,26 @@ int main() { printf("Hello, world!\n"); } - Notice that this is completely equivalent - to the use of &SConscript; that we - learned about in the previous section. + This is completely equivalent + to the use of &f-link-SConscript; with the + variant_dir argument + from earlier in this chapter, + but did require callng the SConscript using the already established + variant directory path to trigger that behavior. + If you use SConscript('src/SConscript') + you would get a normal in-place build in &src;.
                    -
                    +
                    Using &Glob; with &VariantDir; The &f-link-Glob; file name pattern matching function - works just as usual when using &VariantDir;. + works just as usual when using &f-link-VariantDir;. For example, if the src/SConscript looks like this: @@ -496,7 +559,7 @@ const char * f2(); + %scons; @@ -13,36 +19,12 @@ %variables-mod; ]> -
                    -Variant Build Examples - - +Variant Build Examples -- cgit v0.12 From c544c4c2dc2c4fe7c04703479f088377ac7cb0cf Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 14 Dec 2022 14:59:48 -0700 Subject: Clarify build tree separation wording [skip appveyor] Review pointed out that a use of the term "build" wasn't that clear, "build tree" would be more descriptive. Signed-off-by: Mats Wichmann --- doc/user/separate.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/separate.xml b/doc/user/separate.xml index f0636cc..4955d0e 100644 --- a/doc/user/separate.xml +++ b/doc/user/separate.xml @@ -61,7 +61,7 @@ Copyright The SCons Foundation - Having a separated build also helps you keep your source tree clean - + Having a separated build tree also helps you keep your source tree clean - there is less chance of accidentally checking in build products to version control that were not intended to be checked in. You can add a separated build directory to your -- cgit v0.12 From 889ce62c4c053a09bf9c8a029b848cc574297729 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 16 Dec 2022 07:49:43 -0700 Subject: uguide: clarify VariantDir example [skip appvyor] A little more tweaking to show all references have to be to the variant dir (the "implied target" form of the example might let readers miss it applies to targets too). Signed-off-by: Mats Wichmann --- doc/user/separate.xml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/doc/user/separate.xml b/doc/user/separate.xml index 4955d0e..0485a55 100644 --- a/doc/user/separate.xml +++ b/doc/user/separate.xml @@ -35,7 +35,7 @@ Copyright The SCons Foundation - + Consider if you have a project to build an embedded software system for a variety of different controller hardware. @@ -78,7 +78,7 @@ Copyright The SCons Foundation than the sources being built. You set up out of tree builds by establishing what &SCons; calls a variant directory, - a place where you can build a single variant of your software + a place where you can build a single variant of your software (of course you can define more than one of these if you need to). Since &SCons; tracks targets by their path, it is able to distinguish build products like build/A/network.obj @@ -98,7 +98,7 @@ Copyright The SCons Foundation - + The variant directory mechanism does support doing multiple builds in one invocation of &SCons;, but the remainder of this chapter will focus on setting up a single build. You can combine these @@ -173,7 +173,7 @@ int main() { printf("Hello, world!\n"); } build/hello.c file to create the object file, and that file is now seen in &build;. - + @@ -217,7 +217,7 @@ int main() { printf("Hello, world!\n"); } When you set up a variant directory &SCons; conceptually behaves as - if you requested a build in that directory. + if you requested a build in that directory. As noted in the previous chapter, all builds actually happen from the top level directory, but as an aid to understanding how &SCons; operates, think @@ -357,8 +357,8 @@ hello.o - Use the &f-link-VariantDir; function to establish that target - files should be built in a separate directory + You can also use the &f-link-VariantDir; function to establish + that target files should be built in a separate directory tree from the source files: @@ -376,13 +376,18 @@ int main() { printf("Hello, world!\n"); } - Note that when you are not using - an &SConscript; file in the &src; directory, - you must actually specify that - the program must be built from - the build/hello.c - file that &SCons; will duplicate in the - &build; subdirectory. + When using this form, you have to tell &SCons; that + sources and targets are in the variant directory, + and those references will trigger the remapping, + necessary file copying, etc. for an already established + variant directory. Here is the same example in a more + spelled out form to show this more clearly: + + +VariantDir('build', 'src') +env = Environment() +env.Program(target='build/hello', source=['build/hello.c']) + @@ -439,7 +444,7 @@ int main() { printf("Hello, world!\n"); } Even when using the &f-link-VariantDir; function, - it's more natural to use it with + it is more natural to use it with a subsidiary &SConscript; file, because then you don't have to adjust your individual build instructions to use the variant directory path. @@ -492,7 +497,7 @@ int main() { printf("Hello, world!\n"); } from earlier in this chapter, but did require callng the SConscript using the already established variant directory path to trigger that behavior. - If you use SConscript('src/SConscript') + If you call SConscript('src/SConscript') you would get a normal in-place build in &src;. -- cgit v0.12 From 755ab60ef33cf62e2f887cfddb1b084b7f44ba21 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 16 Dec 2022 08:30:40 -0700 Subject: Updated RELEASE to show VariantDir updated [ci skip] Signed-off-by: Mats Wichmann --- RELEASE.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASE.txt b/RELEASE.txt index 2a32831..c158efb 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -105,6 +105,9 @@ DOCUMENTATION - Updated the User Guide chapter on installation: modernized the notes on Python installs, SCons installs, and having multiple SCons versions present on a single system. +- Updated the User Guide chapter on variant directories with more + explanation, and the introduction of terms like "out of tree" that + may help in forming a mental model. DEVELOPMENT ----------- -- cgit v0.12 From dd4694c16d5750aee679b97ce59b7de70f4724c0 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 18 Dec 2022 11:50:11 -0700 Subject: Util splitup: do some work to quiet sider Adds a .flake8 file to somewhat control the checking. Signed-off-by: Mats Wichmann --- .flake8 | 22 ++++++++++++++++++++++ SCons/Util/__init__.py | 17 ++++++++--------- SCons/Util/hashes.py | 6 +++--- SCons/Util/types.py | 5 +++-- 4 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..d10d72d --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +show-source = True +# don't complain about work black has done +max-line-length = 88 +extend-exclude = + bin, + bench, + doc, + src, + template, + testing, + test, + timings, + SCons/Tool/docbook/docbook-xsl-1.76.1, + bootstrap.py, + runtest.py +extend-ignore = + E302, + E305 +per-file-ignores = + # module symbols made available for compat - ignore "unused" warns + SCons/Util/__init__.py: F401 diff --git a/SCons/Util/__init__.py b/SCons/Util/__init__.py index 6b48d05..0281e1e 100644 --- a/SCons/Util/__init__.py +++ b/SCons/Util/__init__.py @@ -383,7 +383,7 @@ def print_tree( cross = BOX_VERT_RIGHT + BOX_HORIZ # sign used to point to the leaf. # check if this is the last leaf of the branch if lastChild: - #if this if the last leaf, then terminate: + # if this if the last leaf, then terminate: cross = BOX_UP_RIGHT + BOX_HORIZ # sign for the last leaf # if this branch has children then split it @@ -395,7 +395,7 @@ def print_tree( cross += BOX_HORIZ_DOWN if prune and rname in visited and children: - sys.stdout.write(''.join(tags + margins + [cross,'[', rname, ']']) + '\n') + sys.stdout.write(''.join(tags + margins + [cross, '[', rname, ']']) + '\n') return sys.stdout.write(''.join(tags + margins + [cross, rname]) + '\n') @@ -892,7 +892,7 @@ class Selector(OrderedDict): # Try to perform Environment substitution on the keys of # the dictionary before giving up. s_dict = {} - for (k,v) in self.items(): + for (k, v) in self.items(): if k is not None: s_k = env.subst(k) if s_k in s_dict: @@ -901,7 +901,7 @@ class Selector(OrderedDict): # and a variable suffix contains this literal, # the literal wins and we don't raise an error. raise KeyError(s_dict[s_k][0], k, s_k) - s_dict[s_k] = (k,v) + s_dict[s_k] = (k, v) try: return s_dict[ext][1] except KeyError: @@ -947,7 +947,6 @@ def adjustixes(fname, pre, suf, ensure_suffix=False) -> str: return fname - # From Tim Peters, # https://code.activestate.com/recipes/52560 # ASPN: Python Cookbook: Remove duplicates from a sequence @@ -1212,9 +1211,9 @@ def make_path_relative(path) -> str: drive_s, path = os.path.splitdrive(path) if not drive_s: - path=re.compile(r"/*(.*)").findall(path)[0] + path = re.compile(r"/*(.*)").findall(path)[0] else: - path=path[1:] + path = path[1:] assert not os.path.isabs(path), path return path @@ -1263,7 +1262,7 @@ def wait_for_process_to_die(pid): if sys.platform == 'win32': import ctypes # pylint: disable=import-outside-toplevel PROCESS_QUERY_INFORMATION = 0x1000 - processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0,pid) + processHandle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION, 0, pid) if processHandle == 0: break ctypes.windll.kernel32.CloseHandle(processHandle) @@ -1290,7 +1289,7 @@ class DispatchingFormatter(Formatter): def sanitize_shell_env(execution_env: dict) -> dict: """Sanitize all values in *execution_env* - The execution environment (typically comes from (env['ENV']) is + The execution environment (typically comes from (env['ENV']) is propagated to the shell, and may need to be cleaned first. Args: diff --git a/SCons/Util/hashes.py b/SCons/Util/hashes.py index 4771c44..b97cd4d 100644 --- a/SCons/Util/hashes.py +++ b/SCons/Util/hashes.py @@ -212,9 +212,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): ) # pylint: disable=import-outside-toplevel raise UserError( - 'Hash format "%s" is not available in your Python interpreter. ' - 'Expected to be supported algorithm by set_allowed_viable_default_hashes, ' - 'Assertion error in SCons.' % hash_format_lower + f'Hash format "{hash_format_lower}" is not available in your ' + 'Python interpreter. Expected to be supported algorithm by ' + 'set_allowed_viable_default_hashes. Assertion error in SCons.' ) else: # Set the default hash format based on what is available, defaulting diff --git a/SCons/Util/types.py b/SCons/Util/types.py index 53dacd1..9aef13e 100644 --- a/SCons/Util/types.py +++ b/SCons/Util/types.py @@ -230,8 +230,9 @@ def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-b except AttributeError: if isinstance(obj, dict): # pprint will output dictionary in key sorted order - # with py3.5 the order was randomized. In general depending on dictionary order - # which was undefined until py3.6 (where it's by insertion order) was not wise. + # with py3.5 the order was randomized. Depending on dict order + # which was undefined until py3.6 (where it's by insertion order) + # was not wise. # TODO: Change code when floor is raised to PY36 return pprint.pformat(obj, width=1000000) return to_String_for_subst(obj) -- cgit v0.12 From c08e9f41c024b72c36f8b4cafaf53a046184a2c9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 18 Dec 2022 12:15:24 -0700 Subject: Fix typo Signed-off-by: Mats Wichmann --- SCons/Util/envs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Util/envs.py b/SCons/Util/envs.py index 9acfe4a..963c963 100644 --- a/SCons/Util/envs.py +++ b/SCons/Util/envs.py @@ -4,7 +4,7 @@ """Various SCons utility functions -Routines for working with environemnts and construction variables +Routines for working with environments and construction variables that don't need the specifics of Environment. """ -- cgit v0.12 From ecdf211966dad20879d802fb66c8822c65062380 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 19 Dec 2022 11:13:11 -0700 Subject: Added RELEASE info for SCons.Util refactor [ci skip] Signed-off-by: Mats Wichmann --- RELEASE.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/RELEASE.txt b/RELEASE.txt index 2a32831..cf9cdc8 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -111,6 +111,13 @@ DEVELOPMENT - Refactored SCons/Taskmaster into a package. Moved SCons/Jobs.py into that package. NOTE: If you hook into SCons.Jobs, you'll have to change that to use SCons.Taskmaster.Jobs +- Refactored SCons.Util, which had grown quite large, to be a package, + not a single-file module. Change should be transparent: the same import + of SCons.Util and public symbols from it will continue to work, however + code which reaches directly in to grab private symbols (starting with + underscore) which moved to a submodule, that code will have to be adjusted, + as those are not imported to the package level (new SCons.Util.hashes has + some of these, which are used by existing unit tests). Thanks to the following contributors listed below for their contributions to this release. ========================================================================================== -- cgit v0.12 From d14d9694f925a04f033e6f0119f36f5fbfbc73f3 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 19 Dec 2022 20:32:18 -0800 Subject: Fixed using --diskcheck=none from command line. It was always broken. SetOption('diskcheck','none') has been working all along. Also refactored the DiskChecker class to have more meaningful properties and not shadow default python objects (list, dir).. --- CHANGES.txt | 2 ++ RELEASE.txt | 2 ++ SCons/Node/FS.py | 41 ++++++++++++++++++++++++----------------- SCons/Script/SConsOptions.py | 5 ++++- test/diskcheck.py | 17 ++++++++++++----- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 04ce9e7..c46f9b6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -34,6 +34,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER read packaging/etc/README.txt if you are interested. - Added --experimental=tm_v2, which enables Andrew Morrow's new NewParallel Job implementation. This should scale much better for highly parallel builds. You can also enable this via SetOption(). + - Fixed setting diskcheck to 'none' via command line works --diskcheck=none. Apparently it has never worked + even though SetOption('diskcheck','none') did work. From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). diff --git a/RELEASE.txt b/RELEASE.txt index 81ee612..40852fe 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -64,6 +64,8 @@ FIXES - Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an execution environment variable set to a python list it would crash. Now it will create a string joining the list with os.pathsep +- Fixed setting diskcheck to 'none' via command line works --diskcheck=none. Apparently it has never worked + even though SetOption('diskcheck','none') did work. IMPROVEMENTS ------------ diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index f5642e2..aadcba5 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -378,20 +378,26 @@ else: return x.upper() - class DiskChecker: - def __init__(self, type, do, ignore): - self.type = type - self.do = do - self.ignore = ignore - self.func = do + """ + This Class will hold various types of logic for checking if a file/dir on disk matches + the type which is expected. And allow Options to decide to enable or disable said check + """ + def __init__(self, disk_check_type, do_check_function, ignore_check_function): + self.disk_check_type = disk_check_type + self.do_check_function = do_check_function + self.ignore_check_function = ignore_check_function + self.func = do_check_function + def __call__(self, *args, **kw): return self.func(*args, **kw) - def set(self, list): - if self.type in list: - self.func = self.do + + def set(self, disk_check_type_list): + if self.disk_check_type in disk_check_type_list: + self.func = self.do_check_function else: - self.func = self.ignore + self.func = self.ignore_check_function + def do_diskcheck_match(node, predicate, errorfmt): result = predicate() @@ -409,24 +415,25 @@ def do_diskcheck_match(node, predicate, errorfmt): if result: raise TypeError(errorfmt % node.get_abspath()) + def ignore_diskcheck_match(node, predicate, errorfmt): pass - diskcheck_match = DiskChecker('match', do_diskcheck_match, ignore_diskcheck_match) diskcheckers = [ diskcheck_match, ] -def set_diskcheck(list): + +def set_diskcheck(enabled_checkers): for dc in diskcheckers: - dc.set(list) + dc.set(enabled_checkers) -def diskcheck_types(): - return [dc.type for dc in diskcheckers] +def diskcheck_types(): + return [dc.disk_check_type for dc in diskcheckers] class EntryProxy(SCons.Util.Proxy): @@ -2403,7 +2410,7 @@ class RootDir(Dir): return Base.must_be_same(self, klass) - def _lookup_abs(self, p, klass, create=1): + def _lookup_abs(self, p, klass, create=True): """ Fast (?) lookup of a *normalized* absolute path. @@ -2428,7 +2435,7 @@ class RootDir(Dir): raise SCons.Errors.UserError(msg) # There is no Node for this path name, and we're allowed # to create it. - dir_name, file_name = p.rsplit('/',1) + dir_name, file_name = p.rsplit('/', 1) dir_node = self._lookup_abs(dir_name, Dir) result = klass(file_name, dir_node, self.fs) diff --git a/SCons/Script/SConsOptions.py b/SCons/Script/SConsOptions.py index a3e3ea8..8391d62 100644 --- a/SCons/Script/SConsOptions.py +++ b/SCons/Script/SConsOptions.py @@ -54,7 +54,10 @@ def diskcheck_convert(value): if v == 'all': result = diskcheck_all elif v == 'none': - result = [] + # Don't use an empty list here as that fails the normal check + # to see if an optparse parser of if parser.argname: + # Changed to ['none'] as diskcheck expects a list value + result = ['none'] elif v in diskcheck_all: result.append(v) else: diff --git a/test/diskcheck.py b/test/diskcheck.py index 36cfa4e..c07dc6b 100644 --- a/test/diskcheck.py +++ b/test/diskcheck.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Test that the --diskcheck option and SetOption('diskcheck') correctly @@ -41,16 +41,23 @@ test.write('file', "file\n") test.write('SConstruct', """ -SetOption('diskcheck', 'none') + +if GetOption('diskcheck') == ['match'] or ARGUMENTS.get('setoption_none',0): + SetOption('diskcheck', 'none') File('subdir') """) -test.run() +test.run(status=2, stderr=None) +test.must_contain_all_lines(test.stderr(), ["found where file expected"]) test.run(arguments='--diskcheck=match', status=2, stderr=None) test.must_contain_all_lines(test.stderr(), ["found where file expected"]) +# Test that setting --diskcheck to none via command line also works. +test.run(arguments='--diskcheck=none') +# Test that SetOption('diskcheck','none') works to override default as well +test.run(arguments='setoption_none=1') test.pass_test() -- cgit v0.12 From d2569dc26751464f9e42df9c4723d53458a86ed2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 20 Dec 2022 10:36:47 -0800 Subject: Address feedback from mwichmann in PR --- CHANGES.txt | 4 ++-- RELEASE.txt | 4 ++-- SCons/Node/FS.py | 15 +++++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c46f9b6..bbc2800 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -34,8 +34,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER read packaging/etc/README.txt if you are interested. - Added --experimental=tm_v2, which enables Andrew Morrow's new NewParallel Job implementation. This should scale much better for highly parallel builds. You can also enable this via SetOption(). - - Fixed setting diskcheck to 'none' via command line works --diskcheck=none. Apparently it has never worked - even though SetOption('diskcheck','none') did work. + - Fixed command line argument --diskcheck: previously a value of 'none' was ignored. + SetOption('diskcheck','none') is unaffected, as it did not have the problem. From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). diff --git a/RELEASE.txt b/RELEASE.txt index 40852fe..32094bf 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -64,8 +64,8 @@ FIXES - Ninja: Fix execution environment sanitation for launching ninja. Previously if you set an execution environment variable set to a python list it would crash. Now it will create a string joining the list with os.pathsep -- Fixed setting diskcheck to 'none' via command line works --diskcheck=none. Apparently it has never worked - even though SetOption('diskcheck','none') did work. +- Fixed command line argument --diskcheck: previously a value of 'none' was ignored. + SetOption('diskcheck','none') is unaffected, as it did not have the problem. IMPROVEMENTS ------------ diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index aadcba5..67e1ff6 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -380,8 +380,10 @@ else: class DiskChecker: """ - This Class will hold various types of logic for checking if a file/dir on disk matches - the type which is expected. And allow Options to decide to enable or disable said check + Implement disk check variation. + + This Class will hold functions to determine what this particular disk + checking implementation should do when enabled or disabled. """ def __init__(self, disk_check_type, do_check_function, ignore_check_function): self.disk_check_type = disk_check_type @@ -392,7 +394,12 @@ class DiskChecker: def __call__(self, *args, **kw): return self.func(*args, **kw) - def set(self, disk_check_type_list): + def enable(self, disk_check_type_list): + """ + If the current object's disk_check_type matches any in the list passed + :param disk_check_type_list: List of disk checks to enable + :return: + """ if self.disk_check_type in disk_check_type_list: self.func = self.do_check_function else: @@ -429,7 +436,7 @@ diskcheckers = [ def set_diskcheck(enabled_checkers): for dc in diskcheckers: - dc.set(enabled_checkers) + dc.enable(enabled_checkers) def diskcheck_types(): -- cgit v0.12 From 17ea1b324363076f6922d80b5d109d30fc77e274 Mon Sep 17 00:00:00 2001 From: Thad Guidry Date: Sat, 31 Dec 2022 08:51:55 +0800 Subject: fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index bf34fd1..6cc89bd 100755 --- a/README.rst +++ b/README.rst @@ -117,7 +117,7 @@ Installation Requirements ========================= SCons has no installation dependencies beyond a compatible version -of Python. The tools which will be used to to actually construct the +of Python. The tools which will be used to actually construct the project, such as compilers, documentation production tools, etc. should of course be installed by the appropriate means. -- cgit v0.12 From 82a4c4c419ec36c090beda64665c775953444724 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 31 Dec 2022 15:41:50 -0800 Subject: Update copyright header text, format files --- test/CompilationDatabase/basic.py | 8 +++++--- test/CompilationDatabase/fixture/SConstruct | 2 +- test/CompilationDatabase/variant_dir.py | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/CompilationDatabase/basic.py b/test/CompilationDatabase/basic.py index b6f1a79..6bf0d1a 100644 --- a/test/CompilationDatabase/basic.py +++ b/test/CompilationDatabase/basic.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,7 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# + """ Test CompilationDatabase and several variations of ways to call it and values of COMPILATIONDB_USE_ABSPATH @@ -46,7 +48,7 @@ rel_files = [ 'compile_commands_target.json', 'compile_commands.json', 'compile_commands_over_rel.json', - 'compile_commands_over_abs_0.json' + 'compile_commands_over_abs_0.json', ] abs_files = [ diff --git a/test/CompilationDatabase/fixture/SConstruct b/test/CompilationDatabase/fixture/SConstruct index ea23bc3..bd2f780 100644 --- a/test/CompilationDatabase/fixture/SConstruct +++ b/test/CompilationDatabase/fixture/SConstruct @@ -7,7 +7,7 @@ env = Environment( LINKFLAGS=[], CC='$PYTHON mygcc.py cc', CXX='$PYTHON mygcc.py c++', - tools=['gcc','g++','gnulink'], + tools=['gcc', 'g++', 'gnulink'], ) env.Tool('compilation_db') diff --git a/test/CompilationDatabase/variant_dir.py b/test/CompilationDatabase/variant_dir.py index b0a9464..21eb78b 100644 --- a/test/CompilationDatabase/variant_dir.py +++ b/test/CompilationDatabase/variant_dir.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,7 +22,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# + """ Test CompilationDatabase and several variations of ways to call it and values of COMPILATIONDB_USE_ABSPATH -- cgit v0.12 From 6bf97c4c32b86cb7509d0d92bd63b671d5681cbc Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 31 Dec 2022 15:42:13 -0800 Subject: Simplify test for commpilation db + TEMPFILE interaction --- test/CompilationDatabase/TEMPFILE.py | 66 +++++++++++++++ .../fixture/SConstruct_tempfile | 27 ++---- test/CompilationDatabase/tmpfile.py | 96 ---------------------- 3 files changed, 71 insertions(+), 118 deletions(-) create mode 100644 test/CompilationDatabase/TEMPFILE.py delete mode 100644 test/CompilationDatabase/tmpfile.py diff --git a/test/CompilationDatabase/TEMPFILE.py b/test/CompilationDatabase/TEMPFILE.py new file mode 100644 index 0000000..45e4521 --- /dev/null +++ b/test/CompilationDatabase/TEMPFILE.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Test that CompilationDatabase works when TEMPFILE is being used to handle long +commandlines for compilers/linkers/etc +""" + +import sys +import os +import os.path +import TestSCons + +test = TestSCons.TestSCons() + +test.file_fixture('mygcc.py') +test.file_fixture('test_main.c') +test.file_fixture('fixture/SConstruct_tempfile', 'SConstruct') + +test.run() + +rel_files = [ + 'compile_commands_only_arg.json', +] + +example_rel_file = """[ + { + "command": "%s mygcc.py cc -o test_main.o -c test_main.c", + "directory": "%s", + "file": "test_main.c", + "output": "test_main.o" + } +] +""" % (sys.executable, test.workdir) + +if sys.platform == 'win32': + example_rel_file = example_rel_file.replace('\\', '\\\\') + +for f in rel_files: + # print("Checking:%s" % f) + test.must_exist(f) + test.must_match(f, example_rel_file, mode='r') + +test.pass_test() diff --git a/test/CompilationDatabase/fixture/SConstruct_tempfile b/test/CompilationDatabase/fixture/SConstruct_tempfile index d7c2efd..2e942db 100644 --- a/test/CompilationDatabase/fixture/SConstruct_tempfile +++ b/test/CompilationDatabase/fixture/SConstruct_tempfile @@ -6,39 +6,22 @@ env = Environment( LINK='$PYTHON mylink.py', LINKFLAGS=[], CC='$PYTHON mygcc.py cc', - CXX='$PYTHON mygcc.py c++', - tools=['gcc','g++','gnulink'], + tools=['gcc'], MAXLINELENGTH=10, ) + # make sure TempFileMunge is used if 'TEMPFILE' not in env['CCCOM']: - env['CCCOM'] = '${TEMPFILE("%s")}'%(env['CCCOM']) + env['CCCOM'] = '${TEMPFILE("%s")}' % (env['CCCOM']) env.Tool('compilation_db') outputs = [] -env_abs = env.Clone(COMPILATIONDB_USE_ABSPATH=True) -outputs+= env_abs.CompilationDatabase('compile_commands_clone_abs.json') - -# Should be relative paths -outputs+= env.CompilationDatabase('compile_commands_only_arg.json') -outputs+= env.CompilationDatabase(target='compile_commands_target.json') - -# Should default name compile_commands.json -outputs+= env.CompilationDatabase() - -# Should be absolute paths -outputs+= env.CompilationDatabase('compile_commands_over_abs.json', COMPILATIONDB_USE_ABSPATH=True) -outputs+= env.CompilationDatabase(target='compile_commands_target_over_abs.json', COMPILATIONDB_USE_ABSPATH=True) # Should be relative paths -outputs+= env.CompilationDatabase('compile_commands_over_rel.json', COMPILATIONDB_USE_ABSPATH=False) - -# Try 1/0 for COMPILATIONDB_USE_ABSPATH -outputs+= env.CompilationDatabase('compile_commands_over_abs_1.json', COMPILATIONDB_USE_ABSPATH=1) -outputs+= env.CompilationDatabase('compile_commands_over_abs_0.json', COMPILATIONDB_USE_ABSPATH=0) +outputs += env.CompilationDatabase('compile_commands_only_arg.json') -env.Program('main', 'test_main.c') +env.Object('test_main.c') # Prevent actual call of $PYTHON @tempfile since "mygcc.py cc ..." is not a proper python statement # Interesting outputs are json databases diff --git a/test/CompilationDatabase/tmpfile.py b/test/CompilationDatabase/tmpfile.py deleted file mode 100644 index ea28a07..0000000 --- a/test/CompilationDatabase/tmpfile.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python -# -# __COPYRIGHT__ -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -""" -Test CompilationDatabase and several variations of ways to call it -and values of COMPILATIONDB_USE_ABSPATH -""" - -import sys -import os -import os.path -import TestSCons - -test = TestSCons.TestSCons() - -test.file_fixture('mylink.py') -test.file_fixture('mygcc.py') - -test.verbose_set(1) -test.file_fixture('fixture/SConstruct_tempfile', 'SConstruct') -test.file_fixture('test_main.c') -test.run() - -rel_files = [ - 'compile_commands_only_arg.json', - 'compile_commands_target.json', - 'compile_commands.json', - 'compile_commands_over_rel.json', - 'compile_commands_over_abs_0.json' -] - -abs_files = [ - 'compile_commands_clone_abs.json', - 'compile_commands_over_abs.json', - 'compile_commands_target_over_abs.json', - 'compile_commands_over_abs_1.json', -] - -example_rel_file = """[ - { - "command": "%s mygcc.py cc -o test_main.o -c test_main.c", - "directory": "%s", - "file": "test_main.c", - "output": "test_main.o" - } -] -""" % (sys.executable, test.workdir) - -if sys.platform == 'win32': - example_rel_file = example_rel_file.replace('\\', '\\\\') - -for f in rel_files: - # print("Checking:%s" % f) - test.must_exist(f) - test.must_match(f, example_rel_file, mode='r') - -example_abs_file = """[ - { - "command": "%s mygcc.py cc -o test_main.o -c test_main.c", - "directory": "%s", - "file": "%s", - "output": "%s" - } -] -""" % (sys.executable, test.workdir, os.path.join(test.workdir, 'test_main.c'), os.path.join(test.workdir, 'test_main.o')) - -if sys.platform == 'win32': - example_abs_file = example_abs_file.replace('\\', '\\\\') - - -for f in abs_files: - test.must_exist(f) - test.must_match(f, example_abs_file, mode='r') - - -test.pass_test() -- cgit v0.12 From dd6ec0687894c028b89dc3f18e1e70daa9e87750 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 31 Dec 2022 15:48:07 -0800 Subject: add content to CHANGES/RELEASE for this PR --- CHANGES.txt | 5 +++++ RELEASE.txt | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index bbc2800..1512700 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -36,6 +36,11 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER This should scale much better for highly parallel builds. You can also enable this via SetOption(). - Fixed command line argument --diskcheck: previously a value of 'none' was ignored. SetOption('diskcheck','none') is unaffected, as it did not have the problem. + - Added overrides argument to SCons.Subst.scons_subst(), subst_list(), subst(), and Action's process(), + strfunction(). This allows passing a dictionary of envvars to override when evaluating subst()'d strings/lists + - Fixed Issue #4275 - when outputting compilation db and TEMPFILE was in use, the compilation db would have + command lines using the generated tempfile for long command lines, instead of the full command line for + the compilation step for the source/target pair. From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). diff --git a/RELEASE.txt b/RELEASE.txt index 32094bf..ecf23c3 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -66,6 +66,10 @@ FIXES will create a string joining the list with os.pathsep - Fixed command line argument --diskcheck: previously a value of 'none' was ignored. SetOption('diskcheck','none') is unaffected, as it did not have the problem. +- Fixed Issue #4275 - when outputting compilation db and TEMPFILE was in use, the compilation db would have + command lines using the generated tempfile for long command lines, instead of the full command line for + the compilation step for the source/target pair. + IMPROVEMENTS ------------ @@ -123,6 +127,9 @@ DEVELOPMENT underscore) which moved to a submodule, that code will have to be adjusted, as those are not imported to the package level (new SCons.Util.hashes has some of these, which are used by existing unit tests). +- Added overrides argument to SCons.Subst.scons_subst(), subst_list(), subst(), and Action's process(), + strfunction(). This allows passing a dictionary of envvars to override when evaluating subst()'d strings/lists + Thanks to the following contributors listed below for their contributions to this release. ========================================================================================== -- cgit v0.12 From 8d99ea7b4a759838d68ea2618afd7f778889685b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 31 Dec 2022 16:37:07 -0800 Subject: Force OBJSUFFIX for TEMPFILE test to be .o so it match expected on all platforms --- test/CompilationDatabase/fixture/SConstruct_tempfile | 1 + 1 file changed, 1 insertion(+) diff --git a/test/CompilationDatabase/fixture/SConstruct_tempfile b/test/CompilationDatabase/fixture/SConstruct_tempfile index 2e942db..6b95977 100644 --- a/test/CompilationDatabase/fixture/SConstruct_tempfile +++ b/test/CompilationDatabase/fixture/SConstruct_tempfile @@ -8,6 +8,7 @@ env = Environment( CC='$PYTHON mygcc.py cc', tools=['gcc'], MAXLINELENGTH=10, + OBJSUFFIX='.o', ) # make sure TempFileMunge is used -- cgit v0.12 From cb1f9ff5aaa2db02d7eb26b11d8e281859ec81f9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 31 Mar 2022 11:00:43 -0600 Subject: Experimental: build a scons-local zipapp [skip appveyor] To use, set do_zipapp to True in site_scons/scons_local_package.py Will produce a build/dist/scons-local-${VERSION}.pyz which can be directly executed - copy to a work dir, and run as python scons-local-4.3.1.pyz or even ./scons-local-4.3.1.pyz Windows should have a file association for .pyz files, otherwise it works there too by calling it as an argument of the interpreter. Signed-off-by: Mats Wichmann --- site_scons/scons_local_package.py | 97 ++++++++++++++++---------- site_scons/zip_utils.py | 141 +++++++++++++++++++++++++------------- 2 files changed, 153 insertions(+), 85 deletions(-) diff --git a/site_scons/scons_local_package.py b/site_scons/scons_local_package.py index aaf058a..0a5d008 100644 --- a/site_scons/scons_local_package.py +++ b/site_scons/scons_local_package.py @@ -23,7 +23,7 @@ from glob import glob import os.path -from zip_utils import zipit +from zip_utils import zipit, zipappit from Utilities import is_windows @@ -36,9 +36,11 @@ def get_local_package_file_list(): # import pdb; pdb.set_trace() non_test = [f for f in s_files if "Tests.py" not in f] - non_test_non_doc = [f for f in non_test if '.xml' not in f or "SCons/Tool/docbook" in f] + non_test_non_doc = [ + f for f in non_test if '.xml' not in f or "SCons/Tool/docbook" in f + ] filtered_list = [f for f in non_test_non_doc if 'pyc' not in f] - filtered_list = [f for f in filtered_list if '__pycache__' not in f ] + filtered_list = [f for f in filtered_list if '__pycache__' not in f] filtered_list = [f for f in filtered_list if not os.path.isdir(f)] return filtered_list @@ -50,47 +52,71 @@ def install_local_package_files(env): files = get_local_package_file_list() target_dir = '#/build/scons-local/scons-local-$VERSION' for f in files: - all_local_installed.extend(env.Install(os.path.join(target_dir, os.path.dirname(f)), - f)) - - basedir_files = ['scripts/scons.bat', - 'scripts/scons.py', - 'scripts/scons-configure-cache.py', - 'scripts/sconsign.py', - 'bin/scons-time.py'] + all_local_installed.extend( + env.Install(os.path.join(target_dir, os.path.dirname(f)), f) + ) + + basedir_files = [ + 'scripts/scons.bat', + 'scripts/scons.py', + 'scripts/scons-configure-cache.py', + 'scripts/sconsign.py', + 'bin/scons-time.py', + ] for bf in basedir_files: fn = os.path.basename(bf) - all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bf)) + all_local_installed.append( + env.SCons_revision(f'#/build/scons-local/{fn}', bf) + ) # Now copy manpages into scons-local package - built_manpage_files = env.Glob('build/doc/man/*.1') + built_manpage_files = env.Glob('build/doc/man/*.1') for bmp in built_manpage_files: fn = os.path.basename(str(bmp)) - all_local_installed.append(env.SCons_revision('#/build/scons-local/%s'%fn, bmp)) - - rename_files = [('scons-${VERSION}.bat', 'scripts/scons.bat'), - ('scons-README', 'README-local'), - ('scons-LICENSE', 'LICENSE-local')] + all_local_installed.append( + env.SCons_revision(f'#/build/scons-local/{fn}', bmp) + ) + + rename_files = [ + ('scons-${VERSION}.bat', 'scripts/scons.bat'), + ('scons-README', 'README-local'), + ('scons-LICENSE', 'LICENSE-local'), + ] for t, f in rename_files: - target_file = "#/build/scons-local/%s"%t + target_file = f"#/build/scons-local/{t}" all_local_installed.append(env.SCons_revision(target_file, f)) return all_local_installed def create_local_packages(env): - # Add SubstFile builder - env.Tool('textfile') [env.Tool(x) for x in ['packaging', 'filesystem', 'zip']] installed_files = install_local_package_files(env) build_local_dir = 'build/scons-local' - package = env.Command('#build/dist/scons-local-${VERSION}.zip', - installed_files, - zipit, - CD=build_local_dir, - PSV='.', - ) + package = env.Command( + '#build/dist/scons-local-${VERSION}.zip', + installed_files, + zipit, + CD=build_local_dir, + PSV='.', + ) + + + do_zipapp = False + if do_zipapp: + # We need to descend into the versioned directory for zipapp, + # but we don't know the version. env.Glob lets us expand that. + # The action isn't going to use the sources here, but including + # them makes sure the deps work out right. + app_dir = env.Glob(f"{build_local_dir}/scons-local-*")[0] + zipapp = env.Command( + target='#build/dist/scons-local-${VERSION}.pyz', + source=installed_files, + action=zipappit, + CD=app_dir, + PSV='SCons', + ) if is_windows(): # avoid problem with tar interpreting c:/ as a remote machine @@ -98,13 +124,12 @@ def create_local_packages(env): else: tar_cargs = '-czf' - env.Command('#build/dist/scons-local-${VERSION}.tar.gz', - installed_files, - "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_local_dir, tar_cargs)) - - print("Package:%s"%package) - - - - + env.Command( + '#build/dist/scons-local-${VERSION}.tar.gz', + installed_files, + "cd %s && tar %s $( ${TARGET.abspath} $) *" % (build_local_dir, tar_cargs), + ) + print(f"Package:{package}") + if do_zipapp: + print(f"Zipapp:{zipapp}") diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index 1a0f843..d9f6a9b 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -1,54 +1,97 @@ +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Actions to zip and unzip, for working with SCons release bundles +Action for creating a zipapp +""" + import os.path +import zipfile +import zipapp -zcat = 'gzip -d -c' +def zipit(env, target, source): + """ + zip *source* into *target*, using some values from *env* -# -# Figure out if we can handle .zip files. -# -zipit = None -unzipit = None -try: - import zipfile - - def zipit(env, target, source): - print("Zipping %s:" % str(target[0])) - def visit(arg, dirname, filenames): - for filename in filenames: - path = os.path.join(dirname, filename) - if os.path.isfile(path): - arg.write(path) - # default ZipFile compression is ZIP_STORED - zf = zipfile.ZipFile(str(target[0]), 'w', compression=zipfile.ZIP_DEFLATED) - olddir = os.getcwd() - os.chdir(env.Dir(env['CD']).abspath) + *env* values: `CD` is the directory to work in, + `PSV` is the directory name to walk down to find the files + """ + print(f"Zipping {target[0]}:") + + def visit(arg, dirname, filenames): + for filename in filenames: + path = os.path.join(dirname, filename) + if os.path.isfile(path): + arg.write(path) + + # default ZipFile compression is ZIP_STORED + zf = zipfile.ZipFile(str(target[0]), 'w', compression=zipfile.ZIP_DEFLATED) + olddir = os.getcwd() + os.chdir(env.Dir(env['CD']).abspath) + try: + for dirname, dirnames, filenames in os.walk(env['PSV']): + visit(zf, dirname, filenames) + finally: + os.chdir(olddir) + zf.close() + + +def unzipit(env, target, source): + print(f"Unzipping {source[0]}:") + zf = zipfile.ZipFile(str(source[0]), 'r') + for name in zf.namelist(): + dest = os.path.join(env['UNPACK_ZIP_DIR'], name) + dir = os.path.dirname(dest) try: - for dirname, dirnames, filenames in os.walk(env['PSV']): - visit(zf, dirname, filenames) - finally: - os.chdir(olddir) - zf.close() - - def unzipit(env, target, source): - print("Unzipping %s:" % str(source[0])) - zf = zipfile.ZipFile(str(source[0]), 'r') - for name in zf.namelist(): - dest = os.path.join(env['UNPACK_ZIP_DIR'], name) - dir = os.path.dirname(dest) - try: - os.makedirs(dir) - except: - pass - print(dest,name) - # if the file exists, then delete it before writing - # to it so that we don't end up trying to write to a symlink: - if os.path.isfile(dest) or os.path.islink(dest): - os.unlink(dest) - if not os.path.isdir(dest): - with open(dest, 'wb') as fp: - fp.write(zf.read(name)) - -except ImportError: - if unzip and zip: - zipit = "cd $CD && $ZIP $ZIPFLAGS $( ${TARGET.abspath} $) $PSV" - unzipit = "$UNZIP $UNZIPFLAGS $SOURCES" + os.makedirs(dir) + except: + pass + print(dest, name) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + with open(dest, 'wb') as fp: + fp.write(zf.read(name)) + + +def zipappit(env, target, source): + """ + Create a zipapp *target* from specified directory. + + *env* values: ``"CD"`` is the Dir node for the place we want to work. + ``"PSV"`` is the directory name to point :meth:`zipapp.create_archive` at + (note *source* is unused here in favor of PSV) + """ + print(f"Creating zipapp {target[0]}:") + dest = target[0].abspath + olddir = os.getcwd() + #os.chdir(env.Dir(env['CD']).abspath) + os.chdir(env['CD'].abspath) + try: + zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") + finally: + os.chdir(olddir) -- cgit v0.12 From 982dbaceb447b275b56ffc2e993845592e4bc01f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Thu, 31 Mar 2022 12:00:16 -0600 Subject: Fix sider complaint Signed-off-by: Mats Wichmann --- site_scons/zip_utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index d9f6a9b..754029c 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -64,10 +64,7 @@ def unzipit(env, target, source): for name in zf.namelist(): dest = os.path.join(env['UNPACK_ZIP_DIR'], name) dir = os.path.dirname(dest) - try: - os.makedirs(dir) - except: - pass + os.makedirs(dir, exist_ok=True) print(dest, name) # if the file exists, then delete it before writing # to it so that we don't end up trying to write to a symlink: @@ -89,7 +86,6 @@ def zipappit(env, target, source): print(f"Creating zipapp {target[0]}:") dest = target[0].abspath olddir = os.getcwd() - #os.chdir(env.Dir(env['CD']).abspath) os.chdir(env['CD'].abspath) try: zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") -- cgit v0.12 From 4938de49dc7681b6ff35916805d31c21035e2523 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 1 Apr 2022 10:59:00 -0600 Subject: Get the zipapp support working correctly [ci skip] Needed to pass a different source and also pass an entry point to the zipapp.create_archive method. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 ++ RELEASE.txt | 5 +++-- site_scons/scons_local_package.py | 15 +++++++-------- site_scons/zip_utils.py | 28 ++++++++++++++++++---------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1512700..ecc82a4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -96,6 +96,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - The single-file Util module was split into a package with a few functional areas getting their own files - Util.py had grown to over 2100 lines. + - Add a zipapp package of scons-local: can use SCons from a local + file which does not need unpacking. diff --git a/RELEASE.txt b/RELEASE.txt index ecf23c3..e9f2d02 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -90,8 +90,9 @@ PACKAGING requirements_pkg.txt are the requirements to do a full build (including docs build) with an intent to create the packages. - Moved rpm and debian directories under packaging -- Added logic to help packagers enable reproducible builds into packaging/etc/. Please - read packaging/etc/README.txt if you are interested. +- Added logic to help packagers enable reproducible builds into packaging/etc/. + Please read packaging/etc/README.txt if you are interested. +- A zipapp of scons-local is now also built. DOCUMENTATION diff --git a/site_scons/scons_local_package.py b/site_scons/scons_local_package.py index 0a5d008..8eca758 100644 --- a/site_scons/scons_local_package.py +++ b/site_scons/scons_local_package.py @@ -28,9 +28,8 @@ from Utilities import is_windows def get_local_package_file_list(): - """ - Get list of all files which should be included in scons-local package - """ + """Get list of all files which should be included in scons-local package""" + s_files = glob("SCons/**", recursive=True) # import pdb; pdb.set_trace() @@ -102,20 +101,20 @@ def create_local_packages(env): PSV='.', ) - - do_zipapp = False + do_zipapp = True # Q: maybe an external way to specify whether to build? if do_zipapp: # We need to descend into the versioned directory for zipapp, # but we don't know the version. env.Glob lets us expand that. - # The action isn't going to use the sources here, but including - # them makes sure the deps work out right. + # The action isn't going to use the sources, but including + # them makes sure SCons has populated the dir we're going to zip. app_dir = env.Glob(f"{build_local_dir}/scons-local-*")[0] zipapp = env.Command( target='#build/dist/scons-local-${VERSION}.pyz', source=installed_files, action=zipappit, CD=app_dir, - PSV='SCons', + PSV='.', + entry='SCons.Script.Main:main', ) if is_windows(): diff --git a/site_scons/zip_utils.py b/site_scons/zip_utils.py index 754029c..a38a68f 100644 --- a/site_scons/zip_utils.py +++ b/site_scons/zip_utils.py @@ -32,11 +32,11 @@ import zipapp def zipit(env, target, source): - """ - zip *source* into *target*, using some values from *env* + """Action function to zip *source* into *target* - *env* values: `CD` is the directory to work in, - `PSV` is the directory name to walk down to find the files + Values extracted from *env*: + *CD*: the directory to work in + *PSV*: the directory name to walk down to find the files """ print(f"Zipping {target[0]}:") @@ -59,6 +59,8 @@ def zipit(env, target, source): def unzipit(env, target, source): + """Action function to unzip *source*""" + print(f"Unzipping {source[0]}:") zf = zipfile.ZipFile(str(source[0]), 'r') for name in zf.namelist(): @@ -76,18 +78,24 @@ def unzipit(env, target, source): def zipappit(env, target, source): - """ - Create a zipapp *target* from specified directory. + """Action function to Create a zipapp *target* from specified directory. - *env* values: ``"CD"`` is the Dir node for the place we want to work. - ``"PSV"`` is the directory name to point :meth:`zipapp.create_archive` at - (note *source* is unused here in favor of PSV) + Values extracted from *env*: + *CD*: the Dir node for the place we want to work. + *PSV*: the directory name to point :meth:`zipapp.create_archive` at + (note *source* is unused here in favor of PSV) + *main*: the entry point for the zipapp """ print(f"Creating zipapp {target[0]}:") dest = target[0].abspath olddir = os.getcwd() os.chdir(env['CD'].abspath) try: - zipapp.create_archive(env['PSV'], dest, "/usr/bin/env python") + zipapp.create_archive( + source=env['PSV'], + target=dest, + main=env['entry'], + interpreter="/usr/bin/env python", + ) finally: os.chdir(olddir) -- cgit v0.12 From 0c2cdf3e2202360d37a3d93b047a67f6e92505bb Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Mon, 9 Jan 2023 14:41:21 -0700 Subject: Docs: update MSVSProject, MSVSSolution [skip appveyor] A bit more description, renaming where warranted, some notes to watch out for. Along with some of the usual convert-known-strings-to-entity-refs stuff. Signed-off-by: Mats Wichmann --- RELEASE.txt | 1 + SCons/Tool/docbook/__init__.py | 2 +- SCons/Tool/msvs.xml | 574 ++++++++++++++++++++++++----------------- 3 files changed, 346 insertions(+), 231 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index e9f2d02..0ffe50b 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -115,6 +115,7 @@ DOCUMENTATION - Updated the User Guide chapter on variant directories with more explanation, and the introduction of terms like "out of tree" that may help in forming a mental model. +- Additional explanations for MSVSProject and MSVSSolution builders. DEVELOPMENT ----------- diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 5cf5e61..52e2911 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -69,7 +69,7 @@ re_refname = re.compile(r"([^<]*)") # lxml etree XSLT global max traversal depth # -lmxl_xslt_global_max_depth = 3100 +lmxl_xslt_global_max_depth = 3600 if has_lxml and lmxl_xslt_global_max_depth: def __lxml_xslt_set_global_max_depth(max_depth): diff --git a/SCons/Tool/msvs.xml b/SCons/Tool/msvs.xml index 55f0401..864d9a2 100644 --- a/SCons/Tool/msvs.xml +++ b/SCons/Tool/msvs.xml @@ -1,10 +1,12 @@ + %scons; @@ -20,7 +22,7 @@ See its __doc__ string for a discussion of the format. - Sets construction variables for Microsoft Visual Studio. + Sets &consvars; for Microsoft Visual Studio. MSVSPROJECTCOM @@ -39,70 +41,106 @@ See its __doc__ string for a discussion of the format. - Builds a Microsoft Visual Studio project file, and by default - builds a solution file as well. + Build a Microsoft Visual C++ project file and solution file. - This builds a Visual Studio project file, based on the - version of Visual Studio that is configured (either the - latest installed version, or the version specified by - &cv-link-MSVS_VERSION; in the Environment constructor). For - Visual Studio 6, it will generate a .dsp - file. For Visual Studio 7, 8, and 9, it will - generate a .vcproj file. For Visual - Studio 10 and later, it will generate a - .vcxproj file. - - - By default, this also generates a solution file for the - specified project, a .dsw file for - Visual Studio 6 or a .sln file for - Visual Studio 7 and later. This behavior may be disabled by - specifying auto_build_solution=0 when you - call &b-MSVSProject;, in which case you presumably want to - build the solution file(s) by calling the &b-MSVSSolution; - Builder (see below). - - - The &b-MSVSProject; builder takes several lists of filenames - to be placed into the project file. These are currently - limited to srcs, incs, - localincs, resources, and - misc. These are pretty self-explanatory, - but it should be noted that these lists are added to the - &cv-link-SOURCES; construction variable as strings, NOT as - SCons File Nodes. This is because they represent file names - to be added to the project file, not the source files used - to build the project file. - - - The above filename lists are all optional, although at least - one must be specified for the resulting project file to + Builds a C++ project file based on the + version of Visual Studio (or to be more precise, of MSBuild) + that is configured: either the latest installed version, + or the version specified by + &cv-link-MSVC_VERSION; in the current &consenv;. + For Visual Studio 6.0 a .dsp file is generated. + For Visual Studio versions 2002-2008, + a .vcproj file is generated. + For Visual Studio 2010 and later a .vcxproj + file is generated. + Note there are multiple versioning schemes involved in + the Microsoft compilation environment - + see the description of &cv-link-MSVC_VERSION; for equivalences. + &SCons; does not know how to construct project files for + other languages (such as .csproj for C#, + .vbproj for Visual Basic or + .pyproject for Python)). + + + For the .vcxproj file, the underlying + format is the MSBuild XML Schema, and the details conform to: + + https://learn.microsoft.com/en-us/cpp/build/reference/vcxproj-file-structure. + The generated solution file enables Visual Studio to + understand the project structure, and allows building it + using MSBuild to call back to &SCons;. + The project file encodes a toolset version that has been + selected by &SCons; as described above. Since recent Visual + Studio versions support multiple concurrent toolsets, + use &cv-link-MSVC_VERSION; to select the desired one if + it does not match the &SCons; default. + The project file also includes entries which describe + how to call &SCons; to build the project from within Visual Studio + (or from an MSBuild command line). + In some situations &SCons; may generate this incorrectly - + notably when using the scons-local + distribution, which is not installed in a way that that + matches the default invocation line. + If so, the &cv-link-SCONS_HOME; &consvar; can be used to describe + the right way to locate the &SCons; code so that it can be imported. + + + By default, a matching solution file for the project is also generated. + This behavior may be disabled by + specifying auto_build_solution=0 + to the &b-MSVSProject; builder. + The solution file can also be independently + generated by calling the &b-MSVSSolution; builder, + such as in the case where a solution should describe + multiple projects. + See the &b-link-MSVSSolution; description for further information. + + + The &b-MSVSProject; builder accepts several keyword arguments + describing lists of filenames to be placed into the project file. + Currently, + srcs, + incs, + localincs, + resources, + and misc + are recognized. + The names are intended to be self-explanatory, but note that the + filenames need to be specified as strings, not + as &SCons; File Nodes + (for example if you generate files for inclusion by using the + &f-link-Glob; function, the results should be converted to + a list of strings before passing them to &b-MSVSProject;). + This is because Visual Studio and MSBuild know nothing about &SCons; + Node types. + Each of the filename lists are individually optional, but at + least one list must be specified for the resulting project file to be non-empty. In addition to the above lists of values, the following values - may be specified: + may be specified as keyword arguments: - target + target The name of the target .dsp or .vcproj file. The correct suffix for the version of Visual Studio must be used, but the &cv-link-MSVSPROJECTSUFFIX; - construction variable will be defined to the correct + &consvar; will be defined to the correct value (see example below). - variant + variant - The name of this particular variant. For Visual Studio 7 + The name of this particular variant. Except for Visual Studio 6 projects, this can also be a list of variant names. These are typically things like "Debug" or "Release", but really can be anything you want. For Visual Studio @@ -117,145 +155,184 @@ See its __doc__ string for a discussion of the format. - cmdargs + cmdargs Additional command line arguments for the different variants. The number of - cmdargs entries must match the number - of variant entries, or be empty (not + cmdargs entries must match the number + of variant entries, or be empty (not specified). If you give only one, it will automatically be propagated to all variants. - cppdefines + cppdefines Preprocessor definitions for the different variants. - The number of cppdefines entries - must match the number of variant + The number of cppdefines entries + must match the number of variant entries, or be empty (not specified). If you give only one, it will automatically be propagated to all - variants. If you don't give this parameter, SCons + variants. If you don't give this parameter, &SCons; will use the invoking environment's - CPPDEFINES entry for all variants. + &cv-link-CPPDEFINES; entry for all variants. - cppflags + cppflags Compiler flags for the different variants. - If a /std:c++ flag is found then /Zc:__cplusplus is - appended to the flags if not already found, this - ensures that intellisense uses the /std:c++ switch. - The number of cppflags entries - must match the number of variant + If a flag is found then + is appended to the + flags if not already found, this ensures that Intellisense + uses the switch. + The number of cppflags entries + must match the number of variant entries, or be empty (not specified). If you give only one, it will automatically be propagated to all variants. If you don't give this parameter, SCons will combine the invoking environment's - CCFLAGS, CXXFLAGS, - CPPFLAGS entries for all variants. + &cv-link-CCFLAGS;, &cv-link-CXXFLAGS;, + &cv-link-CPPFLAGS; entries for all variants. - cpppaths + cpppaths Compiler include paths for the different variants. - The number of cpppaths entries - must match the number of variant + The number of cpppaths entries + must match the number of variant entries, or be empty (not specified). If you give only one, it will automatically be propagated to all variants. If you don't give this parameter, SCons will use the invoking environment's - CPPPATH entry for all variants. + &cv-link-CPPPATH; entry for all variants. - buildtarget + buildtarget An optional string, node, or list of strings or nodes (one per build variant), to tell the Visual Studio debugger what output target to use in what build variant. The number of - buildtarget entries must match the - number of variant entries. + buildtarget entries must match the + number of variant entries. - runfile + runfile The name of the file that Visual Studio 7 and later will run and debug. This appears as the - value of the Output field in the - resulting Visual Studio project file. If this is not + value of the Output field in the + resulting Visual C++ project file. If this is not specified, the default is the same as the specified - buildtarget value. + buildtarget value. + + + &SCons; and Microsoft Visual Studio understand projects in + different ways, and the mapping is sometimes imperfect: + - Note that because &SCons; always executes its build commands + Because &SCons; always executes its build commands from the directory in which the &SConstruct; file is located, if you generate a project file in a different directory - than the &SConstruct; directory, users will not be able to + than the directory of the &SConstruct; file, users will not be able to double-click on the file name in compilation error messages displayed in the Visual Studio console output window. This can - be remedied by adding the Visual C/C++ /FC + be remedied by adding the Visual C/C++ compiler option to the &cv-link-CCFLAGS; variable so that the compiler will print the full path name of any files that cause compilation errors. + + If the project file is only used to teach the Visual Studio + project browser about the file layout there should be no issues, + However, Visual Studio should not be used to make changes + to the project structure, build options, etc. as these will + (a) not feed back to the &SCons; description of the project + and (b) be lost if &SCons; regenerates the project file. + The SConscript files should remain the definitive description + of the build. + + + If the project file is used to drive MSBuild (such as selecting + "build" from the Visual Studio interface) you lose the direct + control of target selection and command-line options you would + have if launching the build directly from &SCons;, + because these will be hardcoded in the project file to the + values specified in the &b-MSVSProject; call. + You can regain some of this control by defining multiple variants, + using multiple &b-MSVSProject; calls to arrange different build + targets, arguments, defines, flags and paths for different variants. + + + If the build is divided into a solution with multiple MSBuild + projects the mapping is further strained. In this case, + it is important not to set Visual Studio to do parallel builds, + as it will then launch the separate project builds in parallel, + and &SCons; does not work well if called that way. + Instead you can set up the &SCons; build for parallel building - + see the &f-link-SetOption; function for how to do this with + num_jobs. + + + Example usage: barsrcs = ['bar.cpp'] barincs = ['bar.h'] barlocalincs = ['StdAfx.h'] -barresources = ['bar.rc','resource.h'] +barresources = ['bar.rc', 'resource.h'] barmisc = ['bar_readme.txt'] -dll = env.SharedLibrary(target='bar.dll', - source=barsrcs) +dll = env.SharedLibrary(target='bar.dll', source=barsrcs) buildtarget = [s for s in dll if str(s).endswith('dll')] -env.MSVSProject(target='Bar' + env['MSVSPROJECTSUFFIX'], - srcs=barsrcs, - incs=barincs, - localincs=barlocalincs, - resources=barresources, - misc=barmisc, - buildtarget=buildtarget, - variant='Release') +env.MSVSProject( + target='Bar' + env['MSVSPROJECTSUFFIX'], + srcs=barsrcs, + incs=barincs, + localincs=barlocalincs, + resources=barresources, + misc=barmisc, + buildtarget=buildtarget, + variant='Release', +) - - Starting with version 2.4 of SCons it is - also possible to specify the optional argument - DebugSettings, which creates files - for debugging under Visual Studio: - + - DebugSettings + DebugSettings A dictionary of debug settings that get written to the .vcproj.user or the .vcxproj.user file, depending on the - version installed. As it is done for cmdargs (see above), + version installed. As for cmdargs, you can specify a DebugSettings dictionary per variant. If you give only one, it will be propagated to all variants. + + Changed in version 2.4: + Added the optional DebugSettings parameter. + @@ -279,12 +356,17 @@ msvcver = vars.args.get('vc', '9') # Check command args to force one Microsoft Visual Studio version if msvcver == '9' or msvcver == '11': - env = Environment(MSVC_VERSION=msvcver+'.0', MSVC_BATCH=False) + env = Environment(MSVC_VERSION=msvcver + '.0', MSVC_BATCH=False) else: - env = Environment() + env = Environment() -AddOption('--userfile', action='store_true', dest='userfile', default=False, - help="Create Visual Studio Project user file") +AddOption( + '--userfile', + action='store_true', + dest='userfile', + default=False, + help="Create Visual C++ project file", +) # # 1. Configure your Debug Setting dictionary with options you want in the list @@ -292,28 +374,28 @@ AddOption('--userfile', action='store_true', dest='userfile', default=False, # a specific application for testing your dll with Microsoft Visual Studio 2008 (v9): # V9DebugSettings = { - 'Command':'c:\\myapp\\using\\thisdll.exe', + 'Command': 'c:\\myapp\\using\\thisdll.exe', 'WorkingDirectory': 'c:\\myapp\\using\\', 'CommandArguments': '-p password', -# 'Attach':'false', -# 'DebuggerType':'3', -# 'Remote':'1', -# 'RemoteMachine': None, -# 'RemoteCommand': None, -# 'HttpUrl': None, -# 'PDBPath': None, -# 'SQLDebugging': None, -# 'Environment': '', -# 'EnvironmentMerge':'true', -# 'DebuggerFlavor': None, -# 'MPIRunCommand': None, -# 'MPIRunArguments': None, -# 'MPIRunWorkingDirectory': None, -# 'ApplicationCommand': None, -# 'ApplicationArguments': None, -# 'ShimCommand': None, -# 'MPIAcceptMode': None, -# 'MPIAcceptFilter': None, + # 'Attach':'false', + # 'DebuggerType':'3', + # 'Remote':'1', + # 'RemoteMachine': None, + # 'RemoteCommand': None, + # 'HttpUrl': None, + # 'PDBPath': None, + # 'SQLDebugging': None, + # 'Environment': '', + # 'EnvironmentMerge':'true', + # 'DebuggerFlavor': None, + # 'MPIRunCommand': None, + # 'MPIRunArguments': None, + # 'MPIRunWorkingDirectory': None, + # 'ApplicationCommand': None, + # 'ApplicationArguments': None, + # 'ShimCommand': None, + # 'MPIAcceptMode': None, + # 'MPIAcceptFilter': None, } # @@ -327,28 +409,28 @@ V10DebugSettings = { 'LocalDebuggerCommand': 'c:\\myapp\\using\\thisdll.exe', 'LocalDebuggerWorkingDirectory': 'c:\\myapp\\using\\', 'LocalDebuggerCommandArguments': '-p password', -# 'LocalDebuggerEnvironment': None, -# 'DebuggerFlavor': 'WindowsLocalDebugger', -# 'LocalDebuggerAttach': None, -# 'LocalDebuggerDebuggerType': None, -# 'LocalDebuggerMergeEnvironment': None, -# 'LocalDebuggerSQLDebugging': None, -# 'RemoteDebuggerCommand': None, -# 'RemoteDebuggerCommandArguments': None, -# 'RemoteDebuggerWorkingDirectory': None, -# 'RemoteDebuggerServerName': None, -# 'RemoteDebuggerConnection': None, -# 'RemoteDebuggerDebuggerType': None, -# 'RemoteDebuggerAttach': None, -# 'RemoteDebuggerSQLDebugging': None, -# 'DeploymentDirectory': None, -# 'AdditionalFiles': None, -# 'RemoteDebuggerDeployDebugCppRuntime': None, -# 'WebBrowserDebuggerHttpUrl': None, -# 'WebBrowserDebuggerDebuggerType': None, -# 'WebServiceDebuggerHttpUrl': None, -# 'WebServiceDebuggerDebuggerType': None, -# 'WebServiceDebuggerSQLDebugging': None, + # 'LocalDebuggerEnvironment': None, + # 'DebuggerFlavor': 'WindowsLocalDebugger', + # 'LocalDebuggerAttach': None, + # 'LocalDebuggerDebuggerType': None, + # 'LocalDebuggerMergeEnvironment': None, + # 'LocalDebuggerSQLDebugging': None, + # 'RemoteDebuggerCommand': None, + # 'RemoteDebuggerCommandArguments': None, + # 'RemoteDebuggerWorkingDirectory': None, + # 'RemoteDebuggerServerName': None, + # 'RemoteDebuggerConnection': None, + # 'RemoteDebuggerDebuggerType': None, + # 'RemoteDebuggerAttach': None, + # 'RemoteDebuggerSQLDebugging': None, + # 'DeploymentDirectory': None, + # 'AdditionalFiles': None, + # 'RemoteDebuggerDeployDebugCppRuntime': None, + # 'WebBrowserDebuggerHttpUrl': None, + # 'WebBrowserDebuggerDebuggerType': None, + # 'WebServiceDebuggerHttpUrl': None, + # 'WebServiceDebuggerDebuggerType': None, + # 'WebServiceDebuggerSQLDebugging': None, } # @@ -370,71 +452,85 @@ else: barsrcs = ['bar.cpp', 'dllmain.cpp', 'stdafx.cpp'] barincs = ['targetver.h'] barlocalincs = ['StdAfx.h'] -barresources = ['bar.rc','resource.h'] +barresources = ['bar.rc', 'resource.h'] barmisc = ['ReadMe.txt'] -dll = env.SharedLibrary(target='bar.dll', - source=barsrcs) +dll = env.SharedLibrary(target='bar.dll', source=barsrcs) -env.MSVSProject(target='Bar' + env['MSVSPROJECTSUFFIX'], - srcs=barsrcs, - incs=barincs, - localincs=barlocalincs, - resources=barresources, - misc=barmisc, - buildtarget=[dll[0]] * 2, - variant=('Debug|Win32', 'Release|Win32'), - cmdargs='vc=%s' % msvcver, - DebugSettings=(dbgSettings, {})) +env.MSVSProject( + target='Bar' + env['MSVSPROJECTSUFFIX'], + srcs=barsrcs, + incs=barincs, + localincs=barlocalincs, + resources=barresources, + misc=barmisc, + buildtarget=[dll[0]] * 2, + variant=('Debug|Win32', 'Release|Win32'), + cmdargs=f'vc={msvcver}', + DebugSettings=(dbgSettings, {}), +) - Builds a Microsoft Visual Studio solution file. + Build a Microsoft Visual Studio Solution file. - This builds a Visual Studio solution file, based on the - version of Visual Studio that is configured (either the + Builds a Visual Studio solution file based on the + version of Visual Studio that is configured: either the latest installed version, or the version specified by - &cv-link-MSVS_VERSION; in the construction environment). For - Visual Studio 6, it will generate a .dsw - file. For Visual Studio 7 (.NET), it will generate a - .sln file. + &cv-link-MSVC_VERSION; in the &consenv;. For + Visual Studio 6, a .dsw file is generated. + For Visual Studio .NET 2002 and later, + it will generate a .sln file. + Note there are multiple versioning schemes involved in + the Microsoft compilation environment - + see the description of &cv-link-MSVC_VERSION; for equivalences. + + + The solution file is a container for one or more projects, + and follows the format described at + + https://learn.microsoft.com/en-us/visualstudio/extensibility/internals/solution-dot-sln-file. The following values must be specified: - target + target - The name of the target .dsw or .sln file. The correct + The name of the target .dsw or + .sln file. The correct suffix for the version of Visual Studio must be used, but the value &cv-link-MSVSSOLUTIONSUFFIX; will be defined to the correct value (see example below). - - variant + + + variant + The name of this particular variant, or a list of variant names (the latter is only supported for MSVS 7 solutions). These are typically things like "Debug" or "Release", but really can be anything you want. For MSVS 7 they may also specify target platform, like this - "Debug|Xbox". Default platform is Win32. + "Debug|Xbox". Default platform is Win32. - - projects + + + projects + A list of project file names, or Project nodes returned - by calls to the &b-MSVSProject; Builder, to be placed - into the solution file. It should be noted that these - file names are NOT added to the $SOURCES environment - variable in form of files, but rather as strings. - This is because they represent file names to be added - to the solution file, not the source files used to - build the solution file. + by calls to the &b-link-MSVSProject; Builder, to be placed + into the solution file. + Note that these filenames need to be specified as strings, + NOT as &SCons; File Nodes. + This is because the solution file will be interpreted by MSBuild + and by Visual Studio, which know nothing about &SCons; Node types. @@ -456,28 +552,39 @@ env.MSVSSolution( - VERSION + VERSION + the version of MSVS being used (can be set via - &cv-link-MSVS_VERSION;) + &cv-link-MSVC_VERSION;) - - VERSIONS + + + VERSIONS + the available versions of MSVS installed - - VCINSTALLDIR + + + VCINSTALLDIR + installed directory of Visual C++ - - VSINSTALLDIR + + + VSINSTALLDIR + installed directory of Visual Studio - - FRAMEWORKDIR + + + FRAMEWORKDIR + installed directory of the .NET framework - - FRAMEWORKVERSIONS + + + FRAMEWORKVERSIONS + list of installed versions of the .NET framework, sorted latest to oldest. @@ -514,7 +621,12 @@ env.MSVSSolution( - If a value is not set, it was not available in the registry. + If a value is not set, it was not available in the registry. + Visual Studio 2017 and later do not use the registry for + primary storage of this information, so typically for these + versions only PROJECTSUFFIX and + SOLUTIONSUFFIX will be set. + @@ -534,7 +646,7 @@ env.MSVSSolution( The string placed in a generated -Microsoft Visual Studio project file as the value of the +Microsoft Visual C++ project file as the value of the ProjectGUID attribute. There is no default value. If not defined, a new GUID is generated. @@ -545,9 +657,9 @@ defined, a new GUID is generated. The path name placed in a generated -Microsoft Visual Studio project file as the value of the +Microsoft Visual C++ project file as the value of the SccAuxPath attribute if the - MSVS_SCC_PROVIDER construction variable is + MSVS_SCC_PROVIDER &consvar; is also set. There is no default value. @@ -559,7 +671,7 @@ no default value. The root path of projects in your SCC workspace, i.e the path under which all project and solution files will be generated. It is used as a reference path from which the - relative paths of the generated Microsoft Visual Studio project + relative paths of the generated Microsoft Visual C++ project and solution files are computed. The relative project file path is placed as the value of the SccLocalPath attribute of the project file and as the values of the @@ -572,7 +684,7 @@ no default value. to the number of projects in the solution) attributes of the GlobalSection(SourceCodeControl) section of the Microsoft Visual Studio solution file. This is used only if - the MSVS_SCC_PROVIDER construction variable is + the MSVS_SCC_PROVIDER &consvar; is also set. The default value is the current working directory. @@ -580,9 +692,9 @@ no default value. The project name placed in a generated Microsoft - Visual Studio project file as the value of the + Visual C++ project file as the value of the SccProjectName attribute if the - MSVS_SCC_PROVIDER construction variable + MSVS_SCC_PROVIDER &consvar; is also set. In this case the string is also placed in the SccProjectName0 attribute of the GlobalSection(SourceCodeControl) section @@ -594,7 +706,7 @@ no default value. The string placed in a generated Microsoft - Visual Studio project file as the value of the + Visual C++ project file as the value of the SccProvider attribute. The string is also placed in the SccProvider0 attribute of the GlobalSection(SourceCodeControl) @@ -604,23 +716,25 @@ no default value. - Sets the preferred version of Microsoft Visual Studio to use. + Set the preferred version of Microsoft Visual Studio to use. If &cv-MSVS_VERSION; is not set, &SCons; will (by default) select the latest version of Visual Studio installed on your system. So, if you have version 6 and version 7 (MSVS .NET) installed, it will prefer version 7. You can override this by - specifying the MSVS_VERSION variable in the - Environment initialization, setting it to the appropriate + specifying the &cv-link-MSVS_VERSION; variable when + initializing the Environment, setting it to the appropriate version ('6.0' or '7.0', for example). If the specified version isn't installed, tool initialization will fail. - This is obsolete: use &cv-MSVC_VERSION; instead. If - &cv-MSVS_VERSION; is set and &cv-MSVC_VERSION; is - not, &cv-MSVC_VERSION; will be set automatically to - &cv-MSVS_VERSION;. If both are set to different values, - scons will raise an error. + Deprecated since 1.3.0: + &cv-MSVS_VERSION; is deprecated in favor of &cv-link-MSVC_VERSION;. + As a transitional aid, if &cv-MSVS_VERSION; is set + and &cv-MSVC_VERSION; is not, + &cv-MSVC_VERSION; will be initialized to the value + of &cv-MSVS_VERSION;. + An error is raised if If both are set and have different values, @@ -628,8 +742,8 @@ no default value. The build command line placed in a generated Microsoft Visual - Studio project file. The default is to have Visual Studio - invoke SCons with any specified build targets. + C++ project file. The default is to have Visual Studio + invoke &SCons; with any specified build targets. @@ -637,33 +751,34 @@ no default value. The clean command line placed in a generated Microsoft Visual - Studio project file. The default is to have Visual Studio - invoke SCons with the -c option to remove any specified - targets. + C++ project file. The default is to have Visual Studio + invoke &SCons; with the option to remove + any specified targets. The encoding string placed in a generated Microsoft - Visual Studio project file. The default is encoding + Visual C++ project file. The default is encoding Windows-1252. - The action used to generate Microsoft Visual Studio project files. + The action used to generate Microsoft Visual C++ project files. - The suffix used for Microsoft Visual Studio project (DSP) - files. The default value is .vcproj - when using Visual Studio version 7.x (.NET) or later version, - and .dsp when using earlier versions of - Visual Studio. + The suffix used for Microsoft Visual C++ project (DSP) + files. The default value is + .vcxproj when using Visual Studio 2010 + and later, .vcproj + when using Visual Studio versions between 2002 and 2008, + and .dsp when using Visual Studio 6.0. @@ -671,8 +786,8 @@ no default value. The rebuild command line placed in a generated Microsoft - Visual Studio project file. The default is to have Visual - Studio invoke SCons with any specified rebuild targets. + Visual C++ project file. The default is to have Visual + Studio invoke &SCons; with any specified rebuild targets. @@ -680,8 +795,8 @@ no default value. - The SCons used in generated Microsoft Visual Studio project - files. The default is the version of SCons being used to + The &SCons; used in generated Microsoft Visual C++ project + files. The default is the version of &SCons; being used to generate the project file. @@ -689,15 +804,15 @@ no default value. - The SCons flags used in generated Microsoft Visual Studio project files. + The &SCons; flags used in generated Microsoft Visual C++ project files. - The default SCons command used in generated Microsoft Visual - Studio project files. + The default &SCons; command used in generated Microsoft Visual + C++ project files. @@ -705,10 +820,10 @@ no default value. The sconscript file (that is, &SConstruct; or &SConscript; - file) that will be invoked by Visual Studio project files + file) that will be invoked by Visual C++ project files (through the &cv-link-MSVSSCONSCOM; variable). The default is the same sconscript file that contains the call to - &b-MSVSProject; to build the project file. + &b-link-MSVSProject; to build the project file. @@ -721,20 +836,19 @@ no default value. The suffix used for Microsoft Visual Studio solution (DSW) files. The default value is .sln - when using Visual Studio version 7.x (.NET), and - .dsw when using earlier versions of - Visual Studio. + when using Visual Studio version 7.x (.NET 2002) and later, + and .dsw when using Visual Studio 6.0. - The (optional) path to the SCons library directory, + The (optional) path to the &SCons; library directory, initialized from the external environment. If set, this is used to construct a shorter and more efficient search path in the &cv-link-MSVSSCONS; command line executed from Microsoft - Visual Studio project files. + Visual C++ project files. -- cgit v0.12 From 31abe6c20100c318ad9d3e5117c0e3db0f34ad79 Mon Sep 17 00:00:00 2001 From: djh <1810493+djh82@users.noreply.github.com> Date: Fri, 13 Jan 2023 15:02:15 +0000 Subject: feat: adds JAVAPROCESSORPATH construction variable; updates JavaScanner to scan JAVAPROCESSORPATH --- CHANGES.txt | 4 ++ SCons/Scanner/Java.py | 9 ++-- SCons/Scanner/JavaTests.py | 64 ++++++++++++++++++++++++++++ SCons/Tool/javac.py | 4 +- SCons/Tool/javac.xml | 25 +++++++++++ test/Java/JAVAPROCESSORPATH.py | 95 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+), 5 deletions(-) create mode 100644 test/Java/JAVAPROCESSORPATH.py diff --git a/CHANGES.txt b/CHANGES.txt index ecc82a4..55cf185 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -42,6 +42,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER command lines using the generated tempfile for long command lines, instead of the full command line for the compilation step for the source/target pair. + From David H: + - Added JAVAPROCESSORPATH construction variable which populates -processorpath. + - Updated JavaScanner to scan JAVAPROCESSORPATH. + From Dan Mezhiborsky: - Add newline to end of compilation db (compile_commands.json). diff --git a/SCons/Scanner/Java.py b/SCons/Scanner/Java.py index 8c31bc1..e6c2db9 100644 --- a/SCons/Scanner/Java.py +++ b/SCons/Scanner/Java.py @@ -59,9 +59,9 @@ def _collect_classes(classlist, dirname, files): def scan(node, env, libpath=()) -> list: - """Scan for files on the JAVACLASSPATH. + """Scan for files both on JAVACLASSPATH and JAVAPROCESSORPATH. - JAVACLASSPATH path can contain: + JAVACLASSPATH/JAVAPROCESSORPATH path can contain: - Explicit paths to JAR/Zip files - Wildcards (*) - Directories which contain classes in an unnamed package @@ -70,8 +70,9 @@ def scan(node, env, libpath=()) -> list: Class path entries that are neither directories nor archives (.zip or JAR files) nor the asterisk (*) wildcard character are ignored. """ - classpath = env.get('JAVACLASSPATH', []) - classpath = _subst_paths(env, classpath) + classpath = [] + for var in ['JAVACLASSPATH', 'JAVAPROCESSORPATH']: + classpath += _subst_paths(env, env.get(var, [])) result = [] for path in classpath: diff --git a/SCons/Scanner/JavaTests.py b/SCons/Scanner/JavaTests.py index 77cd560..faa0c49 100644 --- a/SCons/Scanner/JavaTests.py +++ b/SCons/Scanner/JavaTests.py @@ -179,6 +179,70 @@ class JavaScannerSearchPathClasspath(unittest.TestCase): deps_match(self, deps, expected) +class JavaScannerEmptyProcessorpath(unittest.TestCase): + def runTest(self): + path = [] + env = DummyEnvironment(JAVASUFFIXES=['.java'], JAVAPROCESSORPATH=path) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = [] + deps_match(self, deps, expected) + + +class JavaScannerProcessorpath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment(JAVASUFFIXES=['.java'], + JAVAPROCESSORPATH=[test.workpath('classpath.jar')]) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['classpath.jar'] + deps_match(self, deps, expected) + + +class JavaScannerWildcardProcessorpath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment(JAVASUFFIXES=['.java'], + JAVAPROCESSORPATH=[test.workpath('*')]) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['bootclasspath.jar', 'classpath.jar', 'Test.class'] + deps_match(self, deps, expected) + + +class JavaScannerDirProcessorpath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment(JAVASUFFIXES=['.java'], + JAVAPROCESSORPATH=[test.workpath()]) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['Test.class', 'com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + + +class JavaScannerNamedDirProcessorpath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVAPROCESSORPATH=[test.workpath('com'), test.workpath('java space')], + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + + +class JavaScannerSearchPathProcessorpath(unittest.TestCase): + def runTest(self): + env = DummyEnvironment( + JAVASUFFIXES=['.java'], + JAVAPROCESSORPATH=os.pathsep.join([test.workpath('com'), test.workpath('java space')]), + ) + s = SCons.Scanner.Java.JavaScanner() + deps = s(DummyNode('dummy'), env) + expected = ['com/Test.class', 'java space/Test.class'] + deps_match(self, deps, expected) + + if __name__ == "__main__": unittest.main() diff --git a/SCons/Tool/javac.py b/SCons/Tool/javac.py index 9e18964..1b33125 100644 --- a/SCons/Tool/javac.py +++ b/SCons/Tool/javac.py @@ -231,13 +231,15 @@ def generate(env): JAVABOOTCLASSPATH=[], JAVACLASSPATH=[], JAVASOURCEPATH=[], + JAVAPROCESSORPATH=[], ) env['_javapathopt'] = pathopt env['_JAVABOOTCLASSPATH'] = '${_javapathopt("-bootclasspath", "JAVABOOTCLASSPATH")} ' + env['_JAVAPROCESSORPATH'] = '${_javapathopt("-processorpath", "JAVAPROCESSORPATH")} ' env['_JAVACLASSPATH'] = '${_javapathopt("-classpath", "JAVACLASSPATH")} ' env['_JAVASOURCEPATH'] = '${_javapathopt("-sourcepath", "JAVASOURCEPATH", "_JAVASOURCEPATHDEFAULT")} ' env['_JAVASOURCEPATHDEFAULT'] = '${TARGET.attributes.java_sourcedir}' - env['_JAVACCOM'] = '$JAVAC $JAVACFLAGS $_JAVABOOTCLASSPATH $_JAVACLASSPATH -d ${TARGET.attributes.java_classdir} $_JAVASOURCEPATH $SOURCES' + env['_JAVACCOM'] = '$JAVAC $JAVACFLAGS $_JAVABOOTCLASSPATH $_JAVAPROCESSORPATH $_JAVACLASSPATH -d ${TARGET.attributes.java_classdir} $_JAVASOURCEPATH $SOURCES' env['JAVACCOM'] = "${TEMPFILE('$_JAVACCOM','$JAVACCOMSTR')}" def exists(env): diff --git a/SCons/Tool/javac.xml b/SCons/Tool/javac.xml index 014d905..9001e64 100644 --- a/SCons/Tool/javac.xml +++ b/SCons/Tool/javac.xml @@ -152,6 +152,31 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' + + + + Specifies the location of the annotation processor class files. + Can be specified as a string or Node object, + or as a list of strings or Node objects. + + + The value will be added to the JDK command lines + via the option, + which requires a system-specific search path separator. + This will be supplied by &SCons; as needed when it + constructs the command line if &cv-JAVAPROCESSORPATH; is + provided in list form. + If &cv-JAVAPROCESSORPATH; is a single string containing + search path separator characters + (: for POSIX systems or + ; for Windows), it will not be modified; + and so is inherently system-specific; + to supply the path in a system-independent manner, + give &cv-JAVAPROCESSORPATH; as a list of paths instead. + + + + diff --git a/test/Java/JAVAPROCESSORPATH.py b/test/Java/JAVAPROCESSORPATH.py new file mode 100644 index 0000000..2b8f04d --- /dev/null +++ b/test/Java/JAVAPROCESSORPATH.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Verify that use of $JAVAPROCESSORPATH sets the -processorpath option +on javac compilations. +""" + +import os + +import TestSCons + +test = TestSCons.TestSCons() + +where_javac, java_version = test.java_where_javac() + +test.write('SConstruct', """ +DefaultEnvironment(tools=[]) +env = Environment(tools=['javac'], JAVAPROCESSORPATH=['dir1', 'dir2']) +j1 = env.Java(target='class', source='com/Example1.java') +j2 = env.Java(target='class', source='com/Example2.java') +""") + +test.subdir('com') + +test.write(['com', 'Example1.java'], """\ +package com; + +public class Example1 +{ + + public static void main(String[] args) + { + + } + +} +""") + +test.write(['com', 'Example2.java'], """\ +package com; + +public class Example2 +{ + + public static void main(String[] args) + { + + } + +} +""") + +# Setting -processorpath messes with the Java runtime environment, so +# we'll just take the easy way out and examine the -n output to see if +# the expected option shows up on the command line. + +processorpath = os.pathsep.join(['dir1', 'dir2']) + +expect = """\ +javac -processorpath %(processorpath)s -d class -sourcepath com com.Example1\\.java +javac -processorpath %(processorpath)s -d class -sourcepath com com.Example2\\.java +""" % locals() + +test.run(arguments = '-Q -n .', stdout = expect, match=TestSCons.match_re) + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 854c3bdd06e995ceadceccc7d99f773e80cb9707 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 13 Jan 2023 10:26:36 -0700 Subject: Fix problem where Java inner classes cannot cache Generated files contained a '$' in filename and this blew up subst. Situation arose because of a need to fetch the FS entry of the source for finding permissions. Now we use the permissions of the cached target to decide whether to chmod to add write permission, this avoids the need to call File() on the source. Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 +- RELEASE.txt | 2 + SCons/CacheDir.py | 40 ++++++++++++-------- test/Java/inner-cacheable-live.py | 77 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 test/Java/inner-cacheable-live.py diff --git a/CHANGES.txt b/CHANGES.txt index ecc82a4..087464b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -98,7 +98,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER over 2100 lines. - Add a zipapp package of scons-local: can use SCons from a local file which does not need unpacking. - + - Fix a problem (4.4 only) where a Java inner class could not be cached + because the emitted filename contained a '$' and ended up generating + a Python SyntaxError because is was passed through scons_subst(). RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index e9f2d02..76b51d1 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -69,6 +69,8 @@ FIXES - Fixed Issue #4275 - when outputting compilation db and TEMPFILE was in use, the compilation db would have command lines using the generated tempfile for long command lines, instead of the full command line for the compilation step for the source/target pair. +- A refactor in the caching logic for version 4.4 left Java inner classes + failing with an exception when a CacheDir was enabled. This is now corrected. IMPROVEMENTS diff --git a/SCons/CacheDir.py b/SCons/CacheDir.py index 14e52ad..70c4f38 100644 --- a/SCons/CacheDir.py +++ b/SCons/CacheDir.py @@ -211,35 +211,42 @@ class CacheDir: (self.requests, self.hits, self.misses, self.hit_ratio)) @classmethod - def copy_from_cache(cls, env, src, dst): + def copy_from_cache(cls, env, src, dst) -> str: + """Copy a file from cache.""" if env.cache_timestamp_newer: return env.fs.copy(src, dst) else: return env.fs.copy2(src, dst) @classmethod - def copy_to_cache(cls, env, src, dst): + def copy_to_cache(cls, env, src, dst) -> str: + """Copy a file to cache. + + Just use the FS copy2 ("with metadata") method, except do an additional + check and if necessary a chmod to ensure the cachefile is writeable, + to forestall permission problems if the cache entry is later updated. + """ try: result = env.fs.copy2(src, dst) - fs = env.File(src).fs - st = fs.stat(src) - fs.chmod(dst, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + st = stat.S_IMODE(os.stat(result).st_mode) + if not st | stat.S_IWRITE: + os.chmod(dst, st | stat.S_IWRITE) return result except AttributeError as ex: raise EnvironmentError from ex @property - def hit_ratio(self): + def hit_ratio(self) -> float: return (100.0 * self.hits / self.requests if self.requests > 0 else 100) @property - def misses(self): + def misses(self) -> int: return self.requests - self.hits - def is_enabled(self): + def is_enabled(self) -> bool: return cache_enabled and self.path is not None - def is_readonly(self): + def is_readonly(self) -> bool: return cache_readonly def get_cachedir_csig(self, node): @@ -247,18 +254,21 @@ class CacheDir: if cachefile and os.path.exists(cachefile): return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize) - def cachepath(self, node): - """ + def cachepath(self, node) -> tuple: + """Return where to cache a file. + + Given a Node, obtain the configured cache directory and + the path to the cached file, which is generated from the + node's build signature. If caching is not enabled for the + None, return a tuple of None. """ if not self.is_enabled(): return None, None sig = node.get_cachedir_bsig() - subdir = sig[:self.config['prefix_len']].upper() - - dir = os.path.join(self.path, subdir) - return dir, os.path.join(dir, sig) + cachedir = os.path.join(self.path, subdir) + return cachedir, os.path.join(cachedir, sig) def retrieve(self, node): """ diff --git a/test/Java/inner-cacheable-live.py b/test/Java/inner-cacheable-live.py new file mode 100644 index 0000000..9f70291 --- /dev/null +++ b/test/Java/inner-cacheable-live.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Test Java inner classes can be cached. Requires a working JDK. + +Regression test: one iteration of CacheDir left it unable to deal +with class names from the emitter which contained an embedded '$'. +Led to error like: + +SyntaxError `invalid syntax (, line 1)' trying to evaluate `$Inner.class' +""" + +import TestSCons + +test = TestSCons.TestSCons() +where_javac, java_version = test.java_where_javac() + +# Work around javac 1.4 not reporting its version: +java_version = java_version or "1.4" + +# Skip this test as SCons doesn't (currently) predict the generated +# inner/anonymous class generated .class files generated by gcj +# and so will always fail. +if test.javac_is_gcj: + test.skip_test('Test not valid for gcj (gnu java); skipping test(s).\n') + +test.write( + 'SConstruct', + """ +env = Environment() +env.CacheDir("cache") +env.Java("classes", "source") +""" + % locals(), +) + +test.subdir('source') + +test.write( + ['source', 'Test.java'], + """\ +class Test { class Inner {} } +""", +) + +test.run(arguments='.') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 511282de0cbb82bd931681b7ca8a6c83755af4d9 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 13 Jan 2023 13:08:53 -0700 Subject: Remove unneeded code in new Java test java inner class cache teest: String didn't need to interpolate from locals() as there were no variables to fill in. Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 ++++--- test/Java/inner-cacheable-live.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 087464b..32e1e5b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -98,9 +98,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER over 2100 lines. - Add a zipapp package of scons-local: can use SCons from a local file which does not need unpacking. - - Fix a problem (4.4 only) where a Java inner class could not be cached - because the emitted filename contained a '$' and ended up generating - a Python SyntaxError because is was passed through scons_subst(). + - Fix a problem (present in 4.4.0 only) where a Java inner class could + not be cached because the emitted filename contained a '$' and when + looked up through a node ended up generating a Python SyntaxError + because it was passed through scons_subst(). RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/test/Java/inner-cacheable-live.py b/test/Java/inner-cacheable-live.py index 9f70291..e0391d2 100644 --- a/test/Java/inner-cacheable-live.py +++ b/test/Java/inner-cacheable-live.py @@ -50,11 +50,11 @@ if test.javac_is_gcj: test.write( 'SConstruct', """ +DefaultEnvironment(tools=[]) env = Environment() env.CacheDir("cache") env.Java("classes", "source") -""" - % locals(), +""", ) test.subdir('source') -- cgit v0.12 From 20614c573806d5c651d772bec7767f50d16645c3 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 16 Jan 2023 18:37:52 -0500 Subject: [ci skip] Add notation that JAVAPROCESSORPATH is new in version 4.5.0 (The next planned release version string) --- SCons/Tool/javac.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/SCons/Tool/javac.xml b/SCons/Tool/javac.xml index 9001e64..6f48356 100644 --- a/SCons/Tool/javac.xml +++ b/SCons/Tool/javac.xml @@ -174,6 +174,9 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' to supply the path in a system-independent manner, give &cv-JAVAPROCESSORPATH; as a list of paths instead. + + New in version 4.5.0 + -- cgit v0.12 From 57939580405dfb45c3b2de058ffe007325a63e5b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 16 Jan 2023 22:47:29 -0500 Subject: changed which version the cachedir java issue occurs in in RELEASE.txt to 4.4.0 from 4.4 --- RELEASE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.txt b/RELEASE.txt index 76b51d1..35fd845 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -69,7 +69,7 @@ FIXES - Fixed Issue #4275 - when outputting compilation db and TEMPFILE was in use, the compilation db would have command lines using the generated tempfile for long command lines, instead of the full command line for the compilation step for the source/target pair. -- A refactor in the caching logic for version 4.4 left Java inner classes +- A refactor in the caching logic for version 4.4.0 left Java inner classes failing with an exception when a CacheDir was enabled. This is now corrected. -- cgit v0.12 From d439d26cad5d8829d756bf6e1b8cab58924f5044 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 17 Jan 2023 11:14:43 -0700 Subject: Tweak gfortran tool to respect tool setting If one of the Fortran compiler environment values *other than* FORTRAN is set when calling Environment, if the tool is gfortran that choice was not respected. Now uses the supplied values of, for example, F77, SHF77, F95 or SHF95. The setting of FORTRAN/SHFORTRAN was already respected. Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 +++ SCons/Tool/fortran.xml | 8 ++--- SCons/Tool/gfortran.py | 23 ++++++------ SCons/Tool/linkCommon/__init__.py | 13 ++++--- test/Fortran/F95FLAGS.py | 72 ++++++++++++++++++------------------- test/Fortran/SHF95FLAGS.py | 75 +++++++++++++++++++-------------------- test/Fortran/link-with-cxx.py | 57 +++++++++++++---------------- 7 files changed, 125 insertions(+), 127 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 279ef2e..c2bb504 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -106,6 +106,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER not be cached because the emitted filename contained a '$' and when looked up through a node ended up generating a Python SyntaxError because it was passed through scons_subst(). + - Have the gfortran tool do a better job of honoring user preferences + for the dialect tools (F95, SHF95, etc.). Previously set those + unconditionally to 'gfortran'. Cleaned a few Fortran tests - + behavior does not change. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/SCons/Tool/fortran.xml b/SCons/Tool/fortran.xml index 5bb1bd2..e91f659 100644 --- a/SCons/Tool/fortran.xml +++ b/SCons/Tool/fortran.xml @@ -111,9 +111,8 @@ contain (or similar) include or module search path options that scons generates automatically from &cv-link-FORTRANPATH;. See -&cv-link-_FORTRANINCFLAGS; and &cv-link-_FORTRANMODFLAG;, -below, -for the variables that expand those options. +&cv-link-_FORTRANINCFLAGS; and &cv-link-_FORTRANMODFLAG; +for the &consvars; that expand those options. @@ -123,8 +122,9 @@ for the variables that expand those options. General user-specified options that are passed to the Fortran compiler. Similar to &cv-link-FORTRANFLAGS;, -but this variable is applied to all dialects. +but is &consvar; is applied to all dialects. +New in version 4.4.
                    diff --git a/SCons/Tool/gfortran.py b/SCons/Tool/gfortran.py index 3c7e8b5..f9c0a45 100644 --- a/SCons/Tool/gfortran.py +++ b/SCons/Tool/gfortran.py @@ -29,24 +29,27 @@ It will usually be imported through the generic SCons.Tool.Tool() selection method. """ -import SCons.Util +from SCons.Util import CLVar from . import fortran def generate(env): - """Add Builders and construction variables for gfortran to an - Environment.""" + """Add Builders and construction variables for gfortran.""" fortran.generate(env) - for dialect in ['F77', 'F90', 'FORTRAN', 'F95', 'F03', 'F08']: - env[f'{dialect}'] = 'gfortran' - env[f'SH{dialect}'] = f'${dialect}' - if env['PLATFORM'] in ['cygwin', 'win32']: - env[f'SH{dialect}FLAGS'] = SCons.Util.CLVar(f'${dialect}FLAGS') - else: - env[f'SH{dialect}FLAGS'] = SCons.Util.CLVar(f'${dialect}FLAGS -fPIC') + # fill in other dialects (FORTRAN dialect set by fortran.generate(), + # but don't overwrite if they have been set manually. + for dialect in ['F77', 'F90', 'F95', 'F03', 'F08']: + if dialect not in env: + env[f'{dialect}'] = 'gfortran' + if f'SH{dialect}' not in env: + env[f'SH{dialect}'] = f'${dialect}' + # The fortran module always sets the shlib FLAGS, but does not + # include -fPIC, which is needed for the GNU tools. Rewrite if needed. + if env['PLATFORM'] not in ['cygwin', 'win32']: + env[f'SH{dialect}FLAGS'] = CLVar(f'${dialect}FLAGS -fPIC') env[f'INC{dialect}PREFIX'] = "-I" env[f'INC{dialect}SUFFIX'] = "" diff --git a/SCons/Tool/linkCommon/__init__.py b/SCons/Tool/linkCommon/__init__.py index 7aaffab..5461ad3 100644 --- a/SCons/Tool/linkCommon/__init__.py +++ b/SCons/Tool/linkCommon/__init__.py @@ -137,11 +137,14 @@ def smart_link(source, target, env, for_signature): if has_cplusplus and has_fortran and not has_d: global issued_mixed_link_warning if not issued_mixed_link_warning: - msg = "Using $CXX to link Fortran and C++ code together.\n\t" + \ - "This may generate a buggy executable if the '%s'\n\t" + \ - "compiler does not know how to deal with Fortran runtimes." - SCons.Warnings.warn(SCons.Warnings.FortranCxxMixWarning, - msg % env.subst('$CXX')) + msg = ( + "Using $CXX to link Fortran and C++ code together.\n" + " This may generate a buggy executable if the '%s'\n" + " compiler does not know how to deal with Fortran runtimes." + ) + SCons.Warnings.warn( + SCons.Warnings.FortranCxxMixWarning, msg % env.subst('$CXX') + ) issued_mixed_link_warning = True return '$CXX' elif has_d: diff --git a/test/Fortran/F95FLAGS.py b/test/Fortran/F95FLAGS.py index 2853cc9..e706264 100644 --- a/test/Fortran/F95FLAGS.py +++ b/test/Fortran/F95FLAGS.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -31,26 +30,30 @@ _python_ = TestSCons._python_ test = TestSCons.TestSCons() _exe = TestSCons._exe +# ref: test/fixture/mylink.py test.file_fixture('mylink.py') +# ref: test/Fortran/fixture/myfortran_flags.py test.file_fixture(['fixture', 'myfortran_flags.py']) test.write('SConstruct', """ -env = Environment(LINK = r'%(_python_)s mylink.py', - LINKFLAGS = [], - F95 = r'%(_python_)s myfortran_flags.py g95', - F95FLAGS = '-x', - FORTRAN = r'%(_python_)s myfortran_flags.py fortran', - FORTRANFLAGS = '-y') -env.Program(target = 'test01', source = 'test01.f') -env.Program(target = 'test02', source = 'test02.F') -env.Program(target = 'test03', source = 'test03.for') -env.Program(target = 'test04', source = 'test04.FOR') -env.Program(target = 'test05', source = 'test05.ftn') -env.Program(target = 'test06', source = 'test06.FTN') -env.Program(target = 'test07', source = 'test07.fpp') -env.Program(target = 'test08', source = 'test08.FPP') -env.Program(target = 'test13', source = 'test13.f95') -env.Program(target = 'test14', source = 'test14.F95') +env = Environment( + LINK=r'%(_python_)s mylink.py', + LINKFLAGS=[], + F95=r'%(_python_)s myfortran_flags.py g95', + F95FLAGS='-x', + FORTRAN=r'%(_python_)s myfortran_flags.py fortran', + FORTRANFLAGS='-y', +) +env.Program(target='test01', source='test01.f') +env.Program(target='test02', source='test02.F') +env.Program(target='test03', source='test03.for') +env.Program(target='test04', source='test04.FOR') +env.Program(target='test05', source='test05.ftn') +env.Program(target='test06', source='test06.FTN') +env.Program(target='test07', source='test07.fpp') +env.Program(target='test08', source='test08.FPP') +env.Program(target='test13', source='test13.f95') +env.Program(target='test14', source='test14.F95') """ % locals()) test.write('test01.f', "This is a .f file.\n#link\n#fortran\n") @@ -80,24 +83,22 @@ test.must_match('test14' + _exe, " -c -x\nThis is a .F95 file.\n") fc = 'f95' g95 = test.detect_tool(fc) - - if g95: test.subdir('x') - test.write(['x','dummy.i'], """ # Exists only such that -Ix finds the directory... """) + # ref: test/fixture/wrapper.py test.file_fixture('wrapper.py') test.write('SConstruct', """ -foo = Environment(F95 = '%(fc)s') +foo = Environment(F95='%(fc)s') f95 = foo.Dictionary('F95') -bar = foo.Clone(F95 = r'%(_python_)s wrapper.py ' + f95, F95FLAGS = '-Ix') -foo.Program(target = 'foo', source = 'foo.f95') -bar.Program(target = 'bar', source = 'bar.f95') +bar = foo.Clone(F95=r'%(_python_)s wrapper.py ' + f95, F95FLAGS='-Ix') +foo.Program(target='foo', source='foo.f95') +bar.Program(target='bar', source='bar.f95') """ % locals()) test.write('foo.f95', r""" @@ -114,21 +115,18 @@ bar.Program(target = 'bar', source = 'bar.f95') END """) - - test.run(arguments = 'foo' + _exe, stderr = None) - - test.run(program = test.workpath('foo'), stdout = " foo.f95\n") - + test.run(arguments='foo' + _exe, stderr=None) + test.run(program=test.workpath('foo'), stdout=" foo.f95\n") test.must_not_exist('wrapper.out') import sys - if sys.platform[:5] == 'sunos': - test.run(arguments = 'bar' + _exe, stderr = None) - else: - test.run(arguments = 'bar' + _exe) - test.run(program = test.workpath('bar'), stdout = " bar.f95\n") + if sys.platform.startswith('sunos'): + test.run(arguments='bar' + _exe, stderr=None) + else: + test.run(arguments='bar' + _exe) + test.run(program=test.workpath('bar'), stdout=" bar.f95\n") test.must_match('wrapper.out', "wrapper.py\n") test.pass_test() diff --git a/test/Fortran/SHF95FLAGS.py b/test/Fortran/SHF95FLAGS.py index 56744d6..7aaca56 100644 --- a/test/Fortran/SHF95FLAGS.py +++ b/test/Fortran/SHF95FLAGS.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -32,23 +31,25 @@ _obj = TestSCons._shobj obj_ = TestSCons.shobj_ test = TestSCons.TestSCons() +# ref: test/Fortran/fixture/myfortran_flags.py test.file_fixture(['fixture', 'myfortran_flags.py']) test.write('SConstruct', """ -env = Environment(SHF95 = r'%(_python_)s myfortran_flags.py g95', - SHFORTRAN = r'%(_python_)s myfortran_flags.py fortran') -env.Append(SHF95FLAGS = '-x', - SHFORTRANFLAGS = '-y') -env.SharedObject(target = 'test01', source = 'test01.f') -env.SharedObject(target = 'test02', source = 'test02.F') -env.SharedObject(target = 'test03', source = 'test03.for') -env.SharedObject(target = 'test04', source = 'test04.FOR') -env.SharedObject(target = 'test05', source = 'test05.ftn') -env.SharedObject(target = 'test06', source = 'test06.FTN') -env.SharedObject(target = 'test07', source = 'test07.fpp') -env.SharedObject(target = 'test08', source = 'test08.FPP') -env.SharedObject(target = 'test13', source = 'test13.f95') -env.SharedObject(target = 'test14', source = 'test14.F95') +env = Environment( + SHF95=r'%(_python_)s myfortran_flags.py g95', + SHFORTRAN=r'%(_python_)s myfortran_flags.py fortran', +) +env.Append(SHF95FLAGS='-x', SHFORTRANFLAGS='-y') +env.SharedObject(target='test01', source='test01.f') +env.SharedObject(target='test02', source='test02.F') +env.SharedObject(target='test03', source='test03.for') +env.SharedObject(target='test04', source='test04.FOR') +env.SharedObject(target='test05', source='test05.ftn') +env.SharedObject(target='test06', source='test06.FTN') +env.SharedObject(target='test07', source='test07.fpp') +env.SharedObject(target='test08', source='test08.FPP') +env.SharedObject(target='test13', source='test13.f95') +env.SharedObject(target='test14', source='test14.F95') """ % locals()) test.write('test01.f', "This is a .f file.\n#fortran\n") @@ -62,8 +63,7 @@ test.write('test08.FPP', "This is a .FPP file.\n#fortran\n") test.write('test13.f95', "This is a .f95 file.\n#g95\n") test.write('test14.F95', "This is a .F95 file.\n#g95\n") -test.run(arguments = '.', stderr = None) - +test.run(arguments='.', stderr=None) test.must_match(obj_ + 'test01' + _obj, " -c -y\nThis is a .f file.\n") test.must_match(obj_ + 'test02' + _obj, " -c -y\nThis is a .F file.\n") test.must_match(obj_ + 'test03' + _obj, " -c -y\nThis is a .for file.\n") @@ -75,29 +75,30 @@ test.must_match(obj_ + 'test08' + _obj, " -c -y\nThis is a .FPP file.\n") test.must_match(obj_ + 'test13' + _obj, " -c -x\nThis is a .f95 file.\n") test.must_match(obj_ + 'test14' + _obj, " -c -x\nThis is a .F95 file.\n") - - fc = 'f95' g95 = test.detect_tool(fc) - if g95: - test.subdir('x') - test.write(['x','dummy.i'], """ # Exists only such that -Ix finds the directory... """) + # ref: test/fixture/wrapper.py test.file_fixture('wrapper.py') - test.write('SConstruct', """ -foo = Environment(SHF95 = '%(fc)s') +foo = Environment(SHF95='%(fc)s') shf95 = foo.Dictionary('SHF95') -bar = foo.Clone(SHF95 = r'%(_python_)s wrapper.py ' + shf95) -bar.Append(SHF95FLAGS = '-Ix') -foo.SharedLibrary(target = 'foo/foo', source = 'foo.f95') -bar.SharedLibrary(target = 'bar/bar', source = 'bar.f95') +#print(f"foo SHF95={foo.Dictionary('SHF95')}", file=sys.stderr) +#print(f"foo F95FLAGS={foo.Dictionary('F95FLAGS')}", file=sys.stderr) +#print(f"foo SHF95FLAGS={foo.Dictionary('SHF95FLAGS')}", file=sys.stderr) +bar = foo.Clone(SHF95=r'%(_python_)s wrapper.py ' + shf95) +bar.Append(SHF95FLAGS='-Ix') +#print(f"bar SHF95={bar.Dictionary('SHF95')}", file=sys.stderr) +#print(f"bar F95FLAGS={bar.Dictionary('F95FLAGS')}", file=sys.stderr) +#print(f"bar SHF95FLAGS={bar.Dictionary('SHF95FLAGS')}", file=sys.stderr) +foo.SharedLibrary(target='foo/foo', source='foo.f95') +bar.SharedLibrary(target='bar/bar', source='bar.f95') """ % locals()) test.write('foo.f95', r""" @@ -114,17 +115,15 @@ bar.SharedLibrary(target = 'bar/bar', source = 'bar.f95') END """) - - test.run(arguments = 'foo', stderr = None) - + test.run(arguments='foo', stderr=None) test.must_not_exist('wrapper.out') import sys - if sys.platform[:5] == 'sunos': - test.run(arguments = 'bar', stderr = None) - else: - test.run(arguments = 'bar') + if sys.platform.startswith('sunos'): + test.run(arguments='bar', stderr=None) + else: + test.run(arguments='bar') test.must_match('wrapper.out', "wrapper.py\n") test.pass_test() diff --git a/test/Fortran/link-with-cxx.py b/test/Fortran/link-with-cxx.py index 22d9081..2f19e82 100644 --- a/test/Fortran/link-with-cxx.py +++ b/test/Fortran/link-with-cxx.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" """ Verify the smart_link() warning messages used when attempting to link @@ -41,6 +40,7 @@ test = TestSCons.TestSCons(match = TestSCons.match_re) test.write('test_linker.py', """\ import sys + if sys.argv[1] == '-o': with open(sys.argv[2], 'wb') as ofp: for infile in sys.argv[3:]: @@ -54,9 +54,9 @@ elif sys.argv[1][:5] == '/OUT:': sys.exit(0) """) - test.write('test_fortran.py', """\ import sys + with open(sys.argv[2], 'wb') as ofp: for infile in sys.argv[4:]: with open(infile, 'rb') as ifp: @@ -64,35 +64,35 @@ with open(sys.argv[2], 'wb') as ofp: sys.exit(0) """) - test.write('SConstruct', """ import SCons.Tool.link + def copier(target, source, env): s = str(source[0]) t = str(target[0]) with open(t, 'wb') as ofp, open(s, 'rb') as ifp: ofp.write(ifp.read()) -env = Environment(CXX = r'%(_python_)s test_linker.py', - CXXCOM = Action(copier), - SMARTLINK = SCons.Tool.link.smart_link, - LINK = r'$SMARTLINK', - LINKFLAGS = '', - # We want to re-define this as follows (so as to - # not rely on a real Fortran compiler) but can't - # because $FORTRANCOM is defined with an extra space - # so it ends up as a CommandAction, not a LazyAction. - # Must look into changing that after 1.0 is out. - #FORTRANCOM = Action(copier)) - FORTRAN = r'%(_python_)s test_fortran.py') +env = Environment( + CXX=r'%(_python_)s test_linker.py', + CXXCOM=Action(copier), + SMARTLINK=SCons.Tool.link.smart_link, + LINK=r'$SMARTLINK', + LINKFLAGS='', + # We want to re-define this as follows (so as to + # not rely on a real Fortran compiler) but can't + # because $FORTRANCOM is defined with an extra space + # so it ends up as a CommandAction, not a LazyAction. + # Must look into changing that after 1.0 is out. + # FORTRANCOM = Action(copier)) + FORTRAN=r'%(_python_)s test_fortran.py', +) env.Program('prog1.exe', ['f1.cpp', 'f2.f']) env.Program('prog2.exe', ['f1.cpp', 'f2.f']) if ARGUMENTS.get('NO_LINK'): - # Can remove no-deprecated when we drop Python1.5 - SetOption('warn', ['no-link', 'no-deprecated']) + SetOption('warn', ['no-link']) if ARGUMENTS.get('NO_MIX'): - # Can remove no-deprecated when we drop Python1.5 - SetOption('warn', ['no-fortran-cxx-mix', 'no-deprecated']) + SetOption('warn', ['no-fortran-cxx-mix']) """ % locals()) test.write('f1.cpp', "f1.cpp\n") @@ -100,52 +100,43 @@ test.write('f2.f', "f2.f\n") expect = (""" scons: warning: Using \\$CXX to link Fortran and C\\+\\+ code together. -\tThis may generate a buggy executable if the '%s test_linker.py' -\tcompiler does not know how to deal with Fortran runtimes. + This may generate a buggy executable if the '%s test_linker.py' + compiler does not know how to deal with Fortran runtimes. """ % re.escape(_python_)) + TestSCons.file_expr test.run(arguments = '.', stderr=expect) - test.must_match('prog1.exe', "f1.cpp\nf2.f\n") test.must_match('prog2.exe', "f1.cpp\nf2.f\n") test.run(arguments = '-c .', stderr=expect) - test.must_not_exist('prog1.exe') test.must_not_exist('prog2.exe') test.run(arguments = '--warning=no-link .') - test.must_match('prog1.exe', "f1.cpp\nf2.f\n") test.must_match('prog2.exe', "f1.cpp\nf2.f\n") test.run(arguments = '-c .', stderr=expect) - test.must_not_exist('prog1.exe') test.must_not_exist('prog2.exe') test.run(arguments = '--warning=no-fortran-cxx-mix .') - test.must_match('prog1.exe', "f1.cpp\nf2.f\n") test.must_match('prog2.exe', "f1.cpp\nf2.f\n") test.run(arguments = '-c .', stderr=expect) - test.must_not_exist('prog1.exe') test.must_not_exist('prog2.exe') test.run(arguments = 'NO_LINK=1 .') - test.must_match('prog1.exe', "f1.cpp\nf2.f\n") test.must_match('prog2.exe', "f1.cpp\nf2.f\n") test.run(arguments = '-c .', stderr=expect) - test.must_not_exist('prog1.exe') test.must_not_exist('prog2.exe') test.run(arguments = 'NO_MIX=1 .') - test.must_match('prog1.exe', "f1.cpp\nf2.f\n") test.must_match('prog2.exe', "f1.cpp\nf2.f\n") -- cgit v0.12 From eaaeab30cc6a559cf458c6ee60ebc25aa919c6fc Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 18 Jan 2023 08:19:27 -0700 Subject: Doc: update msvc, add version table [skip appveyor] The changes in 4.4 didn't marke new construction variables with the now preferred version add marker, so added these. The references to different versions gets so confusing that to try to help added a version correspondence table to the MSVC_VERSION construction varaiable doc. Signed-off-by: Mats Wichmann --- RELEASE.txt | 2 + SCons/Tool/msvc.xml | 305 ++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 241 insertions(+), 66 deletions(-) diff --git a/RELEASE.txt b/RELEASE.txt index 35fd845..7b08898 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -117,6 +117,8 @@ DOCUMENTATION - Updated the User Guide chapter on variant directories with more explanation, and the introduction of terms like "out of tree" that may help in forming a mental model. +- Updated MSVC documentation - adds "version added" annotations on recently + added construction variables and provides a version-mapping table. DEVELOPMENT ----------- diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index c26c20d..bf2e267 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -1,31 +1,10 @@ -Sets construction variables for the Microsoft Visual C/C++ compiler. +Sets &consvars; for the Microsoft Visual C/C++ compiler. @@ -96,7 +75,17 @@ Sets construction variables for the Microsoft Visual C/C++ compiler. PCH PCHSTOP PDB +MSVC_VERSION +MSVC_USE_SCRIPT +MSVC_USE_SCRIPT_ARGS +MSVC_USE_SETTINGS MSVC_NOTFOUND_POLICY +MSVC_SCRIPTERROR_POLICY +MSVC_SCRIPT_ARGS +MSVC_SDK_VERSION +MSVC_TOOLSET_VERSION +MSVC_SPECTRE_LIBS + @@ -110,7 +99,7 @@ file as the second element. Normally the object file is ignored. This builder is only provided when Microsoft Visual C++ is being used as the compiler. The &b-PCH; builder is generally used in -conjunction with the &cv-link-PCH; construction variable to force object files to use +conjunction with the &cv-link-PCH; &consvar; to force object files to use the precompiled header: @@ -148,7 +137,7 @@ Options added to the compiler command line to support building with precompiled headers. The default value expands expands to the appropriate Microsoft Visual C++ command-line options -when the &cv-link-PCH; construction variable is set. +when the &cv-link-PCH; &consvar; is set.                     @@ -161,7 +150,7 @@ to support storing debugging information in a Microsoft Visual C++ PDB file. The default value expands expands to appropriate Microsoft Visual C++ command-line options -when the &cv-link-PDB; construction variable is set. +when the &cv-link-PDB; &consvar; is set. @@ -208,11 +197,11 @@ compilation of object files when calling the Microsoft Visual C/C++ compiler. All compilations of source files from the same source directory that generate target files in a same output directory -and were configured in SCons using the same construction environment +and were configured in SCons using the same &consenv; will be built in a single call to the compiler. Only source files that have changed since their object files were built will be passed to each compiler invocation -(via the &cv-link-CHANGED_SOURCES; construction variable). +(via the &cv-link-CHANGED_SOURCES; &consvar;). Any compilations where the object (target) file base name (minus the .obj) does not match the source file base name @@ -261,9 +250,9 @@ If this is not set, then &cv-link-PCHCOM; (the command line) is displayed. -A construction variable that, when expanded, +A &consvar; that, when expanded, adds the flag to the command line -only if the &cv-link-PDB; construction variable is set. +only if the &cv-link-PDB; &consvar; is set. @@ -324,7 +313,7 @@ The flags passed to the resource compiler by the &b-link-RES; builder. -An automatically-generated construction variable +An automatically-generated &consvar; containing the command-line options for specifying directories to be searched by the resource compiler. @@ -343,7 +332,7 @@ of each directory in &cv-link-CPPPATH;. The prefix (flag) used to specify an include directory on the resource compiler command line. This will be prepended to the beginning of each directory -in the &cv-link-CPPPATH; construction variable +in the &cv-link-CPPPATH; &consvar; when the &cv-link-RCINCFLAGS; variable is expanded. @@ -355,7 +344,7 @@ when the &cv-link-RCINCFLAGS; variable is expanded. The suffix used to specify an include directory on the resource compiler command line. This will be appended to the end of each directory -in the &cv-link-CPPPATH; construction variable +in the &cv-link-CPPPATH; &consvar; when the &cv-link-RCINCFLAGS; variable is expanded.                     @@ -365,12 +354,10 @@ when the &cv-link-RCINCFLAGS; variable is expanded. Sets the preferred version of Microsoft Visual C/C++ to use. - - - +If the specified version is unavailable (not installed, +or not discoverable), tool initialization will fail. If &cv-MSVC_VERSION; is not set, SCons will (by default) select the -latest version of Visual C/C++ installed on your system. If the -specified version isn't installed, tool initialization will fail. +latest version of Visual C/C++ installed on your system. @@ -383,28 +370,186 @@ loaded into the environment. -Valid values for Windows are -14.3, -14.2, -14.1, -14.1Exp, -14.0, -14.0Exp, -12.0, -12.0Exp, -11.0, -11.0Exp, -10.0, -10.0Exp, -9.0, -9.0Exp, -8.0, -8.0Exp, -7.1, -7.0, -and 6.0. -Versions ending in Exp refer to "Express" or -"Express for Desktop" editions. +The valid values for &cv-MSVC_VERSION; represent major versions +of the compiler, except that versions ending in Exp +refer to "Express" or "Express for Desktop" Visual Studio editions, +which require distict entries because they use a different +filesystem layout and have some feature limitations compared to +the full version. +The following table shows correspondence +of the selector string to various version indicators +('x' is used as a placeholder for +a single digit that can vary). +Note that it is not necessary to install Visual Studio +to build with &SCons; (for example, you can install only +Build Tools), but if Visual Studio is installed, +additional builders such as &b-link-MSVSSolution; and +&b-link-MSVSProject; become avaialable and will +correspond to the indicated versions. + + + + + + + + + + + + SCons Key + MSVC++ Version + _MSVC_VER + VS Product + MSBuild/VS Version + + + + + 14.3 + 14.3x + 193x + Visual Studio 2022 + 17.x + + + 14.2 + 14.2x + 192x + Visual Studio 2019 + 16.x, 16.1x + + + 14.1 + 14.1 or 14.1x + 191x + Visual Studio 2017 + 15.x + + + 14.1Exp + 14.1 + 1910 + Visual Studio 2017 Express + 15.0 + + + 14.0 + 14.0 + 1900 + Visual Studio 2015 + 14.0 + + + 14.0Exp + 14.0 + 1900 + Visual Studio 2015 Express + 14.0 + + + 12.0 + 12.0 + 1800 + Visual Studio 2013 + 12.0 + + + 12.0Exp + 12.0 + 1800 + Visual Studio 2013 Express + 12.0 + + + 11.0 + 11.0 + 1700 + Visual Studio 2012 + 11.0 + + + 11.0Exp + 11.0 + 1700 + Visual Studio 2012 Express + 11.0 + + + 10.0 + 10.0 + 1600 + Visual Studio 2010 + 10.0 + + + 10.0Exp + 10.0 + 1600 + Visual C++ Express 2010 + 10.0 + + + 9.0 + 9.0 + 1500 + Visual Studio 2008 + 9.0 + + + 9.0Exp + 9.0 + 1500 + Visual C++ Express 2008 + 9.0 + + + 8.0 + 8.0 + 1400 + Visual Studio 2005 + 8.0 + + + 8.0Exp + 8.0 + 1400 + Visual C++ Express 2005 + 8.0 + + + 7.1 + 7.1 + 1300 + Visual Studio .NET 2003 + 7.1 + + + 7.0 + 7.0 + 1200 + Visual Studio .NET 2002 + 7.0 + + + 6.0 + 6.0 + 1100 + Visual Studio 6.0 + 6.0 + + + + + + +The compilation environment can be further or more precisely specified through the +use of several other &consvars;: see the descriptions of +&cv-link-MSVC_TOOLSET_VERSION;, +&cv-link-MSVC_SDK_VERSION;, +&cv-link-MSVC_USE_SCRIPT;, +&cv-link-MSVC_USE_SCRIPT_ARGS;, +and &cv-link-MSVC_USE_SETTINGS;. @@ -433,7 +578,7 @@ This can be useful to force the use of a compiler version that Setting &cv-MSVC_USE_SCRIPT; to None bypasses the Visual Studio autodetection entirely; -use this if you are running SCons in a Visual Studio cmd +use this if you are running &SCons; in a Visual Studio cmd window and importing the shell's environment variables - that is, if you are sure everything is set correctly already and you don't want &SCons; to change anything. @@ -441,6 +586,12 @@ you don't want &SCons; to change anything. &cv-MSVC_USE_SCRIPT; ignores &cv-link-MSVC_VERSION; and &cv-link-TARGET_ARCH;. + +Changed in version 4.4: +new &cv-link-MSVC_USE_SCRIPT_ARGS; provides a +way to pass arguments. + +                     @@ -449,6 +600,9 @@ you don't want &SCons; to change anything. Provides arguments passed to the script &cv-link-MSVC_USE_SCRIPT;. + +New in version 4.4 + @@ -529,11 +683,15 @@ therefore may change at any time. The burden is on the user to ensure the dictionary contents are minimally sufficient to ensure successful builds. - + + + +New in version 4.4 + @@ -780,6 +938,8 @@ When &cv-MSVC_NOTFOUND_POLICY; is not specified, the default &scons; behavior is subject to the conditions listed above. The default &scons; behavior may change in the future. +New in version 4.4 + @@ -831,6 +991,9 @@ Issue a warning when msvc batch file errors are detected. Suppress msvc batch file error messages. + +New in version 4.4 + @@ -905,6 +1068,8 @@ when setting the script error policy to raise an exception (e.g., 'Erro +New in version 4.4 + @@ -916,8 +1081,8 @@ Pass user-defined arguments to the Visual C++ batch file determined via autodete &cv-MSVC_SCRIPT_ARGS; is available for msvc batch file arguments that do not have first-class support -via construction variables or when there is an issue with the appropriate construction variable validation. -When available, it is recommended to use the appropriate construction variables (e.g., &cv-link-MSVC_TOOLSET_VERSION;) +via &consvars; or when there is an issue with the appropriate &consvar; validation. +When available, it is recommended to use the appropriate &consvars; (e.g., &cv-link-MSVC_TOOLSET_VERSION;) rather than &cv-MSVC_SCRIPT_ARGS; arguments. @@ -1041,6 +1206,8 @@ and compatible with the version of msvc selected. +New in version 4.4 + @@ -1159,6 +1326,8 @@ specify a Windows 10 SDK (e.g., '10.0.20348.0') for the build +New in version 4.4 + @@ -1329,6 +1498,8 @@ The burden is on the user to ensure the requisite toolset target architecture bu +New in version 4.4 + @@ -1409,6 +1580,8 @@ The burden is on the user to ensure the requisite libraries with spectre mitigat +New in version 4.4 + -- cgit v0.12 From 8b793169a1bab12e9912a54c4d53875e8347aca0 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 18 Jan 2023 08:39:21 -0700 Subject: Fix editing mistake in fortran doc [skip appveyor] Signed-off-by: Mats Wichmann --- SCons/Tool/fortran.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/fortran.xml b/SCons/Tool/fortran.xml index e91f659..4a092ec 100644 --- a/SCons/Tool/fortran.xml +++ b/SCons/Tool/fortran.xml @@ -122,7 +122,7 @@ for the &consvars; that expand those options. General user-specified options that are passed to the Fortran compiler. Similar to &cv-link-FORTRANFLAGS;, -but is &consvar; is applied to all dialects. +but this &consvar; is applied to all dialects. New in version 4.4. -- cgit v0.12 From 704ad77c0648a026def09c17e99dd365300ca2a7 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Wed, 18 Jan 2023 14:12:42 -0700 Subject: Adjust the appveyor build Skip lxml install, our Windows CI doesn't need it Skip installing coverage if not a coverage run Drop an uneeded test skip Signed-off-by: Mats Wichmann --- .appveyor.yml | 47 +++++++++++++++++++++++-------------------- .appveyor/disable_msvc_10.ps1 | 5 ----- .appveyor/exclude_tests.ps1 | 8 ++++++++ .appveyor/install-cov.bat | 2 ++ .appveyor/install.bat | 12 ++++++----- 5 files changed, 42 insertions(+), 32 deletions(-) delete mode 100644 .appveyor/disable_msvc_10.ps1 create mode 100644 .appveyor/exclude_tests.ps1 create mode 100644 .appveyor/install-cov.bat diff --git a/.appveyor.yml b/.appveyor.yml index 0302122..a8db5e9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -17,31 +17,33 @@ cache: - C:\ProgramData\chocolatey\lib -> appveyor.yml install: - # add python and python user-base to path for pip installs + # direct choco install supposed to work, but not? still doing in install.bat + #- cinst: dmd ldc swig vswhere ixsltproc winflexbison3 - cmd: .\.appveyor\install.bat + - cmd: if %COVERAGE% equ 1 .\.appveyor\install-cov.bat -# build matrix will be number of images multiplied by each '-' below, -# less any exclusions. -# split builds into sets of four jobs due to appveyor per-job time limit -# Leaving the Coverage build on VS2017 for build-time reasons (1hr time limit) -# Maybe move this one somewhere else in future to restore some flexibility. +# Build matrix will be number of images multiplied by #entries in matrix:, +# less any excludes. +# +# "Build" is kind of a misnomer - we are actually running the test suite, +# and this is slow on Windows, so keep the matrix as small as possible. +# Leaving the Coverage build on VS2017 for build-time reasons (1hr time limit). +# maybe move coverage to github in future to restore some flexibility? environment: + COVERAGE: 0 + SCONS_CACHE_MSVC_CONFIG: "true" matrix: - + # Test oldest and newest supported Pythons, and a subset in between. + # Skipping 3.7 and 3.9 at this time - WINPYTHON: "Python311" - COVERAGE: 0 - WINPYTHON: "Python310" - COVERAGE: 0 - WINPYTHON: "Python38" - COVERAGE: 0 - + - WINPYTHON: "Python36" COVERAGE: 1 - # skipping 3.7 and 3.9 at this time - # remove sets of build jobs based on criteria below # to fine tune the number and platforms tested matrix: @@ -68,21 +70,23 @@ matrix: - image: Visual Studio 2022 WINPYTHON: "Python38" -# remove some binaries we don't want to be found +# Remove some binaries we don't want to be found +# Note this is no longer needed, git-windows bin/ is quite minimal now. before_build: - ps: .\.appveyor\ignore_git_bins.ps1 build: off build_script: - - # exclude VS 10.0 because it hangs the testing until this is resolved: - # https://help.appveyor.com/discussions/problems/19283-visual-studio-2010-trial-license-has-expired - - ps: .\.appveyor\disable_msvc_10.ps1 + # Image version-based excludes: + # No excludes at the moment, but the exclude script generates the + # (possibly empty) exclude_list.txt which is used in the following step, + # so leave the scheme in place in case we need to put back excludes later. + - ps: .\.appveyor\exclude_tests.ps1 # setup coverage by creating the coverage config file, and adding coverage # to the sitecustomize so that all python processes start with coverage - - ps: .\.appveyor\coverage_setup.ps1 + - ps: if ($env:COVERAGE -eq 1) { .\.appveyor\coverage_setup.ps1 } # NOTE: running powershell from cmd is intended because # it formats the output correctly @@ -90,8 +94,7 @@ build_script: # run coverage even if there was a test failure on_finish: - - ps: .\.appveyor\coverage_report.ps1 - # running codecov in powershell causes an error so running in platform - # shells + - ps: if ($env:COVERAGE -eq 1) { .\.appveyor\coverage_report.ps1 } + # running codecov in powershell causes an error so running in cmd - cmd: if %COVERAGE% equ 1 codecov -X gcov --file coverage_xml.xml diff --git a/.appveyor/disable_msvc_10.ps1 b/.appveyor/disable_msvc_10.ps1 deleted file mode 100644 index 086f1e4..0000000 --- a/.appveyor/disable_msvc_10.ps1 +++ /dev/null @@ -1,5 +0,0 @@ -New-Item -Name exclude_list.txt -ItemType File; -$workaround_image = "Visual Studio 2015"; -if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq $workaround_image) { - Add-Content -Path 'exclude_list.txt' -Value 'test\MSVS\vs-10.0-exec.py'; -} diff --git a/.appveyor/exclude_tests.ps1 b/.appveyor/exclude_tests.ps1 new file mode 100644 index 0000000..6006a5c --- /dev/null +++ b/.appveyor/exclude_tests.ps1 @@ -0,0 +1,8 @@ +New-Item -Name exclude_list.txt -ItemType File; + +# exclude VS 10.0 because it hangs the testing until this is resolved: +# https://help.appveyor.com/discussions/problems/19283-visual-studio-2010-trial-license-has-expired +$workaround_image = "Visual Studio 2015"; +if ($env:APPVEYOR_BUILD_WORKER_IMAGE -eq $workaround_image) { + Add-Content -Path 'exclude_list.txt' -Value 'test\MSVS\vs-10.0-exec.py'; +} diff --git a/.appveyor/install-cov.bat b/.appveyor/install-cov.bat new file mode 100644 index 0000000..7dbc945 --- /dev/null +++ b/.appveyor/install-cov.bat @@ -0,0 +1,2 @@ +for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys.path[-1])"') do (set PYSITEDIR=%%g) +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off coverage codecov diff --git a/.appveyor/install.bat b/.appveyor/install.bat index b014dc7..561faac 100644 --- a/.appveyor/install.bat +++ b/.appveyor/install.bat @@ -1,12 +1,14 @@ C:\\%WINPYTHON%\\python.exe --version for /F "tokens=*" %%g in ('C:\\%WINPYTHON%\\python.exe -c "import sys; print(sys.path[-1])"') do (set PYSITEDIR=%%g) REM use mingw 32 bit until #3291 is resolved +REM add python and python user-base to path for pip installs set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH% C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off coverage codecov -set STATIC_DEPS=true & C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off lxml -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements-dev.txt -REM install 3rd party tools to test with +REM No real use for lxml on Windows (and some versions don't have it): +REM We don't install the docbook bits so those tests won't run anyway +REM The Windows builds don't attempt to make the docs +REM Adjust this as requirements-dev.txt changes. +REM C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements-dev.txt +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off ninja psutil choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc winflexbison3 -set SCONS_CACHE_MSVC_CONFIG=true set -- cgit v0.12 From f30704396e0a6d4dd7c49c85494de445df68655a Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 22 Jan 2023 07:44:28 -0700 Subject: gfortran - remove some debug prints [skip appveyor] One of the tests had some debug fluff left over - cleaned. Reworded the CHANGES blurb and added one to RELEASE. Signed-off-by: Mats Wichmann --- CHANGES.txt | 7 ++++--- RELEASE.txt | 6 ++++++ test/Fortran/SHF95FLAGS.py | 6 ------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c2bb504..aaa44a0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -107,9 +107,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER looked up through a node ended up generating a Python SyntaxError because it was passed through scons_subst(). - Have the gfortran tool do a better job of honoring user preferences - for the dialect tools (F95, SHF95, etc.). Previously set those - unconditionally to 'gfortran'. Cleaned a few Fortran tests - - behavior does not change. + for the dialect tools (F95, SHF95, etc.). Previously these were + unconditionally forced to 'gfortran'; the change should be more + in line with expectations of how these variables should work. + Also cleaned a few Fortran tests - test behavior does not change. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 35fd845..703bf7b 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -71,6 +71,12 @@ FIXES the compilation step for the source/target pair. - A refactor in the caching logic for version 4.4.0 left Java inner classes failing with an exception when a CacheDir was enabled. This is now corrected. +- When using the gfortran tool (the default on most platforms as long as a GNU + toolchain is installed), the user setting of the "dialect" compilers + (F77, F90, F03 and F09, as well as the shared-library complements SHF77, + SHF90, SHF03, SHF09) is now honored; previously the tool forced the + settings to 'gfortran', which made it difficult reference a cross-compile + version for dialects. IMPROVEMENTS diff --git a/test/Fortran/SHF95FLAGS.py b/test/Fortran/SHF95FLAGS.py index 7aaca56..dcec49b 100644 --- a/test/Fortran/SHF95FLAGS.py +++ b/test/Fortran/SHF95FLAGS.py @@ -89,14 +89,8 @@ if g95: test.write('SConstruct', """ foo = Environment(SHF95='%(fc)s') shf95 = foo.Dictionary('SHF95') -#print(f"foo SHF95={foo.Dictionary('SHF95')}", file=sys.stderr) -#print(f"foo F95FLAGS={foo.Dictionary('F95FLAGS')}", file=sys.stderr) -#print(f"foo SHF95FLAGS={foo.Dictionary('SHF95FLAGS')}", file=sys.stderr) bar = foo.Clone(SHF95=r'%(_python_)s wrapper.py ' + shf95) bar.Append(SHF95FLAGS='-Ix') -#print(f"bar SHF95={bar.Dictionary('SHF95')}", file=sys.stderr) -#print(f"bar F95FLAGS={bar.Dictionary('F95FLAGS')}", file=sys.stderr) -#print(f"bar SHF95FLAGS={bar.Dictionary('SHF95FLAGS')}", file=sys.stderr) foo.SharedLibrary(target='foo/foo', source='foo.f95') bar.SharedLibrary(target='bar/bar', source='bar.f95') """ % locals()) -- cgit v0.12 From 2b81cdb47534a9c4faf71d750e7b0baeef97fd67 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Sun, 22 Jan 2023 11:40:27 -0700 Subject: Add Python 3.12 support * Updated one testcase which now generates a warning, failing the test (which expects no stderr). * Updated ActionTests.py to know about 3.12, and uses the current bytecode sequences (these might change later in the 3.12 cycle) * Added 3.11 and 3.12 to setup.cfg so tools which query "what Pythons does SCons support" from pypi metadata won't be fooled into thinking 3.11 isn't supported (or 3.12, though that's preliminary). Signed-off-by: Mats Wichmann --- CHANGES.txt | 4 ++++ RELEASE.txt | 1 + SCons/ActionTests.py | 18 ++++++++++++++++-- setup.cfg | 2 ++ test/rebuild-generated.py | 2 +- 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 279ef2e..a110bd1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -106,6 +106,10 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER not be cached because the emitted filename contained a '$' and when looked up through a node ended up generating a Python SyntaxError because it was passed through scons_subst(). + - Add Python 3.12 support, and indicate 3.11/3.12 support in package. + 3.12 is in alpha for this SCons release, the bytecode sequences + embedded in SCons/ActionTests.py may need to change later, but + based on what is known now, 3.12 itself should work with this release. RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index 35fd845..6c4cd53 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -43,6 +43,7 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY octal modes using the modern Python syntax (0o755 rather than 0755). - Migrated logging logic for --taskmastertrace to use Python's logging module. Added logging to NewParallel Job class (Andrew Morrow's new parallel job implementation) +- Preliminary support for Python 3.12. FIXES diff --git a/SCons/ActionTests.py b/SCons/ActionTests.py index 101953b..88bb36f 100644 --- a/SCons/ActionTests.py +++ b/SCons/ActionTests.py @@ -1541,6 +1541,7 @@ class CommandGeneratorActionTestCase(unittest.TestCase): (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), + (3, 12): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } meth_matches = [ @@ -1719,6 +1720,7 @@ class FunctionActionTestCase(unittest.TestCase): (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), + (3, 12): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } @@ -1730,6 +1732,7 @@ class FunctionActionTestCase(unittest.TestCase): (3, 9): bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'1, 1, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 11): bytearray(b'1, 1, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), + (3, 12): bytearray(b'1, 1, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } def factory(act, **kw): @@ -1974,6 +1977,7 @@ class LazyActionTestCase(unittest.TestCase): (3, 9): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 10): bytearray(b'0, 0, 0, 0,(),(),(d\x00S\x00),(),()'), (3, 11): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), + (3, 12): bytearray(b'0, 0, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()'), } meth_matches = [ @@ -2039,6 +2043,7 @@ class ActionCallerTestCase(unittest.TestCase): (3, 9): b'd\x00S\x00', (3, 10): b'd\x00S\x00', (3, 11): b'\x97\x00d\x00S\x00', + (3, 12): b'\x97\x00d\x00S\x00', } af = SCons.Action.ActionFactory(GlobalFunc, strfunc) @@ -2250,6 +2255,7 @@ class ObjectContentsTestCase(unittest.TestCase): bytearray(b'3, 3, 0, 0,(),(),(|\x00S\x00),(),()'), ), # 3.10.1, 3.10.2 (3, 11): bytearray(b'3, 3, 0, 0,(),(),(\x97\x00|\x00S\x00),(),()'), + (3, 12): bytearray(b'3, 3, 0, 0,(),(),(\x97\x00|\x00S\x00),(),()'), } c = SCons.Action._function_contents(func1) @@ -2288,7 +2294,11 @@ class ObjectContentsTestCase(unittest.TestCase): b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(d\x01|\x00_\x00d\x02|\x00_\x01d\x00S\x00),(),(),2, 2, 0, 0,(),(),(d\x00S\x00),(),()}}{{{a=a,b=b}}}" ), (3, 11): bytearray( - b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(\x97\x00d\x01|\x00_\x00\x00\x00\x00\x00\x00\x00\x00\x00d\x02|\x00_\x01\x00\x00\x00\x00\x00\x00\x00\x00d\x00S\x00),(),(),2, 2, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()}}{{{a=a,b=b}}}"), + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(\x97\x00d\x01|\x00_\x00\x00\x00\x00\x00\x00\x00\x00\x00d\x02|\x00_\x01\x00\x00\x00\x00\x00\x00\x00\x00d\x00S\x00),(),(),2, 2, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), + (3, 12): bytearray( + b"{TestClass:__main__}[[[(, ()), [(, (,))]]]]{{1, 1, 0, 0,(a,b),(a,b),(\x97\x00d\x01|\x00_\x00\x00\x00\x00\x00\x00\x00\x00\x00d\x02|\x00_\x01\x00\x00\x00\x00\x00\x00\x00\x00d\x00S\x00),(),(),2, 2, 0, 0,(),(),(\x97\x00d\x00S\x00),(),()}}{{{a=a,b=b}}}" + ), } assert c == expected[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( @@ -2322,7 +2332,11 @@ class ObjectContentsTestCase(unittest.TestCase): b'0, 0, 0, 0,(Hello, World!),(print),(e\x00d\x00\x83\x01\x01\x00d\x01S\x00)' ), (3, 11): bytearray( - b'0, 0, 0, 0,(Hello, World!),(print),(\x97\x00\x02\x00e\x00d\x00\xa6\x01\x00\x00\xab\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00d\x01S\x00)'), + b'0, 0, 0, 0,(Hello, World!),(print),(\x97\x00\x02\x00e\x00d\x00\xa6\x01\x00\x00\xab\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00d\x01S\x00)' + ), + (3, 12): bytearray( + b'0, 0, 0, 0,(Hello, World!),(print),(\x97\x00\x02\x00e\x00d\x00\xab\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00d\x01S\x00)' + ), } assert c == expected[sys.version_info[:2]], f"Got\n{c!r}\nExpected\n" + repr( diff --git a/setup.cfg b/setup.cfg index 941db34..f177d6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,8 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Environment :: Console Intended Audience :: Developers License :: OSI Approved :: MIT License diff --git a/test/rebuild-generated.py b/test/rebuild-generated.py index 0b3659e..91b4e1e 100644 --- a/test/rebuild-generated.py +++ b/test/rebuild-generated.py @@ -83,7 +83,7 @@ env = Environment() kernelDefines = env.Command("header.hh", "header.hh.in", Copy('$TARGET', '$SOURCE')) kernelImporterSource = env.Command("generated.cc", ["%s"], "%s") kernelImporter = env.Program(kernelImporterSource + ["main.cc"]) -kernelImports = env.Command("KernelImport.hh", kernelImporter, ".%s$SOURCE > $TARGET") +kernelImports = env.Command("KernelImport.hh", kernelImporter, r".%s$SOURCE > $TARGET") osLinuxModule = env.StaticObject(["target.cc"]) """ % (generator_name, kernel_action, sep)) -- cgit v0.12 From a10124dc776002f2c59ccb79923c79a930781d72 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 09:28:20 -0800 Subject: Skip lxml if using py 3.11+ and you're on windows as there's currently not a lxml binary wheel for such. Remove when this changes --- requirements-dev.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6f9855f..1b12a75 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,9 @@ # for now keep pinning "known working" lxml, # it's been a troublesome component in the past. -lxml==4.9.1 +# Skip lxml for python 3.11+ on win32 as there's no binary wheel as of 01/22/2023 +lxml==4.9.1 ; python_version < '3.11' and sys_platform != 'win32' + ninja # Needed for test/Parallel/failed-build/failed-build.py -- cgit v0.12 From dda03dbc3c35fce1b3fe5e0e65fc59d0339817a8 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 09:30:39 -0800 Subject: Skip lxml if using py 3.11+ and you're on windows as there's currently not a lxml binary wheel for such. Remove when this changes --- .appveyor/install.bat | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.appveyor/install.bat b/.appveyor/install.bat index 561faac..95f5115 100644 --- a/.appveyor/install.bat +++ b/.appveyor/install.bat @@ -4,11 +4,10 @@ REM use mingw 32 bit until #3291 is resolved REM add python and python user-base to path for pip installs set PATH=C:\\%WINPYTHON%;C:\\%WINPYTHON%\\Scripts;C:\\ProgramData\\chocolatey\\bin;C:\\MinGW\\bin;C:\\MinGW\\msys\\1.0\\bin;C:\\cygwin\\bin;C:\\msys64\\usr\\bin;C:\\msys64\\mingw64\\bin;%PATH% C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off pip setuptools wheel -REM No real use for lxml on Windows (and some versions don't have it): -REM We don't install the docbook bits so those tests won't run anyway -REM The Windows builds don't attempt to make the docs -REM Adjust this as requirements-dev.txt changes. -REM C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements-dev.txt -C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off ninja psutil + +REM requirements-dev.txt will skip installing lxml for windows and py 3.11+, where there's +REM no current binary wheel +C:\\%WINPYTHON%\\python.exe -m pip install -U --progress-bar off -r requirements-dev.txt + choco install --allow-empty-checksums dmd ldc swig vswhere xsltproc winflexbison3 set -- cgit v0.12 From 1f2864183e491181224fc1531e3ac4cb024c33a3 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 10:34:57 -0800 Subject: set max version for sphinx to be 6.0, and bump lxml up 1 version and prohibit installing it on windows with py3.12 for now --- requirements-dev.txt | 3 ++- requirements-pkg.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1b12a75..1a0ef84 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,8 @@ # for now keep pinning "known working" lxml, # it's been a troublesome component in the past. # Skip lxml for python 3.11+ on win32 as there's no binary wheel as of 01/22/2023 -lxml==4.9.1 ; python_version < '3.11' and sys_platform != 'win32' +lxml==4.9.2 ; python_version < '3.12' and sys_platform == 'win32' +lxml==4.9.2 ; sys_platform != 'win32' ninja diff --git a/requirements-pkg.txt b/requirements-pkg.txt index 10b5393..ae71cde 100644 --- a/requirements-pkg.txt +++ b/requirements-pkg.txt @@ -8,6 +8,6 @@ readme-renderer # sphinx pinned because it has broken several times on new releases -sphinx>=5.1.1 +sphinx < 6.0 sphinx-book-theme rst2pdf -- cgit v0.12 From a844619dc360ea95baa72369db33727a22ba5058 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 10:49:22 -0800 Subject: revise based on mwichmann's review --- requirements-dev.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a0ef84..e168ccb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,8 @@ # for now keep pinning "known working" lxml, # it's been a troublesome component in the past. -# Skip lxml for python 3.11+ on win32 as there's no binary wheel as of 01/22/2023 -lxml==4.9.2 ; python_version < '3.12' and sys_platform == 'win32' -lxml==4.9.2 ; sys_platform != 'win32' +# Skip lxml for win32 as no tests which require it currently pass on win32 +lxml==4.9.2; python_version < 3.12 and sys_platform != 'win32' ninja -- cgit v0.12 From 3b3a8bd1a1927c6e1ffc7aa381eda4d8c2beae96 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 10:58:26 -0800 Subject: Quote python_version value --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index e168ccb..82faa28 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,7 +5,7 @@ # for now keep pinning "known working" lxml, # it's been a troublesome component in the past. # Skip lxml for win32 as no tests which require it currently pass on win32 -lxml==4.9.2; python_version < 3.12 and sys_platform != 'win32' +lxml==4.9.2; python_version < '3.12' and sys_platform != 'win32' ninja -- cgit v0.12 From f293dd97feafeb9faf9ac805323d173bc8173176 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 11:11:36 -0800 Subject: [ci skip] Add blurb to CHANGES.txt --- CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.txt b/CHANGES.txt index 279ef2e..90b0aee 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -106,6 +106,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER not be cached because the emitted filename contained a '$' and when looked up through a node ended up generating a Python SyntaxError because it was passed through scons_subst(). + - Updated MSVC documentation - adds "version added" annotations on recently + added construction variables and provides a version-mapping table. + RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 -- cgit v0.12 From d108866d7cea119ca73e87a0ea131c5b25612033 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 11:16:16 -0800 Subject: [ci skip] Updates to CHANGES/RELEASE --- CHANGES.txt | 5 +++-- RELEASE.txt | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index aaa44a0..c7944ba 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -107,8 +107,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER looked up through a node ended up generating a Python SyntaxError because it was passed through scons_subst(). - Have the gfortran tool do a better job of honoring user preferences - for the dialect tools (F95, SHF95, etc.). Previously these were - unconditionally forced to 'gfortran'; the change should be more + for the dialect tools (F77, F90, F03 and F09, as well as the shared-library + equivalents SHF77, SHF90, SHF03, SHF09). Previously these were + unconditionally overwritten to 'gfortran'; the change should be more in line with expectations of how these variables should work. Also cleaned a few Fortran tests - test behavior does not change. diff --git a/RELEASE.txt b/RELEASE.txt index 703bf7b..c78bcb2 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -73,8 +73,8 @@ FIXES failing with an exception when a CacheDir was enabled. This is now corrected. - When using the gfortran tool (the default on most platforms as long as a GNU toolchain is installed), the user setting of the "dialect" compilers - (F77, F90, F03 and F09, as well as the shared-library complements SHF77, - SHF90, SHF03, SHF09) is now honored; previously the tool forced the + (F77, F90, F03 and F09, as well as the shared-library equivalents SHF77, + SHF90, SHF03, SHF09) is now honored; previously the tool overwrote the settings to 'gfortran', which made it difficult reference a cross-compile version for dialects. -- cgit v0.12 From 186ccb05d5fd4765392546a15d3a6e71c4ced14b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 11:21:12 -0800 Subject: [ci skip] add blurb to CHANGES --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index ecc82a4..eb393d0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -98,6 +98,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER over 2100 lines. - Add a zipapp package of scons-local: can use SCons from a local file which does not need unpacking. + - Additional explanations for MSVSProject and MSVSSolution builders. -- cgit v0.12 From c96a660e4d38ec2d3c5fd25fa74a04bc47c62845 Mon Sep 17 00:00:00 2001 From: Lukas Schrangl Date: Sun, 4 Aug 2019 12:11:33 +0200 Subject: Run latex after bibtex/biber only when necessary Although comments in src/engine/SCons/Tool/tex.py indicated that latex should only be run after biber/bibtex if the .bbl file had changed, it was always run. --- CHANGES.txt | 3 +++ SCons/Tool/tex.py | 4 ++-- test/TEX/biber_biblatex2.py | 17 ++++++++++++++--- test/TEX/bibtex-latex-rerun.py | 16 +++++++++++++--- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cae5783..2308a2f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1066,6 +1066,9 @@ RELEASE 3.1.1 - Mon, 07 Aug 2019 20:09:12 -0500 - JSON encoding errors for CacheDir config - JSON decoding errors for CacheDir config + From Lukas Schrangl: + - Run LaTeX after biber/bibtex only if necessary + RELEASE 3.1.0 - Mon, 20 Jul 2019 16:59:23 -0700 diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index 6d0a5fb..b9dd29e 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -345,7 +345,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None result = BibTeXAction(bibfile, bibfile, env) if result != 0: check_file_error_message(env['BIBTEX'], 'blg') - must_rerun_latex = True + check_MD5(suffix_nodes[".bbl"], ".bbl") # Now decide if biber will need to be run. # When the backend for biblatex is biber (by choice or default) the @@ -369,7 +369,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None result = BiberAction(bibfile, bibfile, env) if result != 0: check_file_error_message(env['BIBER'], 'blg') - must_rerun_latex = True + check_MD5(suffix_nodes[".bbl"], ".bbl") # Now decide if latex will need to be run again due to index. if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex): diff --git a/test/TEX/biber_biblatex2.py b/test/TEX/biber_biblatex2.py index 92c7ea9..049f06e 100644 --- a/test/TEX/biber_biblatex2.py +++ b/test/TEX/biber_biblatex2.py @@ -72,7 +72,7 @@ sources_bib_content = r""" """ test.write(['ref.bib'],sources_bib_content % '2013' ) -test.write(['bibertest.tex'],r""" +sources_tex_content = r""" \documentclass{article} \usepackage{biblatex} @@ -80,13 +80,14 @@ test.write(['bibertest.tex'],r""" \begin{document} -Hello. This is boring. +Hello. This is %s boring. \cite{mybook} And even more boring. \printbibliography \end{document} -""") +""" +test.write(['bibertest.tex'], sources_tex_content % "") test.run() @@ -110,7 +111,17 @@ for f in files: pdf_output_1 = test.read('bibertest.pdf') +# Change tex, but don't change bib. In this case, pdf should still be rebuilt. +test.write(['bibertest.tex'], sources_tex_content % "very") +test.run() +pdf_output_1a = test.read('bibertest.pdf') +# If the PDF file is the same as it was previously, then it didn't +# pick up the change in the tex file, so fail. +test.fail_test(pdf_output_1 == pdf_output_1a) + +# Change bib. +test.write(['bibertest.tex'], sources_tex_content % "") test.write(['ref.bib'],sources_bib_content % '1982') test.run() diff --git a/test/TEX/bibtex-latex-rerun.py b/test/TEX/bibtex-latex-rerun.py index f0f8c34..7334f1e 100644 --- a/test/TEX/bibtex-latex-rerun.py +++ b/test/TEX/bibtex-latex-rerun.py @@ -47,14 +47,14 @@ env = Environment(tools=['pdftex', 'tex']) env.PDF( 'bibtest.tex' ) """) -test.write(['bibtest.tex'], r""" +sources_tex_content = r""" \documentclass{article} \begin{document} -Learn about cool math in \cite{koblitz:elliptic_curves}. +Learn about %s cool math in \cite{koblitz:elliptic_curves}. \bibliographystyle{alpha} \bibliography{sources} \end{document} -""") +""" sources_bib_content = r""" @book{koblitz:elliptic_curves, @@ -67,14 +67,24 @@ sources_bib_content = r""" +test.write(['bibtest.tex'], sources_tex_content % "") test.write('sources.bib', sources_bib_content % '1981') test.run() pdf_output_1 = test.read('bibtest.pdf') +# Change tex, but don't change bib. In this case, pdf should still be rebuilt. +test.write(['bibtest.tex'], sources_tex_content % "really") +test.run() +pdf_output_1a = test.read('bibtest.pdf') +# If the PDF file is the same as it was previously, then it didn't +# pick up the change in the tex file, so fail. +test.fail_test(pdf_output_1 == pdf_output_1a) +# Change bib. +test.write(['bibtest.tex'], sources_tex_content % "") test.write('sources.bib', sources_bib_content % '1982') test.run() -- cgit v0.12 From dc28694b2088f5b102692c89b049fae56947a39b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 20:30:24 -0800 Subject: Added blurb to RELEASE.txt. Renamed function check_content_hash() from check_MD5(). --- CHANGES.txt | 7 +++---- RELEASE.txt | 3 ++- SCons/Tool/tex.py | 25 ++++++++++++------------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2308a2f..2475bdd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -63,6 +63,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Ryan Saunders: - Fixed runtest.py failure on Windows caused by excessive escaping of the path to python.exe. + From Lukas Schrangl: + - Run LaTeX after biber/bibtex only if necessary + From Flaviu Tamas: - Added -fsanitize support to ParseFlags(). This will propagate to CCFLAGS and LINKFLAGS. @@ -1066,10 +1069,6 @@ RELEASE 3.1.1 - Mon, 07 Aug 2019 20:09:12 -0500 - JSON encoding errors for CacheDir config - JSON decoding errors for CacheDir config - From Lukas Schrangl: - - Run LaTeX after biber/bibtex only if necessary - - RELEASE 3.1.0 - Mon, 20 Jul 2019 16:59:23 -0700 From Joseph Brill: diff --git a/RELEASE.txt b/RELEASE.txt index 302e494..146f371 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -44,6 +44,7 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - Migrated logging logic for --taskmastertrace to use Python's logging module. Added logging to NewParallel Job class (Andrew Morrow's new parallel job implementation) - Preliminary support for Python 3.12. +- Run LaTeX after biber/bibtex only if necessary FIXES @@ -53,7 +54,7 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. -- The Java Scanner processing of JAVACLASSPATH for dependencies (behavior +- The Java Scanner processing of JAVACLASSPATH for dep qendencies (behavior that was introduced in SCons 4.4.0) is adjusted to split on the system's search path separator instead of on a space. The previous behavior meant that a path containing spaces (e.g. r"C:\somepath\My Classes") would diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index b9dd29e..0a688f5 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -253,10 +253,10 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None # .aux files already processed by BibTex already_bibtexed = [] - # - # routine to update MD5 hash and compare - # - def check_MD5(filenode, suffix): + def check_content_hash(filenode, suffix): + """ + Routine to update content hash and compare + """ global must_rerun_latex # two calls to clear old csig filenode.clear_memoized_values() @@ -295,7 +295,6 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None with open(logfilename, "rb") as f: logContent = f.read().decode(errors='replace') - # Read the fls file to find all .aux files flsfilename = targetbase + '.fls' flsContent = '' @@ -345,7 +344,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None result = BibTeXAction(bibfile, bibfile, env) if result != 0: check_file_error_message(env['BIBTEX'], 'blg') - check_MD5(suffix_nodes[".bbl"], ".bbl") + check_content_hash(suffix_nodes[".bbl"], ".bbl") # Now decide if biber will need to be run. # When the backend for biblatex is biber (by choice or default) the @@ -369,10 +368,10 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None result = BiberAction(bibfile, bibfile, env) if result != 0: check_file_error_message(env['BIBER'], 'blg') - check_MD5(suffix_nodes[".bbl"], ".bbl") + check_content_hash(suffix_nodes[".bbl"], ".bbl") # Now decide if latex will need to be run again due to index. - if check_MD5(suffix_nodes['.idx'],'.idx') or (count == 1 and run_makeindex): + if check_content_hash(suffix_nodes['.idx'], '.idx') or (count == 1 and run_makeindex): # We must run makeindex if Verbose: print("Need to run makeindex") @@ -387,10 +386,10 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None # Harder is case is where an action needs to be called -- that should be rare (I hope?) for index in check_suffixes: - check_MD5(suffix_nodes[index],index) + check_content_hash(suffix_nodes[index], index) # Now decide if latex will need to be run again due to nomenclature. - if check_MD5(suffix_nodes['.nlo'],'.nlo') or (count == 1 and run_nomenclature): + if check_content_hash(suffix_nodes['.nlo'], '.nlo') or (count == 1 and run_nomenclature): # We must run makeindex if Verbose: print("Need to run makeindex for nomenclature") @@ -402,7 +401,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None #return result # Now decide if latex will need to be run again due to glossary. - if check_MD5(suffix_nodes['.glo'],'.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary): + if check_content_hash(suffix_nodes['.glo'], '.glo') or (count == 1 and run_glossaries) or (count == 1 and run_glossary): # We must run makeindex if Verbose: print("Need to run makeindex for glossary") @@ -414,7 +413,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None #return result # Now decide if latex will need to be run again due to acronyms. - if check_MD5(suffix_nodes['.acn'],'.acn') or (count == 1 and run_acronyms): + if check_content_hash(suffix_nodes['.acn'], '.acn') or (count == 1 and run_acronyms): # We must run makeindex if Verbose: print("Need to run makeindex for acronyms") @@ -427,7 +426,7 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None # Now decide if latex will need to be run again due to newglossary command. for ng in newglossary_suffix: - if check_MD5(suffix_nodes[ng[2]], ng[2]) or (count == 1): + if check_content_hash(suffix_nodes[ng[2]], ng[2]) or (count == 1): # We must run makeindex if Verbose: print("Need to run makeindex for newglossary") -- cgit v0.12 From ac3fd37000f26fc40a89a2b33a43c9a74cf1ccf1 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 23 Jan 2023 20:32:13 -0800 Subject: Fix typo --- RELEASE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.txt b/RELEASE.txt index 146f371..f74a2b0 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -54,7 +54,7 @@ FIXES - A list argument as the source to the Copy() action function is now handled. Both the implementation and the strfunction which prints the progress message were adjusted. -- The Java Scanner processing of JAVACLASSPATH for dep qendencies (behavior +- The Java Scanner processing of JAVACLASSPATH for dependencies (behavior that was introduced in SCons 4.4.0) is adjusted to split on the system's search path separator instead of on a space. The previous behavior meant that a path containing spaces (e.g. r"C:\somepath\My Classes") would -- cgit v0.12 From 378004ff41c6ed8937a35257b3fc67027b1bd3fd Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Fri, 20 Jan 2023 16:48:17 -0700 Subject: Add "append" kwarg to two configure checks Add append=True/False to CheckLib, CheckLibWithHeader in SConf. The "implementation", Conftest.CheckLib, already accepted this kwarg, but it could not be passed from an SConscript using the offical API. Updated manpage to describe and expanded a unit test to check. Fixes #2767 Additionally, clarified some things in manpage, including a recent user confusion about how to call CheckFunc. Signed-off-by: Mats Wichmann --- CHANGES.txt | 2 + RELEASE.txt | 4 ++ SCons/SConf.py | 14 ++-- SCons/SConfTests.py | 17 ++++- doc/man/scons.xml | 179 +++++++++++++++++++++++++++++----------------------- 5 files changed, 128 insertions(+), 88 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2475bdd..93be68d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -122,6 +122,8 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER 3.12 is in alpha for this SCons release, the bytecode sequences embedded in SCons/ActionTests.py may need to change later, but based on what is known now, 3.12 itself should work with this release. + - Add "append" keyword argument to Configure context's CheckLib and + CheckLibWithHeader to control whether to append or prepend (issue #2767) RELEASE 4.4.0 - Sat, 30 Jul 2022 14:08:29 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index f74a2b0..01bf63b 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -45,6 +45,10 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY to NewParallel Job class (Andrew Morrow's new parallel job implementation) - Preliminary support for Python 3.12. - Run LaTeX after biber/bibtex only if necessary +- Configure context methods CheckLib and CheckLibWithHeader now expose + an additional keyword argument 'append' which controls whether to append + (the default) or prepend discovered libraries to $LIBS. The functionality + was always present but prepending could not be requested via the offical API. FIXES diff --git a/SCons/SConf.py b/SCons/SConf.py index fe14a0c..051c14c 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -1067,7 +1067,7 @@ def CheckCXXHeader(context, header, include_quotes = '""'): def CheckLib(context, library = None, symbol = "main", - header = None, language = None, autoadd = 1): + header = None, language = None, autoadd=True, append=True,) -> bool: """ A test for a library. See also CheckLibWithHeader. Note that library may also be None to test whether the given symbol @@ -1082,15 +1082,16 @@ def CheckLib(context, library = None, symbol = "main", # ToDo: accept path for the library res = SCons.Conftest.CheckLib(context, library, symbol, header = header, - language = language, autoadd = autoadd) - context.did_show_result = 1 + language = language, autoadd = autoadd, + append=append) + context.did_show_result = True return not res # XXX # Bram: Can only include one header and can't use #ifdef HAVE_HEADER_H. def CheckLibWithHeader(context, libs, header, language, - call = None, autoadd = 1): + call = None, autoadd=True, append=True) -> bool: # ToDo: accept path for library. Support system header files. """ Another (more sophisticated) test for a library. @@ -1099,8 +1100,7 @@ def CheckLibWithHeader(context, libs, header, language, As in CheckLib, we support library=None, to test if the call compiles without extra link flags. """ - prog_prefix, dummy = \ - createIncludesFromHeaders(header, 0) + prog_prefix, dummy = createIncludesFromHeaders(header, 0) if not libs: libs = [None] @@ -1108,7 +1108,7 @@ def CheckLibWithHeader(context, libs, header, language, libs = [libs] res = SCons.Conftest.CheckLib(context, libs, None, prog_prefix, - call = call, language = language, autoadd = autoadd) + call = call, language = language, autoadd=autoadd, append=append) context.did_show_result = 1 return not res diff --git a/SCons/SConfTests.py b/SCons/SConfTests.py index a06b227..172fba7 100644 --- a/SCons/SConfTests.py +++ b/SCons/SConfTests.py @@ -572,14 +572,27 @@ int main(void) { env = sconf.env.Clone() try: - r = sconf.CheckLibWithHeader( existing_lib, "math.h", "C", autoadd=1 ) + r = sconf.CheckLibWithHeader( + existing_lib, "math.h", "C", autoadd=True, append=True + ) assert r, "did not find math.h with %s" % existing_lib expect = libs(env) + [existing_lib] got = libs(sconf.env) assert got == expect, "LIBS: expected %s, got %s" % (expect, got) sconf.env = env.Clone() - r = sconf.CheckLibWithHeader( existing_lib, "math.h", "C", autoadd=0 ) + r = sconf.CheckLibWithHeader( + existing_lib, "math.h", "C", autoadd=True, append=False + ) + assert r, "did not find math.h with %s" % existing_lib + expect = [existing_lib] + libs(env) + got = libs(sconf.env) + assert got == expect, "LIBS: expected %s, got %s" % (expect, got) + + sconf.env = env.Clone() + r = sconf.CheckLibWithHeader( + existing_lib, "math.h", "C", autoadd=False + ) assert r, "did not find math.h with %s" % existing_lib expect = libs(env) got = libs(sconf.env) diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 824cd41..15270e2 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -174,14 +174,14 @@ looks for a file named in the current directory and reads the build configuration from that file (other names are allowed, -see +see and the option for more information). The build may be structured in a hierarchical manner: the &SConstruct; file may specify subsidiary configuration files by calling the -&f-link-SConscript; function, +&f-link-SConscript; function, and these may, in turn, do the same. By convention, these subsidiary files are named @@ -3665,7 +3665,7 @@ does not maintain an explicit cache of the tested values (this is different than &Autoconf;), but uses its normal dependency tracking to keep the checked values up to date. However, users may override this behaviour with the - + command line option. @@ -3795,13 +3795,16 @@ env = conf.Finish() A &configure_context; has the following predefined methods which can be used to perform checks. Where -language is a required or -optional parameter, the choice can currently -be C or C++. The spellings accepted for +language is an optional parameter, +it specifies the compiler to use for the check, +currently a choice of C or C++. +The spellings accepted for C are C or c; for C++ the value can be CXX, cxx, C++ or c++. +If language is omitted, +C is assumed. @@ -3810,7 +3813,7 @@ or c++. Checks if header -is usable in the specified language. +is usable in the specified language. header may be a list, in which case the last item in the list @@ -3826,14 +3829,8 @@ must be a two character string, where the first character denotes the opening quote and the second character denotes the closing quote. By default, both characters are " (double quote). -The optional argument -language -should be either -C -or -C++ -and selects the compiler to be used for the check. -Returns a boolean indicating success or failure. + +Returns a boolean indicating success or failure. @@ -3894,25 +3891,23 @@ Returns a boolean indicating success or failure. context.CheckFunc(function_name, [header, language]) -Checks if the specified -C or C++ library function is available based on the -context's local environment settings (that is, using -the values of &cv-link-CFLAGS;, &cv-link-CPPFLAGS;, &cv-link-LIBS; -or other relevant &consvars;). +Checks if function_name is usable +in the context's local environment using the compiler +specified by language - that is, +can a check referencing it be compiled using the current values +of &cv-link-CFLAGS;, &cv-link-CPPFLAGS;, +&cv-link-LIBS; or other relevant &consvars;. -function_name -is the name of the function to check for. The optional header -argument is a string -that will be -placed at the top -of the test file -that will be compiled -to check if the function exists; -the default is: +argument is a string representing a code fragment +to place at the top of the test program +that will be compiled to check if the function exists. +If omitted, the default stanza will be +(with function_name appropriately substituted): + #ifdef __cplusplus @@ -3922,61 +3917,79 @@ char function_name(); -Returns an empty string on success, a string containing -an error message on failure. +Note: do not use header +to include the standard header file that declares +function_name - successful +compilation of the test program depends on using +a dummy prototype for it, +to avoid probems with compilers which object to +function signature mismatches. + +Returns a boolean indicating success or failure. - context.CheckLib([library, symbol, header, language, autoadd=True]) + context.CheckLib([library, symbol, header, language, autoadd=True, append=True]) Checks if library provides -symbol. -If -autoadd -is true (the default) and the library provides the specified -symbol, -appends the library to the LIBS &consvar; -library -may also be None (the default), -in which case -symbol -is checked with the current LIBS variable, -or a list of library names, -in which case each library in the list -will be checked for -symbol. -If -symbol -is not set or is -None, -then -CheckLib +symbol by compiling a simple stub program +with the compiler selected by language, +and optionally adds that library to the context. +If supplied, the text of header is included at the +top of the stub. +If autoadd is true (the default), +and the library provides the specified +symbol (as defined by successfully +linking the stub program), +it is added to the &cv-link-LIBS; &consvar; in the context. +if append is true (the default), +the library is appended, otherwise it is prepended. + + +library can be a list of library names, +or None (the default if the argument is omitted). +If the former, symbol is checked against +each library name in order, returning on the first +successful test; if the latter, +it is checked with the current value of &cv-LIBS; +(in this case no library name would be added). +If symbol +is omitted or None, +then CheckLib just checks if you can link against the specified -library. +library, Note though it is legal syntax, it would not be very useful to call this method with library and symbol both -omitted or None. -Returns a boolean indicating success or failure. +omitted or None - +at least one should be supplied. + +Returns a boolean indicating success or failure. + +Changed in version 4.5.0: added the +append parameter. + - context.CheckLibWithHeader(library, header, language, [call, autoadd=True]) + context.CheckLibWithHeader(library, header, [language, call, autoadd=True, append=True]) -Provides a more sophisticated way to check against libraries then the -CheckLib call. +Provides an alternative to the +CheckLib method +for checking for libraries usable in a build. library -specifies the library or a list of libraries to check. +specifies a library or list of libraries to check. header -specifies a header to check for. +specifies a header to include in the test program, +and language indicates the compiler to use. header may be a list, in which case the last item in the list @@ -3986,18 +3999,25 @@ header files whose #include lines should precede the header line being checked for. -call -can be any valid expression (with a trailing ';'). -If -call -is not set, -the default simply checks that you -can link against the specified +A code fragment +(must be a a valid expression, including a trailing semicolon) +to serve as the test can be supplied in +call; +if not supplied, +the default checks the ability to link against the specified library. -autoadd (default true) -specifies whether to add the library to the environment if the check -succeeds. -Returns a boolean indicating success or failure. +If autoadd is true (the default), +the first library that passes the check +is added to the &cv-link-LIBS; &consvar; in the context +and the method returns. +If append is true (the default), +the library is appended, otherwise prepended. + +Returns a boolean indicating success or failure. + +Changed in version 4.5.0: added the +append parameter. + @@ -4019,11 +4039,7 @@ Example: sconf.CheckType('foo_type', '#include "my_types.h"', 'C++')                     - -Returns an empty string on success, a string containing -an error message on failure. - - +Returns a boolean indicating success or failure. @@ -4050,7 +4066,7 @@ is supplied, it should be an integer size; type_name is actually that size. Returns the size in bytes, or zero if the type was not found -(or if the size did not match expect). +(or if the size did not match optional expect). For example, @@ -4081,6 +4097,7 @@ for C source files, so by setting relevant &consvars; it can be used to detect if particular compiler flags will be accepted or rejected by the compiler. +Returns a boolean indicating success or failure. @@ -4101,6 +4118,7 @@ for C++ source files, so by setting relevant &consvars; it can be used to detect if particular compiler flags will be accepted or rejected by the compiler. +Returns a boolean indicating success or failure. @@ -4123,6 +4141,7 @@ be accepted or rejected by the compiler. Note this does not check whether a shared library/dll can be created. +Returns a boolean indicating success or failure. @@ -4145,6 +4164,7 @@ be accepted or rejected by the compiler. Note this does not check whether a shared library/dll can be created. +Returns a boolean indicating success or failure. @@ -4171,7 +4191,8 @@ is a string containing one or more #include lines that will be inserted into the program that will be run to test for the existence of the symbol. -Returns a boolean indicating success or failure. + +Returns a boolean indicating success or failure. -- cgit v0.12