diff options
| author | William Deegan <bill@baddogconsulting.com> | 2020-04-27 00:05:14 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-04-27 00:05:14 (GMT) |
| commit | 5e70c65f633dcecc035751c9f0c6f894088df8a0 (patch) | |
| tree | 40ab00e0e5d3606906f8503664c06b918ee023ad /src/engine/SCons | |
| parent | e2f22641ceeb8a61215e5f649d8e97d3d67dd4a8 (diff) | |
| parent | b235883c0be3ae674634d2ce8b24250732273432 (diff) | |
| download | SCons-5e70c65f633dcecc035751c9f0c6f894088df8a0.zip SCons-5e70c65f633dcecc035751c9f0c6f894088df8a0.tar.gz SCons-5e70c65f633dcecc035751c9f0c6f894088df8a0.tar.bz2 | |
Merge branch 'master' into c-conditional-scanner
Diffstat (limited to 'src/engine/SCons')
39 files changed, 1300 insertions, 337 deletions
diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py index 0b7282c..5bc85d5 100644 --- a/src/engine/SCons/Action.py +++ b/src/engine/SCons/Action.py @@ -105,6 +105,7 @@ import pickle import re import sys import subprocess +from subprocess import DEVNULL import itertools import inspect from collections import OrderedDict @@ -751,30 +752,20 @@ def get_default_ENV(env): return default_ENV -def _subproc(scons_env, cmd, error = 'ignore', **kw): - """Do common setup for a subprocess.Popen() call +def _subproc(scons_env, cmd, error='ignore', **kw): + """Wrapper for subprocess which pulls from construction env. - This function is still in draft mode. We're going to need something like - it in the long run as more and more places use subprocess, but I'm sure - it'll have to be tweaked to get the full desired functionality. - one special arg (so far?), 'error', to tell what to do with exceptions. + Use for calls to subprocess which need to interpolate values from + an SCons construction environment into the environment passed to + subprocess. Adds an an error-handling argument. Adds ability + to specify std{in,out,err} with "'devnull'" tag. """ - # allow std{in,out,err} to be "'devnull'". This is like - # subprocess.DEVNULL, which does not exist for Py2. Use the - # subprocess one if possible. - # Clean this up when Py2 support is dropped - try: - from subprocess import DEVNULL - except ImportError: - DEVNULL = None - + # TODO: just uses subprocess.DEVNULL now, we can drop the "devnull" + # string now - it is a holdover from Py2, which didn't have DEVNULL. for stream in 'stdin', 'stdout', 'stderr': io = kw.get(stream) if is_String(io) and io == 'devnull': - if DEVNULL: - kw[stream] = DEVNULL - else: - kw[stream] = open(os.devnull, "r+") + kw[stream] = DEVNULL # Figure out what shell environment to use ENV = kw.get('env', None) @@ -968,11 +959,33 @@ class CommandAction(_ActionAction): return env.subst_target_source(cmd, SUBST_SIG, target, source) def get_implicit_deps(self, target, source, env, executor=None): + """Return the implicit dependencies of this action's command line.""" icd = env.get('IMPLICIT_COMMAND_DEPENDENCIES', True) if is_String(icd) and icd[:1] == '$': icd = env.subst(icd) - if not icd or icd in ('0', 'None'): + + if not icd or str(icd).lower in ('0', 'none', 'false', 'no', 'off'): return [] + + try: + icd_int = int(icd) + except ValueError: + icd_int = None + + if (icd_int and icd_int > 1) or icd == 'all': + # An integer value greater than 1 specifies the number of entries + # to scan. "all" means to scan all. + return self._get_implicit_deps_heavyweight(target, source, env, executor, icd_int) + else: + # Everything else (usually 1 or True) means that we want + # lightweight dependency scanning. + return self._get_implicit_deps_lightweight(target, source, env, executor) + + def _get_implicit_deps_lightweight(self, target, source, env, executor): + """ + Lightweight dependency scanning involves only scanning the first entry + in an action string, even if it contains &&. + """ from SCons.Subst import SUBST_SIG if executor: cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, executor=executor) @@ -990,6 +1003,65 @@ class CommandAction(_ActionAction): res.append(env.fs.File(d)) return res + def _get_implicit_deps_heavyweight(self, target, source, env, executor, + icd_int): + """ + Heavyweight dependency scanning involves scanning more than just the + first entry in an action string. The exact behavior depends on the + value of icd_int. Only files are taken as implicit dependencies; + directories are ignored. + + If icd_int is an integer value, it specifies the number of entries to + scan for implicit dependencies. Action strings are also scanned after + a &&. So for example, if icd_int=2 and the action string is + "cd <some_dir> && $PYTHON $SCRIPT_PATH <another_path>", the implicit + dependencies would be the path to the python binary and the path to the + script. + + If icd_int is None, all entries are scanned for implicit dependencies. + """ + + # Avoid circular and duplicate dependencies by not providing source, + # target, or executor to subst_list. This causes references to + # $SOURCES, $TARGETS, and all related variables to disappear. + from SCons.Subst import SUBST_SIG + cmd_list = env.subst_list(self.cmd_list, SUBST_SIG, conv=lambda x: x) + res = [] + + for cmd_line in cmd_list: + if cmd_line: + entry_count = 0 + for entry in cmd_line: + d = str(entry) + if ((icd_int is None or entry_count < icd_int) and + not d.startswith(('&', '-', '/') if os.name == 'nt' + else ('&', '-'))): + m = strip_quotes.match(d) + if m: + d = m.group(1) + + if d: + # Resolve the first entry in the command string using + # PATH, which env.WhereIs() looks in. + # For now, only match files, not directories. + p = os.path.abspath(d) if os.path.isfile(d) else None + if not p and entry_count == 0: + p = env.WhereIs(d) + + if p: + res.append(env.fs.File(p)) + + entry_count = entry_count + 1 + else: + entry_count = 0 if d == '&&' else entry_count + 1 + + # Despite not providing source and target to env.subst() above, we + # can still end up with sources in this list. For example, files in + # LIBS will still resolve in env.subst(). This won't result in + # circular dependencies, but it causes problems with cache signatures + # changing between full and incremental builds. + return [r for r in res if r not in target and r not in source] + class CommandGeneratorAction(ActionBase): """Class for command-generator actions.""" diff --git a/src/engine/SCons/Action.xml b/src/engine/SCons/Action.xml index 7a8194e..b519cdc 100644 --- a/src/engine/SCons/Action.xml +++ b/src/engine/SCons/Action.xml @@ -35,31 +35,88 @@ executed to build targets. By default, SCons will add to each target an implicit dependency on the command -represented by the first argument on any -command line it executes. +represented by the first argument of any +command line it executes (which is typically +the command itself). By setting such +a dependency, &SCons; can determine that +a target should be rebuilt if the command changes, +such as when a compiler is upgraded to a new version. The specific file for the dependency is found by searching the <varname>PATH</varname> variable in the -<varname>ENV</varname> -environment used to execute the command. +<varname>ENV</varname> dictionary +in the &consenv; used to execute the command. +The default is the same as +setting the &consvar; +&cv-IMPLICIT_COMMAND_DEPENDENCIES; +to a True-like value (<quote>true</quote>, +<quote>yes</quote>, +or <quote>1</quote> - but not a number +greater than one, as that has a different meaning). </para> <para> -If the construction variable -&cv-IMPLICIT_COMMAND_DEPENDENCIES; -is set to a false value -(<literal>None</literal>, -<literal>False</literal>, -<literal>0</literal>, +Action strings can be segmented by the +use of an AND operator, <literal>&&</literal>. +In a segemented string, each segment is a separate +<quote>command line</quote>, these are run +sequentially until one fails or the entire +sequence has been executed. If an +action string is segmented, then the selected +behavior of &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is applied to each segment. +</para> + +<para> +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is set to a False-like value +(<quote>none</quote>, +<quote>false</quote>, +<quote>no</quote>, +<quote>0</quote>, etc.), then the implicit dependency will not be added to the targets -built with that construction environment. +built with that &consenv;. +</para> + +<para> +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is set to <quote>2</quote> or higher, +then that number of arguments in the command line +will be scanned for relative or absolute paths. +If any are present, they will be added as +implicit dependencies to the targets built +with that &consenv;. +The first argument in the command line will be +searched for using the <varname>PATH</varname> +variable in the <varname>ENV</varname> dictionary +in the &consenv; used to execute the command. +The other arguments will only be found if they +are absolute paths or valid paths relative +to the working directory. +</para> + +<para> +If &cv-IMPLICIT_COMMAND_DEPENDENCIES; +is set to <quote>all</quote>, +then all arguments in the command line will be +scanned for relative or absolute paths. +If any are present, they will be added as +implicit dependencies to the targets built +with that &consenv;. +The first argument in the command line will be +searched for using the <varname>PATH</varname> +variable in the <varname>ENV</varname> dictionary +in the &consenv; used to execute the command. +The other arguments will only be found if they +are absolute paths or valid paths relative +to the working directory. </para> <example_commands> -env = Environment(IMPLICIT_COMMAND_DEPENDENCIES = 0) +env = Environment(IMPLICIT_COMMAND_DEPENDENCIES=False) </example_commands> </summary> </cvar> diff --git a/src/engine/SCons/CacheDir.py b/src/engine/SCons/CacheDir.py index 9bb7ef3..117f6b0 100644 --- a/src/engine/SCons/CacheDir.py +++ b/src/engine/SCons/CacheDir.py @@ -135,9 +135,6 @@ def CachePushFunc(target, source, env): CachePush = SCons.Action.Action(CachePushFunc, None) -# Nasty hack to cut down to one warning for each cachedir path that needs -# upgrading. -warned = dict() class CacheDir(object): @@ -159,12 +156,12 @@ class CacheDir(object): if path is None: return - self._readconfig3(path) + self._readconfig(path) - def _readconfig3(self, path): + def _readconfig(self, path): """ - Python3 version of reading the cache config. + Read the cache config. If directory or config file do not exist, create. Take advantage of Py3 capability in os.makedirs() and in file open(): just try @@ -201,64 +198,6 @@ class CacheDir(object): raise SCons.Errors.SConsEnvironmentError(msg) - def _readconfig2(self, path): - """ - Python2 version of reading cache config. - - See if there is a config file in the cache directory. If there is, - use it. If there isn't, and the directory exists and isn't empty, - produce a warning. If the directory does not exist or is empty, - write a config file. - - :param path: path to the cache directory - """ - config_file = os.path.join(path, 'config') - if not os.path.exists(config_file): - # A note: There is a race hazard here if two processes start and - # attempt to create the cache directory at the same time. However, - # Python 2.x does not give you the option to do exclusive file - # creation (not even the option to error on opening an existing - # file for writing...). The ordering of events here is an attempt - # to alleviate this, on the basis that it's a pretty unlikely - # occurrence (would require two builds with a brand new cache - # directory) - if os.path.isdir(path) and any(f != "config" for f in os.listdir(path)): - self.config['prefix_len'] = 1 - # When building the project I was testing this on, the warning - # was output over 20 times. That seems excessive - global warned - if self.path not in warned: - msg = "Please upgrade your cache by running " +\ - "scons-configure-cache.py " + self.path - SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) - warned[self.path] = True - else: - if not os.path.isdir(path): - try: - os.makedirs(path) - except OSError: - # If someone else is trying to create the directory at - # the same time as me, bad things will happen - msg = "Failed to create cache directory " + path - raise SCons.Errors.SConsEnvironmentError(msg) - - self.config['prefix_len'] = 2 - if not os.path.exists(config_file): - try: - with open(config_file, 'w') as config: - json.dump(self.config, config) - except Exception: - msg = "Failed to write cache configuration for " + path - raise SCons.Errors.SConsEnvironmentError(msg) - else: - try: - with open(config_file) as config: - self.config = json.load(config) - except ValueError: - msg = "Failed to read cache configuration for " + path - raise SCons.Errors.SConsEnvironmentError(msg) - - def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: if cache_debug == '-': diff --git a/src/engine/SCons/CacheDirTests.py b/src/engine/SCons/CacheDirTests.py index ff22d01..a3114c1 100644 --- a/src/engine/SCons/CacheDirTests.py +++ b/src/engine/SCons/CacheDirTests.py @@ -168,7 +168,7 @@ class ExceptionTestCase(unittest.TestCase): os.remove(old_config) try: - self._CacheDir._readconfig3(self._CacheDir.path) + self._CacheDir._readconfig(self._CacheDir.path) assert False, "Should have raised exception and did not" except SCons.Errors.SConsEnvironmentError as e: assert str(e) == "Failed to write cache configuration for {}".format(self._CacheDir.path) diff --git a/src/engine/SCons/Defaults.xml b/src/engine/SCons/Defaults.xml index 22f46fe..06fad1c 100644 --- a/src/engine/SCons/Defaults.xml +++ b/src/engine/SCons/Defaults.xml @@ -268,12 +268,8 @@ into a list of Dir instances relative to the target being built. <para> The list of suffixes of files that will be scanned for imported D package files. -The default list is: +The default list is <literal>['.d']</literal>. </para> - -<example_commands> -['.d'] -</example_commands> </summary> </cvar> @@ -584,9 +580,12 @@ in order to execute many of the global functions in this list from source code management systems. The default environment is a singleton, so the keyword arguments affect it only on the first call, on subsequent -calls the already-constructed object is returned. +calls the already-constructed object is returned and +any arguments are ignored. The default environment can be modified in the same way as any &consenv;. +Modifying the &defenv; has no effect on the environment +constructed by a subsequent &f-Environment; call. </para> </summary> </scons_function> diff --git a/src/engine/SCons/Environment.xml b/src/engine/SCons/Environment.xml index e1d5e74..5dea66e 100644 --- a/src/engine/SCons/Environment.xml +++ b/src/engine/SCons/Environment.xml @@ -844,7 +844,7 @@ env3 = env.Clone(CCFLAGS = '-g') <para> Additionally, a list of tools and a toolpath may be specified, as in -the Environment constructor: +the &f-link-Environment; constructor: </para> <example_commands> @@ -1836,9 +1836,10 @@ Examples: </para> <example_commands> -Program('foo', Glob('*.c')) -Zip('/tmp/everything', Glob('.??*') + Glob('*')) -sources = Glob('*.cpp', exclude=['os_*_specific_*.cpp']) + Glob('os_%s_specific_*.cpp'%currentOS) +Program("foo", Glob("*.c")) +Zip("/tmp/everything", Glob(".??*") + Glob("*")) +sources = Glob("*.cpp", exclude=["os_*_specific_*.cpp"]) + \ + Glob( "os_%s_specific_*.cpp" % currentOS) </example_commands> </summary> </scons_function> @@ -3016,31 +3017,29 @@ source_nodes = env.subst('$EXPAND_TO_NODELIST', <scons_function name="Tool"> <arguments> -(string, [toolpath, **kw]) +(name, [toolpath, **kwargs]) </arguments> <summary> + <para> -The -&f-Tool; -form of the function -returns a callable object -that can be used to initialize -a construction environment using the -tools keyword of the Environment() method. -The object may be called with a construction -environment as an argument, -in which case the object will -add the necessary variables -to the construction environment -and the name of the tool will be added to the -&cv-link-TOOLS; -construction variable. +Runs the tool identified by +<parameter>name</parameter>, which is +searched for in standard locations and any +paths specified by the optional +<parameter>toolpath</parameter>, +to update a &consenv; with &consvars; +needed to use the mechanisms that tool describes. +Any additional keyword arguments +<parameter>kwargs</parameter> are passed +on to the tool module's <function>generate</function> function. </para> <para> -Additional keyword arguments are passed to the tool's -<function>generate</function>() -method. +When called as a &consenv; method, +the tool module is called to update the +&consenv; and the name of the tool is +appended to the &cv-link-TOOLS; +&consvar; in that environment. </para> <para> @@ -3048,33 +3047,36 @@ Examples: </para> <example_commands> -env = Environment(tools = [ Tool('msvc') ]) - -env = Environment() -t = Tool('msvc') -t(env) # adds 'msvc' to the TOOLS variable -u = Tool('opengl', toolpath = ['tools']) -u(env) # adds 'opengl' to the TOOLS variable +env.Tool('gcc') +env.Tool('opengl', toolpath=['build/tools']) </example_commands> <para> -The -&f-env-Tool; -form of the function -applies the callable object for the specified tool -<varname>string</varname> -to the environment through which the method was called. +When called as a global function, +returns a callable tool object; +the tool is not called at this time, +as it lacks the context of an environment to update. +This tool object can be passed to an +&f-link-Environment; or &f-link-Clone; call +as part of the <parameter>tools</parameter> keyword argument, +or it can be called directly, +passing a &consenv; to update as the argument. +Either approach will also update the +<varname>TOOLS</varname> &consvar;. </para> <para> -Additional keyword arguments are passed to the tool's -<function>generate</function>() -method. +Examples: </para> <example_commands> -env.Tool('gcc') -env.Tool('opengl', toolpath = ['build/tools']) +env = Environment(tools=[Tool('msvc')]) + +env = Environment() +t = Tool('msvc') +t(env) # adds 'msvc' to the TOOLS variable +u = Tool('opengl', toolpath = ['tools']) +u(env) # adds 'opengl' to the TOOLS variable </example_commands> </summary> </scons_function> diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index bb965db..b7f6abe 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2246,7 +2246,7 @@ class RootDir(Dir): this directory. """ - __slots__ = ('_lookupDict', ) + __slots__ = ('_lookupDict', 'abspath', 'path') def __init__(self, drive, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.RootDir') @@ -2288,6 +2288,12 @@ class RootDir(Dir): self._tpath = dirname self.dirname = dirname + # EntryProxy interferes with this class and turns drive paths on + # Windows such as "C:" into "C:\C:". Avoid this problem by setting + # commonly-accessed attributes directly. + self.abspath = self._abspath + self.path = self._path + self._morph() self.duplicate = 0 diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index ce948a0..62cd908 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -430,10 +430,11 @@ class QuestionTask(SCons.Taskmaster.AlwaysTask): class TreePrinter(object): - def __init__(self, derived=False, prune=False, status=False): + def __init__(self, derived=False, prune=False, status=False, sLineDraw=False): self.derived = derived self.prune = prune self.status = status + self.sLineDraw = sLineDraw def get_all_children(self, node): return node.all_children() def get_derived_children(self, node): @@ -445,7 +446,7 @@ class TreePrinter(object): else: func = self.get_all_children s = self.status and 2 or 0 - SCons.Util.print_tree(t, func, prune=self.prune, showtags=s) + SCons.Util.print_tree(t, func, prune=self.prune, showtags=s, lastChild=True, singleLineDraw=self.sLineDraw) def python_version_string(): diff --git a/src/engine/SCons/Script/SConsOptions.py b/src/engine/SCons/Script/SConsOptions.py index 8ec8ccf..0c82faf 100644 --- a/src/engine/SCons/Script/SConsOptions.py +++ b/src/engine/SCons/Script/SConsOptions.py @@ -834,7 +834,7 @@ def Parser(version): help="Trace Node evaluation to FILE.", metavar="FILE") - tree_options = ["all", "derived", "prune", "status"] + tree_options = ["all", "derived", "prune", "status", "linedraw"] def opt_tree(option, opt, value, parser, tree_options=tree_options): from . import Main @@ -848,6 +848,8 @@ def Parser(version): tp.prune = True elif o == 'status': tp.status = True + elif o == 'linedraw': + tp.sLineDraw = True else: raise OptionValueError(opt_invalid('--tree', o, tree_options)) parser.values.tree_printers.append(tp) diff --git a/src/engine/SCons/Script/__init__.py b/src/engine/SCons/Script/__init__.py index 8f526be..0a672fc 100644 --- a/src/engine/SCons/Script/__init__.py +++ b/src/engine/SCons/Script/__init__.py @@ -288,8 +288,8 @@ def set_missing_sconscript_error(flag=1): _no_missing_sconscript = flag return old -# -def Variables(files=[], args=ARGUMENTS): + +def Variables(files=None, args=ARGUMENTS): return SCons.Variables.Variables(files, args) diff --git a/src/engine/SCons/SubstTests.py b/src/engine/SCons/SubstTests.py index 574e714..f2def65 100644 --- a/src/engine/SCons/SubstTests.py +++ b/src/engine/SCons/SubstTests.py @@ -577,10 +577,6 @@ class scons_subst_TestCase(SubstTestCase): scons_subst("${NONE[2]}", env, gvars={'NONE':None}) except SCons.Errors.UserError as e: expect = [ - # Python 2.3, 2.4 - "TypeError `unsubscriptable object' trying to evaluate `${NONE[2]}'", - # Python 2.5, 2.6 - "TypeError `'NoneType' object is unsubscriptable' trying to evaluate `${NONE[2]}'", # Python 2.7 and later "TypeError `'NoneType' object is not subscriptable' trying to evaluate `${NONE[2]}'", # Python 2.7 and later under Fedora @@ -596,9 +592,6 @@ class scons_subst_TestCase(SubstTestCase): scons_subst("${func(1)}", env, gvars={'func':func}) except SCons.Errors.UserError as e: expect = [ - # Python 2.3, 2.4, 2.5 - "TypeError `func() takes exactly 3 arguments (1 given)' trying to evaluate `${func(1)}'", - # Python 3.5 (and 3.x?) "TypeError `func() missing 2 required positional arguments: 'b' and 'c'' trying to evaluate `${func(1)}'" ] diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py index 1711de1..c6c19ae 100644 --- a/src/engine/SCons/Tool/JavaCommon.py +++ b/src/engine/SCons/Tool/JavaCommon.py @@ -404,7 +404,7 @@ if java_parsing: def parse_java_file(fn, version=default_java_version): - with open(fn, 'r') as f: + with open(fn, 'r', encoding='utf-8') as f: data = f.read() return parse_java(data, version) diff --git a/src/engine/SCons/Tool/JavaCommonTests.py b/src/engine/SCons/Tool/JavaCommonTests.py index 9242624..b0a788e 100644 --- a/src/engine/SCons/Tool/JavaCommonTests.py +++ b/src/engine/SCons/Tool/JavaCommonTests.py @@ -68,6 +68,30 @@ public class Foo assert classes == ['Foo'], classes + def test_file_parser(self): + """Test the file parser""" + input = """\ +package com.sub.bar; + +public class Foo +{ + public static void main(String[] args) + { + /* This tests that unicde is handled . */ + String hello1 = new String("ఎత్తువెడల్పు"); + } +} +""" + file_name = 'test_file_parser.java' + with open(file_name, 'w', encoding='UTF-8') as jf: + print(input, file=jf) + + pkg_dir, classes = SCons.Tool.JavaCommon.parse_java_file(file_name) + if os.path.exists(file_name): + os.remove(file_name) + assert pkg_dir == os.path.join('com', 'sub', 'bar'), pkg_dir + assert classes == ['Foo'], classes + def test_dollar_sign(self): """Test class names with $ in them""" diff --git a/src/engine/SCons/Tool/MSCommon/__init__.py b/src/engine/SCons/Tool/MSCommon/__init__.py index c87bf71..be7720a 100644 --- a/src/engine/SCons/Tool/MSCommon/__init__.py +++ b/src/engine/SCons/Tool/MSCommon/__init__.py @@ -42,7 +42,8 @@ from SCons.Tool.MSCommon.sdk import mssdk_exists, \ from SCons.Tool.MSCommon.vc import msvc_exists, \ msvc_setup_env, \ msvc_setup_env_once, \ - msvc_version_to_maj_min + msvc_version_to_maj_min, \ + msvc_find_vswhere from SCons.Tool.MSCommon.vs import get_default_version, \ get_vs_by_version, \ diff --git a/src/engine/SCons/Tool/MSCommon/common.py b/src/engine/SCons/Tool/MSCommon/common.py index 505136e..b2b5de9 100644 --- a/src/engine/SCons/Tool/MSCommon/common.py +++ b/src/engine/SCons/Tool/MSCommon/common.py @@ -64,8 +64,7 @@ def read_script_env_cache(): try: with open(CONFIG_CACHE, 'r') as f: envcache = json.load(f) - # TODO can use more specific FileNotFoundError when py2 dropped - except IOError: + except FileNotFoundError: # don't fail if no cache file, just proceed without it pass return envcache @@ -157,15 +156,14 @@ def normalize_env(env, keys, force=False): if k in os.environ and (force or k not in normenv): normenv[k] = os.environ[k] - # This shouldn't be necessary, since the default environment should include system32, - # but keep this here to be safe, since it's needed to find reg.exe which the MSVC - # bat scripts use. - sys32_dir = os.path.join(os.environ.get("SystemRoot", - os.environ.get("windir", r"C:\Windows\system32")), - "System32") - - if sys32_dir not in normenv['PATH']: - normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_dir + # add some things to PATH to prevent problems: + # Shouldn't be necessary to add system32, since the default environment + # should include it, but keep this here to be safe (needed for reg.exe) + sys32_dir = os.path.join( + os.environ.get("SystemRoot", os.environ.get("windir", r"C:\Windows")), "System32" +) + if sys32_dir not in normenv["PATH"]: + normenv["PATH"] = normenv["PATH"] + os.pathsep + sys32_dir # Without Wbem in PATH, vcvarsall.bat has a "'wmic' is not recognized" # error starting with Visual Studio 2017, although the script still @@ -174,8 +172,14 @@ def normalize_env(env, keys, force=False): if sys32_wbem_dir not in normenv['PATH']: normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_wbem_dir - debug("PATH: %s" % normenv['PATH']) + # Without Powershell in PATH, an internal call to a telemetry + # function (starting with a VS2019 update) can fail + # Note can also set VSCMD_SKIP_SENDTELEMETRY to avoid this. + sys32_ps_dir = os.path.join(sys32_dir, r'WindowsPowerShell\v1.0') + if sys32_ps_dir not in normenv['PATH']: + normenv['PATH'] = normenv['PATH'] + os.pathsep + sys32_ps_dir + debug("PATH: %s" % normenv['PATH']) return normenv @@ -186,16 +190,21 @@ def get_output(vcbat, args=None, env=None): # Create a blank environment, for use in launching the tools env = SCons.Environment.Environment(tools=[]) - # TODO: This is a hard-coded list of the variables that (may) need - # to be imported from os.environ[] for v[sc]*vars*.bat file - # execution to work. This list should really be either directly - # controlled by vc.py, or else derived from the common_tools_var - # settings in vs.py. + # TODO: Hard-coded list of the variables that (may) need to be + # imported from os.environ[] for the chain of development batch + # files to execute correctly. One call to vcvars*.bat may + # end up running a dozen or more scripts, changes not only with + # each release but with what is installed at the time. We think + # in modern installations most are set along the way and don't + # need to be picked from the env, but include these for safety's sake. + # Any VSCMD variables definitely are picked from the env and + # control execution in interesting ways. + # Note these really should be unified - either controlled by vs.py, + # or synced with the the common_tools_var # settings in vs.py. vs_vc_vars = [ - 'COMSPEC', - # VS100 and VS110: Still set, but modern MSVC setup scripts will - # discard these if registry has values. However Intel compiler setup - # script still requires these as of 2013/2014. + 'COMSPEC', # path to "shell" + 'VS160COMNTOOLS', # path to common tools for given version + 'VS150COMNTOOLS', 'VS140COMNTOOLS', 'VS120COMNTOOLS', 'VS110COMNTOOLS', @@ -205,6 +214,8 @@ def get_output(vcbat, args=None, env=None): 'VS71COMNTOOLS', 'VS70COMNTOOLS', 'VS60COMNTOOLS', + 'VSCMD_DEBUG', # enable logging and other debug aids + 'VSCMD_SKIP_SENDTELEMETRY', ] env['ENV'] = normalize_env(env['ENV'], vs_vc_vars, force=False) @@ -235,21 +246,35 @@ def get_output(vcbat, args=None, env=None): # debug('get_output():stdout:%s'%stdout) # debug('get_output():stderr:%s'%stderr) + # Ongoing problems getting non-corrupted text led to this + # changing to "oem" from "mbcs" - the scripts run presumably + # attached to a console, so some particular rules apply. + # Unfortunately, "oem" not defined in Python 3.5, so get another way + if sys.version_info.major == 3 and sys.version_info.minor < 6: + from ctypes import windll + + OEM = "cp{}".format(windll.kernel32.GetConsoleOutputCP()) + else: + OEM = "oem" if stderr: # TODO: find something better to do with stderr; # this at least prevents errors from getting swallowed. - sys.stderr.write(stderr) + sys.stderr.write(stderr.decode(OEM)) if popen.wait() != 0: - raise IOError(stderr.decode("mbcs")) + raise IOError(stderr.decode(OEM)) - output = stdout.decode("mbcs") - return output + return stdout.decode(OEM) -KEEPLIST = ("INCLUDE", "LIB", "LIBPATH", "PATH", 'VSCMD_ARG_app_plat', - 'VCINSTALLDIR', # needed by clang -VS 2017 and newer - 'VCToolsInstallDir', # needed by clang - VS 2015 and older - ) +KEEPLIST = ( + "INCLUDE", + "LIB", + "LIBPATH", + "PATH", + "VSCMD_ARG_app_plat", + "VCINSTALLDIR", # needed by clang -VS 2017 and newer + "VCToolsInstallDir", # needed by clang - VS 2015 and older +) def parse_output(output, keep=KEEPLIST): diff --git a/src/engine/SCons/Tool/MSCommon/vc.py b/src/engine/SCons/Tool/MSCommon/vc.py index 82fb6b9..58be07c 100644 --- a/src/engine/SCons/Tool/MSCommon/vc.py +++ b/src/engine/SCons/Tool/MSCommon/vc.py @@ -233,16 +233,16 @@ _VCVER = ["14.2", "14.1", "14.1Exp", "14.0", "14.0Exp", "12.0", "12.0Exp", "11.0 # if using vswhere, a further mapping is needed _VCVER_TO_VSWHERE_VER = { - '14.2' : '[16.0, 17.0)', - '14.1' : '[15.0, 16.0)', + '14.2': '[16.0, 17.0)', + '14.1': '[15.0, 16.0)', } _VCVER_TO_PRODUCT_DIR = { - '14.2' : [ + '14.2': [ (SCons.Util.HKEY_LOCAL_MACHINE, r'')], # not set by this version - '14.1' : [ + '14.1': [ (SCons.Util.HKEY_LOCAL_MACHINE, r'')], # not set by this version - '14.1Exp' : [ + '14.1Exp': [ (SCons.Util.HKEY_LOCAL_MACHINE, r'')], # not set by this version '14.0' : [ (SCons.Util.HKEY_LOCAL_MACHINE, r'Microsoft\VisualStudio\14.0\Setup\VC\ProductDir')], @@ -290,6 +290,7 @@ _VCVER_TO_PRODUCT_DIR = { ] } + def msvc_version_to_maj_min(msvc_version): msvc_version_numeric = get_msvc_version_numeric(msvc_version) @@ -303,6 +304,7 @@ def msvc_version_to_maj_min(msvc_version): except ValueError as e: raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) + def is_host_target_supported(host_target, msvc_version): """Check if (host, target) pair is supported for a VC version. @@ -321,7 +323,30 @@ def is_host_target_supported(host_target, msvc_version): return True -def find_vc_pdir_vswhere(msvc_version): +VSWHERE_PATHS = [os.path.join(p,'vswhere.exe') for p in [ + os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer"), + os.path.expandvars(r"%ProgramFiles%\Microsoft Visual Studio\Installer"), + os.path.expandvars(r"%ChocolateyInstall%\bin"), +]] + +def msvc_find_vswhere(): + """ + Find the location of vswhere + """ + # For bug 3333: support default location of vswhere for both + # 64 and 32 bit windows installs. + # For bug 3542: also accommodate not being on C: drive. + # NB: this gets called from testsuite on non-Windows platforms. + # Whether that makes sense or not, don't break it for those. + vswhere_path = None + for pf in VSWHERE_PATHS: + if os.path.exists(pf): + vswhere_path = pf + break + + return vswhere_path + +def find_vc_pdir_vswhere(msvc_version, env=None): """ Find the MSVC product directory using the vswhere program. @@ -336,26 +361,15 @@ def find_vc_pdir_vswhere(msvc_version): debug("Unknown version of MSVC: %s" % msvc_version) raise UnsupportedVersion("Unknown version %s" % msvc_version) - # For bug 3333: support default location of vswhere for both - # 64 and 32 bit windows installs. - # For bug 3542: also accommodate not being on C: drive. - # NB: this gets called from testsuite on non-Windows platforms. - # Whether that makes sense or not, don't break it for those. - # TODO: requested to add a user-specified path to vswhere - # and have this routine set the same var if it finds it. - pfpaths = [ - os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer"), - os.path.expandvars(r"%ProgramFiles%\Microsoft Visual Studio\Installer"), - os.path.expandvars(r"%ChocolateyInstall%\bin"), - ] - for pf in pfpaths: - vswhere_path = os.path.join(pf, "vswhere.exe") - if os.path.exists(vswhere_path): - break + if env is None or not env.get('VSWHERE'): + vswhere_path = msvc_find_vswhere() else: - # No vswhere on system, no install info available this way + vswhere_path = env.subst('$VSWHERE') + + if vswhere_path is None: return None + debug('find_vc_pdir_vswhere(): VSWHERE = %s'%vswhere_path) vswhere_cmd = [ vswhere_path, "-products", "*", @@ -363,6 +377,8 @@ def find_vc_pdir_vswhere(msvc_version): "-property", "installationPath", ] + debug("find_vc_pdir_vswhere(): running: %s" % vswhere_cmd) + #cp = subprocess.run(vswhere_cmd, capture_output=True) # 3.7+ only cp = subprocess.run(vswhere_cmd, stdout=PIPE, stderr=PIPE) @@ -379,7 +395,7 @@ def find_vc_pdir_vswhere(msvc_version): return None -def find_vc_pdir(msvc_version): +def find_vc_pdir(env, msvc_version): """Find the MSVC product directory for the given version. Tries to look up the path using a registry key from the table @@ -410,7 +426,7 @@ def find_vc_pdir(msvc_version): try: comps = None if not key: - comps = find_vc_pdir_vswhere(msvc_version) + comps = find_vc_pdir_vswhere(msvc_version, env) if not comps: debug('find_vc_pdir_vswhere(): no VC found for version {}'.format(repr(msvc_version))) raise SCons.Util.WinError @@ -448,7 +464,7 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): so use that and return an indication we don't need the argument we would have computed to run vcvarsall.bat. """ - pdir = find_vc_pdir(msvc_version) + pdir = find_vc_pdir(env, msvc_version) if pdir is None: raise NoVersionFound("No version of Visual Studio found") debug('find_batch_file() in {}'.format(pdir)) @@ -633,7 +649,7 @@ def get_installed_vcs(env=None): for ver in _VCVER: debug('trying to find VC %s' % ver) try: - VC_DIR = find_vc_pdir(ver) + VC_DIR = find_vc_pdir(env, ver) if VC_DIR: debug('found VC %s' % ver) if _check_cl_exists_in_vc_dir(env, VC_DIR, ver): diff --git a/src/engine/SCons/Tool/MSCommon/vcTests.py b/src/engine/SCons/Tool/MSCommon/vcTests.py index 09991f5..eb09def 100644 --- a/src/engine/SCons/Tool/MSCommon/vcTests.py +++ b/src/engine/SCons/Tool/MSCommon/vcTests.py @@ -45,12 +45,42 @@ MSVCUnsupportedTargetArch = SCons.Tool.MSCommon.vc.MSVCUnsupportedTargetArch MS_TOOLS_VERSION='1.1.1' +class VswhereTestCase(unittest.TestCase): + @staticmethod + def _createVSWhere(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write("Created:%s"%f) + + def testDefaults(self): + """ + Verify that msvc_find_vswhere() find's files in the specified paths + """ + # import pdb; pdb.set_trace() + vswhere_dirs = [os.path.splitdrive(p)[1] for p in SCons.Tool.MSCommon.vc.VSWHERE_PATHS] + base_dir = test.workpath('fake_vswhere') + test_vswhere_dirs = [os.path.join(base_dir,d[1:]) for d in vswhere_dirs] + + SCons.Tool.MSCommon.vc.VSWHERE_PATHS = test_vswhere_dirs + for vsw in test_vswhere_dirs: + VswhereTestCase._createVSWhere(vsw) + find_path = SCons.Tool.MSCommon.vc.msvc_find_vswhere() + self.assertTrue(vsw == find_path, "Didn't find vswhere in %s found in %s"%(vsw, find_path)) + os.remove(vsw) + + # def specifiedVswherePathTest(self): + # "Verify that msvc.generate() respects VSWHERE Specified" + + + + + class MSVcTestCase(unittest.TestCase): @staticmethod def _createDummyCl(path, add_bin=True): """ - Creates a dummy cl.ex in the correct directory. + Creates a dummy cl.exe in the correct directory. It will create all missing parent directories as well Args: @@ -74,8 +104,6 @@ class MSVcTestCase(unittest.TestCase): ct.write('created') - - def runTest(self): """ Check that all proper HOST_PLATFORM and TARGET_PLATFORM are handled. diff --git a/src/engine/SCons/Tool/MSCommon/vs.py b/src/engine/SCons/Tool/MSCommon/vs.py index e13f52f..e71eb27 100644 --- a/src/engine/SCons/Tool/MSCommon/vs.py +++ b/src/engine/SCons/Tool/MSCommon/vs.py @@ -64,22 +64,22 @@ class VisualStudio(object): return None return batch_file - def find_vs_dir_by_vc(self): - SCons.Tool.MSCommon.vc.get_installed_vcs() - dir = SCons.Tool.MSCommon.vc.find_vc_pdir(self.vc_version) + def find_vs_dir_by_vc(self, env): + SCons.Tool.MSCommon.vc.get_installed_vcs(env) + dir = SCons.Tool.MSCommon.vc.find_vc_pdir(env, self.vc_version) if not dir: debug('find_vs_dir_by_vc(): no installed VC %s' % self.vc_version) return None return os.path.abspath(os.path.join(dir, os.pardir)) - def find_vs_dir_by_reg(self): + def find_vs_dir_by_reg(self, env): root = 'Software\\' if is_win64(): root = root + 'Wow6432Node\\' for key in self.hkeys: if key=='use_dir': - return self.find_vs_dir_by_vc() + return self.find_vs_dir_by_vc(env) key = root + key try: comps = read_reg(key) @@ -90,19 +90,19 @@ class VisualStudio(object): return comps return None - def find_vs_dir(self): + def find_vs_dir(self, env): """ Can use registry or location of VC to find vs dir First try to find by registry, and if that fails find via VC dir """ - vs_dir=self.find_vs_dir_by_reg() + vs_dir=self.find_vs_dir_by_reg(env) if not vs_dir: - vs_dir = self.find_vs_dir_by_vc() + vs_dir = self.find_vs_dir_by_vc(env) debug('find_vs_dir(): found VS in ' + str(vs_dir )) return vs_dir - def find_executable(self): - vs_dir = self.get_vs_dir() + def find_executable(self, env): + vs_dir = self.get_vs_dir(env) if not vs_dir: debug('find_executable(): no vs_dir ({})'.format(vs_dir)) return None @@ -121,21 +121,21 @@ class VisualStudio(object): self._cache['batch_file'] = batch_file return batch_file - def get_executable(self): + def get_executable(self, env=None): try: debug('get_executable using cache:%s'%self._cache['executable']) return self._cache['executable'] except KeyError: - executable = self.find_executable() + executable = self.find_executable(env) self._cache['executable'] = executable debug('get_executable not in cache:%s'%executable) return executable - def get_vs_dir(self): + def get_vs_dir(self, env): try: return self._cache['vs_dir'] except KeyError: - vs_dir = self.find_vs_dir() + vs_dir = self.find_vs_dir(env) self._cache['vs_dir'] = vs_dir return vs_dir @@ -413,7 +413,7 @@ for vs in SupportedVSList: InstalledVSList = None InstalledVSMap = None -def get_installed_visual_studios(): +def get_installed_visual_studios(env=None): global InstalledVSList global InstalledVSMap if InstalledVSList is None: @@ -421,7 +421,7 @@ def get_installed_visual_studios(): InstalledVSMap = {} for vs in SupportedVSList: debug('trying to find VS %s' % vs.version) - if vs.get_executable(): + if vs.get_executable(env): debug('found VS %s' % vs.version) InstalledVSList.append(vs) InstalledVSMap[vs.version] = vs @@ -472,8 +472,8 @@ def reset_installed_visual_studios(): # for variable, directory in env_tuple_list: # env.PrependENVPath(variable, directory) -def msvs_exists(): - return (len(get_installed_visual_studios()) > 0) +def msvs_exists(env=None): + return (len(get_installed_visual_studios(env)) > 0) def get_vs_by_version(msvs): global InstalledVSMap diff --git a/src/engine/SCons/Tool/applelink.xml b/src/engine/SCons/Tool/applelink.xml index fc0cf63..977f2ce 100644 --- a/src/engine/SCons/Tool/applelink.xml +++ b/src/engine/SCons/Tool/applelink.xml @@ -171,7 +171,7 @@ See its __doc__ string for a discussion of the format. </para> <example_commands> - env.AppendUnique(FRAMEWORKS=Split('System Cocoa SystemConfiguration')) +env.AppendUnique(FRAMEWORKS=Split('System Cocoa SystemConfiguration')) </example_commands> </summary> @@ -213,7 +213,7 @@ See its __doc__ string for a discussion of the format. </para> <example_commands> - env.AppendUnique(FRAMEWORKPATH='#myframeworkdir') +env.AppendUnique(FRAMEWORKPATH='#myframeworkdir') </example_commands> <para> @@ -221,7 +221,7 @@ See its __doc__ string for a discussion of the format. </para> <example_commands> - ... -Fmyframeworkdir +... -Fmyframeworkdir </example_commands> <para> diff --git a/src/engine/SCons/Tool/docbook/__init__.py b/src/engine/SCons/Tool/docbook/__init__.py index c4fdfba..7f47e9d 100644 --- a/src/engine/SCons/Tool/docbook/__init__.py +++ b/src/engine/SCons/Tool/docbook/__init__.py @@ -351,11 +351,16 @@ def __build_lxml(target, source, env): else: result = transform(doc) + # we'd like the resulting output to be readably formatted, + # so try pretty-print. Sometimes (esp. if the output is + # not an xml file) we end up with a None type somewhere in + # the transformed tree and tostring throws TypeError, + # so provide a fallback. try: with open(str(target[0]), "wb") as of: of.write(etree.tostring(result, pretty_print=True)) - except: - pass + except TypeError: + result.write_output(str(target[0])) return None diff --git a/src/engine/SCons/Tool/gnulink.py b/src/engine/SCons/Tool/gnulink.py index b1d5088..5372322 100644 --- a/src/engine/SCons/Tool/gnulink.py +++ b/src/engine/SCons/Tool/gnulink.py @@ -35,9 +35,7 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.Util import SCons.Tool -import os import sys -import re from . import link @@ -57,21 +55,22 @@ def generate(env): # OpenBSD doesn't usually use SONAME for libraries use_soname = not sys.platform.startswith('openbsd') - link._setup_versioned_lib_variables(env, tool = 'gnulink', use_soname = use_soname) + link._setup_versioned_lib_variables(env, tool='gnulink', use_soname=use_soname) env['LINKCALLBACKS'] = link._versioned_lib_callbacks() - # For backward-compatibility with older SCons versions - env['SHLIBVERSIONFLAGS'] = SCons.Util.CLVar('-Wl,-Bsymbolic') - + # # For backward-compatibility with older SCons versions + # env['SHLIBVERSIONFLAGS'] = SCons.Util.CLVar('') + + def exists(env): # TODO: sync with link.smart_link() to choose a linker - linkers = { 'CXX': ['g++'], 'CC': ['gcc'] } + linkers = {'CXX': ['g++'], 'CC': ['gcc']} alltools = [] for langvar, linktools in linkers.items(): - if langvar in env: # use CC over CXX when user specified CC but not CXX + if langvar in env: # use CC over CXX when user specified CC but not CXX return SCons.Tool.FindTool(linktools, env) alltools.extend(linktools) - return SCons.Tool.FindTool(alltools, env) # find CXX or CC + return SCons.Tool.FindTool(alltools, env) # find CXX or CC # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Tool/jar.xml b/src/engine/SCons/Tool/jar.xml index 30c51f8..151dda1 100644 --- a/src/engine/SCons/Tool/jar.xml +++ b/src/engine/SCons/Tool/jar.xml @@ -120,7 +120,7 @@ If this is not set, then &cv-link-JARCOM; (the command line) is displayed. </para> <example_commands> -env = Environment(JARCOMSTR = "JARchiving $SOURCES into $TARGET") +env = Environment(JARCOMSTR="JARchiving $SOURCES into $TARGET") </example_commands> </summary> </cvar> diff --git a/src/engine/SCons/Tool/javac.xml b/src/engine/SCons/Tool/javac.xml index 893130b..b1548c7 100644 --- a/src/engine/SCons/Tool/javac.xml +++ b/src/engine/SCons/Tool/javac.xml @@ -174,7 +174,7 @@ env['ENV']['LANG'] = 'en_GB.UTF-8' </para> <example_commands> -env = Environment(JAVACCOMSTR = "Compiling class files $TARGETS from $SOURCES") +env = Environment(JAVACCOMSTR="Compiling class files $TARGETS from $SOURCES") </example_commands> </summary> </cvar> diff --git a/src/engine/SCons/Tool/javah.xml b/src/engine/SCons/Tool/javah.xml index 4d436b1..5a2840f 100644 --- a/src/engine/SCons/Tool/javah.xml +++ b/src/engine/SCons/Tool/javah.xml @@ -77,17 +77,18 @@ Examples: <example_commands> # builds java_native.h -classes = env.Java(target = 'classdir', source = 'src') -env.JavaH(target = 'java_native.h', source = classes) +classes = env.Java(target="classdir", source="src") +env.JavaH(target="java_native.h", source=classes) # builds include/package_foo.h and include/package_bar.h -env.JavaH(target = 'include', - source = ['package/foo.class', 'package/bar.class']) +env.JavaH(target="include", source=["package/foo.class", "package/bar.class"]) # builds export/foo.h and export/bar.h -env.JavaH(target = 'export', - source = ['classes/foo.class', 'classes/bar.class'], - JAVACLASSDIR = 'classes') +env.JavaH( + target="export", + source=["classes/foo.class", "classes/bar.class"], + JAVACLASSDIR="classes", +) </example_commands> </summary> </builder> @@ -120,7 +121,7 @@ If this is not set, then &cv-link-JAVAHCOM; (the command line) is displayed. </para> <example_commands> -env = Environment(JAVAHCOMSTR = "Generating header/stub file(s) $TARGETS from $SOURCES") +env = Environment(JAVAHCOMSTR="Generating header/stub file(s) $TARGETS from $SOURCES") </example_commands> </summary> </cvar> diff --git a/src/engine/SCons/Tool/linkloc.py b/src/engine/SCons/Tool/linkloc.py index c73852b..ad189b2 100644 --- a/src/engine/SCons/Tool/linkloc.py +++ b/src/engine/SCons/Tool/linkloc.py @@ -101,7 +101,7 @@ def generate(env): addPharLapPaths(env) def exists(env): - if msvs_exists(): + if msvs_exists(env): return env.Detect('linkloc') else: return 0 diff --git a/src/engine/SCons/Tool/mingw.py b/src/engine/SCons/Tool/mingw.py index e5f472a..d6a27ae 100644 --- a/src/engine/SCons/Tool/mingw.py +++ b/src/engine/SCons/Tool/mingw.py @@ -47,6 +47,7 @@ mingw_paths = [ r'c:\MinGW\bin', r'C:\cygwin64\bin', r'C:\msys64', + r'C:\msys64\mingw64\bin', r'C:\cygwin\bin', r'C:\msys', ] diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py index 65c0e91..463e372 100644 --- a/src/engine/SCons/Tool/msvc.py +++ b/src/engine/SCons/Tool/msvc.py @@ -48,7 +48,7 @@ import SCons.Util import SCons.Warnings import SCons.Scanner.RC -from .MSCommon import msvc_exists, msvc_setup_env_once, msvc_version_to_maj_min +from .MSCommon import msvc_exists, msvc_setup_env_once, msvc_version_to_maj_min, msvc_find_vswhere CSuffixes = ['.c', '.C'] CXXSuffixes = ['.cc', '.cpp', '.cxx', '.c++', '.C++'] @@ -214,6 +214,7 @@ ShCXXAction = SCons.Action.Action("$SHCXXCOM", "$SHCXXCOMSTR", batch_key=msvc_batch_key, targets='$CHANGED_TARGETS') + def generate(env): """Add Builders and construction variables for MSVC++ to an Environment.""" static_obj, shared_obj = SCons.Tool.createObjBuilders(env) @@ -276,6 +277,9 @@ def generate(env): # without it for lex generation env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd") + # Get user specified vswhere location or locate. + env['VSWHERE'] = env.get('VSWHERE', msvc_find_vswhere()) + # Set-up ms tools paths msvc_setup_env_once(env) diff --git a/src/engine/SCons/Tool/msvc.xml b/src/engine/SCons/Tool/msvc.xml index 100c84c..e92fa9b 100644 --- a/src/engine/SCons/Tool/msvc.xml +++ b/src/engine/SCons/Tool/msvc.xml @@ -475,4 +475,54 @@ Valid values are '1' or '0' </summary> </cvar> +<cvar name="VSWHERE"> +<summary> +<para> +Specify the location of <filename>vswhere.exe</filename>. +</para> + +<para> + The <filename>vswhere.exe</filename> executable is distributed with Microsoft Visual Studio and Build + Tools since the 2017 edition, but is also available standalone. + It provides full information about installations of 2017 and later editions. + With the <option>-legacy</option> argument, <filename>vswhere.exe</filename> can detect installations of the 2010 through 2015 + editions with limited data returned. +If <envar>VSWHERE</envar> is set, SCons will use that location. +</para> +<para> + Otherwise SCons will look in the following locations and set <envar>VSWHERE</envar> to the path of the first <filename>vswhere.exe</filename> +located. +</para> + +<itemizedlist> +<listitem><para><literal>%ProgramFiles(x86)%\Microsoft Visual Studio\Installer</literal></para></listitem> +<listitem><para><literal>%ProgramFiles%\Microsoft Visual Studio\Installer</literal></para></listitem> +<listitem><para><literal>%ChocolateyInstall%\bin</literal></para></listitem> +</itemizedlist> + +<para> + Note that <envar>VSWHERE</envar> must be set at the same time or prior to any of &t-link-msvc;, &t-link-msvs; , and/or &t-link-mslink; &f-link-Tool; being initialized. + + Either set it as follows +<programlisting> +env = Environment(VSWHERE='c:/my/path/to/vswhere') +</programlisting> + +or if your &consenv; is created specifying an empty tools list +(or a list of tools which omits all of default, msvs, msvc, and mslink), +and also before &f-link-env-Tool; is called to ininitialize any of those tools: + +<programlisting> + env = Environment(tools=[]) + env['VSWHERE'] = r'c:/my/vswhere/install/location/vswhere.exe' + env.Tool('msvc') + env.Tool('mslink') + env.Tool('msvs') + </programlisting> +</para> + +</summary> +</cvar> + + </sconsdoc> diff --git a/src/engine/SCons/Tool/msvs.xml b/src/engine/SCons/Tool/msvs.xml index b1f79b2..f6c9a39 100644 --- a/src/engine/SCons/Tool/msvs.xml +++ b/src/engine/SCons/Tool/msvs.xml @@ -422,7 +422,11 @@ env.MSVSProject(target='Bar' + env['MSVSPROJECTSUFFIX'], </variablelist> <para>Example Usage:</para> <example_commands> -env.MSVSSolution(target='Bar' + env['MSVSSOLUTIONSUFFIX'], projects=['bar' + env['MSVSPROJECTSUFFIX']], variant='Release') +env.MSVSSolution( + target="Bar" + env["MSVSSOLUTIONSUFFIX"], + projects=["bar" + env["MSVSPROJECTSUFFIX"]], + variant="Release", +) </example_commands> </summary> </builder> <cvar name="MSVS"> diff --git a/src/engine/SCons/Tool/msvsTests.py b/src/engine/SCons/Tool/msvsTests.py index b3373ea..1bacc2c 100644 --- a/src/engine/SCons/Tool/msvsTests.py +++ b/src/engine/SCons/Tool/msvsTests.py @@ -580,7 +580,7 @@ def DummyQueryValue(key, value): def DummyExists(path): return 1 -def DummyVsWhere(msvc_version): +def DummyVsWhere(msvc_version, env): # not testing versions with vswhere, so return none return None diff --git a/src/engine/SCons/Tool/packaging/__init__.py b/src/engine/SCons/Tool/packaging/__init__.py index 5795396..b6dd42e 100644 --- a/src/engine/SCons/Tool/packaging/__init__.py +++ b/src/engine/SCons/Tool/packaging/__init__.py @@ -27,6 +27,9 @@ SCons Packaging Tool. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import importlib +from inspect import getfullargspec + import SCons.Defaults import SCons.Environment from SCons.Variables import * @@ -34,8 +37,6 @@ from SCons.Errors import * from SCons.Util import is_List, make_path_relative from SCons.Warnings import warn, Warning -import os -import importlib __all__ = [ 'src_targz', 'src_tarbz2', 'src_tarxz', 'src_zip', @@ -168,13 +169,7 @@ def Package(env, target=None, source=None, **kw): # this exception means that a needed argument for the packager is # missing. As our packagers get their "tags" as named function # arguments we need to find out which one is missing. - #TODO: getargspec deprecated in Py3. cleanup when Py2.7 dropped. - try: - from inspect import getfullargspec - argspec = getfullargspec(packager.package) - except ImportError: - from inspect import getargspec - argspec = getargspec(packager.package) + argspec = getfullargspec(packager.package) args = argspec.args if argspec.defaults: # throw away arguments with default values diff --git a/src/engine/SCons/Tool/packaging/__init__.xml b/src/engine/SCons/Tool/packaging/__init__.xml index 66a7aa0..37e97ff 100644 --- a/src/engine/SCons/Tool/packaging/__init__.xml +++ b/src/engine/SCons/Tool/packaging/__init__.xml @@ -62,17 +62,21 @@ option or with the &cv-PACKAGETYPE; construction variable. Currently the following packagers available: </para> -<para><literal>msi</literal> - Microsoft Installer</para> -<para><literal>rpm</literal> - RPM Package Manger</para> -<para><literal>ipkg</literal> - Itsy Package Management System</para> -<para><literal>tarbz2</literal> - bzip2 compressed tar</para> -<para><literal>targz</literal> - gzip compressed tar</para> -<para><literal>tarxz</literal> - xz compressed tar</para> -<para><literal>zip</literal> - zip file</para> -<para><literal>src_tarbz2</literal> - bzip2 compressed tar source</para> -<para><literal>src_targz</literal> - gzip compressed tar source</para> -<para><literal>src_tarxz</literal> - xz compressed tar source</para> -<para><literal>src_zip</literal> - zip file source</para> +<blockquote> +<simplelist type='vert' columns='1'> +<member><literal>msi</literal> - Microsoft Installer</member> +<member><literal>rpm</literal> - RPM Package Manger</member> +<member><literal>ipkg</literal> - Itsy Package Management System</member> +<member><literal>tarbz2</literal> - bzip2 compressed tar</member> +<member><literal>targz</literal> - gzip compressed tar</member> +<member><literal>tarxz</literal> - xz compressed tar</member> +<member><literal>zip</literal> - zip file</member> +<member><literal>src_tarbz2</literal> - bzip2 compressed tar source</member> +<member><literal>src_targz</literal> - gzip compressed tar source</member> +<member><literal>src_tarxz</literal> - xz compressed tar source</member> +<member><literal>src_zip</literal> - zip file source</member> +</simplelist> +</blockquote> <para> An updated list is always available under the @@ -82,18 +86,19 @@ on a project that has packaging activated. </para> <example_commands> -env = Environment(tools=['default', 'packaging']) -env.Install('/bin/', 'my_program') -env.Package( NAME = 'foo', - VERSION = '1.2.3', - PACKAGEVERSION = 0, - PACKAGETYPE = 'rpm', - LICENSE = 'gpl', - SUMMARY = 'balalalalal', - DESCRIPTION = 'this should be really really long', - X_RPM_GROUP = 'Application/fu', - SOURCE_URL = 'http://foo.org/foo-1.2.3.tar.gz' - ) +env = Environment(tools=["default", "packaging"]) +env.Install("/bin/", "my_program") +env.Package( + NAME="foo", + VERSION="1.2.3", + PACKAGEVERSION=0, + PACKAGETYPE="rpm", + LICENSE="gpl", + SUMMARY="balalalalal", + DESCRIPTION="this should be really really long", + X_RPM_GROUP="Application/fu", + SOURCE_URL="http://foo.org/foo-1.2.3.tar.gz", +) </example_commands> </summary> </builder> @@ -199,17 +204,21 @@ placed if applicable. The default value is "$NAME-$VERSION". Selects the package type to build. Currently these are available: </para> -<para><literal>msi</literal> - Microsoft Installer</para> -<para><literal>rpm</literal> - RPM Package Manger</para> -<para><literal>ipkg</literal> - Itsy Package Management System</para> -<para><literal>tarbz2</literal> - bzip2 compressed tar</para> -<para><literal>targz</literal> - gzip compressed tar</para> -<para><literal>tarxz</literal> - xz compressed tar</para> -<para><literal>zip</literal> - zip file</para> -<para><literal>src_tarbz2</literal> - bzip2 compressed tar source</para> -<para><literal>src_targz</literal> - gzip compressed tar source</para> -<para><literal>src_tarxz</literal> - xz compressed tar source</para> -<para><literal>src_zip</literal> - zip file source</para> +<blockquote> +<simplelist type='vert' columns='1'> +<member><literal>msi</literal> - Microsoft Installer</member> +<member><literal>rpm</literal> - RPM Package Manger</member> +<member><literal>ipkg</literal> - Itsy Package Management System</member> +<member><literal>tarbz2</literal> - bzip2 compressed tar</member> +<member><literal>targz</literal> - gzip compressed tar</member> +<member><literal>tarxz</literal> - xz compressed tar</member> +<member><literal>zip</literal> - zip file</member> +<member><literal>src_tarbz2</literal> - bzip2 compressed tar source</member> +<member><literal>src_targz</literal> - gzip compressed tar source</member> +<member><literal>src_tarxz</literal> - xz compressed tar source</member> +<member><literal>src_zip</literal> - zip file source</member> +</simplelist> +</blockquote> <para> This may be overridden with the <option>package_type</option> @@ -495,13 +504,14 @@ Added in version 3.1. <example_commands> env.Package( - NAME = 'foo', -... - X_RPM_EXTRADEFS = [ - '%define _unpackaged_files_terminate_build 0' - '%define _missing_doc_files_terminate_build 0' + NAME="foo", + ... + X_RPM_EXTRADEFS=[ + "%define _unpackaged_files_terminate_build 0" + "%define _missing_doc_files_terminate_build 0" ], -... ) + ... +) </example_commands> </summary> diff --git a/src/engine/SCons/Tool/textfile.py b/src/engine/SCons/Tool/textfile.py index c233658..b404304 100644 --- a/src/engine/SCons/Tool/textfile.py +++ b/src/engine/SCons/Tool/textfile.py @@ -171,7 +171,7 @@ _text_builder = SCons.Builder.Builder( suffix='$TEXTFILESUFFIX', ) -_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'TEXTFILESUFFIX'] +_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'SUBSTFILESUFFIX'] _subst_builder = SCons.Builder.Builder( action=SCons.Action.Action(_action, _strfunc, varlist=_subst_varlist), source_factory=SCons.Node.FS.File, diff --git a/src/engine/SCons/Util.py b/src/engine/SCons/Util.py index 22df6fa..588d02f 100644 --- a/src/engine/SCons/Util.py +++ b/src/engine/SCons/Util.py @@ -253,8 +253,18 @@ def render_tree(root, child_func, prune=0, margin=[0], visited=None): IDX = lambda N: N and 1 or 0 - -def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): +# unicode line drawing chars: +BOX_HORIZ = chr(0x2500) # '─' +BOX_VERT = chr(0x2502) # '│' +BOX_UP_RIGHT = chr(0x2514) # '└' +BOX_DOWN_RIGHT = chr(0x250c) # '┌' +BOX_DOWN_LEFT = chr(0x2510) # '┐' +BOX_UP_LEFT = chr(0x2518) # '┘' +BOX_VERT_RIGHT = chr(0x251c) # '├' +BOX_HORIZ_DOWN = chr(0x252c) # '┬' + + +def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None, lastChild=False, singleLineDraw=False): """ Print a tree of nodes. This is like render_tree, except it prints lines directly instead of creating a string representation in memory, @@ -267,6 +277,7 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): - `showtags` - print status information to the left of each node line - `margin` - the format of the left margin to use for children of root. 1 results in a pipe, and 0 results in no pipe. - `visited` - a dictionary of visited nodes in the current branch if not prune, or in the whole tree if prune. + - `singleLineDraw` - use line-drawing characters rather than ASCII. """ rname = str(root) @@ -300,7 +311,8 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): [0, 1][IDX(root.has_explicit_builder())] + [0, 2][IDX(root.has_builder())] ], - ' S'[IDX(root.side_effect)], ' P'[IDX(root.precious)], + ' S'[IDX(root.side_effect)], + ' P'[IDX(root.precious)], ' A'[IDX(root.always_build)], ' C'[IDX(root.is_up_to_date())], ' N'[IDX(root.noclean)], @@ -312,27 +324,51 @@ def print_tree(root, child_func, prune=0, showtags=0, margin=[0], visited=None): tags = [] def MMM(m): - return [" ","| "][m] + if singleLineDraw: + return [" ", BOX_VERT + " "][m] + else: + return [" ", "| "][m] + margins = list(map(MMM, margin[:-1])) children = child_func(root) + + cross = "+-" + if singleLineDraw: + cross = BOX_VERT_RIGHT + BOX_HORIZ # sign used to point to the leaf. + # check if this is the last leaf of the branch + if lastChild: + #if this if the last leaf, then terminate: + cross = BOX_UP_RIGHT + BOX_HORIZ # sign for the last leaf + + # if this branch has children then split it + if children: + # if it's a leaf: + if prune and rname in visited and children: + cross += BOX_HORIZ + else: + cross += BOX_HORIZ_DOWN + if prune and rname in visited and children: - sys.stdout.write(''.join(tags + margins + ['+-[', rname, ']']) + '\n') + sys.stdout.write(''.join(tags + margins + [cross,'[', rname, ']']) + '\n') return - sys.stdout.write(''.join(tags + margins + ['+-', rname]) + '\n') + sys.stdout.write(''.join(tags + margins + [cross, rname]) + '\n') visited[rname] = 1 + # if this item has children: if children: - margin.append(1) + margin.append(1) # Initialize margin with 1 for vertical bar. idx = IDX(showtags) + _child = 0 # Initialize this for the first child. for C in children[:-1]: - print_tree(C, child_func, prune, idx, margin, visited) - margin[-1] = 0 - print_tree(children[-1], child_func, prune, idx, margin, visited) - margin.pop() + _child = _child + 1 # number the children + print_tree(C, child_func, prune, idx, margin, visited, (len(children) - _child) <= 0 ,singleLineDraw) + margin[-1] = 0 # margins are with space (index 0) because we arrived to the last child. + print_tree(children[-1], child_func, prune, idx, margin, visited, True ,singleLineDraw) # for this call child and nr of children needs to be set 0, to signal the second phase. + margin.pop() # destroy the last margin added # Functions for deciding if things are like various types, mainly to diff --git a/src/engine/SCons/Utilities/ConfigureCache.py b/src/engine/SCons/Utilities/ConfigureCache.py new file mode 100644 index 0000000..80783a9 --- /dev/null +++ b/src/engine/SCons/Utilities/ConfigureCache.py @@ -0,0 +1,182 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Show or convert the configuration of an SCons cache directory. + +A cache of derived files is stored by file signature. +The files are split into directories named by the first few +digits of the signature. The prefix length used for directory +names can be changed by this script. +""" + +import argparse +import glob +import json +import os + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + + +def rearrange_cache_entries(current_prefix_len, new_prefix_len): + """Move cache files if prefix length changed. + + Move the existing cache files to new directories of the + appropriate name length and clean up the old directories. + """ + print('Changing prefix length from', current_prefix_len, + 'to', new_prefix_len) + dirs = set() + old_dirs = set() + for file in glob.iglob(os.path.join('*', '*')): + name = os.path.basename(file) + dname = name[:current_prefix_len].upper() + if dname not in old_dirs: + print('Migrating', dname) + old_dirs.add(dname) + dname = name[:new_prefix_len].upper() + if dname not in dirs: + os.mkdir(dname) + dirs.add(dname) + os.rename(file, os.path.join(dname, name)) + + # Now delete the original directories + for dname in old_dirs: + os.rmdir(dname) + + +# The configuration dictionary should have one entry per entry in the +# cache config. The value of each entry should include the following: +# implicit - (optional) This is to allow adding a new config entry and also +# changing the behaviour of the system at the same time. This +# indicates the value the config entry would have had if it had +# been specified. +# default - The value the config entry should have if it wasn't previously +# specified +# command-line - parameters to pass to ArgumentParser.add_argument +# converter - (optional) Function to call if conversion is required +# if this configuration entry changes +config_entries = { + 'prefix_len': { + 'implicit': 1, + 'default': 2, + 'command-line': { + 'help': 'Length of cache file name used as subdirectory prefix', + 'metavar': '<number>', + 'type': int + }, + 'converter': rearrange_cache_entries + } +} + + +def main(): + parser = argparse.ArgumentParser( + description='Modify the configuration of an scons cache directory', + epilog=''' + Unspecified options will not be changed unless they are not + set at all, in which case they are set to an appropriate default. + ''') + + parser.add_argument('cache-dir', help='Path to scons cache directory') + for param in config_entries: + parser.add_argument('--' + param.replace('_', '-'), + **config_entries[param]['command-line']) + parser.add_argument('--version', + action='version', + version='%(prog)s 1.0') + parser.add_argument('--show', + action="store_true", + help="show current configuration") + + # Get the command line as a dict without any of the unspecified entries. + args = dict([x for x in vars(parser.parse_args()).items() if x[1]]) + + # It seems somewhat strange to me, but positional arguments don't get the - + # in the name changed to _, whereas optional arguments do... + cache = args['cache-dir'] + if not os.path.isdir(cache): + raise RuntimeError("There is no cache directory named %s" % cache) + os.chdir(cache) + del args['cache-dir'] + + if not os.path.exists('config'): + # old config dirs did not have a 'config' file. Try to update. + # Validate the only files in the directory are directories 0-9, a-f + expected = ['{:X}'.format(x) for x in range(0, 16)] + if not set(os.listdir('.')).issubset(expected): + raise RuntimeError( + "%s does not look like a valid version 1 cache directory" % cache) + config = dict() + else: + with open('config') as conf: + config = json.load(conf) + + if args.get('show', None): + print("Current configuration in '%s':" % cache) + print(json.dumps(config, sort_keys=True, + indent=4, separators=(',', ': '))) + # in case of the show argument, emit some stats as well + file_count = 0 + for _, _, files in os.walk('.'): + file_count += len(files) + if file_count: # skip config file if it exists + file_count -= 1 + print("Cache contains %s files" % file_count) + del args['show'] + + # Find any keys that are not currently set but should be + for key in config_entries: + if key not in config: + if 'implicit' in config_entries[key]: + config[key] = config_entries[key]['implicit'] + else: + config[key] = config_entries[key]['default'] + if key not in args: + args[key] = config_entries[key]['default'] + + # Now go through each entry in args to see if it changes an existing config + # setting. + for key in args: + if args[key] != config[key]: + if 'converter' in config_entries[key]: + config_entries[key]['converter'](config[key], args[key]) + config[key] = args[key] + + # and write the updated config file + with open('config', 'w') as conf: + json.dump(config, conf) + +if __name__ == "__main__": + main()
\ No newline at end of file diff --git a/src/engine/SCons/Utilities/__init__.py b/src/engine/SCons/Utilities/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/engine/SCons/Utilities/__init__.py diff --git a/src/engine/SCons/Utilities/sconsign.py b/src/engine/SCons/Utilities/sconsign.py new file mode 100644 index 0000000..89084ba --- /dev/null +++ b/src/engine/SCons/Utilities/sconsign.py @@ -0,0 +1,507 @@ +#! /usr/bin/env python +# +# SCons - a Software Constructor +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +__version__ = "__VERSION__" + +__build__ = "__BUILD__" + +__buildsys__ = "__BUILDSYS__" + +__date__ = "__DATE__" + +__developer__ = "__DEVELOPER__" + +import getopt +import os +import sys +from dbm import whichdb + +import time +import pickle + +import SCons.compat +import SCons.SConsign + + +def my_whichdb(filename): + if filename[-7:] == ".dblite": + return "SCons.dblite" + try: + with open(filename + ".dblite", "rb"): + return "SCons.dblite" + except IOError: + pass + return whichdb(filename) + + +def my_import(mname): + import imp + + if '.' in mname: + i = mname.rfind('.') + parent = my_import(mname[:i]) + fp, pathname, description = imp.find_module(mname[i+1:], + parent.__path__) + else: + fp, pathname, description = imp.find_module(mname) + return imp.load_module(mname, fp, pathname, description) + + +class Flagger(object): + default_value = 1 + + def __setitem__(self, item, value): + self.__dict__[item] = value + self.default_value = 0 + + def __getitem__(self, item): + return self.__dict__.get(item, self.default_value) + + +Do_Call = None +Print_Directories = [] +Print_Entries = [] +Print_Flags = Flagger() +Verbose = 0 +Readable = 0 +Warns = 0 + + +def default_mapper(entry, name): + """ + Stringify an entry that doesn't have an explicit mapping. + + Args: + entry: entry + name: field name + + Returns: str + + """ + try: + val = eval("entry." + name) + except AttributeError: + val = None + if sys.version_info.major >= 3 and isinstance(val, bytes): + # This is a dirty hack for py 2/3 compatibility. csig is a bytes object + # in Python3 while Python2 bytes are str. Hence, we decode the csig to a + # Python3 string + val = val.decode() + return str(val) + + +def map_action(entry, _): + """ + Stringify an action entry and signature. + + Args: + entry: action entry + second argument is not used + + Returns: str + + """ + try: + bact = entry.bact + bactsig = entry.bactsig + except AttributeError: + return None + return '%s [%s]' % (bactsig, bact) + + +def map_timestamp(entry, _): + """ + Stringify a timestamp entry. + + Args: + entry: timestamp entry + second argument is not used + + Returns: str + + """ + try: + timestamp = entry.timestamp + except AttributeError: + timestamp = None + if Readable and timestamp: + return "'" + time.ctime(timestamp) + "'" + else: + return str(timestamp) + + +def map_bkids(entry, _): + """ + Stringify an implicit entry. + + Args: + entry: + second argument is not used + + Returns: str + + """ + try: + bkids = entry.bsources + entry.bdepends + entry.bimplicit + bkidsigs = entry.bsourcesigs + entry.bdependsigs + entry.bimplicitsigs + except AttributeError: + return None + + if len(bkids) != len(bkidsigs): + global Warns + Warns += 1 + # add warning to result rather than direct print so it will line up + msg = "Warning: missing information, {} ids but {} sigs" + result = [msg.format(len(bkids), len(bkidsigs))] + else: + result = [] + result += [nodeinfo_string(bkid, bkidsig, " ") + for bkid, bkidsig in zip(bkids, bkidsigs)] + if not result: + return None + return "\n ".join(result) + + +map_field = { + 'action' : map_action, + 'timestamp' : map_timestamp, + 'bkids' : map_bkids, +} + +map_name = { + 'implicit' : 'bkids', +} + + +def field(name, entry, verbose=Verbose): + if not Print_Flags[name]: + return None + fieldname = map_name.get(name, name) + mapper = map_field.get(fieldname, default_mapper) + val = mapper(entry, name) + if verbose: + val = name + ": " + val + return val + + +def nodeinfo_raw(name, ninfo, prefix=""): + """ + This just formats the dictionary, which we would normally use str() + to do, except that we want the keys sorted for deterministic output. + """ + d = ninfo.__getstate__() + try: + keys = ninfo.field_list + ['_version_id'] + except AttributeError: + keys = sorted(d.keys()) + values = [] + for key in keys: + values.append('%s: %s' % (repr(key), repr(d.get(key)))) + if '\n' in name: + name = repr(name) + return name + ': {' + ', '.join(values) + '}' + + +def nodeinfo_cooked(name, ninfo, prefix=""): + try: + field_list = ninfo.field_list + except AttributeError: + field_list = [] + if '\n' in name: + name = repr(name) + outlist = [name + ':'] + [ + f for f in [field(x, ninfo, Verbose) for x in field_list] if f + ] + if Verbose: + sep = '\n ' + prefix + else: + sep = ' ' + return sep.join(outlist) + + +nodeinfo_string = nodeinfo_cooked + + +def printfield(name, entry, prefix=""): + outlist = field("implicit", entry, 0) + if outlist: + if Verbose: + print(" implicit:") + print(" " + outlist) + outact = field("action", entry, 0) + if outact: + if Verbose: + print(" action: " + outact) + else: + print(" " + outact) + + +def printentries(entries, location): + if Print_Entries: + for name in Print_Entries: + try: + entry = entries[name] + except KeyError: + err = "sconsign: no entry `%s' in `%s'\n" % (name, location) + sys.stderr.write(err) + else: + try: + ninfo = entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + else: + for name in sorted(entries.keys()): + entry = entries[name] + try: + entry.ninfo + except AttributeError: + print(name + ":") + else: + print(nodeinfo_string(name, entry.ninfo)) + printfield(name, entry.binfo) + + +class Do_SConsignDB(object): + def __init__(self, dbm_name, dbm): + self.dbm_name = dbm_name + self.dbm = dbm + + def __call__(self, fname): + # The *dbm modules stick their own file suffixes on the names + # that are passed in. This causes us to jump through some + # hoops here. + try: + # Try opening the specified file name. Example: + # SPECIFIED OPENED BY self.dbm.open() + # --------- ------------------------- + # .sconsign => .sconsign.dblite + # .sconsign.dblite => .sconsign.dblite.dblite + db = self.dbm.open(fname, "r") + except (IOError, OSError) as e: + print_e = e + try: + # That didn't work, so try opening the base name, + # so that if they actually passed in 'sconsign.dblite' + # (for example), the dbm module will put the suffix back + # on for us and open it anyway. + db = self.dbm.open(os.path.splitext(fname)[0], "r") + except (IOError, OSError): + # That didn't work either. See if the file name + # they specified even exists (independent of the dbm + # suffix-mangling). + try: + with open(fname, "rb"): + pass # this is a touch only, we don't use it here. + except (IOError, OSError) as e: + # Nope, that file doesn't even exist, so report that + # fact back. + print_e = e + sys.stderr.write("sconsign: %s\n" % print_e) + return + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s'\n" + % (self.dbm_name, fname)) + return + except Exception as e: + sys.stderr.write("sconsign: ignoring invalid `%s' file `%s': %s\n" + % (self.dbm_name, fname, e)) + exc_type, _, _ = sys.exc_info() + if exc_type.__name__ == "ValueError" and sys.version_info < (3,0,0): + sys.stderr.write("Python 2 only supports pickle protocols 0-2.\n") + return + + if Print_Directories: + for dir in Print_Directories: + try: + val = db[dir] + except KeyError: + err = "sconsign: no dir `%s' in `%s'\n" % (dir, args[0]) + sys.stderr.write(err) + else: + self.printentries(dir, val) + else: + for dir in sorted(db.keys()): + self.printentries(dir, db[dir]) + + @staticmethod + def printentries(dir, val): + try: + print('=== ' + dir + ':') + except TypeError: + print('=== ' + dir.decode() + ':') + printentries(pickle.loads(val), dir) + + +def Do_SConsignDir(name): + try: + with open(name, 'rb') as fp: + try: + sconsign = SCons.SConsign.Dir(fp) + except KeyboardInterrupt: + raise + except pickle.UnpicklingError: + err = "sconsign: ignoring invalid .sconsign file `%s'\n" % name + sys.stderr.write(err) + return + except Exception as e: + err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e) + sys.stderr.write(err) + return + printentries(sconsign.entries, args[0]) + except (IOError, OSError) as e: + sys.stderr.write("sconsign: %s\n" % e) + return + + +############################################################################## +def main(): + global Do_Call + global nodeinfo_string + global args + global Verbose + global Readable + + helpstr = """\ + Usage: sconsign [OPTIONS] [FILE ...] + Options: + -a, --act, --action Print build action information. + -c, --csig Print content signature information. + -d DIR, --dir=DIR Print only info about DIR. + -e ENTRY, --entry=ENTRY Print only info about ENTRY. + -f FORMAT, --format=FORMAT FILE is in the specified FORMAT. + -h, --help Print this message and exit. + -i, --implicit Print implicit dependency information. + -r, --readable Print timestamps in human-readable form. + --raw Print raw Python object representations. + -s, --size Print file sizes. + -t, --timestamp Print timestamp information. + -v, --verbose Verbose, describe each field. + """ + + try: + opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv", + ['act', 'action', + 'csig', 'dir=', 'entry=', + 'format=', 'help', 'implicit', + 'raw', 'readable', + 'size', 'timestamp', 'verbose']) + except getopt.GetoptError as err: + sys.stderr.write(str(err) + '\n') + print(helpstr) + sys.exit(2) + + for o, a in opts: + if o in ('-a', '--act', '--action'): + Print_Flags['action'] = 1 + elif o in ('-c', '--csig'): + Print_Flags['csig'] = 1 + elif o in ('-d', '--dir'): + Print_Directories.append(a) + elif o in ('-e', '--entry'): + Print_Entries.append(a) + elif o in ('-f', '--format'): + # Try to map the given DB format to a known module + # name, that we can then try to import... + Module_Map = {'dblite': 'SCons.dblite', 'sconsign': None} + dbm_name = Module_Map.get(a, a) + if dbm_name: + try: + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + except ImportError: + sys.stderr.write("sconsign: illegal file format `%s'\n" % a) + print(helpstr) + sys.exit(2) + Do_Call = Do_SConsignDB(a, dbm) + else: + Do_Call = Do_SConsignDir + elif o in ('-h', '--help'): + print(helpstr) + sys.exit(0) + elif o in ('-i', '--implicit'): + Print_Flags['implicit'] = 1 + elif o in ('--raw',): + nodeinfo_string = nodeinfo_raw + elif o in ('-r', '--readable'): + Readable = 1 + elif o in ('-s', '--size'): + Print_Flags['size'] = 1 + elif o in ('-t', '--timestamp'): + Print_Flags['timestamp'] = 1 + elif o in ('-v', '--verbose'): + Verbose = 1 + + if Do_Call: + for a in args: + Do_Call(a) + else: + if not args: + args = [".sconsign.dblite"] + for a in args: + dbm_name = my_whichdb(a) + if dbm_name: + Map_Module = {'SCons.dblite': 'dblite'} + if dbm_name != "SCons.dblite": + dbm = my_import(dbm_name) + else: + import SCons.dblite + + dbm = SCons.dblite + # Ensure that we don't ignore corrupt DB files, + # this was handled by calling my_import('SCons.dblite') + # again in earlier versions... + SCons.dblite.ignore_corrupt_dbfiles = 0 + Do_SConsignDB(Map_Module.get(dbm_name, dbm_name), dbm)(a) + else: + Do_SConsignDir(a) + + if Warns: + print("NOTE: there were %d warnings, please check output" % Warns) + + +if __name__ == "__main__": + main() + sys.exit(0) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/src/engine/SCons/Variables/PathVariable.py b/src/engine/SCons/Variables/PathVariable.py index 0ffcbc5..eb9b299 100644 --- a/src/engine/SCons/Variables/PathVariable.py +++ b/src/engine/SCons/Variables/PathVariable.py @@ -78,11 +78,13 @@ import SCons.Errors class _PathVariableClass(object): - def PathAccept(self, key, val, env): + @staticmethod + def PathAccept(key, val, env): """Accepts any path, no checking done.""" pass - def PathIsDir(self, key, val, env): + @staticmethod + def PathIsDir(key, val, env): """Validator to check if Path is a directory.""" if not os.path.isdir(val): if os.path.isfile(val): @@ -91,7 +93,8 @@ class _PathVariableClass(object): m = 'Directory path for option %s does not exist: %s' raise SCons.Errors.UserError(m % (key, val)) - def PathIsDirCreate(self, key, val, env): + @staticmethod + def PathIsDirCreate(key, val, env): """Validator to check if Path is a directory, creating it if it does not exist.""" if os.path.isfile(val): @@ -100,7 +103,8 @@ class _PathVariableClass(object): if not os.path.isdir(val): os.makedirs(val) - def PathIsFile(self, key, val, env): + @staticmethod + def PathIsFile(key, val, env): """Validator to check if Path is a file""" if not os.path.isfile(val): if os.path.isdir(val): @@ -109,7 +113,8 @@ class _PathVariableClass(object): m = 'File path for option %s does not exist: %s' raise SCons.Errors.UserError(m % (key, val)) - def PathExists(self, key, val, env): + @staticmethod + def PathExists(key, val, env): """Validator to check if Path exists""" if not os.path.exists(val): m = 'Path for option %s does not exist: %s' diff --git a/src/engine/SCons/Variables/__init__.py b/src/engine/SCons/Variables/__init__.py index 4ff11d0..7bd4cd9 100644 --- a/src/engine/SCons/Variables/__init__.py +++ b/src/engine/SCons/Variables/__init__.py @@ -45,21 +45,20 @@ from .PathVariable import PathVariable # okay class Variables(object): - instance=None - """ Holds all the options, updates the environment with the variables, and renders the help text. """ + instance=None + def __init__(self, files=None, args=None, is_global=1): """ files - [optional] List of option configuration files to load (backward compatibility) If a single string is passed it is - automatically placed in a file list + automatically placed in a file list + args - dictionary to override values set from files. """ - # initialize arguments - if files is None: - files = [] + if args is None: args = {} self.options = [] @@ -74,7 +73,7 @@ class Variables(object): # create the singleton instance if is_global: - self=Variables.instance + self = Variables.instance if not Variables.instance: Variables.instance=self |