diff options
54 files changed, 1803 insertions, 748 deletions
diff --git a/CHANGES.txt b/CHANGES.txt index 80fee1c..ecd4c60 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,18 +12,37 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER From Joseph Brill: - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception when the user specified msvc script does not exist. + - Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and + did not explicitly request the mingw tool, mingw tool initialization would fail and set the + default compiler to MSVC which wasn't installed, yielding broken build. + Updated mingw tool so that the generate and exists methods use the same mingw search paths + (issue #4134). + - Update the debug output written to stdout for MSVC initialization which is enabled by setting + SCONS_MSCOMMON_DEBUG=- to use the logging module. Also changed the debug output format + written to stdout to include more information about the source for each message of MSVC + initialization debugging output. A single space was added before the message for all + debugging output records written to stdout and to files. + - Refactor the data definitions for msvc configurations to allow derived data structures to be + constructed during initialization that removes the need for special case handling during + runtime execution. Special case handling of host/target combinations is eliminated and + replaced with pre-computed search lists that implicitly handle the differences between full + versions and express versions of msvc. This fixes an issue where Express versions of the MSVC + compiler were not detected due to differences in initial msvc detection and msvc batch file + determination when configuring the build environment. This could lead to build failures when + only an MSVC Express instance is installed and the MSVC version is not explicitly specified + (issue #2668 and issue #2697). From William Deegan: - Fix check for unsupported Python version. It was broken. Also now the error message will include what is the minimum supported version of Python - Fix ActionTests to work with python 3.10.1 (and higher) NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) - Action._subproc() can now be used as a python context manager to ensure that the - POpen object is properly closed. + POpen object is properly closed. (Thanks to Mats Wichmann for catching that DummyPopen needed additional logic) - Added project_url for mailing lists and Discord - Updated project url in steup.cfg to be https instead of http @@ -44,6 +63,29 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Ninja: Fix issue where Configure files weren't being properly processed when build run via ninja. - Added ninja mingw support and improved ninja CommandGeneratorAction support. + - Update ninja file generation to only create response files for build commands + which exceed MAXLINELENGTH + - Ninja: Added NINJA_GENERATED_SOURCE_ALIAS_NAME which allows user to specify an + Alias() which the ninja tool can use to determine which files are generated sources. + If this is not set by the user then the ninja tool will still dynamically determine + which files are generated sources based on NINJA_GENERATED_SOURCE_SUFFIXES, and create + a phony target _ninja_generated_sources. Generated sources will be built first by + ninja. This is needed because ninja cannot determine which generated sources are + required by other build targets. Code contributed by MongoDB + The downstream commit is here: + https://github.com/mongodb/mongo/commit/2fef432fa6e7cf3fd4f22ba3b193222c2887f14f + - Added special case for ninja scons daemon to work in win32 python3.6 environments. + This particular environment does a bad job managing popen standard file handles, so + some special workarounds are needed. + - Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected + format. Compiler tools will also configure the variable automatically. + - Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define + a function which will be called to generate or alter the execution environment which will + be used in the shell command of some Action. + - Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. + - Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry + to connect to the server during start up. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to @@ -81,6 +123,13 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER or the super() two-argument syntax. - Renamed ParseFlag's internal data structure to "mapping" instead of "dict" (avoid redefining builtin) + - Fix an old use-before-set bug in tex tool (issue #2888) + - Fix a test harness exception returning stderr if a wait_for timed out. + - ParseConfig now correctly passes the *unique* flag to a user-supplied + flag-merging function. + - Restore the ability of the content-timestamp decider to see that a + a source which is a symlink has changed if the file-system target of + that link has been modified (issue #3880) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. diff --git a/RELEASE.txt b/RELEASE.txt index aed58b8..15b78dc 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -7,7 +7,7 @@ on the SCons download page: Here is a summary of the changes since 4.3.1: NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), you may - see unexpected rebuilds. This is due to Python internals changing which changed + see unexpected rebuilds. This is due to Python internals changing which changed the signature of a Python Action Function. @@ -16,6 +16,9 @@ NEW FUNCTIONALITY - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. - Added Configure.CheckMember() checker to check if struct/class has the specified member. +- Added SHELL_ENV_GENERATOR construction variables. This variable allows the user to Define + a function which will be called to generate or alter the execution environment which will + be used in the shell command of some Action. DEPRECATED FUNCTIONALITY @@ -49,7 +52,13 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY changes have happened. The cache file can still be manually removed if there are issues to force a regen. The default cache filename now has a .json suffix - the contents have always been json. - +- Update ninja file generation to only create response files for build commands + which exceed MAXLINELENGTH +- Update the debug output written to stdout for MSVC initialization which is enabled + by setting SCONS_MSCOMMON_DEBUG=- to use the logging module. Also changed the debug + output format written to stdout to include more information about the source for each + message of MSVC initialization debugging output. A single space was added before the + message for all debugging output records written to stdout and to files. FIXES ----- @@ -65,6 +74,33 @@ FIXES with no defaults, so old usage where _defines() was called without source and target arguments would yield an exception. This issue was found via qt4 and qt5 tools in scons-contrib https://github.com/SCons/scons-contrib/issues/45 +- Fix issue where if you only had mingw installed on a Windows system and no MSVC compiler, and + did not explicitly request the mingw tool, mingw tool initialization would fail and set the + default compiler to MSVC which wasn't installed, yielding broken build. + Updated mingw tool so that the generate and exists methods use the same mingw search paths + (issue #4134). +- Ninja: Added NINJA_GENERATED_SOURCE_ALIAS_NAME which allows user to specify an + Alias() which the ninja tool can use to determine which files are generated sources. + If this is not set by the user then the ninja tool will still dynamically determine + which files are generated sources based on NINJA_GENERATED_SOURCE_SUFFIXES, and create + a phony target _ninja_generated_sources. Generated sources will be built first by + ninja. This is needed because ninja cannot determine which generated sources are + required by other build targets. Code contributed by MongoDB. +- Added special case for ninja scons daemon to work in win32 python3.6 environments. + This particular environment does a bad job managing popen standard file handles, so + some special workarounds are needed. +- Added user configurable setting of ninja depfile format via NINJA_DEPFILE_PARSE_FORMAT. + Now setting NINJA_DEPFILE_PARSE_FORMAT to [msvc,gcc,clang] can force the ninja expected + format. Compiler tools will also configure the variable automatically. +- Fix issue where Express versions of the MSVC compiler were not detected due to differences + in initial msvc detection and msvc batch file determination when configuring the build + environment. This could lead to build failures when only an MSVC Express instance is installed + and the MSVC version is not explicitly specified (issue #2668 and issue #2697). +- Restore the ability of the content-timestamp decider to see that a + a source which is a symlink has changed if the file-system target of + that link has been modified (issue #3880) +- Fix typo in ninja scons daemon startup which causes ConnectionRefusedError to not retry + to connect to the server during start up. IMPROVEMENTS ------------ @@ -75,6 +111,11 @@ IMPROVEMENTS - Added ninja mingw support and improved ninja CommandGeneratorAction support. - Command-line help is now sensitive to the size of the terminal window: the width of the help text will scale for terminals other than 80 chars wide. +- Refactor the msvc code so that the same data structures are used during initial msvc detection + and msvc batch file determination when configuring the build environment. Simplify the msvc + code by eliminating special case handling primarily due to the differences between the full + versions and express versions of visual studio. +- Updated ninja scons daemon scripts to output errors to stderr as well as the daemon log. PACKAGING --------- diff --git a/SCons/Action.py b/SCons/Action.py index 81dc033..0849178 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -732,7 +732,7 @@ def _string_from_cmd_list(cmd_list): default_ENV = None -def get_default_ENV(env): +def get_default_ENV(env, target=None, source=None): """ A fiddlin' little function that has an 'import SCons.Environment' which can't be moved to the top level without creating an import loop. Since @@ -799,25 +799,25 @@ def _subproc(scons_env, cmd, error='ignore', **kw): if error == 'raise': raise # return a dummy Popen instance that only returns error class dummyPopen: - def __init__(self, e): + def __init__(self, e): self.exception = e # Add the following two to enable using the return value as a context manager - # for example + # for example # with Action._subproc(...) as po: # logic here which uses po - def __enter__(self): + def __enter__(self): return self - def __exit__(self, *args): + def __exit__(self, *args): pass - def communicate(self, input=None): + def communicate(self, input=None): return ('', '') - def wait(self): + def wait(self): return -self.exception.errno - + stdin = None class f: def read(self): return '' @@ -924,9 +924,10 @@ class CommandAction(_ActionAction): escape = env.get('ESCAPE', lambda x: x) - ENV = get_default_ENV(env) + ENV = env.get('SHELL_ENV_GENERATOR', get_default_ENV)(env, target, source) # Ensure that the ENV values are all strings: + for key, value in ENV.items(): if not is_String(value): if is_List(value): diff --git a/SCons/Action.xml b/SCons/Action.xml index 0e3ef33..2c18d55 100644 --- a/SCons/Action.xml +++ b/SCons/Action.xml @@ -200,4 +200,34 @@ in which the command should be executed. </summary> </cvar> +<cvar name="SHELL_ENV_GENERATOR"> + <summary> + <para> +A function to generate or alter the environment dictionary which will be used +when executing the &cv-link-SPAWN; function. This primarily give the +user a chance to customize the execution environment for particular Actions. +It must return a dictionary containing the environment variables as +keys and the values as values. + </para> + + <example_commands> +def custom_shell_env(env, target, source): + </example_commands> + + <para> + <varname>env</varname> +The SCons construction environment from which the +execution environment can be derived from. + </para> + <para> + <varname>target</varname> +The list of targets associated with this action. + </para> + <para> + <varname>source</varname> +The list of sources associated with this action. + </para> + </summary> +</cvar> + </sconsdoc> diff --git a/SCons/Environment.py b/SCons/Environment.py index 3507263..c38b51c 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -558,20 +558,38 @@ class SubstitutionEnvironment: subst_target_source = subst - def backtick(self, command): + + def backtick(self, command) -> str: + """Emulate command substitution. + + Provides behavior conceptually like POSIX Shell notation + for running a command in backquotes (backticks) by running + ``command`` and returning the resulting output string. + + This is not really a public API any longer, it is provided for the + use of :meth:`ParseFlags` (which supports it using a syntax of + !command) and :meth:`ParseConfig`. + + Raises: + OSError: if the external command returned non-zero exit status. + """ + import subprocess + # common arguments - kw = { 'stdin' : 'devnull', - 'stdout' : subprocess.PIPE, - 'stderr' : subprocess.PIPE, - 'universal_newlines' : True, - } + kw = { + "stdin": "devnull", + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "universal_newlines": True, + } # if the command is a list, assume it's been quoted # othewise force a shell - if not is_List(command): kw['shell'] = True + if not is_List(command): + kw["shell"] = True # run constructed command p = SCons.Action._subproc(self, command, **kw) - out,err = p.communicate() + out, err = p.communicate() status = p.wait() if err: sys.stderr.write("" + err) @@ -579,6 +597,7 @@ class SubstitutionEnvironment: raise OSError("'%s' exited %d" % (command, status)) return out + def AddMethod(self, function, name=None): """ Adds the specified function as a method of this construction @@ -622,12 +641,12 @@ class SubstitutionEnvironment: env.MergeFlags(merges) return env - def ParseFlags(self, *flags): + def ParseFlags(self, *flags) -> dict: """Return a dict of parsed flags. Parse ``flags`` and return a dict with the flags distributed into the appropriate construction variable names. The flags are treated - as a typical set of command-line flags for a GNU-like toolchain, + as a typical set of command-line flags for a GNU-style toolchain, such as might have been generated by one of the {foo}-config scripts, and used to populate the entries based on knowledge embedded in this method - the choices are not expected to be portable to other @@ -815,18 +834,19 @@ class SubstitutionEnvironment: do_parse(arg) return mapping - def MergeFlags(self, args, unique=True): + def MergeFlags(self, args, unique=True) -> None: """Merge flags into construction variables. Merges the flags from ``args`` into this construction environent. - If ``args`` is not a dict, it is first converted to a dictionary with + If ``args`` is not a dict, it is first converted to one with flags distributed into appropriate construction variables. See :meth:`ParseFlags`. Args: args: flags to merge - unique: merge flags rather than appending (default: True) - + unique: merge flags rather than appending (default: True). + When merging, path variables are retained from the front, + other construction variables from the end. """ if not is_Dict(args): args = self.ParseFlags(args) @@ -1627,25 +1647,32 @@ class Base(SubstitutionEnvironment): if name[:len(prefix)] == prefix and name[-len(suffix):] == suffix: return path + def ParseConfig(self, command, function=None, unique=True): - """ - Use the specified function to parse the output of the command - in order to modify the current environment. The 'command' can - be a string or a list of strings representing a command and - its arguments. 'Function' is an optional argument that takes - the environment, the output of the command, and the unique flag. - If no function is specified, MergeFlags, which treats the output - as the result of a typical 'X-config' command (i.e. gtk-config), - will merge the output into the appropriate variables. + """Parse the result of running a command to update construction vars. + + Use ``function`` to parse the output of running ``command`` + in order to modify the current environment. + + Args: + command: a string or a list of strings representing a command + and its arguments. + function: called to process the result of ``command``, which will + be passed as ``args``. If ``function`` is omitted or ``None``, + :meth:`MergeFlags` is used. Takes 3 args ``(env, args, unique)`` + unique: whether no duplicate values are allowed (default true) """ if function is None: + def parse_conf(env, cmd, unique=unique): return env.MergeFlags(cmd, unique) + function = parse_conf if is_List(command): command = ' '.join(command) command = self.subst(command) - return function(self, self.backtick(command)) + return function(self, self.backtick(command), unique) + def ParseDepends(self, filename, must_exist=None, only_one=False): """ diff --git a/SCons/Environment.xml b/SCons/Environment.xml index 5b41f22..3a6df97 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -2204,12 +2204,8 @@ not as separate arguments to </para> <para> -By default, -duplicate values are eliminated; -you can, however, specify -<literal>unique=False</literal> -to allow duplicate -values to be added. +If <literal>unique</literal> is true (the default), +duplicate values are not stored. When eliminating duplicate values, any &consvars; that end with the string @@ -2217,6 +2213,8 @@ the string keep the left-most unique value. All other &consvars; keep the right-most unique value. +If <literal>unique</literal> is false, +values are added even if they are duplicates. </para> <para> @@ -2233,9 +2231,13 @@ env.MergeFlags(['!pkg-config gtk+-2.0 --cflags', '-O3']) # Combine an optimization flag with the flags returned from running pkg-config # twice and merge the result into the construction variables. -env.MergeFlags(['-O3', - '!pkg-config gtk+-2.0 --cflags --libs', - '!pkg-config libpng12 --cflags --libs']) +env.MergeFlags( + [ + '-O3', + '!pkg-config gtk+-2.0 --cflags --libs', + '!pkg-config libpng12 --cflags --libs', + ] +) </example_commands> </summary> </scons_function> @@ -2347,15 +2349,13 @@ NoClean(env.Program('hello', 'hello.c')) <summary> <para> Updates the current &consenv; with the values extracted -from the output from running external <parameter>command</parameter>, -by calling a helper function <parameter>function</parameter> -which understands -the output of <parameter>command</parameter>. +from the output of running external <parameter>command</parameter>, +by passing it to a helper <parameter>function</parameter>. <parameter>command</parameter> may be a string or a list of strings representing the command and its arguments. If <parameter>function</parameter> -is not given, +is omitted or <constant>None</constant>, &f-link-env-MergeFlags; is used. By default, duplicate values are not @@ -2366,33 +2366,32 @@ to allow duplicate values to be added. </para> <para> -If &f-env-MergeFlags; is used, -it expects a response in the style of a -<command>*-config</command> -command typical of the POSIX programming environment -(for example, -<application>gtk-config</application>) -and adds the options -to the appropriate construction variables. -Interpreted options -and the construction variables they affect -are as specified for the -&f-link-env-ParseFlags; -method (which -&f-env-MergeFlags; calls). -See that method's description -for a table of options and corresponding construction variables. +<parameter>command</parameter> is executed using the +SCons execution environment (that is, the &consvar; +&cv-link-ENV; in the current &consenv;). +If <parameter>command</parameter> needs additional information +to operate properly, that needs to be set in the execution environment. +For example, <command>pkg-config</command> +may need a custom value set in the <envar>PKG_CONFIG_PATH</envar> +environment variable. </para> <para> -If &f-env-MergeFlags; cannot interpret the results of +&f-env-MergeFlags; needs to understand +the output produced by <parameter>command</parameter> +in order to distribute it to appropriate &consvars;. +&f-env-MergeFlags; uses a separate function to +do that processing - +see &f-link-env-ParseFlags; for the details, including a +a table of options and corresponding construction variables. +To provide alternative processing of the output of <parameter>command</parameter>, you can suppply a custom -<parameter>function</parameter> to do so. -<parameter>function</parameter> -must accept three arguments: -the &consenv; to modify, the string returned -by running <parameter>command</parameter>, +<parameter>function</parameter>, +which must accept three arguments: +the &consenv; to modify, +a string argument containing the output from running +<parameter>command</parameter>, and the optional <parameter>unique</parameter> flag. </para> @@ -2405,8 +2404,7 @@ and the optional </arguments> <summary> <para> -Parses the contents of the specified -<parameter>filename</parameter> +Parses the contents of <parameter>filename</parameter> as a list of dependencies in the style of &Make; or @@ -2417,27 +2415,21 @@ and explicitly establishes all of the listed dependencies. <para> By default, it is not an error -if the specified -<parameter>filename</parameter> +if <parameter>filename</parameter> does not exist. The optional <parameter>must_exist</parameter> -argument may be set to a non-zero -value to have -scons -throw an exception and -generate an error if the file does not exist, +argument may be set to <constant>True</constant> +to have &SCons; +raise an exception if the file does not exist, or is otherwise inaccessible. </para> <para> The optional <parameter>only_one</parameter> -argument may be set to a non-zero -value to have -scons -thrown an exception and -generate an error +argument may be set to <constant>True</constant> +to have &SCons; raise an exception if the file contains dependency information for more than one target. This can provide a small sanity check @@ -2453,7 +2445,6 @@ file. </para> <para> -The <parameter>filename</parameter> and all of the files listed therein will be interpreted relative to @@ -2473,10 +2464,10 @@ function. <summary> <para> Parses one or more strings containing -typical command-line flags for GCC tool chains +typical command-line flags for GCC-style tool chains and returns a dictionary with the flag values separated into the appropriate SCons construction variables. -This is intended as a companion to the +Intended as a companion to the &f-link-env-MergeFlags; method, but allows for the values in the returned dictionary to be modified, if necessary, @@ -2491,11 +2482,20 @@ directly unless you want to manipulate the values.) <para> If the first character in any string is -an exclamation mark (!), +an exclamation mark (<literal>!</literal>), the rest of the string is executed as a command, and the output from the command is parsed as GCC tool chain command-line flags and added to the resulting dictionary. +This can be used to call a <filename>*-config</filename> +command typical of the POSIX programming environment +(for example, +<command>pkg-config</command>). +Note that such a comamnd is executed using the +SCons execution environment; +if the command needs additional information, +that information needs to be explcitly provided. +See &f-link-ParseConfig; for more details. </para> <para> diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index 8359405..8bf327a 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -2075,6 +2075,10 @@ def generate(env): orig_backtick = env.backtick class my_backtick: + """mocked backtick routine so command is not actually issued. + + Just returns the string it was given. + """ def __init__(self, save_command, output): self.save_command = save_command self.output = output @@ -2138,6 +2142,32 @@ def generate(env): finally: env.backtick = orig_backtick + # check that we can pass our own function, + # and that it works for both values of unique + + def my_function(myenv, flags, unique=True): + import json + + args = json.loads(flags) + if unique: + myenv.AppendUnique(**args) + else: + myenv.Append(**args) + + json_str = '{"LIBS": ["yyy", "xxx", "yyy"]}' + + env = Environment(LIBS=['xxx']) + env2 = env.Clone() + env.backtick = my_backtick([], json_str) + env2.backtick = my_backtick([], json_str) + + env.ParseConfig("foo", my_function) + assert env['LIBS'] == ['xxx', 'yyy'], env['LIBS'] + + env2.ParseConfig("foo2", my_function, unique=False) + assert env2['LIBS'] == ['xxx', 'yyy', 'xxx', 'yyy'], env2['LIBS'] + + def test_ParseDepends(self): """Test the ParseDepends() method""" test = TestCmd.TestCmd(workdir = '') diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index c1bfd6a..b4de337 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -255,7 +255,7 @@ else: def _copy_func(fs, src, dest): shutil.copy2(src, dest) st = fs.stat(src) - fs.chmod(dest, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) + fs.chmod(dest, stat.S_IMODE(st.st_mode) | stat.S_IWRITE) Valid_Duplicates = ['hard-soft-copy', 'soft-hard-copy', @@ -733,10 +733,7 @@ class Base(SCons.Node.Node): return SCons.Node._rexists_map[self._func_rexists](self) def getmtime(self): - if self.islink(): - st = self.lstat() - else: - st = self.stat() + st = self.stat() if st: return st[stat.ST_MTIME] @@ -744,28 +741,25 @@ class Base(SCons.Node.Node): return None def getsize(self): - if self.islink(): - st = self.lstat() - else: - st = self.stat() + st = self.stat() if st: - return st[stat.ST_SIZE] + return st.st_size else: return None def isdir(self): st = self.stat() - return st is not None and stat.S_ISDIR(st[stat.ST_MODE]) + return st is not None and stat.S_ISDIR(st.st_mode) def isfile(self): st = self.stat() - return st is not None and stat.S_ISREG(st[stat.ST_MODE]) + return st is not None and stat.S_ISREG(st.st_mode) if hasattr(os, 'symlink'): def islink(self): st = self.lstat() - return st is not None and stat.S_ISLNK(st[stat.ST_MODE]) + return st is not None and stat.S_ISLNK(st.st_mode) else: def islink(self): return False # no symlinks @@ -1939,7 +1933,7 @@ class Dir(Base): return self.srcdir return Base.srcnode(self) - def get_timestamp(self): + def get_timestamp(self) -> int: """Return the latest timestamp from among our children""" stamp = 0 for kid in self.children(): @@ -1947,11 +1941,11 @@ class Dir(Base): stamp = kid.get_timestamp() return stamp - def get_abspath(self): + def get_abspath(self) -> str: """Get the absolute path of the file.""" return self._abspath - def get_labspath(self): + def get_labspath(self) -> str: """Get the absolute path of the file.""" return self._labspath diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index b93efc8..a2400f2 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -408,8 +408,8 @@ class VariantDirTestCase(unittest.TestCase): None) os.chmod(test.workpath('src/foo'), stat.S_IRUSR | stat.S_IWRITE) st = os.stat(test.workpath('build/foo')) - assert (stat.S_IMODE(st[stat.ST_MODE]) & stat.S_IWRITE), \ - stat.S_IMODE(st[stat.ST_MODE]) + assert (stat.S_IMODE(st.st_mode) & stat.S_IWRITE), \ + stat.S_IMODE(st.st_mode) # This used to generate a UserError when we forbid the source # directory from being outside the top-level SConstruct dir. @@ -771,7 +771,7 @@ class FileNodeInfoTestCase(_tempdirTestCase): mtime = st[stat.ST_MTIME] assert ni.timestamp == mtime, (ni.timestamp, mtime) - size = st[stat.ST_SIZE] + size = st.st_size assert ni.size == size, (ni.size, size) import time @@ -783,7 +783,7 @@ class FileNodeInfoTestCase(_tempdirTestCase): mtime = st[stat.ST_MTIME] assert ni.timestamp != mtime, (ni.timestamp, mtime) - size = st[stat.ST_SIZE] + size = st.st_size assert ni.size != size, (ni.size, size) diff --git a/SCons/Tool/MSCommon/common.py b/SCons/Tool/MSCommon/common.py index be042f8..bc7cad3 100644 --- a/SCons/Tool/MSCommon/common.py +++ b/SCons/Tool/MSCommon/common.py @@ -39,14 +39,7 @@ import SCons.Util # SCONS_MSCOMMON_DEBUG is internal-use so undocumented: # set to '-' to print to console, else set to filename to log to LOGFILE = os.environ.get('SCONS_MSCOMMON_DEBUG') -if LOGFILE == '-': - def debug(message, *args): - if args: - print(message % args) - else: - print(message) - -elif LOGFILE: +if LOGFILE: import logging modulelist = ( # root module and parent/root module @@ -73,17 +66,24 @@ elif LOGFILE: relfilename = relfilename.replace('\\', '/') record.relfilename = relfilename return True + # Log format looks like: + # 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [file] + # debug: 00109ms:MSCommon/vc.py:find_vc_pdir#447: VC found '14.3' [stdout] + log_format=( + '%(relativeCreated)05dms' + ':%(relfilename)s' + ':%(funcName)s' + '#%(lineno)s' + ': %(message)s' + ) + if LOGFILE == '-': + log_format = 'debug: ' + log_format + log_handler = logging.StreamHandler(sys.stdout) + else: + log_handler = logging.FileHandler(filename=LOGFILE) logging.basicConfig( - # This looks like: - # 00109ms:MSCommon/vc.py:find_vc_pdir#447:VC found '14.3' - format=( - '%(relativeCreated)05dms' - ':%(relfilename)s' - ':%(funcName)s' - '#%(lineno)s' - ':%(message)s' - ), - filename=LOGFILE, + format=log_format, + handlers=[log_handler], level=logging.DEBUG) logger = logging.getLogger(name=__name__) logger.addFilter(_Debug_Filter()) diff --git a/SCons/Tool/MSCommon/vc.py b/SCons/Tool/MSCommon/vc.py index 64d5d38..525051c 100644 --- a/SCons/Tool/MSCommon/vc.py +++ b/SCons/Tool/MSCommon/vc.py @@ -46,6 +46,11 @@ import platform from pathlib import Path from string import digits as string_digits from subprocess import PIPE +import re +from collections import ( + namedtuple, + OrderedDict, +) import SCons.Util import SCons.Warnings @@ -97,84 +102,302 @@ _ARCH_TO_CANONICAL = { "aarch64" : "arm64", } -# Starting with 14.1 (aka VS2017), the tools are organized by host directory. -# subdirs for each target. They are now in .../VC/Auxuiliary/Build. -# Note 2017 Express uses Hostx86 even if it's on 64-bit Windows, -# not reflected in this table. -_HOST_TARGET_TO_CL_DIR_GREATER_THAN_14 = { - ("amd64","amd64") : ("Hostx64","x64"), - ("amd64","x86") : ("Hostx64","x86"), - ("amd64","arm") : ("Hostx64","arm"), - ("amd64","arm64") : ("Hostx64","arm64"), - ("x86","amd64") : ("Hostx86","x64"), - ("x86","x86") : ("Hostx86","x86"), - ("x86","arm") : ("Hostx86","arm"), - ("x86","arm64") : ("Hostx86","arm64"), -} +# The msvc batch files report errors via stdout. The following +# regular expression attempts to match known msvc error messages +# written to stdout. +re_script_output_error = re.compile( + r'^(' + r'|'.join([ + r'VSINSTALLDIR variable is not set', # 2002-2003 + r'The specified configuration type is missing', # 2005+ + r'Error in script usage', # 2005+ + r'ERROR\:', # 2005+ + r'\!ERROR\!', # 2015-2015 + r'\[ERROR\:', # 2017+ + r'\[ERROR\]', # 2017+ + r'Syntax\:', # 2017+ + ]) + r')' +) + +# Lists of compatible host/target combinations are derived from a set of defined +# constant data structures for each host architecture. The derived data structures +# implicitly handle the differences in full versions and express versions of visual +# studio. The host/target combination search lists are contructed in order of +# preference. The construction of the derived data structures is independent of actual +# visual studio installations. The host/target configurations are used in both the +# initial msvc detection and when finding a valid batch file for a given host/target +# combination. +# +# HostTargetConfig description: +# +# label: +# Name used for identification. +# +# host_all_hosts: +# Defined list of compatible architectures for each host architecture. +# +# host_all_targets: +# Defined list of target architectures for each host architecture. +# +# host_def_targets: +# Defined list of default target architectures for each host architecture. +# +# all_pairs: +# Derived list of all host/target combination tuples. +# +# host_target_map: +# Derived list of all compatible host/target combinations for each +# supported host/target combination. +# +# host_all_targets_map: +# Derived list of all compatible host/target combinations for each +# supported host. This is used in the initial check that cl.exe exists +# in the requisite visual studio vc host/target directory for a given host. +# +# host_def_targets_map: +# Derived list of default compatible host/target combinations for each +# supported host. This is used for a given host when the user does not +# request a target archicture. +# +# target_host_map: +# Derived list of compatible host/target combinations for each supported +# target/host combination. This is used for a given host and target when +# the user requests a target architecture. + +_HOST_TARGET_CONFIG_NT = namedtuple("HostTargetConfig", [ + # defined + "label", # name for debugging/output + "host_all_hosts", # host_all_hosts[host] -> host_list + "host_all_targets", # host_all_targets[host] -> target_list + "host_def_targets", # host_def_targets[host] -> target_list + # derived + "all_pairs", # host_target_list + "host_target_map", # host_target_map[host][target] -> host_target_list + "host_all_targets_map", # host_all_targets_map[host][target] -> host_target_list + "host_def_targets_map", # host_def_targets_map[host][target] -> host_target_list + "target_host_map", # target_host_map[target][host] -> host_target_list +]) + +def _host_target_config_factory(*, label, host_all_hosts, host_all_targets, host_def_targets): + + def _make_host_target_map(all_hosts, all_targets): + # host_target_map[host][target] -> host_target_list + host_target_map = {} + for host, host_list in all_hosts.items(): + host_target_map[host] = {} + for host_platform in host_list: + for target_platform in all_targets[host_platform]: + if target_platform not in host_target_map[host]: + host_target_map[host][target_platform] = [] + host_target_map[host][target_platform].append((host_platform, target_platform)) + return host_target_map + + def _make_host_all_targets_map(all_hosts, host_target_map, all_targets): + # host_all_target_map[host] -> host_target_list + # special host key '_all_' contains all (host,target) combinations + all = '_all_' + host_all_targets_map = {} + host_all_targets_map[all] = [] + for host, host_list in all_hosts.items(): + host_all_targets_map[host] = [] + for host_platform in host_list: + # all_targets[host_platform]: all targets for compatible host + for target in all_targets[host_platform]: + for host_target in host_target_map[host_platform][target]: + for host_key in (host, all): + if host_target not in host_all_targets_map[host_key]: + host_all_targets_map[host_key].append(host_target) + return host_all_targets_map + + def _make_host_def_targets_map(all_hosts, host_target_map, def_targets): + # host_def_targets_map[host] -> host_target_list + host_def_targets_map = {} + for host, host_list in all_hosts.items(): + host_def_targets_map[host] = [] + for host_platform in host_list: + # def_targets[host]: default targets for true host + for target in def_targets[host]: + for host_target in host_target_map[host_platform][target]: + if host_target not in host_def_targets_map[host]: + host_def_targets_map[host].append(host_target) + return host_def_targets_map + + def _make_target_host_map(all_hosts, host_all_targets_map): + # target_host_map[target][host] -> host_target_list + target_host_map = {} + for host_platform in all_hosts.keys(): + for host_target in host_all_targets_map[host_platform]: + _, target = host_target + if target not in target_host_map: + target_host_map[target] = {} + if host_platform not in target_host_map[target]: + target_host_map[target][host_platform] = [] + if host_target not in target_host_map[target][host_platform]: + target_host_map[target][host_platform].append(host_target) + return target_host_map + + host_target_map = _make_host_target_map(host_all_hosts, host_all_targets) + host_all_targets_map = _make_host_all_targets_map(host_all_hosts, host_target_map, host_all_targets) + host_def_targets_map = _make_host_def_targets_map(host_all_hosts, host_target_map, host_def_targets) + target_host_map = _make_target_host_map(host_all_hosts, host_all_targets_map) + + all_pairs = host_all_targets_map['_all_'] + del host_all_targets_map['_all_'] + + host_target_cfg = _HOST_TARGET_CONFIG_NT( + label = label, + host_all_hosts = dict(host_all_hosts), + host_all_targets = host_all_targets, + host_def_targets = host_def_targets, + all_pairs = all_pairs, + host_target_map = host_target_map, + host_all_targets_map = host_all_targets_map, + host_def_targets_map = host_def_targets_map, + target_host_map = target_host_map, + ) + + return host_target_cfg + +# 14.1 (VS2017) and later + +# Given a (host, target) tuple, return a tuple containing the batch file to +# look for and a tuple of path components to find cl.exe. We can't rely on returning +# an arg to use for vcvarsall.bat, because that script will run even if given +# a host/target pair that isn't installed. +# +# Starting with 14.1 (VS2017), the batch files are located in directory +# <VSROOT>/VC/Auxiliary/Build. The batch file name is the first value of the +# stored tuple. +# +# The build tools are organized by host and target subdirectories under each toolset +# version directory. For example, <VSROOT>/VC/Tools/MSVC/14.31.31103/bin/Hostx64/x64. +# The cl path fragment under the toolset version folder is the second value of +# the stored tuple. -# before 14.1 (VS2017): the original x86 tools are in the tools dir, -# any others are in a subdir named by the host/target pair, -# or just a single word if host==target -_HOST_TARGET_TO_CL_DIR = { - ("amd64","amd64") : "amd64", - ("amd64","x86") : "amd64_x86", - ("amd64","arm") : "amd64_arm", - ("amd64","arm64") : "amd64_arm64", - ("x86","amd64") : "x86_amd64", - ("x86","x86") : "", - ("x86","arm") : "x86_arm", - ("x86","arm64") : "x86_arm64", - ("arm","arm") : "arm", -} +_GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS = { + + ('amd64', 'amd64') : ('vcvars64.bat', ('bin', 'Hostx64', 'x64')), + ('amd64', 'x86') : ('vcvarsamd64_x86.bat', ('bin', 'Hostx64', 'x86')), + ('amd64', 'arm') : ('vcvarsamd64_arm.bat', ('bin', 'Hostx64', 'arm')), + ('amd64', 'arm64') : ('vcvarsamd64_arm64.bat', ('bin', 'Hostx64', 'arm64')), + + ('x86', 'amd64') : ('vcvarsx86_amd64.bat', ('bin', 'Hostx86', 'x64')), + ('x86', 'x86') : ('vcvars32.bat', ('bin', 'Hostx86', 'x86')), + ('x86', 'arm') : ('vcvarsx86_arm.bat', ('bin', 'Hostx86', 'arm')), + ('x86', 'arm64') : ('vcvarsx86_arm64.bat', ('bin', 'Hostx86', 'arm64')), -# 14.1 (VS2017) and later: -# Given a (host, target) tuple, return the batch file to look for. -# We can't rely on returning an arg to use for vcvarsall.bat, -# because that script will run even if given a pair that isn't installed. -# Targets that already look like a pair are pseudo targets that -# effectively mean to skip whatever the host was specified as. -_HOST_TARGET_TO_BAT_ARCH_GT14 = { - ("amd64", "amd64"): "vcvars64.bat", - ("amd64", "x86"): "vcvarsamd64_x86.bat", - ("amd64", "x86_amd64"): "vcvarsx86_amd64.bat", - ("amd64", "x86_x86"): "vcvars32.bat", - ("amd64", "arm"): "vcvarsamd64_arm.bat", - ("amd64", "x86_arm"): "vcvarsx86_arm.bat", - ("amd64", "arm64"): "vcvarsamd64_arm64.bat", - ("amd64", "x86_arm64"): "vcvarsx86_arm64.bat", - ("x86", "x86"): "vcvars32.bat", - ("x86", "amd64"): "vcvarsx86_amd64.bat", - ("x86", "x86_amd64"): "vcvarsx86_amd64.bat", - ("x86", "arm"): "vcvarsx86_arm.bat", - ("x86", "x86_arm"): "vcvarsx86_arm.bat", - ("x86", "arm64"): "vcvarsx86_arm64.bat", - ("x86", "x86_arm64"): "vcvarsx86_arm64.bat", } -# before 14.1 (VS2017): -# Given a (host, target) tuple, return the argument for the bat file; -# Both host and target should be canoncalized. -# If the target already looks like a pair, return it - these are -# pseudo targets (mainly used by Express versions) -_HOST_TARGET_ARCH_TO_BAT_ARCH = { - ("x86", "x86"): "x86", - ("x86", "amd64"): "x86_amd64", - ("x86", "x86_amd64"): "x86_amd64", - ("amd64", "x86_amd64"): "x86_amd64", # This is present in (at least) VS2012 express - ("amd64", "amd64"): "amd64", - ("amd64", "x86"): "x86", - ("amd64", "x86_x86"): "x86", - ("x86", "ia64"): "x86_ia64", # gone since 14.0 - ("x86", "arm"): "x86_arm", # since 14.0 - ("x86", "arm64"): "x86_arm64", # since 14.1 - ("amd64", "arm"): "amd64_arm", # since 14.0 - ("amd64", "arm64"): "amd64_arm64", # since 14.1 - ("x86", "x86_arm"): "x86_arm", # since 14.0 - ("x86", "x86_arm64"): "x86_arm64", # since 14.1 - ("amd64", "x86_arm"): "x86_arm", # since 14.0 - ("amd64", "x86_arm64"): "x86_arm64", # since 14.1 +_GE2017_HOST_TARGET_CFG = _host_target_config_factory( + + label = 'GE2017', + + host_all_hosts = OrderedDict([ + ('amd64', ['amd64', 'x86']), + ('x86', ['x86']), + ('arm64', ['amd64', 'x86']), + ('arm', ['x86']), + ]), + + host_all_targets = { + 'amd64': ['amd64', 'x86', 'arm64', 'arm'], + 'x86': ['x86', 'amd64', 'arm', 'arm64'], + 'arm64': [], + 'arm': [], + }, + + host_def_targets = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm64': ['arm64', 'arm'], + 'arm': ['arm'], + }, + +) + +# debug("_GE2017_HOST_TARGET_CFG: %s", _GE2017_HOST_TARGET_CFG) + +# 14.0 (VS2015) to 8.0 (VS2005) + +# Given a (host, target) tuple, return a tuple containing the argument for +# the batch file and a tuple of the path components to find cl.exe. +# +# In 14.0 (VS2015) and earlier, the original x86 tools are in the tools +# bin directory (i.e., <VSROOT>/VC/bin). Any other tools are in subdirectory +# named for the the host/target pair or a single name if the host==target. + +_LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS = { + + ('amd64', 'amd64') : ('amd64', ('bin', 'amd64')), + ('amd64', 'x86') : ('amd64_x86', ('bin', 'amd64_x86')), + ('amd64', 'arm') : ('amd64_arm', ('bin', 'amd64_arm')), + + ('x86', 'amd64') : ('x86_amd64', ('bin', 'x86_amd64')), + ('x86', 'x86') : ('x86', ('bin', )), + ('x86', 'arm') : ('x86_arm', ('bin', 'x86_arm')), + ('x86', 'ia64') : ('x86_ia64', ('bin', 'x86_ia64')), + + ('arm', 'arm') : ('arm', ('bin', 'arm')), + ('ia64', 'ia64') : ('ia64', ('bin', 'ia64')), + } +_LE2015_HOST_TARGET_CFG = _host_target_config_factory( + + label = 'LE2015', + + host_all_hosts = OrderedDict([ + ('amd64', ['amd64', 'x86']), + ('x86', ['x86']), + ('arm', ['arm']), + ('ia64', ['ia64']), + ]), + + host_all_targets = { + 'amd64': ['amd64', 'x86', 'arm'], + 'x86': ['x86', 'amd64', 'arm', 'ia64'], + 'arm': ['arm'], + 'ia64': ['ia64'], + }, + + host_def_targets = { + 'amd64': ['amd64', 'x86'], + 'x86': ['x86'], + 'arm': ['arm'], + 'ia64': ['ia64'], + }, + +) + +# debug("_LE2015_HOST_TARGET_CFG: %s", _LE2015_HOST_TARGET_CFG) + +# 7.1 (VS2003) and earlier + +# For 7.1 (VS2003) and earlier, there are only x86 targets and the batch files +# take no arguments. + +_LE2003_HOST_TARGET_CFG = _host_target_config_factory( + + label = 'LE2003', + + host_all_hosts = OrderedDict([ + ('amd64', ['x86']), + ('x86', ['x86']), + ]), + + host_all_targets = { + 'amd64': ['x86'], + 'x86': ['x86'], + }, + + host_def_targets = { + 'amd64': ['x86'], + 'x86': ['x86'], + }, + +) + +# debug("_LE2003_HOST_TARGET_CFG: %s", _LE2003_HOST_TARGET_CFG) + _CL_EXE_NAME = 'cl.exe' def get_msvc_version_numeric(msvc_version): @@ -193,44 +416,100 @@ def get_msvc_version_numeric(msvc_version): """ return ''.join([x for x in msvc_version if x in string_digits + '.']) -def get_host_target(env): - host_platform = env.get('HOST_ARCH') - debug("HOST_ARCH:%s", str(host_platform)) - if not host_platform: - host_platform = platform.machine() +def get_host_platform(host_platform): + + host_platform = host_platform.lower() # Solaris returns i86pc for both 32 and 64 bit architectures - if host_platform == "i86pc": + if host_platform == 'i86pc': if platform.architecture()[0] == "64bit": host_platform = "amd64" else: host_platform = "x86" - # Retain user requested TARGET_ARCH - req_target_platform = env.get('TARGET_ARCH') - debug("TARGET_ARCH:%s", str(req_target_platform)) - if req_target_platform: - # If user requested a specific platform then only try that one. - target_platform = req_target_platform - else: - target_platform = host_platform - try: - host = _ARCH_TO_CANONICAL[host_platform.lower()] + host =_ARCH_TO_CANONICAL[host_platform] except KeyError: msg = "Unrecognized host architecture %s" raise MSVCUnsupportedHostArch(msg % repr(host_platform)) from None - try: - target = _ARCH_TO_CANONICAL[target_platform.lower()] - except KeyError: - all_archs = str(list(_ARCH_TO_CANONICAL.keys())) - raise MSVCUnsupportedTargetArch( - "Unrecognized target architecture %s\n\tValid architectures: %s" - % (target_platform, all_archs) - ) from None + return host + +_native_host_platform = None + +def get_native_host_platform(): + global _native_host_platform + + if _native_host_platform is None: + + _native_host_platform = get_host_platform(platform.machine()) + + return _native_host_platform + +def get_host_target(env, msvc_version, all_host_targets=False): + + vernum = float(get_msvc_version_numeric(msvc_version)) - return (host, target, req_target_platform) + if vernum > 14: + # 14.1 (VS2017) and later + host_target_cfg = _GE2017_HOST_TARGET_CFG + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) + host_target_cfg = _LE2015_HOST_TARGET_CFG + else: + # 7.1 (VS2003) and earlier + host_target_cfg = _LE2003_HOST_TARGET_CFG + + host_arch = env.get('HOST_ARCH') if env else None + debug("HOST_ARCH:%s", str(host_arch)) + + if host_arch: + host_platform = get_host_platform(host_arch) + else: + host_platform = get_native_host_platform() + + target_arch = env.get('TARGET_ARCH') if env else None + debug("TARGET_ARCH:%s", str(target_arch)) + + if target_arch: + + try: + target_platform = _ARCH_TO_CANONICAL[target_arch.lower()] + except KeyError: + all_archs = str(list(_ARCH_TO_CANONICAL.keys())) + raise MSVCUnsupportedTargetArch( + "Unrecognized target architecture %s\n\tValid architectures: %s" + % (repr(target_arch), all_archs) + ) from None + + target_host_map = host_target_cfg.target_host_map + + try: + host_target_list = target_host_map[target_platform][host_platform] + except KeyError: + host_target_list = [] + warn_msg = "unsupported host, target combination ({}, {}) for MSVC version {}".format( + repr(host_platform), repr(target_platform), msvc_version + ) + SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) + debug(warn_msg) + + else: + + target_platform = None + + if all_host_targets: + host_targets_map = host_target_cfg.host_all_targets_map + else: + host_targets_map = host_target_cfg.host_def_targets_map + + try: + host_target_list = host_targets_map[host_platform] + except KeyError: + msg = "Unrecognized host architecture %s for version %s" + raise MSVCUnsupportedHostArch(msg % (repr(host_platform), msvc_version)) from None + + return (host_platform, target_platform, host_target_list) # If you update this, update SupportedVSList in Tool/MSCommon/vs.py, and the # MSVC_VERSION documentation in Tool/msvc.xml. @@ -337,29 +616,6 @@ def msvc_version_to_maj_min(msvc_version): raise ValueError("Unrecognized version %s (%s)" % (msvc_version,msvc_version_numeric)) from None -def is_host_target_supported(host_target, msvc_version): - """Check if (host, target) pair is supported for a VC version. - - Only checks whether a given version *may* support the given - (host, target) pair, not that the toolchain is actually on the machine. - - Args: - host_target: canonalized host-target pair, e.g. - ("x86", "amd64") for cross compilation from 32- to 64-bit Windows. - msvc_version: Visual C++ version (major.minor), e.g. "10.0" - - Returns: - True or False - - """ - # We assume that any Visual Studio version supports x86 as a target - if host_target[1] != "x86": - maj, min = msvc_version_to_maj_min(msvc_version) - if maj < 8: - return False - return True - - VSWHERE_PATHS = [os.path.join(p,'vswhere.exe') for p in [ os.path.expandvars(r"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer"), os.path.expandvars(r"%ProgramFiles%\Microsoft Visual Studio\Installer"), @@ -494,15 +750,14 @@ def find_vc_pdir(env, msvc_version): raise MissingConfiguration("registry dir {} not found on the filesystem".format(comps)) return None -def find_batch_file(env,msvc_version,host_arch,target_arch): +def find_batch_file(env, msvc_version, host_arch, target_arch): """ Find the location of the batch script which should set up the compiler for any TARGET_ARCH whose compilers were installed by Visual Studio/VCExpress In newer (2017+) compilers, make use of the fact there are vcvars scripts named with a host_target pair that calls vcvarsall.bat properly, - so use that and return an indication we don't need the argument - we would have computed to run vcvarsall.bat. + so use that and return an empty argument. """ pdir = find_vc_pdir(env, msvc_version) if pdir is None: @@ -511,21 +766,26 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): # filter out e.g. "Exp" from the version name msvc_ver_numeric = get_msvc_version_numeric(msvc_version) - use_arg = True vernum = float(msvc_ver_numeric) - if 7 <= vernum < 8: + + arg = '' + if vernum > 14: + # 14.1 (VS2017) and later + batfiledir = os.path.join(pdir, "Auxiliary", "Build") + batfile, _ = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host_arch, target_arch)] + batfilename = os.path.join(batfiledir, batfile) + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) + arg, _ = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host_arch, target_arch)] + batfilename = os.path.join(pdir, "vcvarsall.bat") + elif 8 > vernum >= 7: + # 7.1 (VS2003) to 7.0 (VS2003) pdir = os.path.join(pdir, os.pardir, "Common7", "Tools") batfilename = os.path.join(pdir, "vsvars32.bat") - elif vernum < 7: + else: + # 6.0 (VS6) and earlier pdir = os.path.join(pdir, "Bin") batfilename = os.path.join(pdir, "vcvars32.bat") - elif 8 <= vernum <= 14: - batfilename = os.path.join(pdir, "vcvarsall.bat") - else: # vernum >= 14.1 VS2017 and above - batfiledir = os.path.join(pdir, "Auxiliary", "Build") - targ = _HOST_TARGET_TO_BAT_ARCH_GT14[(host_arch, target_arch)] - batfilename = os.path.join(batfiledir, targ) - use_arg = False if not os.path.exists(batfilename): debug("Not found: %s", batfilename) @@ -533,16 +793,16 @@ def find_batch_file(env,msvc_version,host_arch,target_arch): installed_sdks = get_installed_sdks() for _sdk in installed_sdks: - sdk_bat_file = _sdk.get_sdk_vc_script(host_arch,target_arch) + sdk_bat_file = _sdk.get_sdk_vc_script(host_arch, target_arch) if not sdk_bat_file: debug("batch file not found:%s", _sdk) else: - sdk_bat_file_path = os.path.join(pdir,sdk_bat_file) + sdk_bat_file_path = os.path.join(pdir, sdk_bat_file) if os.path.exists(sdk_bat_file_path): debug('sdk_bat_file_path:%s', sdk_bat_file_path) - return (batfilename, use_arg, sdk_bat_file_path) - return (batfilename, use_arg, None) + return (batfilename, arg, sdk_bat_file_path) + return (batfilename, arg, None) __INSTALLED_VCS_RUN = None _VC_TOOLS_VERSION_FILE_PATH = ['Auxiliary', 'Build', 'Microsoft.VCToolsVersion.default.txt'] @@ -553,8 +813,8 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): Locates cl in the vc_dir depending on TARGET_ARCH, HOST_ARCH and the msvc version. TARGET_ARCH and HOST_ARCH can be extracted from the - passed env, unless it is None, in which case the native platform is - assumed for both host and target. + passed env, unless the env is None, in which case the native platform is + assumed for the host and all associated targets. Args: env: Environment @@ -571,24 +831,16 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): """ - # determine if there is a specific target platform we want to build for and - # use that to find a list of valid VCs, default is host platform == target platform - # and same for if no env is specified to extract target platform from - if env: - (host_platform, target_platform, req_target_platform) = get_host_target(env) - else: - host_platform = platform.machine().lower() - target_platform = host_platform - - host_platform = _ARCH_TO_CANONICAL[host_platform] - target_platform = _ARCH_TO_CANONICAL[target_platform] + # Find the host, target, and all candidate (host, target) platform combinations: + platforms = get_host_target(env, msvc_version, all_host_targets=True) + debug("host_platform %s, target_platform %s host_target_list %s", *platforms) + host_platform, target_platform, host_target_list = platforms - debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - - ver_num = float(get_msvc_version_numeric(msvc_version)) + vernum = float(get_msvc_version_numeric(msvc_version)) # make sure the cl.exe exists meaning the tool is installed - if ver_num > 14: + if vernum > 14: + # 14.1 (VS2017) and later # 2017 and newer allowed multiple versions of the VC toolset to be # installed at the same time. This changes the layout. # Just get the default tool version for now @@ -604,63 +856,54 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('failed to find MSVC version in %s', default_toolset_file) return False - host_trgt_dir = _HOST_TARGET_TO_CL_DIR_GREATER_THAN_14.get((host_platform, target_platform), None) - if host_trgt_dir is None: - debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) - return False + for host_platform, target_platform in host_target_list: + + debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - cl_path = os.path.join(vc_dir, 'Tools','MSVC', vc_specific_version, 'bin', host_trgt_dir[0], host_trgt_dir[1], _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) - if os.path.exists(cl_path): - debug('found %s!', _CL_EXE_NAME) - return True - - elif host_platform == "amd64" and host_trgt_dir[0] == "Hostx64": - # Special case: fallback to Hostx86 if Hostx64 was tried - # and failed. This is because VS 2017 Express running on amd64 - # will look to our probe like the host dir should be Hostx64, - # but Express uses Hostx86 anyway. - # We should key this off the "x86_amd64" and related pseudo - # targets, but we don't see those in this function. - host_trgt_dir = ("Hostx86", host_trgt_dir[1]) - cl_path = os.path.join(vc_dir, 'Tools','MSVC', vc_specific_version, 'bin', host_trgt_dir[0], host_trgt_dir[1], _CL_EXE_NAME) + batchfile_clpathcomps = _GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS.get((host_platform, target_platform), None) + if batchfile_clpathcomps is None: + debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) + continue + + _, cl_path_comps = batchfile_clpathcomps + cl_path = os.path.join(vc_dir, 'Tools', 'MSVC', vc_specific_version, *cl_path_comps, _CL_EXE_NAME) debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + if os.path.exists(cl_path): debug('found %s!', _CL_EXE_NAME) return True - elif 14 >= ver_num >= 8: - # Set default value to be -1 as "", which is the value for x86/x86, - # yields true when tested if not host_trgt_dir - host_trgt_dir = _HOST_TARGET_TO_CL_DIR.get((host_platform, target_platform), None) - if host_trgt_dir is None: - debug('unsupported host/target platform combo') - return False + elif 14 >= vernum >= 8: + # 14.0 (VS2015) to 8.0 (VS2005) - cl_path = os.path.join(vc_dir, 'bin', host_trgt_dir, _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + cl_path_prefixes = [None] + if msvc_version == '9.0': + # Visual C++ for Python registry key is installdir (root) not productdir (vc) + cl_path_prefixes.append(('VC',)) - cl_path_exists = os.path.exists(cl_path) - if not cl_path_exists and host_platform == 'amd64': - # older versions of visual studio only had x86 binaries, - # so if the host platform is amd64, we need to check cross - # compile options (x86 binary compiles some other target on a 64 bit os) + for host_platform, target_platform in host_target_list: - # Set default value to be -1 as "" which is the value for x86/x86 yields true when tested - # if not host_trgt_dir - host_trgt_dir = _HOST_TARGET_TO_CL_DIR.get(('x86', target_platform), None) - if host_trgt_dir is None: - return False + debug('host platform %s, target platform %s for version %s', host_platform, target_platform, msvc_version) - cl_path = os.path.join(vc_dir, 'bin', host_trgt_dir, _CL_EXE_NAME) - debug('checking for %s at %s', _CL_EXE_NAME, cl_path) - cl_path_exists = os.path.exists(cl_path) + batcharg_clpathcomps = _LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS.get((host_platform, target_platform), None) + if batcharg_clpathcomps is None: + debug('unsupported host/target platform combo: (%s,%s)', host_platform, target_platform) + continue + + _, cl_path_comps = batcharg_clpathcomps + for cl_path_prefix in cl_path_prefixes: - if cl_path_exists: - debug('found %s', _CL_EXE_NAME) - return True + cl_path_comps_adj = cl_path_prefix + cl_path_comps if cl_path_prefix else cl_path_comps + cl_path = os.path.join(vc_dir, *cl_path_comps_adj, _CL_EXE_NAME) + debug('checking for %s at %s', _CL_EXE_NAME, cl_path) + + if os.path.exists(cl_path): + debug('found %s', _CL_EXE_NAME) + return True + + elif 8 > vernum >= 6: + # 7.1 (VS2003) to 6.0 (VS6) - elif 8 > ver_num >= 6: # quick check for vc_dir/bin and vc_dir/ before walk # need to check root as the walk only considers subdirectories for cl_dir in ('bin', ''): @@ -676,9 +919,10 @@ def _check_cl_exists_in_vc_dir(env, vc_dir, msvc_version): debug('%s found %s', _CL_EXE_NAME, cl_path) return True return False + else: # version not support return false - debug('unsupported MSVC version: %s', str(ver_num)) + debug('unsupported MSVC version: %s', str(vernum)) return False @@ -767,7 +1011,7 @@ def script_env(script, args=None): # Stupid batch files do not set return code: we take a look at the # beginning of the output for an error message instead olines = stdout.splitlines() - if olines[0].startswith("The specified configuration type is missing"): + if re_script_output_error.match(olines[0]): raise BatchFileExecutionError("\n".join(olines[:2])) cache_data = common.parse_output(stdout) @@ -834,57 +1078,20 @@ def msvc_find_valid_batch_script(env, version): get it right. """ - # Find the host, target, and if present the requested target: - platforms = get_host_target(env) - debug("host_platform %s, target_platform %s req_target_platform %s", *platforms) - host_platform, target_platform, req_target_platform = platforms - - # Most combinations of host + target are straightforward. - # While all MSVC / Visual Studio tools are pysically 32-bit, they - # make it look like there are 64-bit tools if the host is 64-bit, - # so you can invoke the environment batch script to set up to build, - # say, amd64 host -> x86 target. Express versions are an exception: - # they always look 32-bit, so the batch scripts with 64-bit - # host parts are absent. We try to fix that up in a couple of ways. - # One is here: we make a table of "targets" to try, with the extra - # targets being tags that tell us to try a different "host" instead - # of the deduced host. - try_target_archs = [target_platform] - if req_target_platform in ('amd64', 'x86_64'): - try_target_archs.append('x86_amd64') - elif req_target_platform in ('x86',): - try_target_archs.append('x86_x86') - elif req_target_platform in ('arm',): - try_target_archs.append('x86_arm') - elif req_target_platform in ('arm64',): - try_target_archs.append('x86_arm64') - elif not req_target_platform: - if target_platform in ('amd64', 'x86_64'): - try_target_archs.append('x86_amd64') - # If the user hasn't specifically requested a TARGET_ARCH, - # and the TARGET_ARCH is amd64 then also try 32 bits - # if there are no viable 64 bit tools installed - try_target_archs.append('x86') - - debug("host_platform: %s, try_target_archs: %s", host_platform, try_target_archs) + # Find the host, target, and all candidate (host, target) platform combinations: + platforms = get_host_target(env, version) + debug("host_platform %s, target_platform %s host_target_list %s", *platforms) + host_platform, target_platform, host_target_list = platforms d = None - for tp in try_target_archs: + for host_arch, target_arch, in host_target_list: # Set to current arch. - env['TARGET_ARCH'] = tp - - debug("trying target_platform:%s", tp) - host_target = (host_platform, tp) - if not is_host_target_supported(host_target, version): - warn_msg = "host, target = %s not supported for MSVC version %s" % \ - (host_target, version) - SCons.Warnings.warn(SCons.Warnings.VisualCMissingWarning, warn_msg) - arg = _HOST_TARGET_ARCH_TO_BAT_ARCH[host_target] + env['TARGET_ARCH'] = target_arch # Try to locate a batch file for this host/target platform combo try: - (vc_script, use_arg, sdk_script) = find_batch_file(env, version, host_platform, tp) - debug('vc_script:%s sdk_script:%s', vc_script, sdk_script) + (vc_script, arg, sdk_script) = find_batch_file(env, version, host_arch, target_arch) + debug('vc_script:%s vc_script_arg:%s sdk_script:%s', vc_script, arg, sdk_script) except VisualCException as e: msg = str(e) debug('Caught exception while looking for batch file (%s)', msg) @@ -899,8 +1106,6 @@ def msvc_find_valid_batch_script(env, version): debug('use_script 2 %s, args:%s', repr(vc_script), arg) found = None if vc_script: - if not use_arg: - arg = '' # bat file will supply platform type # Get just version numbers maj, min = msvc_version_to_maj_min(version) # VS2015+ @@ -934,7 +1139,7 @@ def msvc_find_valid_batch_script(env, version): # If we cannot find a viable installed compiler, reset the TARGET_ARCH # To it's initial value if not d: - env['TARGET_ARCH']=req_target_platform + env['TARGET_ARCH'] = target_platform return d diff --git a/SCons/Tool/MSCommon/vcTests.py b/SCons/Tool/MSCommon/vcTests.py index eb09def..2b6fbe5 100644 --- a/SCons/Tool/MSCommon/vcTests.py +++ b/SCons/Tool/MSCommon/vcTests.py @@ -114,13 +114,14 @@ class MSVcTestCase(unittest.TestCase): check = SCons.Tool.MSCommon.vc._check_cl_exists_in_vc_dir env={'TARGET_ARCH':'x86'} - p = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR[('x86','x86')] - MSVcTestCase._createDummyCl(p) + _, clpathcomps = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[('x86','x86')] + path = os.path.join('.', *clpathcomps) + MSVcTestCase._createDummyCl(path, add_bin=False) # print("retval:%s"%check(env, '.', '8.0')) - # Setup for VC 14+ tests + # Setup for 14.1 (VS2017) and later tests # Create the VC minor/major version file tools_version_file = SCons.Tool.MSCommon.vc._VC_TOOLS_VERSION_FILE @@ -134,25 +135,26 @@ class MSVcTestCase(unittest.TestCase): print("Failed trying to write :%s :%s"%(tools_version_file, e)) - # Now walk all the valid combinations of host/target for VC 14 + - vc_gt_14_map = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR_GREATER_THAN_14 + # Now walk all the valid combinations of host/target for 14.1 (VS2017) and later + vc_ge2017_list = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_CFG.all_pairs - for key, value in vc_gt_14_map.items(): - # print("GT 14 Got: %s -> %s"%(key,value)) + for host, target in vc_ge2017_list: + batfile, clpathcomps = SCons.Tool.MSCommon.vc._GE2017_HOST_TARGET_BATCHFILE_CLPATHCOMPS[(host,target)] + # print("GT 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batfile,clpathcomps)) - env={'TARGET_ARCH':key[1], 'HOST_ARCH':key[0]} - path = os.path.join('.','Tools','MSVC', MS_TOOLS_VERSION, 'bin', value[0], value[1]) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.', 'Tools', 'MSVC', MS_TOOLS_VERSION, *clpathcomps) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '14.1') - # print("for:%s got :%s"%(key[1], result)) - self.assertTrue(result, "Checking host: %s target: %s"%(value[0], value[1])) + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s"%(value[0], value[1])) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedHostArch: pass else: @@ -163,22 +165,24 @@ class MSVcTestCase(unittest.TestCase): try: result=check(env, '.', '14.1') # print("for:%s got :%s"%(env, result)) - self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s"%(value[0], value[1])) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['TARGET_ARCH']) - - # Test >8 < 14 VC versions - vc_map = SCons.Tool.MSCommon.vc._HOST_TARGET_TO_CL_DIR - for key,value in vc_map.items(): - # print("LT 14 Got: %s -> %s"%(key,value)) - env={'TARGET_ARCH':key[1], 'HOST_ARCH':key[0]} - path = os.path.join('.', 'bin', value ) + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) + + # Test 14.0 (VS2015) to 8.0 (VS2005) versions + vc_le2015_list = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_CFG.all_pairs + + for host, target in vc_le2015_list: + batarg, clpathcomps = SCons.Tool.MSCommon.vc._LE2015_HOST_TARGET_BATCHARG_CLPATHCOMPS[(host, target)] + # print("LE 14 Got: (%s, %s) -> (%s, %s)"%(host,target,batarg,clpathcomps)) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.', *clpathcomps) MSVcTestCase._createDummyCl(path, add_bin=False) result=check(env, '.', '9.0') - # print("for:%s got :%s"%(key[1], result)) - self.assertTrue(result, "Checking host: %s target: %s"%(key[0], key[1])) + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) # Now test bogus value for HOST_ARCH env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} @@ -189,7 +193,7 @@ class MSVcTestCase(unittest.TestCase): except MSVCUnsupportedHostArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['HOST_ARCH']) + self.fail('Did not fail when HOST_ARCH specified as: %s' % env['HOST_ARCH']) # Now test bogus value for TARGET_ARCH env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} @@ -200,7 +204,41 @@ class MSVcTestCase(unittest.TestCase): except MSVCUnsupportedTargetArch: pass else: - self.fail('Did not fail when HOST_ARCH specified as: %s'%env['TARGET_ARCH']) + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) + + # Test 7.1 (VS2003) and earlier + vc_le2003_list = SCons.Tool.MSCommon.vc._LE2003_HOST_TARGET_CFG.all_pairs + + for host, target in vc_le2003_list: + # print("LE 7.1 Got: (%s, %s)"%(host,target)) + env={'TARGET_ARCH':target, 'HOST_ARCH':host} + path = os.path.join('.') + MSVcTestCase._createDummyCl(path) + result=check(env, '.', '6.0') + # print("for:(%s, %s) got :%s"%(host, target, result)) + self.assertTrue(result, "Checking host: %s target: %s" % (host, target)) + + # Now test bogus value for HOST_ARCH + env={'TARGET_ARCH':'x86', 'HOST_ARCH':'GARBAGE'} + try: + result=check(env, '.', '6.0') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus HOST_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedHostArch: + pass + else: + self.fail('Did not fail when HOST_ARCH specified as: %s' % env['HOST_ARCH']) + + # Now test bogus value for TARGET_ARCH + env={'TARGET_ARCH':'GARBAGE', 'HOST_ARCH':'x86'} + try: + result=check(env, '.', '6.0') + # print("for:%s got :%s"%(env, result)) + self.assertFalse(result, "Did not fail with bogus TARGET_ARCH host: %s target: %s" % (env['HOST_ARCH'], env['TARGET_ARCH'])) + except MSVCUnsupportedTargetArch: + pass + else: + self.fail('Did not fail when TARGET_ARCH specified as: %s' % env['TARGET_ARCH']) diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 2a12a31..518b09e 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -82,7 +82,7 @@ def generate(env): env['CCVERSION'] = match.group(1) env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' - + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'clang' def exists(env): diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index a78dc6c..07d8378 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -90,6 +90,7 @@ def generate(env): env['CXXVERSION'] = match.group(1) env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'clang' def exists(env): diff --git a/SCons/Tool/docbook/__init__.py b/SCons/Tool/docbook/__init__.py index 6e01d2a..4c3f60c 100644 --- a/SCons/Tool/docbook/__init__.py +++ b/SCons/Tool/docbook/__init__.py @@ -223,6 +223,7 @@ def __xml_scan(node, env, path, arg): # Try to call xsltproc xsltproc = env.subst("$DOCBOOK_XSLTPROC") if xsltproc and xsltproc.endswith('xsltproc'): + # TODO: switch to _subproc or subprocess.run call result = env.backtick(' '.join([xsltproc, xsl_file, str(node)])) depfiles = [x.strip() for x in str(result).splitlines() if x.strip() != "" and not x.startswith("<?xml ")] return depfiles diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 94dfad3..1a25cb4 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -58,6 +58,7 @@ def generate(env): env['CCVERSION'] = version env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index cc93f93..1272997 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -65,6 +65,7 @@ def generate(env): env['CXXVERSION'] = version env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' diff --git a/SCons/Tool/mingw.py b/SCons/Tool/mingw.py index 0d2bd6d..8e7ac2d 100644 --- a/SCons/Tool/mingw.py +++ b/SCons/Tool/mingw.py @@ -41,7 +41,7 @@ import SCons.Defaults import SCons.Tool import SCons.Util -mingw_paths = [ +mingw_base_paths = [ r'c:\MinGW\bin', r'C:\cygwin64\bin', r'C:\msys64', @@ -127,17 +127,24 @@ def find_version_specific_mingw_paths(): One example of default mingw install paths is: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev2\mingw64\bin - Use glob'ing to find such and add to mingw_paths + Use glob'ing to find such and add to mingw_base_paths """ new_paths = glob.glob(r"C:\mingw-w64\*\mingw64\bin") return new_paths +_mingw_all_paths = None + +def get_mingw_paths(): + global _mingw_all_paths + if _mingw_all_paths is None: + _mingw_all_paths = mingw_base_paths + find_version_specific_mingw_paths() + return _mingw_all_paths + def generate(env): - global mingw_paths # Check for reasoanble mingw default paths - mingw_paths += find_version_specific_mingw_paths() + mingw_paths = get_mingw_paths() mingw = SCons.Tool.find_program_path(env, key_program, default_paths=mingw_paths) if mingw: @@ -204,6 +211,7 @@ def generate(env): def exists(env): + mingw_paths = get_mingw_paths() mingw = SCons.Tool.find_program_path(env, key_program, default_paths=mingw_paths) if mingw: mingw_bin_dir = os.path.dirname(mingw) diff --git a/SCons/Tool/msvc.py b/SCons/Tool/msvc.py index f2cd418..58b7463 100644 --- a/SCons/Tool/msvc.py +++ b/SCons/Tool/msvc.py @@ -316,6 +316,7 @@ def generate(env): env['ENV']['SystemRoot'] = SCons.Platform.win32.get_system_root() env['CCDEPFLAGS'] = '/showIncludes' + env["NINJA_DEPFILE_PARSE_FORMAT"] = 'msvc' def exists(env): diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 4c6aa2f..2be576f 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -276,7 +276,10 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): ret_rule = 'TEMPLATE' else: - ret_rule = rule + if len(' '.join(cmd_list)) < env.get('MAXLINELENGTH', 2048): + ret_rule = rule + else: + ret_rule = rule + '_RSP' return ret_rule, variables, [tool_command] diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index fff60be..63ea3a1 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -132,19 +132,19 @@ class NinjaState: # use this to generate a compile_commands.json database # which can't use the shell command as it's compile # command. - "CC": { + "CC_RSP": { "command": "$env$CC @$out.rsp", "description": "Compiling $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, - "CXX": { + "CXX_RSP": { "command": "$env$CXX @$out.rsp", "description": "Compiling $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, - "LINK": { + "LINK_RSP": { "command": "$env$LINK @$out.rsp", "description": "Linking $out", "rspfile": "$out.rsp", @@ -157,7 +157,7 @@ class NinjaState: # Native SCons will perform this operation so we need to force ninja # to do the same. See related for more info: # https://jira.mongodb.org/browse/SERVER-49457 - "AR": { + "AR_RSP": { "command": "{}$env$AR @$out.rsp".format( '' if sys.platform == "win32" else "rm -f $out && " ), @@ -166,6 +166,26 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, + "CC": { + "command": "$env$CC $rspc", + "description": "Compiling $out", + }, + "CXX": { + "command": "$env$CXX $rspc", + "description": "Compiling $out", + }, + "LINK": { + "command": "$env$LINK $rspc", + "description": "Linking $out", + "pool": "local_pool", + }, + "AR": { + "command": "{}$env$AR $rspc".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", + "pool": "local_pool", + }, "SYMLINK": { "command": ( "cmd /c mklink $out $in" @@ -254,20 +274,7 @@ class NinjaState: "description": "Archiving $out", "pool": "local_pool", } - - num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) - self.pools = { - "local_pool": num_jobs, - "install_pool": num_jobs / 2, - "scons_pool": 1, - } - - for rule in ["CC", "CXX"]: - if env["PLATFORM"] == "win32": - self.rules[rule]["deps"] = "msvc" - else: - self.rules[rule]["deps"] = "gcc" - self.rules[rule]["depfile"] = "$out.d" + self.pools = {"scons_pool": 1} def add_build(self, node): if not node.has_builder(): @@ -339,6 +346,34 @@ class NinjaState: if self.__generated: return + num_jobs = self.env.get('NINJA_MAX_JOBS', self.env.GetOption("num_jobs")) + self.pools.update({ + "local_pool": num_jobs, + "install_pool": num_jobs / 2, + }) + + deps_format = self.env.get("NINJA_DEPFILE_PARSE_FORMAT", 'msvc' if self.env['PLATFORM'] == 'win32' else 'gcc') + for rule in ["CC", "CXX"]: + if deps_format == "msvc": + self.rules[rule]["deps"] = "msvc" + elif deps_format == "gcc" or deps_format == "clang": + self.rules[rule]["deps"] = "gcc" + self.rules[rule]["depfile"] = "$out.d" + else: + raise Exception(f"Unknown 'NINJA_DEPFILE_PARSE_FORMAT'={self.env['NINJA_DEPFILE_PARSE_FORMAT']}, use 'mvsc', 'gcc', or 'clang'.") + + + for key, rule in self.env.get(NINJA_RULES, {}).items(): + # make a non response file rule for users custom response file rules. + if rule.get('rspfile') is not None: + self.rules.update({key + '_RSP': rule}) + non_rsp_rule = rule.copy() + del non_rsp_rule['rspfile'] + del non_rsp_rule['rspfile_content'] + self.rules.update({key: non_rsp_rule}) + else: + self.rules.update({key: rule}) + self.rules.update(self.env.get(NINJA_RULES, {})) self.pools.update(self.env.get(NINJA_POOLS, {})) @@ -360,26 +395,58 @@ class NinjaState: kwargs['pool'] = 'local_pool' ninja.rule(rule, **kwargs) - generated_source_files = sorted({ - output - # First find builds which have header files in their outputs. - for build in self.builds.values() - if self.has_generated_sources(build["outputs"]) - for output in build["outputs"] - # Collect only the header files from the builds with them - # in their output. We do this because is_generated_source - # returns True if it finds a header in any of the outputs, - # here we need to filter so we only have the headers and - # not the other outputs. - if self.is_generated_source(output) - }) + # If the user supplied an alias to determine generated sources, use that, otherwise + # determine what the generated sources are dynamically. + generated_sources_alias = self.env.get('NINJA_GENERATED_SOURCE_ALIAS_NAME') + generated_sources_build = None - if generated_source_files: - ninja.build( - outputs="_generated_sources", - rule="phony", - implicit=generated_source_files + if generated_sources_alias: + generated_sources_build = self.builds.get(generated_sources_alias) + if generated_sources_build is None or generated_sources_build["rule"] != 'phony': + raise Exception( + "ERROR: 'NINJA_GENERATED_SOURCE_ALIAS_NAME' set, but no matching Alias object found." + ) + + if generated_sources_alias and generated_sources_build: + generated_source_files = sorted( + [] if not generated_sources_build else generated_sources_build['implicit'] ) + + def check_generated_source_deps(build): + return ( + build != generated_sources_build + and set(build["outputs"]).isdisjoint(generated_source_files) + ) + else: + generated_sources_build = None + generated_source_files = sorted({ + output + # First find builds which have header files in their outputs. + for build in self.builds.values() + if self.has_generated_sources(build["outputs"]) + for output in build["outputs"] + # Collect only the header files from the builds with them + # in their output. We do this because is_generated_source + # returns True if it finds a header in any of the outputs, + # here we need to filter so we only have the headers and + # not the other outputs. + if self.is_generated_source(output) + }) + + if generated_source_files: + generated_sources_alias = "_ninja_generated_sources" + ninja.build( + outputs=generated_sources_alias, + rule="phony", + implicit=generated_source_files + ) + + def check_generated_source_deps(build): + return ( + not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) + ) template_builders = [] scons_compiledb = False @@ -402,9 +469,7 @@ class NinjaState: # cycle. if ( generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + and check_generated_source_deps(build) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -413,7 +478,7 @@ class NinjaState: # sure that all of these sources are generated before # other builds. order_only = build.get("order_only", []) - order_only.append("_generated_sources") + order_only.append(generated_sources_alias) build["order_only"] = order_only if "order_only" in build: build["order_only"].sort() diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 3c72ee4..526db8a 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -226,7 +226,7 @@ def generate(env): pass else: env.Append(CCFLAGS='$CCDEPFLAGS') - + env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") # Provide a way for custom rule authors to easily access command @@ -276,8 +276,8 @@ def generate(env): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - # some of these construction vars could be generators, e.g. - # CommandGeneratorAction, so if the var is not a string, we + # some of these construction vars could be generators, e.g. + # CommandGeneratorAction, so if the var is not a string, we # can't parse the generated string. if isinstance(env.get(var), str): env.NinjaRuleMapping(env.get(var, None), provider) @@ -297,7 +297,7 @@ def generate(env): # requires that all generated sources are added as order_only # dependencies to any builds that *might* use them. # TODO: switch to using SCons to help determine this (Github Issue #3624) - env["NINJA_GENERATED_SOURCE_SUFFIXES"] = [".h", ".hpp"] + env["NINJA_GENERATED_SOURCE_SUFFIXES"] = env.get('NINJA_GENERATED_SOURCE_SUFFIXES', [".h", ".hpp"]) # Force ARCOM so use 's' flag on ar instead of separately running ranlib ninja_hack_arcom(env) diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 51ff435..6b247d0 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -65,7 +65,9 @@ See its __doc__ string for a discussion of the format. <item>NINJA_ENV_VAR_CACHE</item> <item>NINJA_FILE_NAME</item> <item>NINJA_GENERATED_SOURCE_SUFFIXES</item> + <item>NINJA_GENERATED_SOURCE_ALIAS_NAME</item> <item>NINJA_MSVC_DEPS_PREFIX</item> + <item>NINJA_DEPFILE_PARSE_FORMAT</item> <item>NINJA_POOL</item> <item>NINJA_REGENERATE_DEPS</item> <item>NINJA_SYNTAX</item> @@ -198,6 +200,17 @@ python -m pip install ninja </summary> </cvar> + <cvar name="NINJA_GENERATED_SOURCE_ALIAS_NAME"> + <summary> + <para> + A string matching the name of a user defined alias which represents a list of all generated sources. + This will prevent the auto-detection of generated sources from &cv-NINJA_GENERATED_SOURCE_SUFFIXES;. + Then all other source files will be made to depend on this in the &ninja; build file, forcing the + generated sources to be built first. + </para> + </summary> + </cvar> + <cvar name="NINJA_MSVC_DEPS_PREFIX"> <summary> <para> @@ -209,6 +222,17 @@ python -m pip install ninja </summary> </cvar> + <cvar name="NINJA_DEPFILE_PARSE_FORMAT"> + <summary> + <para> + Determines the type of format ninja should expect when parsing header + include depfiles. Can be <option>msvc</option>, <option>gcc</option>, or <option>clang</option>. + The <option>msvc</option> option corresponds to <option>/showIncludes</option> format, and + <option>gcc</option> or <option>clang</option> correspond to <option>-MMD -MF</option>. + </para> + </summary> + </cvar> + <cvar name="NINJA_DIR"> <summary> <para> @@ -262,7 +286,7 @@ python -m pip install ninja </para> <para> - If not explicitly set, &SCons; will generate this dynamically from the + If not explicitly set, &SCons; will generate this dynamically from the execution environment stored in the current &consenv; (e.g. <literal>env['ENV']</literal>) where those values differ from the existing shell.. diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py index f34935b..80aa741 100644 --- a/SCons/Tool/ninja/ninja_daemon_build.py +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -53,6 +53,10 @@ logging.basicConfig( level=logging.DEBUG, ) +def log_error(msg): + logging.debug(msg) + sys.stderr.write(msg) + while True: try: logging.debug(f"Sending request: {sys.argv[3]}") @@ -68,19 +72,19 @@ while True: except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) except http.client.HTTPException: - logging.debug(f"Error: {traceback.format_exc()}") + log_error(f"Error: {traceback.format_exc()}") exit(1) else: msg = response.read() status = response.status if status != 200: - print(msg.decode("utf-8")) + log_error(msg.decode("utf-8")) exit(1) logging.debug(f"Request Done: {sys.argv[3]}") exit(0) except Exception: - logging.debug(f"Failed to send command: {traceback.format_exc()}") + log_error(f"Failed to send command: {traceback.format_exc()}") exit(1) diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py index 057537a..297bcf4 100644 --- a/SCons/Tool/ninja/ninja_run_daemon.py +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -60,6 +60,10 @@ logging.basicConfig( level=logging.DEBUG, ) +def log_error(msg): + logging.debug(msg) + sys.stderr.write(msg) + if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): cmd = [ sys.executable, @@ -67,10 +71,25 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): ] + sys.argv[1:] logging.debug(f"Starting daemon with {' '.join(cmd)}") - p = subprocess.Popen( - cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False - ) - + + # TODO: Remove the following when Python3.6 support is dropped. + if sys.platform == 'win32' and sys.version_info[0] == 3 and sys.version_info[1] == 6: + # on Windows with Python version 3.6, popen does not do a good job disconnecting + # the std handles and this make ninja hang because they stay open to the original + # process ninja launched. Here we can force the handles to be separated. + # See: https://docs.python.org/3.6/library/subprocess.html#subprocess.STARTUPINFO + # See Also: https://docs.python.org/3.6/library/subprocess.html#subprocess.Popen + # Note when you don't specify stdin, stdout, and/or stderr they default to None + # which indicates no output redirection will occur. + si = subprocess.STARTUPINFO() + si.dwFlags = subprocess.STARTF_USESTDHANDLES + p = subprocess.Popen( + cmd, close_fds=True, shell=False, startupinfo=si + ) + else: + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False, + ) with open(daemon_dir / "pidfile", "w") as f: f.write(str(p.pid)) with open(ninja_builddir / "scons_daemon_dirty", "w") as f: @@ -92,14 +111,13 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): except (http.client.RemoteDisconnected, http.client.ResponseNotReady): time.sleep(0.01) except http.client.HTTPException: - logging.debug(f"Error: {traceback.format_exc()}") - sys.stderr.write(error_msg) + log_error(f"Error: {traceback.format_exc()}") exit(1) else: msg = response.read() status = response.status if status != 200: - print(msg.decode("utf-8")) + log_error(msg.decode("utf-8")) exit(1) logging.debug("Server Responded it was ready!") break @@ -107,16 +125,14 @@ if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): except ConnectionRefusedError: logging.debug(f"Server not ready, server PID: {p.pid}") time.sleep(1) - if p.poll is not None: - logging.debug(f"Server process died, aborting: {p.returncode}") + if p.poll() is not None: + log_error(f"Server process died, aborting: {p.returncode}") sys.exit(p.returncode) except ConnectionResetError: - logging.debug("Server ConnectionResetError") - sys.stderr.write(error_msg) + log_error("Server ConnectionResetError") exit(1) except Exception: - logging.debug(f"Error: {traceback.format_exc()}") - sys.stderr.write(error_msg) + log_error(f"Error: {traceback.format_exc()}") exit(1) # Local Variables: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py index 88f3b22..f900343 100644 --- a/SCons/Tool/ninja/ninja_scons_daemon.py +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -56,6 +56,14 @@ ninja_builddir = pathlib.Path(sys.argv[2]) daemon_keep_alive = int(sys.argv[3]) args = sys.argv[4:] +# TODO: Remove the following when Python3.6 support is dropped. +# Windows and Python36 passed nothing for the std handles because of issues with popen +# and its handles so we have to make some fake ones to prevent exceptions. +if sys.platform == 'win32' and sys.version_info[0] == 3 and sys.version_info[1] == 6: + from io import StringIO + sys.stderr = StringIO() + sys.stdout = StringIO() + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) ) diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index d8b694e..6d0a5fb 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -492,21 +492,23 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None return result -def LaTeXAuxAction(target = None, source= None, env=None): - result = InternalLaTeXAuxAction( LaTeXAction, target, source, env ) +def LaTeXAuxAction(target=None, source=None, env=None): + result = InternalLaTeXAuxAction(LaTeXAction, target, source, env) return result + LaTeX_re = re.compile("\\\\document(style|class)") -def is_LaTeX(flist,env,abspath): + +def is_LaTeX(flist, env, abspath) -> bool: """Scan a file list to decide if it's TeX- or LaTeX-flavored.""" # We need to scan files that are included in case the # \documentclass command is in them. # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS'] - savedpath = modify_env_var(env, 'TEXINPUTS', abspath) - paths = env['ENV']['TEXINPUTS'] + savedpath = modify_env_var(env, "TEXINPUTS", abspath) + paths = env["ENV"]["TEXINPUTS"] if SCons.Util.is_List(paths): pass else: @@ -516,54 +518,58 @@ def is_LaTeX(flist,env,abspath): # now that we have the path list restore the env if savedpath is _null: try: - del env['ENV']['TEXINPUTS'] + del env["ENV"]["TEXINPUTS"] except KeyError: - pass # was never set + pass # was never set else: - env['ENV']['TEXINPUTS'] = savedpath + env["ENV"]["TEXINPUTS"] = savedpath if Verbose: - print("is_LaTeX search path ",paths) - print("files to search :",flist) + print("is_LaTeX search path ", paths) + print("files to search: ", flist) # Now that we have the search path and file list, check each one + file_test = False for f in flist: if Verbose: - print(" checking for Latex source ",str(f)) + print(f" checking for Latex source {f}") content = f.get_text_contents() if LaTeX_re.search(content): if Verbose: - print("file %s is a LaTeX file" % str(f)) - return 1 + print(f"file {f} is a LaTeX file") + return True if Verbose: - print("file %s is not a LaTeX file" % str(f)) + print(f"file {f} is not a LaTeX file") # now find included files - inc_files = [ ] - inc_files.extend( include_re.findall(content) ) + inc_files = [] + inc_files.extend(include_re.findall(content)) if Verbose: - print("files included by '%s': "%str(f),inc_files) + print(f"files included by '{f}': ", inc_files) # inc_files is list of file names as given. need to find them # using TEXINPUTS paths. # search the included files for src in inc_files: - srcNode = FindFile(src,['.tex','.ltx','.latex'],paths,env,requireExt=False) + srcNode = FindFile( + src, [".tex", ".ltx", ".latex"], paths, env, requireExt=False + ) # make this a list since is_LaTeX takes a list. - fileList = [srcNode,] + fileList = [srcNode] if Verbose: - print("FindFile found ",srcNode) + print("FindFile found ", srcNode) if srcNode is not None: file_test = is_LaTeX(fileList, env, abspath) # return on first file that finds latex is needed. if file_test: - return file_test + return True if Verbose: - print(" done scanning ",str(f)) + print(f" done scanning {f}") + + return False - return 0 def TeXLaTeXFunction(target = None, source= None, env=None): """A builder for TeX and LaTeX that scans the source file to diff --git a/doc/man/scons.xml b/doc/man/scons.xml index 733db78..a5eff28 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -7247,23 +7247,38 @@ A tool specification module must include two functions: <varlistentry> <term><function>generate</function>(<parameter>env, **kwargs</parameter>)</term> <listitem> -<para>Modifies the &consenv; <parameter>env</parameter> -to set up necessary &consvars; so that the facilities represented by -the tool can be executed. -It may use any keyword arguments -that the user supplies in <parameter>kwargs</parameter> +<para>Modify the &consenv; <parameter>env</parameter> +to set up necessary &consvars;, Builders, Emitters, etc., +so the facilities represented by the tool can be executed. +Care should be taken not to overwrite &consvars; intended +to be settable by the user. For example: +</para> +<programlisting language="python"> +def generate(env): + ... + if 'MYTOOL' not in env: + env['MYTOOL'] = env.Detect("mytool") + if 'MYTOOLFLAGS' not in env: + env['MYTOOLFLAGS'] = SCons.Util.CLVar('--myarg') + ... +</programlisting> + +<para>The <function>generate</function> function +may use any keyword arguments +that the user supplies via <parameter>kwargs</parameter> to vary its initialization.</para> </listitem> </varlistentry> <varlistentry> <term><function>exists</function>(<parameter>env</parameter>)</term> <listitem> -<para>Returns <constant>True</constant> if the tool can +<para>Return a true value if the tool can be called in the context of <parameter>env</parameter>. +else false. Usually this means looking up one or more known programs using the <varname>PATH</varname> from the supplied <parameter>env</parameter>, but the tool can -make the "exists" decision in any way it chooses. +make the <emphasis>exists</emphasis> decision in any way it chooses. </para> </listitem> </varlistentry> diff --git a/doc/user/environments.xml b/doc/user/environments.xml index 03dc89e..7118f21 100644 --- a/doc/user/environments.xml +++ b/doc/user/environments.xml @@ -1618,7 +1618,7 @@ env.PrependUnique(CCFLAGS=['-g']) <para> - Rather than creating a cloned environmant for specific tasks, + Rather than creating a cloned &consenv; for specific tasks, you can <firstterm>override</firstterm> or add construction variables when calling a builder method by passing them as keyword arguments. The values of these overridden or added variables will only be in diff --git a/doc/user/mergeflags.xml b/doc/user/mergeflags.xml index 5a864b1..18c9bb8 100644 --- a/doc/user/mergeflags.xml +++ b/doc/user/mergeflags.xml @@ -2,7 +2,7 @@ <!DOCTYPE sconsdoc [ <!ENTITY % scons SYSTEM "../scons.mod"> %scons; - + <!ENTITY % builders-mod SYSTEM "../generated/builders.mod"> %builders-mod; <!ENTITY % functions-mod SYSTEM "../generated/functions.mod"> @@ -21,7 +21,9 @@ <!-- - __COPYRIGHT__ + MIT License + + Copyright The SCons Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -46,16 +48,16 @@ <para> - &SCons; &consenvs; have an &f-link-env-MergeFlags; method - that merges values from a passed-in argument into the &consenv; + &SCons; &consenvs; have a &f-link-MergeFlags; method + that merges values from a passed-in argument into the &consenv;. If the argument is a dictionary, &MergeFlags; treats each value in the dictionary - as a list of options such as one might pass to a command + as a list of options you would pass to a command (such as a compiler or linker). &MergeFlags; will not duplicate an option - if it already exists in the construction environment variable. + if it already exists in the &consvar;. If the argument is a string, &MergeFlags; calls the - &f-link-env-ParseFlags; method to burst it out into a + &f-link-ParseFlags; method to burst it out into a dictionary first, then acts on the result. </para> @@ -82,7 +84,7 @@ env = Environment() env.Append(CCFLAGS='-option -O3 -O1') flags = {'CCFLAGS': '-whatever -O3'} env.MergeFlags(flags) -print(env['CCFLAGS']) +print("CCFLAGS:", env['CCFLAGS']) </file> </scons_example> @@ -99,7 +101,7 @@ print(env['CCFLAGS']) --> is an internal &SCons; object which automatically converts - the options we specified as a string into a list. + the options you specify as a string into a list. </para> @@ -109,7 +111,7 @@ env = Environment() env.Append(CPPPATH=['/include', '/usr/local/include', '/usr/include']) flags = {'CPPPATH': ['/usr/opt/include', '/usr/local/include']} env.MergeFlags(flags) -print(env['CPPPATH']) +print("CPPPATH:", env['CPPPATH']) </file> </scons_example> @@ -124,9 +126,9 @@ print(env['CPPPATH']) [TODO: for when we make CLVar public] is a Python list, not a <varname>CLVar</varname>, --> - is a normal Python list, - so we must specify its values as a list - in the dictionary we pass to the &MergeFlags; function. + is a normal &Python; list, + so you should give its values as a list + in the dictionary you pass to the &MergeFlags; function. </para> @@ -143,8 +145,8 @@ env = Environment() env.Append(CCFLAGS='-option -O3 -O1') env.Append(CPPPATH=['/include', '/usr/local/include', '/usr/include']) env.MergeFlags('-whatever -I/usr/opt/include -O3 -I/usr/local/include') -print(env['CCFLAGS']) -print(env['CPPPATH']) +print("CCFLAGS:", env['CCFLAGS']) +print("CPPPATH:", env['CPPPATH']) </file> </scons_example> @@ -157,8 +159,8 @@ print(env['CPPPATH']) In the combined example above, &ParseFlags; has sorted the options into their corresponding variables and returned a dictionary for &MergeFlags; to apply - to the construction variables - in the specified construction environment. + to the &consvars; + in the specified &consenv;. </para> diff --git a/doc/user/parseconfig.xml b/doc/user/parseconfig.xml index a07201a..fc9a889 100644 --- a/doc/user/parseconfig.xml +++ b/doc/user/parseconfig.xml @@ -2,7 +2,7 @@ <!DOCTYPE sconsdoc [ <!ENTITY % scons SYSTEM "../scons.mod"> %scons; - + <!ENTITY % builders-mod SYSTEM "../generated/builders.mod"> %builders-mod; <!ENTITY % functions-mod SYSTEM "../generated/functions.mod"> @@ -21,7 +21,9 @@ <!-- - __COPYRIGHT__ + MIT License + + Copyright The SCons Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -48,22 +50,22 @@ Configuring the right options to build programs to work with libraries--especially shared libraries--that are available - on POSIX systems can be very complicated. + on POSIX systems can be complex. To help this situation, various utilies with names that end in <filename>config</filename> return the command-line options for the GNU Compiler Collection (GCC) - that are needed to use these libraries; + that are needed to build and link against those libraries; for example, the command-line options to use a library named <filename>lib</filename> - would be found by calling a utility named <filename>lib-config</filename>. + could be found by calling a utility named <command>lib-config</command>. </para> <para> A more recent convention is that these options - are available from the generic <filename>pkg-config</filename> program, - which has common framework, error handling, and the like, + are available through the generic <command>pkg-config</command> program, + providing a common framework, error handling, and the like, so that all the package creator has to do is provide the set of strings for his particular package. @@ -71,14 +73,12 @@ <para> - &SCons; construction environments have a &ParseConfig; method - that executes a <filename>*config</filename> utility - (either <filename>pkg-config</filename> or a - more specific utility) - and configures the appropriate construction variables - in the environment - based on the command-line options - returned by the specified command. + &SCons; &consvars; have a &f-link-ParseConfig; + method that asks the host system to execute a command + and then configures the appropriate &consvars; based on + the output of that command. + This lets you run a program like <command>pkg-config</command> + or a more specific utility to help set up your build. </para> @@ -87,7 +87,7 @@ env = Environment() env['CPPPATH'] = ['/lib/compat'] env.ParseConfig("pkg-config x11 --cflags --libs") -print(env['CPPPATH']) +print("CPPPATH:", env['CPPPATH']) </file> </scons_example> @@ -112,14 +112,14 @@ print(env['CPPPATH']) <screen> % <userinput>scons -Q</userinput> -['/lib/compat', '/usr/X11/include'] +CPPPATH: ['/lib/compat', '/usr/X11/include'] scons: `.' is up to date. </screen> <para> In the example above, &SCons; has added the include directory to - <varname>CPPPATH</varname>. + &cv-link-CPPPATH; (Depending upon what other flags are emitted by the <filename>pkg-config</filename> command, other variables may have been extended as well.) @@ -129,8 +129,8 @@ scons: `.' is up to date. <para> Note that the options are merged with existing options using - the &MergeFlags; method, - so that each option only occurs once in the construction variable: + the &f-link-MergeFlags; method, + so that each option only occurs once in the &consvar;. </para> @@ -139,7 +139,7 @@ scons: `.' is up to date. env = Environment() env.ParseConfig("pkg-config x11 --cflags --libs") env.ParseConfig("pkg-config x11 --cflags --libs") -print(env['CPPPATH']) +print("CPPPATH:", "CPPPATH:", env['CPPPATH']) </file> </scons_example> @@ -156,7 +156,7 @@ print(env['CPPPATH']) <screen> % <userinput>scons -Q</userinput> -['/usr/X11/include'] +CPPPATH: ['/usr/X11/include'] scons: `.' is up to date. </screen> diff --git a/doc/user/parseflags.xml b/doc/user/parseflags.xml index 1cd0bbf..a1ab7af 100644 --- a/doc/user/parseflags.xml +++ b/doc/user/parseflags.xml @@ -2,7 +2,7 @@ <!DOCTYPE sconsdoc [ <!ENTITY % scons SYSTEM "../scons.mod"> %scons; - + <!ENTITY % builders-mod SYSTEM "../generated/builders.mod"> %builders-mod; <!ENTITY % functions-mod SYSTEM "../generated/functions.mod"> @@ -21,7 +21,9 @@ <!-- - __COPYRIGHT__ + MIT License + + Copyright The SCons Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the @@ -55,10 +57,10 @@ <para> - &SCons; &consenvs; have a &f-link-env-ParseFlags; method + &SCons; &consenvs; have a &f-link-ParseFlags; method that takes a set of typical command-line options - and distributes them into the appropriate construction variables. - Historically, it was created to support the &f-link-env-ParseConfig; method, + and distributes them into the appropriate &consvars; + Historically, it was created to support the &f-link-ParseConfig; method, so it focuses on options used by the GNU Compiler Collection (GCC) for the C and C++ toolchains. @@ -67,8 +69,8 @@ <para> &ParseFlags; returns a dictionary containing the options - distributed into their respective construction variables. - Normally, this dictionary would then be passed to &MergeFlags; + distributed into their respective &consvars;. + Normally, this dictionary would then be passed to &f-link-MergeFlags; to merge the options into a &consenv;, but the dictionary can be edited if desired to provide additional functionality. @@ -164,8 +166,7 @@ void main() { return 0; } <para> - If a string begins with a an exclamation mark (<literal>!</literal>, - sometimes also called a bang), + If a string begins with a an exclamation mark (<literal>!</literal>), the string is passed to the shell for execution. The output of the command is then parsed: diff --git a/site_scons/Utilities.py b/site_scons/Utilities.py index 5bdcbf2..ebc7bce 100644 --- a/site_scons/Utilities.py +++ b/site_scons/Utilities.py @@ -1,10 +1,10 @@ import os import stat import time -import distutils.util +import sysconfig -platform = distutils.util.get_platform() +platform = sysconfig.get_platform() def is_windows(): """ Check if we're on a Windows platform""" diff --git a/test/Actions/append.py b/test/Actions/append.py index b5d4c3a..42a414b 100644 --- a/test/Actions/append.py +++ b/test/Actions/append.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,13 +22,13 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the addition operator of Action objects. -# Using Environment.Prepend() and Environment.Append(), you should be -# able to add new actions to existing ones, effectively adding steps -# to a build process. -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the addition operator of Action objects. +Using Environment.Prepend() and Environment.Append(), you should be +able to add new actions to existing ones, effectively adding steps +to a build process. +""" import os import stat diff --git a/test/Actions/function.py b/test/Actions/function.py index 7f292cf..699cd17 100644 --- a/test/Actions/function.py +++ b/test/Actions/function.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import sys @@ -59,6 +58,7 @@ options.AddVariables( ('literal_in_listcomp', 'Literal inside list comprehension', '2'), ) +DefaultEnvironment(tools=[]) optEnv = Environment(options=options, tools=[]) r = re.compile(optEnv['regexp']) @@ -67,25 +67,24 @@ withClosure = \ r''' def toto(header='%(header)s', trailer='%(trailer)s'): xxx = %(closure_cell_value)s - def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , - header=header, trailer=trailer): + def writeDeps(target, source, env, b=%(b)s, r=r %(extraarg)s , header=header, trailer=trailer): """+'"""%(docstring)s"""'+""" def foo(b=b): return %(nestedfuncexp)s - f = open(str(target[0]),'wb') - f.write(bytearray(header,'utf-8')) - for d in env['ENVDEPS']: - f.write(bytearray(d+'%(separator)s','utf-8')) - f.write(bytearray(trailer+'\\n','utf-8')) - f.write(bytearray(str(foo())+'\\n','utf-8')) - f.write(bytearray(r.match('aaaa').group(1)+'\\n','utf-8')) - f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1,2]]))+'\\n', 'utf-8')) - %(extracode)s - try: - f.write(bytearray(str(xarg),'utf-8')+b'\\n') - except NameError: - pass - f.close() + + with open(str(target[0]), 'wb') as f: + f.write(bytearray(header, 'utf-8')) + for d in env['ENVDEPS']: + f.write(bytearray(d+'%(separator)s', 'utf-8')) + f.write(bytearray(trailer+'\\n', 'utf-8')) + f.write(bytearray(str(foo())+'\\n', 'utf-8')) + f.write(bytearray(r.match('aaaa').group(1)+'\\n', 'utf-8')) + f.write(bytearray(str(sum([x*%(literal_in_listcomp)s for x in [1, 2]]))+'\\n', 'utf-8')) + %(extracode)s + try: + f.write(bytearray(str(xarg), 'utf-8')+b'\\n') + except NameError: + pass return writeDeps ''' @@ -93,13 +92,9 @@ def toto(header='%(header)s', trailer='%(trailer)s'): exec(withClosure % optEnv) genHeaderBld = SCons.Builder.Builder( - action = SCons.Action.Action( - toto(), - 'Generating $TARGET', - varlist=['ENVDEPS'] - ), - suffix = '.gen.h' - ) + action=SCons.Action.Action(toto(), 'Generating $TARGET', varlist=['ENVDEPS']), + suffix='.gen.h', +) DefaultEnvironment(tools=[]) env = Environment(tools=[]) @@ -128,23 +123,22 @@ scons: done building targets. """ def runtest(arguments, expectedOutFile, expectedRebuild=True, stderr=""): - test.run(arguments=arguments, - stdout=expectedRebuild and rebuildstr or nobuildstr, - stderr="") + test.run( + arguments=arguments, + stdout=expectedRebuild and rebuildstr or nobuildstr, + stderr="", + ) sys.stdout.write('First Build.\n') test.must_match('Out.gen.h', expectedOutFile, message="First Build") - # Should not be rebuild when run a second time with the same - # arguments. - + # Should not be rebuild when run a second time with the same arguments. sys.stdout.write('Rebuild.\n') - test.run(arguments = arguments, stdout=nobuildstr, stderr="") + test.run(arguments=arguments, stdout=nobuildstr, stderr="") test.must_match('Out.gen.h', expectedOutFile, message="Should not rebuild") - # We're making this script chatty to prevent timeouts on really really # slow buildbot slaves (*cough* Solaris *cough*). diff --git a/test/Actions/pre-post.py b/test/Actions/pre-post.py index 358aa43..ce8cbdc 100644 --- a/test/Actions/pre-post.py +++ b/test/Actions/pre-post.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,12 +22,11 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# This test exercises the AddPreAction() and AddPostAction() API -# functions, which add pre-build and post-build actions to nodes. -# -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +""" +This test exercises the AddPreAction() and AddPostAction() API +functions, which add pre-build and post-build actions to nodes. +""" import os @@ -56,11 +57,8 @@ def before(env, target, source): def after(env, target, source): t = str(target[0]) a = "after_" + t - fin = open(t, "rb") - fout = open(a, "wb") - fout.write(fin.read()) - fout.close() - fin.close() + with open(t, "rb") as fin, open(a, "wb") as fout: + fout.write(fin.read()) os.chmod(a, os.stat(a)[stat.ST_MODE] | stat.S_IXUSR) foo = env.Program(source='foo.c', target='foo') @@ -103,19 +101,22 @@ test.must_match(['work3', 'dir', 'file'], "build()\n") # work4 start test.write(['work4', 'SConstruct'], """\ - DefaultEnvironment(tools=[]) def pre_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('pre %%s\\n' %% source[0]).encode()) + def post_action(target, source, env): with open(str(target[0]), 'ab') as f: f.write(('post %%s\\n' %% source[0]).encode()) + env = Environment(tools=[]) -o = env.Command(['pre-post', 'file.out'], - 'file.in', - r'%(_python_)s build.py ${TARGETS[1]} $SOURCE') +o = env.Command( + ['pre-post', 'file.out'], + 'file.in', + r'%(_python_)s build.py ${TARGETS[1]} $SOURCE' +) env.AddPreAction(o, pre_action) env.AddPostAction(o, post_action) """ % locals()) diff --git a/test/Actions/subst_shell_env-fixture/SConstruct b/test/Actions/subst_shell_env-fixture/SConstruct new file mode 100644 index 0000000..6e48add --- /dev/null +++ b/test/Actions/subst_shell_env-fixture/SConstruct @@ -0,0 +1,26 @@ +import sys + +def custom_environment_expansion(env, target, source): + ENV = env['ENV'].copy() + ENV['EXPANDED_SHELL_VAR'] = env.subst(env['ENV']['EXPANDED_SHELL_VAR'], target=target, source=source) + return ENV + +def expand_this_generator(env, target, source, for_signature): + return "I_got_expanded_to_" + str(target[0]) + +env = Environment(tools=['textfile']) + +env['SHELL_ENV_GENERATOR'] = custom_environment_expansion + +env['EXPAND_THIS'] = expand_this_generator +env['ENV']['EXPANDED_SHELL_VAR'] = "$EXPAND_THIS" +env['ENV']['NON_EXPANDED_SHELL_VAR'] = "$EXPAND_THIS" + +env.Textfile('expand_script.py', [ + 'import os', + 'print(os.environ["EXPANDED_SHELL_VAR"])', + 'print(os.environ["NON_EXPANDED_SHELL_VAR"])', +]) +env.Command('out.txt', 'expand_script.py', fr'{sys.executable} $SOURCE > $TARGET') + +env.Depends('out.txt', env.Command('out2.txt', 'expand_script.py', fr'{sys.executable} $SOURCE > $TARGET')) diff --git a/test/Actions/subst_shell_env.py b/test/Actions/subst_shell_env.py new file mode 100644 index 0000000..9f5c5db --- /dev/null +++ b/test/Actions/subst_shell_env.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +""" +Verify that shell environment variables can be expanded per target/source +when exectuting actions on the command line. +""" +import os + +import TestSCons + +test = TestSCons.TestSCons() + +test.dir_fixture('subst_shell_env-fixture') + +test.run(arguments = ['-Q']) +test.must_match('out.txt', f"I_got_expanded_to_out.txt{os.linesep}$EXPAND_THIS{os.linesep}") +test.must_match('out2.txt', f"I_got_expanded_to_out2.txt{os.linesep}$EXPAND_THIS{os.linesep}") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Decider/content-timestamp-symlink.py b/test/Decider/content-timestamp-symlink.py new file mode 100644 index 0000000..4e0c3b0 --- /dev/null +++ b/test/Decider/content-timestamp-symlink.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +Test the content-timestamp decider (formerly known as md5-timestamp) +correctly detects modification of a source file which is a symlink. +""" + +import TestSCons + +_python_ = TestSCons._python_ + +test = TestSCons.TestSCons() + +if not test.platform_has_symlink(): + test.skip_test('Symbolic links not reliably available on this platform, skipping test.\n') + +# a dummy "compiler" for the builder +test.write('build.py', r""" +import sys + +with open(sys.argv[1], 'wb') as ofp: + for infile in sys.argv[2:]: + with open (infile, 'rb') as ifp: + ofp.write(ifp.read()) +sys.exit(0) +""") + +test.write('SConstruct', """ +DefaultEnvironment(tools=[]) +Build = Builder(action=r'%(_python_)s build.py $TARGET $SOURCES') +env = Environment(tools=[], BUILDERS={'Build': Build}) +env.Decider('content-timestamp') +env.Build(target='match1.out', source='match1.in') +env.Build(target='match2.out', source='match2.in') +""" % locals()) + +test.write('match1.in', 'match1.in\n') +test.symlink('match1.in', 'match2.in') + +test.run(arguments='.') +test.must_match('match1.out', 'match1.in\n') +test.must_match('match2.out', 'match1.in\n') + +# first make sure some time has elapsed, so a low-granularity timestamp +# doesn't fail to trigger +test.sleep() +# Now update the source file contents, both targets should rebuild +test.write('match1.in', 'match2.in\n') + +test.run(arguments='.') +test.must_match('match1.out', 'match2.in\n', message="match1.out not rebuilt\n") +test.must_match('match2.out', 'match2.in\n', message="match2.out not rebuilt\n") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Dir/source.py b/test/Dir/source.py index f38b07d..c35d169 100644 --- a/test/Dir/source.py +++ b/test/Dir/source.py @@ -106,6 +106,8 @@ test.up_to_date(arguments='cmd-content.out') test.up_to_date(arguments='cmd-tstamp-noscan.out') test.up_to_date(arguments='cmd-content-noscan.out') +test.sleep() + test.write([ 'tstamp', 'foo.txt' ], 'foo.txt 2\n') test.not_up_to_date(arguments='tstamp.out') diff --git a/test/ninja/generated_sources_alias.py b/test/ninja/generated_sources_alias.py new file mode 100644 index 0000000..2c4ed36 --- /dev/null +++ b/test/ninja/generated_sources_alias.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_generated_sources_alias', 'SConstruct') + +for alias_value in [0, 1]: + + # Gen with source alias + test.run(arguments=['--disable-execute-ninja', f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None) + test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) + test.must_not_exist(test.workpath('gen_source' + _exe)) + + # run ninja independently + program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin + test.run(program=program, stdout=None) + test.run(program=test.workpath('gen_source' + _exe), stdout="4") + + # generate simple build + test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}'], stdout=None) + test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) + test.must_contain_all(test.stdout(), 'Executing:') + test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) + test.run(program=test.workpath('gen_source' + _exe), stdout="4") + + # clean build and ninja files + test.run(arguments=[f'USE_GEN_SOURCE_ALIAS={alias_value}', '-c'], stdout=None) + test.must_contain_all_lines(test.stdout(), [ + 'Removed generated_header1.htest', + 'Removed generated_header2.htest', + 'Removed generated_header2.c', + 'Removed build.ninja']) + + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/mingw_depfile_format.py b/test/ninja/mingw_depfile_format.py new file mode 100644 index 0000000..5de7b5f --- /dev/null +++ b/test/ninja/mingw_depfile_format.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os + +import TestSCons + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_mingw_depfile_format', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.must_contain(test.workpath('build.ninja'), 'deps = msvc') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja-fixture/gen_source.c b/test/ninja/ninja-fixture/gen_source.c new file mode 100644 index 0000000..38ac3fe --- /dev/null +++ b/test/ninja/ninja-fixture/gen_source.c @@ -0,0 +1,9 @@ +#include <stdio.h> +#include <stdlib.h> + +#include "generated_header2.htest" + +int main(int argc, char *argv[]) +{ + printf("%d", func2()); +} diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias new file mode 100644 index 0000000..3661bcc --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_generated_sources_alias @@ -0,0 +1,41 @@ +import SCons + +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +vars = Variables() +vars.Add("USE_GEN_SOURCE_ALIAS") + +env = Environment(variables=vars) +my_gen_alias = 'my_ninja_gen_sources' + + +env['SCANNERS'][0].add_scanner('.htest', SCons.Tool.CScanner) +env.Command('out.txt', [], f'echo USE_GEN_SOURCE_ALIAS={env["USE_GEN_SOURCE_ALIAS"]}') + +if env['USE_GEN_SOURCE_ALIAS'] == "1": + env['NINJA_GENERATED_SOURCE_ALIAS_NAME'] = my_gen_alias +else: + env['NINJA_GENERATED_SOURCE_SUFFIXES'] = ['.htest'] + +env.Tool('ninja') + +env.Alias(my_gen_alias, env.Textfile('generated_header1.htest', [ + '#pragma once', + 'int func1(){return 4;};' +])) +alias = env.Alias(my_gen_alias, env.Textfile('generated_header2.htest', [ + '#pragma once', + '', + 'int func2();' +])) +env.Depends(alias, 'out.txt') + +my_gen_alias, env.Textfile('generated_header2.c', [ + '#include "generated_header1.htest"', + '#include "generated_header2.htest"', + '', + 'int func2(){return func1();}' +]) + +env.Program(target='gen_source', source=['gen_source.c', 'generated_header2.c'])
\ No newline at end of file diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format new file mode 100644 index 0000000..b765db5 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_depfile_format @@ -0,0 +1,7 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=['mingw']) +env.Tool('ninja') +env['NINJA_DEPFILE_PARSE_FORMAT'] = 'msvc' + diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_response_file b/test/ninja/ninja_test_sconscripts/sconstruct_response_file new file mode 100644 index 0000000..6b7eb5d --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_response_file @@ -0,0 +1,15 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +vars = Variables() +vars.Add('MLL', default='2048') + +env = Environment(variables=vars) +env['MAXLINELENGTH'] = int(env['MLL']) +env.Tool('ninja') + +env.Program(target='foo', source='foo.c', OBJSUFFIX=env['OBJSUFFIX'] + "1") + +env2 = env.Clone() +env2.Append(CPPPATH='very/long/and/very/fake/path/for/testing') +env2.Program(target='foo2', source='foo.c', OBJSUFFIX=env['OBJSUFFIX'] + "2")
\ No newline at end of file diff --git a/test/ninja/response_file.py b/test/ninja/response_file.py new file mode 100644 index 0000000..3d23c2b --- /dev/null +++ b/test/ninja/response_file.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import sys +import json +from io import StringIO + + +import TestSCons +from TestCmd import IS_WINDOWS + +test = TestSCons.TestSCons() + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe +_obj = TestSCons._obj + +ninja_bin = os.path.abspath(os.path.join( + ninja.__file__, + os.pardir, + 'data', + 'bin', + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_response_file', 'SConstruct') + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('foo' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin + +# now we must extract the line length for a given command by print the command +# to a StringIO internal buffer and calculating the length of the resulting command. +test.run(program=[program, 'compiledb'], stdout=None) +command_length = None +with open('compile_commands.json') as f: + comp_commands = json.loads(f.read()) + for cmd in comp_commands: + if cmd['output'].endswith('1'): + result = StringIO() + old_stdout = sys.stdout + sys.stdout = result + print(cmd['command']) + result_string = result.getvalue() + sys.stdout = old_stdout + if sys.platform == 'win32': + command_length = len(result_string) + else: + command_length = len(result_string.split(';')[-1]) + +# now regen the ninja file with the MAXLINELENGTH so we can force the response file in one case. +test.run(arguments=['--disable-execute-ninja', 'MLL='+str(command_length)], stdout=None) +test.run(program=[program, '-v', '-d', 'keeprsp'], stdout=None) +test.must_not_exist(test.workpath('foo' + _obj + '1.rsp')) +test.must_exist(test.workpath('foo' + _obj + '2.rsp')) + +test.run(program=test.workpath('foo' + _exe), stdout="foo.c") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/option/option--.py b/test/option/option--.py index 8e06260..dbfb3c9 100644 --- a/test/option/option--.py +++ b/test/option/option--.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,16 +33,17 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ -MyBuild = Builder(action = r'%(_python_)s build.py $TARGETS') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = '-f1.out', source = 'f1.in') -env.MyBuild(target = '-f2.out', source = 'f2.in') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGETS') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='-f1.out', source='f1.in') +env.MyBuild(target='-f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") @@ -51,7 +51,7 @@ test.write('f2.in', "f2.in\n") expect = test.wrap_stdout('%(_python_)s build.py -f1.out\n%(_python_)s build.py -f2.out\n' % locals()) -test.run(arguments = '-- -f1.out -f2.out', stdout = expect) +test.run(arguments='-- -f1.out -f2.out', stdout=expect) test.fail_test(not os.path.exists(test.workpath('-f1.out'))) test.fail_test(not os.path.exists(test.workpath('-f2.out'))) diff --git a/test/option/option--Q.py b/test/option/option--Q.py index f3b82f9..ac56d34 100644 --- a/test/option/option--Q.py +++ b/test/option/option--Q.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -35,24 +34,24 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ - AddOption('--use_SetOption', action='store_true', dest='setoption', default=False) -use_setoption=GetOption('setoption') +use_setoption = GetOption('setoption') if use_setoption: SetOption('no_progress', True) - -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') -env = Environment(BUILDERS = { 'MyBuild' : MyBuild }) -env.MyBuild(target = 'f1.out', source = 'f1.in') -env.MyBuild(target = 'f2.out', source = 'f2.in') + +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') +env = Environment(BUILDERS={'MyBuild': MyBuild}) +env.MyBuild(target='f1.out', source='f1.in') +env.MyBuild(target='f2.out', source='f2.in') """ % locals()) test.write('f1.in', "f1.in\n") diff --git a/test/option/option-i.py b/test/option/option-i.py index 9b5212d..e426e1f 100644 --- a/test/option/option-i.py +++ b/test/option/option-i.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,43 +33,46 @@ test = TestSCons.TestSCons() test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) test.write('SConstruct', """ -Succeed = Builder(action = r'%(_python_)s succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s fail.py $TARGETS') -env = Environment(BUILDERS = { 'Succeed' : Succeed, 'Fail' : Fail }) -env.Fail(target = 'aaa.1', source = 'aaa.in') -env.Succeed(target = 'aaa.out', source = 'aaa.1') -env.Fail(target = 'bbb.1', source = 'bbb.in') -env.Succeed(target = 'bbb.out', source = 'bbb.1') +Succeed = Builder(action=r'%(_python_)s succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}) +env.Fail(target='aaa.1', source='aaa.in') +env.Succeed(target='aaa.out', source='aaa.1') +env.Fail(target='bbb.1', source='bbb.in') +env.Succeed(target='bbb.out', source='bbb.1') """ % locals()) test.write('aaa.in', "aaa.in\n") test.write('bbb.in', "bbb.in\n") -test.run(arguments = 'aaa.1 aaa.out bbb.1 bbb.out', - stderr = 'scons: *** [aaa.1] Error 1\n', - status = 2) +test.run( + arguments='aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(os.path.exists(test.workpath('aaa.out'))) test.fail_test(os.path.exists(test.workpath('bbb.1'))) test.fail_test(os.path.exists(test.workpath('bbb.out'))) -test.run(arguments = '-i aaa.1 aaa.out bbb.1 bbb.out', - stderr = - 'scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='-i aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out',mode='r') != "succeed.py: aaa.out\n") @@ -80,9 +82,10 @@ test.fail_test(test.read('bbb.out',mode='r') != "succeed.py: bbb.out\n") test.unlink("aaa.out") test.unlink("bbb.out") -test.run(arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', - stderr='scons: *** [aaa.1] Error 1\n' - 'scons: *** [bbb.1] Error 1\n') +test.run( + arguments='--ignore-errors aaa.1 aaa.out bbb.1 bbb.out', + stderr='scons: *** [aaa.1] Error 1\nscons: *** [bbb.1] Error 1\n', +) test.fail_test(os.path.exists(test.workpath('aaa.1'))) test.fail_test(test.read('aaa.out', mode='r') != "succeed.py: aaa.out\n") diff --git a/test/option/option-k.py b/test/option/option-k.py index d6c81ea..6a7cfcb 100644 --- a/test/option/option-k.py +++ b/test/option/option-k.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import TestSCons @@ -33,18 +32,17 @@ test = TestSCons.TestSCons() test.subdir('work1', 'work2', 'work3') - - test.write('succeed.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("succeed.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("succeed.py: %s\n" % sys.argv[1]) sys.exit(0) """) test.write('fail.py', r""" import sys + sys.exit(1) """) @@ -66,19 +64,23 @@ env.Succeed(target='bbb.out', source='bbb.in') test.write(['work1', 'aaa.in'], "aaa.in\n") test.write(['work1', 'bbb.in'], "bbb.in\n") -test.run(chdir='work1', - arguments='aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) test.must_not_exist(test.workpath('work1', 'bbb.out')) -test.run(chdir='work1', - arguments='-k aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='-k aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -86,10 +88,12 @@ test.must_match(['work1', 'bbb.out'], "succeed.py: bbb.out\n", mode='r') test.unlink(['work1', 'bbb.out']) -test.run(chdir = 'work1', - arguments='--keep-going aaa.out bbb.out', - stderr='scons: *** [aaa.1] Error 1\n', - status=2) +test.run( + chdir='work1', + arguments='--keep-going aaa.out bbb.out', + stderr='scons: *** [aaa.1] Error 1\n', + status=2, +) test.must_not_exist(test.workpath('work1', 'aaa.1')) test.must_not_exist(test.workpath('work1', 'aaa.out')) @@ -131,11 +135,12 @@ env.Succeed('ddd.out', 'ccc.in') test.write(['work2', 'aaa.in'], "aaa.in\n") test.write(['work2', 'ccc.in'], "ccc.in\n") -test.run(chdir='work2', - arguments='-k .', - status=2, - stderr=None, - stdout="""\ +test.run( + chdir='work2', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -143,7 +148,9 @@ scons: Building targets ... %(_python_)s ../succeed.py ccc.out %(_python_)s ../succeed.py ddd.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work2', 'aaa.out']) test.must_not_exist(['work2', 'bbb.out']) @@ -175,19 +182,19 @@ test.must_match(['work2', 'ddd.out'], "succeed.py: ddd.out\n", mode='r') test.write(['work3', 'SConstruct'], """\ DefaultEnvironment(tools=[]) -Succeed = Builder(action = r'%(_python_)s ../succeed.py $TARGETS') -Fail = Builder(action = r'%(_python_)s ../fail.py $TARGETS') -env = Environment(BUILDERS = {'Succeed': Succeed, 'Fail': Fail}, tools=[]) +Succeed = Builder(action=r'%(_python_)s ../succeed.py $TARGETS') +Fail = Builder(action=r'%(_python_)s ../fail.py $TARGETS') +env = Environment(BUILDERS={'Succeed': Succeed, 'Fail': Fail}, tools=[]) a = env.Fail('aaa.out', 'aaa.in') b = env.Succeed('bbb.out', 'bbb.in') c = env.Succeed('ccc.out', 'ccc.in') -a1 = Alias( 'a1', a ) -a2 = Alias( 'a2', a+b) -a4 = Alias( 'a4', c) -a3 = Alias( 'a3', a4+c) +a1 = Alias('a1', a) +a2 = Alias('a2', a + b) +a4 = Alias('a4', c) +a3 = Alias('a3', a4 + c) -Alias('all', a1+a2+a3) +Alias('all', a1 + a2 + a3) """ % locals()) test.write(['work3', 'aaa.in'], "aaa.in\n") @@ -196,36 +203,37 @@ test.write(['work3', 'ccc.in'], "ccc.in\n") # Test tegular build (i.e. without -k) -test.run(chdir = 'work3', - arguments = '.', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='.', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... %(_python_)s ../fail.py aaa.out scons: building terminated because of errors. -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Current directory -test.run(chdir = 'work3', - arguments = '-k .', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k .', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -233,26 +241,27 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) # Single target -test.run(chdir = 'work3', - arguments = '--keep-going all', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='--keep-going all', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -260,26 +269,26 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - -test.run(chdir = 'work3', - arguments = '-c .') +test.run(chdir='work3', arguments='-c .') test.must_not_exist(['work3', 'aaa.out']) test.must_not_exist(['work3', 'bbb.out']) test.must_not_exist(['work3', 'ccc.out']) - # Separate top-level targets -test.run(chdir = 'work3', - arguments = '-k a1 a2 a3', - status = 2, - stderr = None, - stdout = """\ +test.run( + chdir='work3', + arguments='-k a1 a2 a3', + status=2, + stderr=None, + stdout="""\ scons: Reading SConscript files ... scons: done reading SConscript files. scons: Building targets ... @@ -287,13 +296,14 @@ scons: Building targets ... %(_python_)s ../succeed.py bbb.out %(_python_)s ../succeed.py ccc.out scons: done building targets (errors occurred during build). -""" % locals()) +""" + % locals(), +) test.must_not_exist(['work3', 'aaa.out']) test.must_exist(['work3', 'bbb.out']) test.must_exist(['work3', 'ccc.out']) - test.pass_test() # Local Variables: diff --git a/test/option/option-s.py b/test/option/option-s.py index 89a0c62..359c295 100644 --- a/test/option/option-s.py +++ b/test/option/option-s.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # -# __COPYRIGHT__ +# MIT License +# +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,9 +22,6 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import os.path @@ -34,18 +33,19 @@ test = TestSCons.TestSCons() test.write('build.py', r""" import sys -file = open(sys.argv[1], 'w') -file.write("build.py: %s\n" % sys.argv[1]) -file.close() + +with open(sys.argv[1], 'w') as file: + file.write("build.py: %s\n" % sys.argv[1]) +sys.exit(0) """) test.write('SConstruct', """ DefaultEnvironment(tools=[]) -MyBuild = Builder(action = r'%(_python_)s build.py $TARGET') +MyBuild = Builder(action=r'%(_python_)s build.py $TARGET') -silent = ARGUMENTS.get('QUIET',0) +silent = ARGUMENTS.get('QUIET', 0) if silent: - SetOption('silent',True) + SetOption('silent', True) env = Environment(BUILDERS={'MyBuild': MyBuild}, tools=[]) env.MyBuild(target='f1.out', source='f1.in') @@ -86,10 +86,7 @@ test.run(arguments='QUIET=1 f1.out f2.out', test.fail_test(not os.path.exists(test.workpath('f1.out'))) test.fail_test(not os.path.exists(test.workpath('f2.out'))) - - test.pass_test() - # Local Variables: # tab-width:4 diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py index 79b16f5..5759121 100644 --- a/testing/framework/TestCmd.py +++ b/testing/framework/TestCmd.py @@ -311,6 +311,7 @@ import time import traceback from collections import UserList, UserString from subprocess import PIPE, STDOUT +from typing import Optional IS_WINDOWS = sys.platform == 'win32' IS_MACOS = sys.platform == 'darwin' @@ -1641,7 +1642,7 @@ class TestCmd: """ time.sleep(seconds) - def stderr(self, run=None): + def stderr(self, run=None) -> Optional[str]: """Returns the error output from the specified run number. If there is no specified run number, then returns the error @@ -1653,10 +1654,13 @@ class TestCmd: run = len(self._stderr) elif run < 0: run = len(self._stderr) + run - run = run - 1 - return self._stderr[run] + run -= 1 + try: + return self._stderr[run] + except IndexError: + return None - def stdout(self, run=None): + def stdout(self, run=None) -> Optional[str]: """Returns the stored standard output from a given run. Args: @@ -1673,7 +1677,7 @@ class TestCmd: run = len(self._stdout) elif run < 0: run = len(self._stdout) + run - run = run - 1 + run -= 1 try: return self._stdout[run] except IndexError: diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 079e17d..ec82102 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -1665,7 +1665,12 @@ else: alt_cpp_suffix = '.C' return alt_cpp_suffix - def platform_has_symlink(self): + def platform_has_symlink(self) -> bool: + """Retun an indication of whether symlink tests should be run. + + Despite the name, we really mean "are they reliably usable" + rather than "do they exist" - basically the Windows case. + """ if not hasattr(os, 'symlink') or sys.platform == 'win32': return False else: |