diff options
author | William Deegan <bill@baddogconsulting.com> | 2022-03-29 18:59:10 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-03-29 18:59:10 (GMT) |
commit | 73ea66f408c2797464e02fa9bf8e80564ba03dba (patch) | |
tree | 97b9f5dc13c1f069d1a070f5ab3fc348cd2a008a | |
parent | ec58ef74c0be0edc138305ab95ac2f5732bb6cc1 (diff) | |
parent | 2976ed620bfd86804927a9bc3760924f4a3205e2 (diff) | |
download | SCons-73ea66f408c2797464e02fa9bf8e80564ba03dba.zip SCons-73ea66f408c2797464e02fa9bf8e80564ba03dba.tar.gz SCons-73ea66f408c2797464e02fa9bf8e80564ba03dba.tar.bz2 |
Merge branch 'master' into msvc/cachefix
69 files changed, 1593 insertions, 530 deletions
diff --git a/.github/workflows/experimental_tests.yml b/.github/workflows/experimental_tests.yml index aac28d0..3672144 100644 --- a/.github/workflows/experimental_tests.yml +++ b/.github/workflows/experimental_tests.yml @@ -30,6 +30,12 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + - name: Set up MinGW + uses: egor-tensin/setup-mingw@v2 + if: matrix.os == 'windows-latest' + with: + platform: x64 + - name: Set up Python 3.8 ${{ matrix.os }} uses: actions/setup-python@v2 with: diff --git a/CHANGES.txt b/CHANGES.txt index 41b02d9..80fee1c 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -28,11 +28,22 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added project_url for mailing lists and Discord - Updated project url in steup.cfg to be https instead of http - Updated setup.cfg to remove Python 3.5 and add Python 3.10 + - Added default values for source and target arguments to _defines() function. This + is used to expand CPPDEFINES (and others). Previous change added those arguments + with no defaults, so old usage where _defines() was called without source and target + arguments would yield an exception. This issue was found via qt4 and qt5 tools in + scons-contrib https://github.com/SCons/scons-contrib/issues/45 From Daniel Moody: - Add cache-debug messages for push failures. + - Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) + - Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. + - Added ninja mingw support and improved ninja CommandGeneratorAction support. From Mats Wichmann: - Tweak the way default site_scons paths on Windows are expressed to @@ -65,10 +76,15 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER off if there are no arguments. The default cachefile is changed to have a .json suffix, for better recognition on Windows since the contents are json. + - As "code modernization" all of SCons now uses the current super() + zero-argument syntax instead of direct calls to a parent class method + or the super() two-argument syntax. + - Renamed ParseFlag's internal data structure to "mapping" instead of + "dict" (avoid redefining builtin) From Zhichang Yu: - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. - + - Added Configure.CheckMember() checker to check if struct/class has the specified member. RELEASE 4.3.0 - Tue, 16 Nov 2021 18:12:46 -0700 diff --git a/RELEASE.txt b/RELEASE.txt index c021a40..aed58b8 100755 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -14,8 +14,9 @@ NOTE: If you build with Python 3.10.0 and then rebuild with 3.10.1 (or higher), NEW FUNCTIONALITY ----------------- -- List new features (presumably why a checkpoint is being released) - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT. +- Added Configure.CheckMember() checker to check if struct/class has the specified member. + DEPRECATED FUNCTIONALITY ------------------------ @@ -32,8 +33,13 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY "Application Data" subdirectory here. - Action._subproc() can now be used as a python context manager to ensure that the POpen object is properly closed. -- SCons help (-H) no longer prints the "ignored for compatibility" options - to save some space (these are still listed in the manpage). +- SCons help (-H) no longer prints the "ignored for compatibility" options, + which are still listed in the manpage. +- Help is now sensitive to the size of the terminal window: the width of the + help text will scale to wider (or narrower) terminals than 80 characters. +- Ninja: Changed generated build.ninja file to run SCons only build Actions via + a SCons Deamon. Added logic for starting and connecting to SCons daemon (currently + only used for ninja) - The change to "content" and "content-timestamp" Decider names is reflected in the User Guide as well, since the hash function may be other than md5 (tidying up from earlier change) @@ -50,9 +56,15 @@ FIXES - Fix a number of Python ResourceWarnings which are issued when running SCons and/or it's tests with python 3.9 (or higher) +- Ninja: Fix issue where Configure files weren't being properly processed when build run + via ninja. - Fixed crash in C scanner's dictify_CPPDEFINES() function which happens if AppendUnique is called on CPPPATH. (Issue #4108). - +- Added default values for source and target arguments to _defines() function. This + is used to expand CPPDEFINES (and others). Previous change added those arguments + with no defaults, so old usage where _defines() was called without source and target + arguments would yield an exception. This issue was found via qt4 and qt5 tools in + scons-contrib https://github.com/SCons/scons-contrib/issues/45 IMPROVEMENTS ------------ @@ -60,6 +72,7 @@ IMPROVEMENTS - Verify that a user specified msvc script (via MSVC_USE_SCRIPT) exists and raise an exception immediately when the user specified msvc script does not exist. - Add cache-debug messages for push failures. +- Added ninja mingw support and improved ninja CommandGeneratorAction support. - Command-line help is now sensitive to the size of the terminal window: the width of the help text will scale for terminals other than 80 chars wide. @@ -85,4 +98,4 @@ Thanks to the following contributors listed below for their contributions to thi ========================================================================================== .. code-block:: text - git shortlog --no-merges -ns 4.0.1..HEAD + git shortlog --no-merges -ns 4.3.0..HEAD diff --git a/SCons/Action.py b/SCons/Action.py index b7c6bb7..81dc033 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -848,7 +848,7 @@ class CommandAction(_ActionAction): # variables. if SCons.Debug.track_instances: logInstanceCreation(self, 'Action.CommandAction') - _ActionAction.__init__(self, **kw) + super().__init__(**kw) if is_List(cmd): if [c for c in cmd if is_List(c)]: raise TypeError("CommandAction should be given only " @@ -1231,7 +1231,7 @@ class FunctionAction(_ActionAction): # This is weird, just do the best we can. self.funccontents = _object_contents(execfunction) - _ActionAction.__init__(self, **kw) + super().__init__(**kw) def function_name(self): try: diff --git a/SCons/Builder.py b/SCons/Builder.py index 41475ac..ab51c32 100644 --- a/SCons/Builder.py +++ b/SCons/Builder.py @@ -130,8 +130,8 @@ class DictCmdGenerator(SCons.Util.Selector): to return the proper action based on the file suffix of the source file.""" - def __init__(self, dict=None, source_ext_match=1): - SCons.Util.Selector.__init__(self, dict) + def __init__(self, mapping=None, source_ext_match=True): + super().__init__(mapping) self.source_ext_match = source_ext_match def src_suffixes(self): @@ -222,10 +222,11 @@ class OverrideWarner(UserDict): can actually invoke multiple builders. This class only emits the warnings once, no matter how many Builders are invoked. """ - def __init__(self, dict): - UserDict.__init__(self, dict) + def __init__(self, mapping): + super().__init__(mapping) if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.OverrideWarner') self.already_warned = None + def warn(self): if self.already_warned: return @@ -245,7 +246,7 @@ def Builder(**kw): kw['action'] = SCons.Action.CommandGeneratorAction(kw['generator'], {}) del kw['generator'] elif 'action' in kw: - source_ext_match = kw.get('source_ext_match', 1) + source_ext_match = kw.get('source_ext_match', True) if 'source_ext_match' in kw: del kw['source_ext_match'] if SCons.Util.is_Dict(kw['action']): @@ -876,7 +877,7 @@ class CompositeBuilder(SCons.Util.Proxy): def __init__(self, builder, cmdgen): if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.CompositeBuilder') - SCons.Util.Proxy.__init__(self, builder) + super().__init__(builder) # cmdgen should always be an instance of DictCmdGenerator. self.cmdgen = cmdgen diff --git a/SCons/Conftest.py b/SCons/Conftest.py index 16b444f..83175cb 100644 --- a/SCons/Conftest.py +++ b/SCons/Conftest.py @@ -568,6 +568,63 @@ int main(void) "Set to 1 if %s is defined." % symbol) return st + +def CheckMember(context, aggregate_member, header = None, language = None): + """ + Configure check for a C or C++ member "aggregate_member". + Optional "header" can be defined to include a header file. + "language" should be "C" or "C++" and is used to select the compiler. + Default is "C". + Note that this uses the current value of compiler and linker flags, make + sure $CFLAGS, $CPPFLAGS and $LIBS are set correctly. + + Arguments: + aggregate_member : str + the member to check. For example, 'struct tm.tm_gmtoff'. + includes : str + Optional "header" can be defined to include a header file. + language : str + only C and C++ supported. + + Returns the status (0 or False = Passed, True/non-zero = Failed). + """ + + lang, suffix, msg = _lang2suffix(language) + if msg: + context.Display("Cannot check for member %s: %s\n" % (aggregate_member, msg)) + return True + context.Display("Checking for %s member %s... " % (lang, aggregate_member)) + fields = aggregate_member.split('.') + if len(fields) != 2: + msg = "shall contain just one dot, for example 'struct tm.tm_gmtoff'" + context.Display("Cannot check for member %s: %s\n" % (aggregate_member, msg)) + return True + aggregate, member = fields[0], fields[1] + + # Include "confdefs.h" first, so that the header can use HAVE_HEADER_H. + if context.headerfilename: + includetext = '#include "%s"' % context.headerfilename + else: + includetext = '' + if not header: + header = '' + text = ''' +%(include)s +%(header)s + +int main(void) { + if (sizeof ((%(aggregate)s *) 0)->%(member)s) + return 0; +}''' % {'include': includetext, + 'header': header, + 'aggregate': aggregate, + 'member': member} + + ret = context.BuildProg(text, suffix) + _YesNoResult(context, ret, "HAVE_" + aggregate_member, text, + "Define to 1 if the system has the member `%s`." % aggregate_member) + return ret + def CheckLib(context, libs, func_name = None, header = None, extra_libs = None, call = None, language = None, autoadd = 1, append = True): diff --git a/SCons/Defaults.py b/SCons/Defaults.py index 3edfe7b..65bd75a 100644 --- a/SCons/Defaults.py +++ b/SCons/Defaults.py @@ -509,7 +509,7 @@ def processDefines(defs): return l -def _defines(prefix, defs, suffix, env, target, source, c=_concat_ixes): +def _defines(prefix, defs, suffix, env, target=None, source=None, c=_concat_ixes): """A wrapper around _concat_ixes that turns a list or string into a list of C preprocessor command-line definitions. """ diff --git a/SCons/DefaultsTests.py b/SCons/DefaultsTests.py index 6a7ce9c..7fd62ae 100644 --- a/SCons/DefaultsTests.py +++ b/SCons/DefaultsTests.py @@ -23,22 +23,40 @@ import os import unittest +import collections import TestCmd -from SCons.Defaults import mkdir_func +from SCons.Defaults import mkdir_func, _defines + + +class DummyEnvironment(collections.UserDict): + def __init__(self, **kwargs): + super().__init__() + self.data.update(kwargs) + + def subst(self, str_subst, target=None, source=None, conv=None): + if str_subst[0] == '$': + return self.data[str_subst[1:]] + return str_subst + + def subst_list(self, str_subst, target=None, source=None, conv=None): + if str_subst[0] == '$': + return [self.data[str_subst[1:]]] + return [[str_subst]] + class DefaultsTestCase(unittest.TestCase): def test_mkdir_func0(self): - test = TestCmd.TestCmd(workdir = '') + test = TestCmd.TestCmd(workdir='') test.subdir('sub') subdir2 = test.workpath('sub', 'dir1', 'dir2') # Simple smoke test mkdir_func(subdir2) - mkdir_func(subdir2) # 2nd time should be OK too + mkdir_func(subdir2) # 2nd time should be OK too def test_mkdir_func1(self): - test = TestCmd.TestCmd(workdir = '') + test = TestCmd.TestCmd(workdir='') test.subdir('sub') subdir1 = test.workpath('sub', 'dir1') subdir2 = test.workpath('sub', 'dir1', 'dir2') @@ -48,7 +66,7 @@ class DefaultsTestCase(unittest.TestCase): mkdir_func(subdir1) def test_mkdir_func2(self): - test = TestCmd.TestCmd(workdir = '') + test = TestCmd.TestCmd(workdir='') test.subdir('sub') subdir1 = test.workpath('sub', 'dir1') subdir2 = test.workpath('sub', 'dir1', 'dir2') @@ -65,6 +83,26 @@ class DefaultsTestCase(unittest.TestCase): else: self.fail("expected OSError") + def test__defines_no_target_or_source_arg(self): + """ + Verify that _defines() function can handle either or neither source or + target being specified + """ + env = DummyEnvironment() + + # Neither source or target specified + x = _defines('-D', ['A', 'B', 'C'], 'XYZ', env) + self.assertEqual(x, ['-DAXYZ', '-DBXYZ', '-DCXYZ']) + + # only source specified + y = _defines('-D', ['AA', 'BA', 'CA'], 'XYZA', env, 'XYZ') + self.assertEqual(y, ['-DAAXYZA', '-DBAXYZA', '-DCAXYZA']) + + # source and target specified + z = _defines('-D', ['AAB', 'BAB', 'CAB'], 'XYZAB', env, 'XYZ', 'abc') + self.assertEqual(z,['-DAABXYZAB', '-DBABXYZAB', '-DCABXYZAB']) + + if __name__ == "__main__": unittest.main() diff --git a/SCons/Environment.py b/SCons/Environment.py index d0a4baf..3507263 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -275,12 +275,12 @@ class BuilderDict(UserDict): the Builders. We need to do this because every time someone changes the Builders in the Environment's BUILDERS dictionary, we must update the Environment's attributes.""" - def __init__(self, dict, env): + def __init__(self, mapping, env): # Set self.env before calling the superclass initialization, # because it will end up calling our other methods, which will # need to point the values in this dictionary to self.env. self.env = env - UserDict.__init__(self, dict) + super().__init__(mapping) def __semi_deepcopy__(self): # These cannot be copied since they would both modify the same builder object, and indeed @@ -294,15 +294,15 @@ class BuilderDict(UserDict): pass else: self.env.RemoveMethod(method) - UserDict.__setitem__(self, item, val) + super().__setitem__(item, val) BuilderWrapper(self.env, val, item) def __delitem__(self, item): - UserDict.__delitem__(self, item) + super().__delitem__(item) delattr(self.env, item) - def update(self, dict): - for i, v in dict.items(): + def update(self, mapping): + for i, v in mapping.items(): self.__setitem__(i, v) @@ -637,7 +637,7 @@ class SubstitutionEnvironment: it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ - dict = { + mapping = { 'ASFLAGS' : CLVar(''), 'CFLAGS' : CLVar(''), 'CCFLAGS' : CLVar(''), @@ -667,12 +667,12 @@ class SubstitutionEnvironment: arg = self.backtick(arg[1:]) # utility function to deal with -D option - def append_define(name, dict = dict): + def append_define(name, mapping=mapping): t = name.split('=') if len(t) == 1: - dict['CPPDEFINES'].append(name) + mapping['CPPDEFINES'].append(name) else: - dict['CPPDEFINES'].append([t[0], '='.join(t[1:])]) + mapping['CPPDEFINES'].append([t[0], '='.join(t[1:])]) # Loop through the flags and add them to the appropriate option. # This tries to strike a balance between checking for all possible @@ -702,67 +702,67 @@ class SubstitutionEnvironment: append_define(arg) elif append_next_arg_to == '-include': t = ('-include', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-imacros': t = ('-imacros', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-isysroot': t = ('-isysroot', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '-isystem': t = ('-isystem', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-iquote': t = ('-iquote', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-idirafter': t = ('-idirafter', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-arch': t = ('-arch', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '--param': t = ('--param', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) else: - dict[append_next_arg_to].append(arg) + mapping[append_next_arg_to].append(arg) append_next_arg_to = None elif not arg[0] in ['-', '+']: - dict['LIBS'].append(self.fs.File(arg)) + mapping['LIBS'].append(self.fs.File(arg)) elif arg == '-dylib_file': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) append_next_arg_to = 'LINKFLAGS' elif arg[:2] == '-L': if arg[2:]: - dict['LIBPATH'].append(arg[2:]) + mapping['LIBPATH'].append(arg[2:]) else: append_next_arg_to = 'LIBPATH' elif arg[:2] == '-l': if arg[2:]: - dict['LIBS'].append(arg[2:]) + mapping['LIBS'].append(arg[2:]) else: append_next_arg_to = 'LIBS' elif arg[:2] == '-I': if arg[2:]: - dict['CPPPATH'].append(arg[2:]) + mapping['CPPPATH'].append(arg[2:]) else: append_next_arg_to = 'CPPPATH' elif arg[:4] == '-Wa,': - dict['ASFLAGS'].append(arg[4:]) - dict['CCFLAGS'].append(arg) + mapping['ASFLAGS'].append(arg[4:]) + mapping['CCFLAGS'].append(arg) elif arg[:4] == '-Wl,': if arg[:11] == '-Wl,-rpath=': - dict['RPATH'].append(arg[11:]) + mapping['RPATH'].append(arg[11:]) elif arg[:7] == '-Wl,-R,': - dict['RPATH'].append(arg[7:]) + mapping['RPATH'].append(arg[7:]) elif arg[:6] == '-Wl,-R': - dict['RPATH'].append(arg[6:]) + mapping['RPATH'].append(arg[6:]) else: - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:4] == '-Wp,': - dict['CPPFLAGS'].append(arg) + mapping['CPPFLAGS'].append(arg) elif arg[:2] == '-D': if arg[2:]: append_define(arg[2:]) @@ -771,32 +771,32 @@ class SubstitutionEnvironment: elif arg == '-framework': append_next_arg_to = 'FRAMEWORKS' elif arg[:14] == '-frameworkdir=': - dict['FRAMEWORKPATH'].append(arg[14:]) + mapping['FRAMEWORKPATH'].append(arg[14:]) elif arg[:2] == '-F': if arg[2:]: - dict['FRAMEWORKPATH'].append(arg[2:]) + mapping['FRAMEWORKPATH'].append(arg[2:]) else: append_next_arg_to = 'FRAMEWORKPATH' - elif arg in [ + elif arg in ( '-mno-cygwin', '-pthread', '-openmp', '-fmerge-all-constants', '-fopenmp', - ]: - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + ): + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg == '-mwindows': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:5] == '-std=': if '++' in arg[5:]: - key='CXXFLAGS' + key = 'CXXFLAGS' else: - key='CFLAGS' - dict[key].append(arg) + key = 'CFLAGS' + mapping[key].append(arg) elif arg[0] == '+': - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg in [ '-include', '-imacros', @@ -809,11 +809,11 @@ class SubstitutionEnvironment: ]: append_next_arg_to = arg else: - dict['CCFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) for arg in flags: do_parse(arg) - return dict + return mapping def MergeFlags(self, args, unique=True): """Merge flags into construction variables. @@ -2375,7 +2375,7 @@ class OverrideEnvironment(Base): if SCons.Debug.track_instances: logInstanceCreation(self, 'Environment.OverrideEnvironment') self.__dict__['__subject'] = subject if overrides is None: - self.__dict__['overrides'] = dict() + self.__dict__['overrides'] = {} else: self.__dict__['overrides'] = overrides @@ -2472,6 +2472,11 @@ class OverrideEnvironment(Base): self.__dict__['overrides'].update(other) def _update_onlynew(self, other): + """Update a dict with new keys. + + Unlike the .update method, if the key is already present, + it is not replaced. + """ for k, v in other.items(): if k not in self.__dict__['overrides']: self.__dict__['overrides'][k] = v @@ -2516,28 +2521,35 @@ def NoSubstitutionProxy(subject): class _NoSubstitutionProxy(Environment): def __init__(self, subject): self.__dict__['__subject'] = subject + def __getattr__(self, name): return getattr(self.__dict__['__subject'], name) + def __setattr__(self, name, value): return setattr(self.__dict__['__subject'], name, value) + def executor_to_lvars(self, kwdict): if 'executor' in kwdict: kwdict['lvars'] = kwdict['executor'].get_lvars() del kwdict['executor'] else: kwdict['lvars'] = {} - def raw_to_mode(self, dict): + + def raw_to_mode(self, mapping): try: - raw = dict['raw'] + raw = mapping['raw'] except KeyError: pass else: - del dict['raw'] - dict['mode'] = raw + del mapping['raw'] + mapping['mode'] = raw + def subst(self, string, *args, **kwargs): return string + def subst_kw(self, kw, *args, **kwargs): return kw + def subst_list(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2545,6 +2557,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst_list(*nargs, **nkw) + def subst_target_source(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2552,6 +2565,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst(*nargs, **nkw) + return _NoSubstitutionProxy(subject) # Local Variables: diff --git a/SCons/Environment.xml b/SCons/Environment.xml index d314711..5b41f22 100644 --- a/SCons/Environment.xml +++ b/SCons/Environment.xml @@ -2691,7 +2691,7 @@ Prepend values to &consvars; in the current &consenv;, maintaining uniqueness. Works like &f-link-env-Append; (see for details), except that values are added to the front, -rather than the end, of any existing value of the the &consvar;, +rather than the end, of any existing value of the &consvar;, and values already present in the &consvar; will not be added again. If <parameter>delete_existing</parameter> @@ -3139,9 +3139,16 @@ files = Split(""" </arguments> <summary> <para> -Performs construction variable interpolation +Performs &consvar; interpolation +(<firstterm>substitution</firstterm>) on <parameter>input</parameter>, which can be a string or a sequence. +Substitutable elements take the form +<literal>${<replaceable>expression</replaceable>}</literal>, +although if there is no ambiguity in recognizing the element, +the braces can be omitted. +A literal <emphasis role="bold">$</emphasis> can be entered by +using <emphasis role="bold">$$</emphasis>. </para> <para> @@ -3176,7 +3183,7 @@ pairs </para> <para> -If the input is a sequence +If <parameter>input</parameter> is a sequence (list or tuple), the individual elements of the sequence will be expanded, @@ -3388,7 +3395,7 @@ env.Config(target = 'package-config', source = Value(prefix)) def build_value(target, source, env): # A function that "builds" a Python Value by updating - # the the Python value with the contents of the file + # the Python value with the contents of the file # specified as the source of the Builder call ($SOURCE). target[0].write(source[0].get_contents()) diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index c505422..8359405 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -810,11 +810,13 @@ sys.exit(0) "-fmerge-all-constants " "-fopenmp " "-mno-cygwin -mwindows " - "-arch i386 -isysroot /tmp " + "-arch i386 " + "-isysroot /tmp " "-iquote /usr/include/foo1 " "-isystem /usr/include/foo2 " "-idirafter /usr/include/foo3 " "-imacros /usr/include/foo4 " + "-include /usr/include/foo5 " "--param l1-cache-size=32 --param l2-cache-size=6144 " "+DD64 " "-DFOO -DBAR=value -D BAZ " @@ -832,6 +834,7 @@ sys.exit(0) ('-isystem', '/usr/include/foo2'), ('-idirafter', '/usr/include/foo3'), ('-imacros', env.fs.File('/usr/include/foo4')), + ('-include', env.fs.File('/usr/include/foo5')), ('--param', 'l1-cache-size=32'), ('--param', 'l2-cache-size=6144'), '+DD64'], repr(d['CCFLAGS']) assert d['CXXFLAGS'] == ['-std=c++0x'], repr(d['CXXFLAGS']) diff --git a/SCons/Errors.py b/SCons/Errors.py index 42db072..04cea38 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -99,8 +99,8 @@ class BuildError(Exception): self.action = action self.command = command - Exception.__init__(self, node, errstr, status, exitstatus, filename, - executor, action, command, exc_info) + super().__init__(node, errstr, status, exitstatus, filename, + executor, action, command, exc_info) def __str__(self): if self.filename: @@ -128,7 +128,7 @@ class ExplicitExit(Exception): self.node = node self.status = status self.exitstatus = status - Exception.__init__(self, *args) + super().__init__(*args) def convert_to_BuildError(status, exc_info=None): """Convert a return code to a BuildError Exception. diff --git a/SCons/Job.py b/SCons/Job.py index bcdb88a..b398790 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -238,7 +238,7 @@ else: and a boolean indicating whether the task executed successfully. """ def __init__(self, requestQueue, resultsQueue, interrupted): - threading.Thread.__init__(self) + super().__init__() self.daemon = True self.requestQueue = requestQueue self.resultsQueue = resultsQueue diff --git a/SCons/JobTests.py b/SCons/JobTests.py index 5b5a590..54d7fa4 100644 --- a/SCons/JobTests.py +++ b/SCons/JobTests.py @@ -410,13 +410,13 @@ class DummyNodeInfo: class testnode (SCons.Node.Node): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.executed self.ninfo = DummyNodeInfo() class goodnode (testnode): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.up_to_date self.ninfo = DummyNodeInfo() @@ -430,7 +430,7 @@ class slowgoodnode (goodnode): class badnode (goodnode): def __init__(self): - goodnode.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.failed def build(self, **kw): raise Exception('badnode exception') diff --git a/SCons/Memoize.py b/SCons/Memoize.py index 8c3303f..b02c144 100644 --- a/SCons/Memoize.py +++ b/SCons/Memoize.py @@ -159,7 +159,7 @@ class CountDict(Counter): def __init__(self, cls_name, method_name, keymaker): """ """ - Counter.__init__(self, cls_name, method_name) + super().__init__(cls_name, method_name) self.keymaker = keymaker def count(self, *args, **kw): """ Counts whether the computed key value is already present diff --git a/SCons/Node/Alias.py b/SCons/Node/Alias.py index b5e4eb4..1125c22 100644 --- a/SCons/Node/Alias.py +++ b/SCons/Node/Alias.py @@ -99,7 +99,7 @@ class Alias(SCons.Node.Node): BuildInfo = AliasBuildInfo def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.changed_since_last_build = 1 self.store_info = 0 diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a86..c1bfd6a 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -82,7 +82,7 @@ class EntryProxyAttributeError(AttributeError): of the underlying Entry involved in an AttributeError exception. """ def __init__(self, entry_proxy, attribute): - AttributeError.__init__(self) + super().__init__() self.entry_proxy = entry_proxy self.attribute = attribute def __str__(self): @@ -579,7 +579,7 @@ class Base(SCons.Node.Node): signatures.""" if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Base') - SCons.Node.Node.__init__(self) + super().__init__() # Filenames and paths are probably reused and are intern'ed to save some memory. # Filename with extension as it was specified when the object was @@ -982,7 +982,7 @@ class Entry(Base): 'contentsig'] def __init__(self, name, directory, fs): - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._func_exists = 3 self._func_get_contents = 1 @@ -1574,7 +1574,7 @@ class Dir(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Dir') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def _morph(self): @@ -2563,7 +2563,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): if key != 'dependency_map' and hasattr(self, 'dependency_map'): del self.dependency_map - return super(FileBuildInfo, self).__setattr__(key, value) + return super().__setattr__(key, value) def convert_to_sconsign(self): """ @@ -2674,7 +2674,7 @@ class File(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.File') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def Entry(self, name): diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index d78940e..b93efc8 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -2568,7 +2568,7 @@ class FileTestCase(_tempdirTestCase): class ChangedNode(SCons.Node.FS.File): def __init__(self, name, directory=None, fs=None): - SCons.Node.FS.File.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self.name = name self.Tag('found_includes', []) self.stored_info = None @@ -2608,7 +2608,7 @@ class FileTestCase(_tempdirTestCase): class ChangedEnvironment(SCons.Environment.Base): def __init__(self): - SCons.Environment.Base.__init__(self) + super().__init__() self.decide_source = self._changed_timestamp_then_content class FakeNodeInfo: diff --git a/SCons/Node/NodeTests.py b/SCons/Node/NodeTests.py index b36c2e9..ee4d080 100644 --- a/SCons/Node/NodeTests.py +++ b/SCons/Node/NodeTests.py @@ -160,7 +160,7 @@ class NoneBuilder(Builder): class ListBuilder(Builder): def __init__(self, *nodes): - Builder.__init__(self) + super().__init__() self.nodes = nodes def execute(self, target, source, env): if hasattr(self, 'status'): @@ -200,7 +200,7 @@ class MyNode(SCons.Node.Node): simulate a real, functional Node subclass. """ def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.Tag('found_includes', []) def __str__(self): @@ -1226,7 +1226,7 @@ class NodeTestCase(unittest.TestCase): """Test the get_string() method.""" class TestNode(MyNode): def __init__(self, name, sig): - MyNode.__init__(self, name) + super().__init__(name) self.sig = sig def for_signature(self): diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index c6850ab..80d2762 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -83,7 +83,7 @@ class Value(SCons.Node.Node): BuildInfo = ValueBuildInfo def __init__(self, value, built_value=None, name=None): - SCons.Node.Node.__init__(self) + super().__init__() self.value = value self.changed_since_last_build = 6 self.store_info = 0 diff --git a/SCons/SConf.py b/SCons/SConf.py index 0ad712d..4e8d410 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -142,7 +142,7 @@ SCons.Warnings.enableWarningClass(SConfWarning) # some error definitions class SConfError(SCons.Errors.UserError): def __init__(self,msg): - SCons.Errors.UserError.__init__(self,msg) + super().__init__(msg) class ConfigureDryRunError(SConfError): """Raised when a file or directory needs to be updated during a Configure @@ -152,13 +152,13 @@ class ConfigureDryRunError(SConfError): msg = 'Cannot create configure directory "%s" within a dry-run.' % str(target) else: msg = 'Cannot update configure test "%s" within a dry-run.' % str(target) - SConfError.__init__(self,msg) + super().__init__(msg) class ConfigureCacheError(SConfError): """Raised when a use explicitely requested the cache feature, but the test is run the first time.""" def __init__(self,target): - SConfError.__init__(self, '"%s" is not yet built and cache is forced.' % str(target)) + super().__init__('"%s" is not yet built and cache is forced.' % str(target)) # define actions for building text files @@ -450,6 +450,7 @@ class SConfBase: 'CheckFunc' : CheckFunc, 'CheckType' : CheckType, 'CheckTypeSize' : CheckTypeSize, + 'CheckMember' : CheckMember, 'CheckDeclaration' : CheckDeclaration, 'CheckHeader' : CheckHeader, 'CheckCHeader' : CheckCHeader, @@ -992,6 +993,13 @@ def CheckDeclaration(context, declaration, includes = "", language = None): context.did_show_result = 1 return not res +def CheckMember(context, aggregate_member, header = None, language = None): + '''Returns the status (False : failed, True : ok).''' + res = SCons.Conftest.CheckMember(context, aggregate_member, header=header, language=language) + context.did_show_result = 1 + return not res + + def createIncludesFromHeaders(headers, leaveLast, include_quotes = '""'): # used by CheckHeader and CheckLibWithHeader to produce C - #include # statements from the specified header (list) diff --git a/SCons/SConfTests.py b/SCons/SConfTests.py index 1badad8..a06b227 100644 --- a/SCons/SConfTests.py +++ b/SCons/SConfTests.py @@ -725,6 +725,27 @@ int main(void) { finally: sconf.Finish() + def test_CheckMember(self): + """Test SConf.CheckMember() + """ + self._resetSConfState() + sconf = self.SConf.SConf(self.scons_env, + conf_dir=self.test.workpath('config.tests'), + log_file=self.test.workpath('config.log')) + + try: + # CheckMember() + r = sconf.CheckMember('struct timespec.tv_sec', '#include <time.h>') + assert r, "did not find timespec.tv_sec" + r = sconf.CheckMember('struct timespec.tv_nano', '#include <time.h>') + assert not r, "unexpectedly found struct timespec.tv_nano" + r = sconf.CheckMember('hopefullynomember') + assert not r, "unexpectedly found hopefullynomember :%s" % r + + finally: + sconf.Finish() + + def test_(self): """Test SConf.CheckType() """ diff --git a/SCons/SConsign.py b/SCons/SConsign.py index 5b78855..ecca391 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -248,7 +248,7 @@ class DB(Base): determined by the database module. """ def __init__(self, dir): - Base.__init__(self) + super().__init__() self.dir = dir @@ -319,7 +319,7 @@ class Dir(Base): """ fp - file pointer to read entries from """ - Base.__init__(self) + super().__init__() if not fp: return @@ -352,7 +352,7 @@ class DirFile(Dir): fp = None try: - Dir.__init__(self, fp, dir) + super().__init__(fp, dir) except KeyboardInterrupt: raise except Exception: diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 26a8bcd..d694740 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -29,7 +29,7 @@ # of its own, which might or might not be a good thing. Nevertheless, # here are some enhancements that will probably be requested some day # and are worth keeping in mind (assuming this takes off): -# +# # - A command to re-read / re-load the SConscript files. This may # involve allowing people to specify command-line options (e.g. -f, # -I, --no-site-dir) that affect how the SConscript files are read. @@ -257,7 +257,12 @@ version Prints SCons version information. # from SCons.Debug import Trace # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - SCons.SConsign.Reset() + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index a340f5b..ab2dc0e 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,7 +987,6 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. diff --git a/SCons/Subst.py b/SCons/Subst.py index 298df38..7535772 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -132,7 +132,7 @@ class CmdStringHolder(collections.UserString): proper escape sequences inserted. """ def __init__(self, cmd, literal=None): - collections.UserString.__init__(self, cmd) + super().__init__(cmd) self.literal = literal def is_literal(self): @@ -490,7 +490,7 @@ class ListSubber(collections.UserList): internally. """ def __init__(self, env, mode, conv, gvars): - collections.UserList.__init__(self, []) + super().__init__([]) self.env = env self.mode = mode self.conv = conv diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index b956a25..1a203a0 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -121,7 +121,7 @@ class SubstTestCase(unittest.TestCase): class MyNode(DummyNode): """Simple node work-alike with some extra stuff for testing.""" def __init__(self, name): - DummyNode.__init__(self, name) + super().__init__(name) class Attribute: pass self.attribute = Attribute() diff --git a/SCons/Tool/GettextCommon.py b/SCons/Tool/GettextCommon.py index 16900e0..058b997 100644 --- a/SCons/Tool/GettextCommon.py +++ b/SCons/Tool/GettextCommon.py @@ -206,7 +206,7 @@ class _POFileBuilder(BuilderBase): del kw['target_alias'] if 'target_factory' not in kw: kw['target_factory'] = _POTargetFactory(env, alias=alias).File - BuilderBase.__init__(self, **kw) + super().__init__(**kw) def _execute(self, env, target, source, *args, **kw): """ Execute builder's actions. diff --git a/SCons/Tool/MSCommon/sdk.py b/SCons/Tool/MSCommon/sdk.py index d95df91..aa94f4d 100644 --- a/SCons/Tool/MSCommon/sdk.py +++ b/SCons/Tool/MSCommon/sdk.py @@ -131,7 +131,7 @@ class WindowsSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\Microsoft SDKs\Windows\v%s\InstallationFolder' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.version class PlatformSDK(SDKDefinition): @@ -140,7 +140,7 @@ class PlatformSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\MicrosoftSDK\InstalledSDKS\%s\Install Dir' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.uuid # diff --git a/SCons/Tool/cc.xml b/SCons/Tool/cc.xml index a7d6daa..7c8f944 100644 --- a/SCons/Tool/cc.xml +++ b/SCons/Tool/cc.xml @@ -48,6 +48,7 @@ Sets construction variables for generic POSIX C compilers. <item>SHOBJSUFFIX</item> <item><!--STATIC_AND_SHARED_OBJECTS_ARE_THE_SAME--></item> <item>CFILESUFFIX</item> + <item>CCDEPFLAGS</item> </sets> <uses> <item>PLATFORM</item> @@ -215,4 +216,16 @@ See also &cv-link-CFLAGS; for compiling to static objects. </summary> </cvar> +<cvar name="CCDEPFLAGS"> +<summary> +<para> +Options to pass to C or C++ compiler to generate list of dependency files. +</para> + <para> + This is set only by compilers which support this functionality. (&t-link-gcc;, &t-link-clang;, and &t-link-msvc; currently) + </para> +</summary> +</cvar> + + </sconsdoc> diff --git a/SCons/Tool/clang.py b/SCons/Tool/clang.py index 6c9227c..2a12a31 100644 --- a/SCons/Tool/clang.py +++ b/SCons/Tool/clang.py @@ -43,6 +43,7 @@ from SCons.Tool.MSCommon import msvc_setup_env_once compilers = ['clang'] + def generate(env): """Add Builders and construction variables for clang to an Environment.""" SCons.Tool.cc.generate(env) @@ -58,7 +59,6 @@ def generate(env): # Set-up ms tools paths msvc_setup_env_once(env) - env['CC'] = env.Detect(compilers) or 'clang' if env['PLATFORM'] in ['cygwin', 'win32']: env['SHCCFLAGS'] = SCons.Util.CLVar('$CCFLAGS') @@ -67,7 +67,7 @@ def generate(env): # determine compiler version if env['CC']: - #pipe = SCons.Action._subproc(env, [env['CC'], '-dumpversion'], + # pipe = SCons.Action._subproc(env, [env['CC'], '-dumpversion'], pipe = SCons.Action._subproc(env, [env['CC'], '--version'], stdin='devnull', stderr='devnull', @@ -81,6 +81,10 @@ def generate(env): if match: env['CCVERSION'] = match.group(1) + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + + + def exists(env): return env.Detect(compilers) diff --git a/SCons/Tool/clang.xml b/SCons/Tool/clang.xml index 2d989fa..8fdd3c1 100644 --- a/SCons/Tool/clang.xml +++ b/SCons/Tool/clang.xml @@ -1,6 +1,28 @@ <?xml version="1.0"?> <!-- -__COPYRIGHT__ + MIT License + + Copyright The SCons Foundation + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + This file is processed by the bin/SConsDoc.py module. See its __doc__ string for a discussion of the format. @@ -33,6 +55,7 @@ Set construction variables for the Clang C compiler. <item>CC</item> <item>SHCCFLAGS</item> <item>CCVERSION</item> +<item>CCDEPFLAGS</item> </sets> </tool> diff --git a/SCons/Tool/clangxx.py b/SCons/Tool/clangxx.py index 4443b39..a78dc6c 100644 --- a/SCons/Tool/clangxx.py +++ b/SCons/Tool/clangxx.py @@ -89,6 +89,9 @@ def generate(env): if match: env['CXXVERSION'] = match.group(1) + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + + def exists(env): return env.Detect(compilers) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 9c29c22..94dfad3 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -57,6 +57,9 @@ def generate(env): if version: env['CCVERSION'] = version + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + + def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) diff --git a/SCons/Tool/gcc.xml b/SCons/Tool/gcc.xml index c22e00a..933dc84 100644 --- a/SCons/Tool/gcc.xml +++ b/SCons/Tool/gcc.xml @@ -33,6 +33,7 @@ Set construction variables for the &gcc; C compiler. <item>CC</item> <item>SHCCFLAGS</item> <item>CCVERSION</item> +<item>CCDEPFLAGS</item> </sets> </tool> diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index edb45e2..cc93f93 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -64,6 +64,9 @@ def generate(env): if version: env['CXXVERSION'] = version + env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' + + def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) diff --git a/SCons/Tool/msvc.py b/SCons/Tool/msvc.py index a56450c..f2cd418 100644 --- a/SCons/Tool/msvc.py +++ b/SCons/Tool/msvc.py @@ -315,6 +315,9 @@ def generate(env): if 'SystemRoot' not in env['ENV']: # required for dlls in the winsxs folders env['ENV']['SystemRoot'] = SCons.Platform.win32.get_system_root() + env['CCDEPFLAGS'] = '/showIncludes' + + def exists(env): return msvc_exists(env) diff --git a/SCons/Tool/msvc.xml b/SCons/Tool/msvc.xml index 6d67059..f4d33f4 100644 --- a/SCons/Tool/msvc.xml +++ b/SCons/Tool/msvc.xml @@ -30,6 +30,7 @@ Sets construction variables for the Microsoft Visual C/C++ compiler. </para> </summary> <sets> +<item>CCDEPFLAGS</item> <item>CCPDBFLAGS</item> <item>CCPCHFLAGS</item> <item><!--CCCOMFLAGS--></item> @@ -395,7 +396,7 @@ and extract the relevant variables from the result (typically <envar>%PATH%</envar>) for supplying to the build. This can be useful to force the use of a compiler version that &SCons; does not detect. -&cv-MSVC_USE_SCRIPT_ARGS; privides arguments passed to this script. +&cv-link-MSVC_USE_SCRIPT_ARGS; provides arguments passed to this script. </para> <para> diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 7a827f4..887cb59 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -320,7 +320,7 @@ class _GenerateV7User(_UserGenerator): self.usrhead = V9UserHeader self.usrconf = V9UserConfiguration self.usrdebg = V9DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -395,7 +395,7 @@ class _GenerateV10User(_UserGenerator): self.usrhead = V10UserHeader self.usrconf = V10UserConfiguration self.usrdebg = V10DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -1487,7 +1487,7 @@ class _DSWGenerator: class _GenerateV7DSW(_DSWGenerator): """Generates a Solution file for MSVS .NET""" def __init__(self, dswfile, source, env): - _DSWGenerator.__init__(self, dswfile, source, env) + super().__init__(dswfile, source, env) self.file = None self.version = self.env['MSVS_VERSION'] diff --git a/SCons/Tool/ninja/Methods.py b/SCons/Tool/ninja/Methods.py index 073cf71..4c6aa2f 100644 --- a/SCons/Tool/ninja/Methods.py +++ b/SCons/Tool/ninja/Methods.py @@ -134,7 +134,9 @@ def get_command(env, node, action): # pylint: disable=too-many-branches variables = {} - comstr = get_comstr(sub_env, action, tlist, slist) + # since we will check the ninja rule map for this command str, we must make sure + # its string so its hashable. + comstr = str(get_comstr(sub_env, action, tlist, slist)) if not comstr: return None @@ -255,6 +257,10 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom ) cmd, rsp_content = cmd_list[:tool_idx], cmd_list[tool_idx:] + + # Canonicalize the path to have forward (posix style) dir sep characters. + if os.altsep: + rsp_content = [rsp_content_item.replace(os.sep, os.altsep) for rsp_content_item in rsp_content] rsp_content = ['"' + rsp_content_item + '"' for rsp_content_item in rsp_content] rsp_content = " ".join(rsp_content) @@ -266,7 +272,7 @@ def gen_get_response_file_command(env, rule, tool, tool_is_dynamic=False, custom variables["env"] += env.subst( "export %s=%s;" % (key, value), target=targets, source=sources, executor=executor ) + " " - + if node.get_env().get('NINJA_FORCE_SCONS_BUILD'): ret_rule = 'TEMPLATE' else: diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index d7c260e..fff60be 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -24,11 +24,15 @@ import io import os +import pathlib +import signal +import tempfile import shutil import sys from os.path import splitext from tempfile import NamedTemporaryFile import ninja +import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS @@ -83,6 +87,7 @@ class NinjaState: # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) + scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT',-1)) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script @@ -184,10 +189,10 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} $out", + "description": "Defer to SCons to build $out", + "pool": "local_pool", + "restat": 1 }, "SCONS": { "command": "$SCONS_INVOCATION $out", @@ -211,6 +216,29 @@ class NinjaState: # output. "restat": 1, }, + + "SCONS_DAEMON": { + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {env.get('NINJA_DIR').abspath} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "description": "Starting scons daemon...", + "pool": "local_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, "REGENERATE": { "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $self", @@ -245,6 +273,9 @@ class NinjaState: if not node.has_builder(): return False + if isinstance(node, SCons.Node.Python.Value): + return False + if isinstance(node, SCons.Node.Alias.Alias): build = alias_to_ninja_build(node) else: @@ -256,7 +287,28 @@ class NinjaState: node_string = str(node) if node_string in self.builds: - raise InternalError("Node {} added to ninja build state more than once".format(node_string)) + # TODO: If we work out a way to handle Alias() with same name as file this logic can be removed + # This works around adding Alias with the same name as a Node. + # It's not great way to workaround because it force renames the alias, + # but the alternative is broken ninja support. + warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." + if isinstance(node, SCons.Node.Alias.Alias): + for i, output in enumerate(build["outputs"]): + if output == node_string: + build["outputs"][i] += "_alias" + node_string += "_alias" + print(warn_msg) + elif self.builds[node_string]["rule"] == "phony": + for i, output in enumerate(self.builds[node_string]["outputs"]): + if output == node_string: + self.builds[node_string]["outputs"][i] += "_alias" + tmp_build = self.builds[node_string].copy() + del self.builds[node_string] + node_string += "_alias" + self.builds[node_string] = tmp_build + print(warn_msg) + else: + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True @@ -330,8 +382,12 @@ class NinjaState: ) template_builders = [] + scons_compiledb = False for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if "compile_commands.json" in build["outputs"]: + scons_compiledb = True + if build["rule"] == "TEMPLATE": template_builders.append(build) continue @@ -345,10 +401,10 @@ class NinjaState: # generated sources or else we will create a dependency # cycle. if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -422,41 +478,10 @@ class NinjaState: ninja.build(**build) - template_builds = dict() + scons_daemon_dirty = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_dirty") for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - - # Try to clean up any dependency cycles. If we are passing an - # ouptut node to SCons, it will build any dependencys if ninja - # has not already. - for output in template_builds.get("outputs", []): - inputs = template_builds.get('inputs') - if inputs and output in inputs: - inputs.remove(output) - - implicits = template_builds.get('implicit') - if implicits and output in implicits: - implicits.remove(output) - - ninja.build(**template_builds) + template_builder["implicit"] += [scons_daemon_dirty] + ninja.build(**template_builder) # We have to glob the SCons files here to teach the ninja file # how to regenerate itself. We'll never see ourselves in the @@ -483,30 +508,56 @@ class NinjaState: } ) - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), - '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) + if not scons_compiledb: + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], + ["run_ninja_scons_daemon_phony", scons_daemon_dirty], + rule="SCONS_DAEMON", ) + + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + pidfile = None + if os.path.exists(scons_daemon_dirty): + pidfile = scons_daemon_dirty + elif os.path.exists(daemon_dir / 'pidfile'): + pidfile = daemon_dir / 'pidfile' + + if pidfile: + with open(pidfile) as f: + pid = int(f.readline()) + try: + os.kill(pid, signal.SIGINT) + except OSError: + pass + + if os.path.exists(scons_daemon_dirty): + os.unlink(scons_daemon_dirty) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -588,7 +639,13 @@ class SConsToNinjaTranslator: elif isinstance(action, COMMAND_TYPES): build = get_command(env, node, action) else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } if build is not None: build["order_only"] = get_order_only(node) @@ -657,7 +714,7 @@ class SConsToNinjaTranslator: return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) + dependencies = list({dep for build in results for dep in build.get("implicit", [])}) if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" @@ -669,6 +726,9 @@ class SConsToNinjaTranslator: # condition if not cmdstr. So here we strip preceding # and proceeding whitespace to make strings like the # above become empty strings and so will be skipped. + if not cmd.get("variables") or not cmd["variables"].get("cmd"): + continue + cmdstr = cmd["variables"]["cmd"].strip() if not cmdstr: continue @@ -717,4 +777,10 @@ class SConsToNinjaTranslator: "implicit": dependencies, } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 3adbb53..888218d 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -123,7 +123,7 @@ def get_dependencies(node, skip_sources=False): get_path(src_file(child)) for child in filter_ninja_nodes(node.children()) if child not in node.sources - ] + ] return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] @@ -370,7 +370,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + if isinstance(self, SCons.Node.Node) and (self.is_sconscript() or self.is_conftest()): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index 44c2251..3c72ee4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,6 +26,7 @@ import importlib import os +import random import subprocess import sys @@ -187,6 +188,11 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) + env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000, 60000)) + + if GetOption("disable_ninja"): + env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -200,6 +206,7 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] + def ninja_generate_deps(env): """Return a list of SConscripts TODO: Should we also include files loaded from site_scons/*** @@ -214,10 +221,11 @@ def generate(env): # This adds the required flags such that the generated compile # commands will create depfiles as appropriate in the Ninja file. - if env["PLATFORM"] == "win32": - env.Append(CCFLAGS=["/showIncludes"]) + if 'CCDEPFLAGS' not in env: + # Issue some warning here + pass else: - env.Append(CCFLAGS=["-MMD", "-MF", "${TARGET}.d"]) + env.Append(CCFLAGS='$CCDEPFLAGS') env.AddMethod(CheckNinjaCompdbExpand, "CheckNinjaCompdbExpand") @@ -267,7 +275,12 @@ def generate(env): def robust_rule_mapping(var, rule, tool): provider = gen_get_response_file_command(env, rule, tool) env.NinjaRuleMapping("${" + var + "}", provider) - env.NinjaRuleMapping(env.get(var, None), provider) + + # some of these construction vars could be generators, e.g. + # CommandGeneratorAction, so if the var is not a string, we + # can't parse the generated string. + if isinstance(env.get(var), str): + env.NinjaRuleMapping(env.get(var, None), provider) robust_rule_mapping("CCCOM", "CC", "$CC") robust_rule_mapping("SHCCCOM", "CC", "$CC") @@ -423,7 +436,7 @@ def generate(env): return if target.check_attributes('ninja_file') is None: NINJA_STATE.add_build(target) - else: + else: target.build() SCons.Taskmaster.Task.execute = ninja_execute @@ -450,3 +463,6 @@ def generate(env): env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" env["TEMPFILE"] = NinjaNoResponseFiles + + + env.Alias('run-ninja-scons-daemon', 'run_ninja_scons_daemon_phony') diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index 4638efb..51ff435 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -73,6 +73,8 @@ See its __doc__ string for a discussion of the format. <item>_NINJA_REGENERATE_DEPS_FUNC</item> <!-- <item>__NINJA_NO</item>--> <item>IMPLICIT_COMMAND_DEPENDENCIES</item> + <item>NINJA_SCONS_DAEMON_KEEP_ALIVE</item> + <item>NINJA_SCONS_DAEMON_PORT</item> <!-- TODO: Document these --> @@ -90,6 +92,7 @@ See its __doc__ string for a discussion of the format. <item>ARFLAGS</item> <item>CC</item> <item>CCCOM</item> + <item>CCDEPFLAGS</item> <item>CCFLAGS</item> <item>CXX</item> <item>CXXCOM</item> @@ -201,7 +204,7 @@ python -m pip install ninja The <parameter>msvc_deps_prefix</parameter> string. Propagates directly into the generated &ninja; build file. From Ninja's docs: - <quote>defines the string which should be stripped from msvc’s <option>/showIncludes</option> output</quote> + <quote>defines the string which should be stripped from msvc's <option>/showIncludes</option> output</quote> </para> </summary> </cvar> @@ -349,5 +352,24 @@ python -m pip install ninja </summary> </cvar> + <cvar name="NINJA_SCONS_DAEMON_KEEP_ALIVE"> + <summary> + <para> + The number of seconds for the SCons deamon launched by ninja to stay alive. + (Default: 180000) + </para> + </summary> + </cvar> + + <cvar name="NINJA_SCONS_DAEMON_PORT"> + <summary> + <para> + The TCP/IP port for the SCons daemon to listen on. + <emphasis>NOTE: You cannot use a port already being listened to on your build machine.</emphasis> + (Default: random number between 10000,60000) + </para> + </summary> + </cvar> + </sconsdoc> diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py new file mode 100644 index 0000000..f34935b --- /dev/null +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script is intended to execute a single build target. This script should be +called by ninja, passing the port, ninja dir, and build target via arguments. +The script then executes a simple get request to the scons daemon which is listening +on from localhost on the set port. +""" + +import http.client +import sys +import time +import os +import logging +import pathlib +import tempfile +import hashlib +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon_request.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +while True: + try: + logging.debug(f"Sending request: {sys.argv[3]}") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) + conn.request("GET", "/?build=" + sys.argv[3]) + response = None + + while not response: + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException: + logging.debug(f"Error: {traceback.format_exc()}") + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode("utf-8")) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + exit(0) + + except Exception: + logging.debug(f"Failed to send command: {traceback.format_exc()}") + exit(1) + + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py new file mode 100644 index 0000000..057537a --- /dev/null +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script is intended to be called by ninja to start up the scons daemon process. It will +launch the server and attempt to connect to it. This process needs to completely detach +from the spawned process so ninja can consider the build edge completed. It should be passed +the args which should be forwarded to the scons daemon process which could be any number of +# arguments. However the first few arguments are required to be port, ninja dir, and keep alive +timeout in seconds. + +The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over +restarting the server if the server is running. The assumption here is the pidfile should only +exist if the server is running. +""" + +import subprocess +import sys +import os +import pathlib +import tempfile +import hashlib +import logging +import time +import http.client +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): + cmd = [ + sys.executable, + str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py"), + ] + sys.argv[1:] + logging.debug(f"Starting daemon with {' '.join(cmd)}") + + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False + ) + + with open(daemon_dir / "pidfile", "w") as f: + f.write(str(p.pid)) + with open(ninja_builddir / "scons_daemon_dirty", "w") as f: + f.write(str(p.pid)) + + error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" + + while True: + try: + logging.debug("Attempting to connect scons daemon") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) + conn.request("GET", "/?ready=true") + response = None + + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode("utf-8")) + exit(1) + logging.debug("Server Responded it was ready!") + break + + except ConnectionRefusedError: + logging.debug(f"Server not ready, server PID: {p.pid}") + time.sleep(1) + if p.poll is not None: + logging.debug(f"Server process died, aborting: {p.returncode}") + sys.exit(p.returncode) + except ConnectionResetError: + logging.debug("Server ConnectionResetError") + sys.stderr.write(error_msg) + exit(1) + except Exception: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py new file mode 100644 index 0000000..88f3b22 --- /dev/null +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script primarily consists of two threads, the http server thread and the scons interactive +process thread. The http server thread will listen on the passed port for http get request +which should indicate some action for the scons interactive process to take. + +The daemon will keep log files in a tmp directory correlated to the hash of the absolute path +of the ninja build dir passed. The daemon will also use a keep alive time to know when to shut +itself down after the passed timeout of no activity. Any time the server receives a get request, +the keep alive time will be reset. +""" + +import http.server +import socketserver +from urllib.parse import urlparse, parse_qs +import time +from threading import Condition +from subprocess import PIPE, Popen +import sys +import os +import threading +import queue +import pathlib +import logging +from timeit import default_timer as timer +import traceback +import tempfile +import hashlib + +port = int(sys.argv[1]) +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_keep_alive = int(sys.argv[3]) +args = sys.argv[4:] + +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + + +def daemon_log(message): + logging.debug(message) + + +def custom_readlines(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + if buf.endswith("scons>>>"): + yield buf + buf = "" + + +def custom_readerr(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + + +def enqueue_output(out, queue): + for line in iter(custom_readlines(out)): + queue.put(line) + out.close() + + +def enqueue_error(err, queue): + for line in iter(custom_readerr(err)): + queue.put(line) + err.close() + + +input_q = queue.Queue() +output_q = queue.Queue() +error_q = queue.Queue() + +finished_building = [] +error_nodes = [] + +building_cv = Condition() +error_cv = Condition() + +thread_error = False + + +def daemon_thread_func(): + global thread_error + global finished_building + global error_nodes + try: + args_list = args + ["--interactive"] + daemon_log(f"Starting daemon with args: {' '.join(args_list)}") + daemon_log(f"cwd: {os.getcwd()}") + + p = Popen(args_list, stdout=PIPE, stderr=PIPE, stdin=PIPE) + + t = threading.Thread(target=enqueue_output, args=(p.stdout, output_q)) + t.daemon = True + t.start() + + te = threading.Thread(target=enqueue_error, args=(p.stderr, error_q)) + te.daemon = True + te.start() + + daemon_ready = False + building_node = None + + while p.poll() is None: + + while True: + try: + line = output_q.get(block=False, timeout=0.01) + except queue.Empty: + break + else: + daemon_log("output: " + line.strip()) + + if "scons: building terminated because of errors." in line: + error_output = "" + while True: + try: + error_output += error_q.get(block=False, timeout=0.01) + except queue.Empty: + break + error_nodes += [{"node": building_node, "error": error_output}] + daemon_ready = True + building_node = None + with building_cv: + building_cv.notify() + + elif line == "scons>>>": + with error_q.mutex: + error_q.queue.clear() + daemon_ready = True + with building_cv: + building_cv.notify() + building_node = None + + while daemon_ready and not input_q.empty(): + + try: + building_node = input_q.get(block=False, timeout=0.01) + except queue.Empty: + break + if "exit" in building_node: + p.stdin.write("exit\n".encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + raise + + else: + input_command = "build " + building_node + "\n" + daemon_log("input: " + input_command.strip()) + + p.stdin.write(input_command.encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + + time.sleep(0.01) + except Exception: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + +daemon_thread = threading.Thread(target=daemon_thread_func) +daemon_thread.daemon = True +daemon_thread.start() + +logging.debug( + f"Starting request server on port {port}, keep alive: {daemon_keep_alive}" +) + +keep_alive_timer = timer() +httpd = None + + +def server_thread_func(): + class S(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global thread_error + global keep_alive_timer + global error_nodes + + try: + gets = parse_qs(urlparse(self.path).query) + build = gets.get("build") + if build: + keep_alive_timer = timer() + + daemon_log(f"Got request: {build[0]}") + input_q.put(build[0]) + + def pred(): + return build[0] in finished_building + + with building_cv: + building_cv.wait_for(pred) + + for error_node in error_nodes: + if error_node["node"] == build[0]: + self.send_response(500) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(error_node["error"].encode()) + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + return + + exitbuild = gets.get("exit") + if exitbuild: + input_q.put("exit") + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + except Exception: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + def log_message(self, format, *args): + return + + httpd = socketserver.TCPServer(("127.0.0.1", port), S) + httpd.serve_forever() + + +server_thread = threading.Thread(target=server_thread_func) +server_thread.daemon = True +server_thread.start() + +while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: + time.sleep(1) + +if thread_error: + daemon_log(f"Shutting server on port {port} down because thread error.") +else: + daemon_log( + f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" + ) + +# if there are errors, don't immediately shut down the daemon +# the process which started the server is attempt to connect to +# the daemon before allowing jobs to start being sent. If the daemon +# shuts down too fast, the launch script will think it has not +# started yet and sit and wait. If the launch script is able to connect +# and then the connection is dropped, it will immediately exit with fail. +time.sleep(5) + +if os.path.exists(ninja_builddir / "scons_daemon_dirty"): + os.unlink(ninja_builddir / "scons_daemon_dirty") +if os.path.exists(daemon_dir / "pidfile"): + os.unlink(daemon_dir / "pidfile") + +httpd.shutdown() +server_thread.join() + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Util.py b/SCons/Util.py index 31202cc..093ca41 100644 --- a/SCons/Util.py +++ b/SCons/Util.py @@ -1213,7 +1213,9 @@ class CLVar(UserList): return super().__iadd__(CLVar(other)) def __str__(self): - return ' '.join(self.data) + # Some cases the data can contain Nodes, so make sure they + # processed to string before handing them over to join. + return ' '.join([str(d) for d in self.data]) class Selector(OrderedDict): @@ -1684,10 +1686,10 @@ def _attempt_init_of_python_3_9_hash_object(hash_function_object, sys_used=sys): """ if hash_function_object is None: return None - + # https://stackoverflow.com/a/11887885 details how to check versions with the "packaging" library. # however, for our purposes checking the version is greater than or equal to 3.9 is good enough, as - # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See + # the API is guaranteed to have support for the 'usedforsecurity' flag in 3.9. See # https://docs.python.org/3/library/hashlib.html#:~:text=usedforsecurity for the version support notes. if (sys_used.version_info.major > 3) or (sys_used.version_info.major == 3 and sys_used.version_info.minor >= 9): return hash_function_object(usedforsecurity=False) @@ -1716,7 +1718,7 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): # note: if you call this method repeatedly, example using timeout, this is needed. # otherwise it keeps appending valid formats to the string ALLOWED_HASH_FORMATS = [] - + for test_algorithm in DEFAULT_HASH_FORMATS: _test_hash = getattr(hashlib_used, test_algorithm, None) # we know hashlib claims to support it... check to see if we can call it. @@ -1731,7 +1733,7 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys): except ValueError as e: _last_error = e continue - + if len(ALLOWED_HASH_FORMATS) == 0: from SCons.Errors import SConsEnvironmentError # pylint: disable=import-outside-toplevel # chain the exception thrown with the most recent error from hashlib. @@ -1817,7 +1819,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): 'is reporting; SCons supports: %s. Your local system only ' 'supports: %s' % (hash_format_lower, - ', '.join(DEFAULT_HASH_FORMATS), + ', '.join(DEFAULT_HASH_FORMATS), ', '.join(ALLOWED_HASH_FORMATS)) ) @@ -1825,7 +1827,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): # function did not throw, or when it threw, the exception was caught and ignored, or # the global ALLOWED_HASH_FORMATS was changed by an external user. _HASH_FUNCTION = _attempt_get_hash_function(hash_format_lower, hashlib_used, sys_used) - + if _HASH_FUNCTION is None: from SCons.Errors import UserError # pylint: disable=import-outside-toplevel @@ -1842,7 +1844,7 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): # disabled. for choice in ALLOWED_HASH_FORMATS: _HASH_FUNCTION = _attempt_get_hash_function(choice, hashlib_used, sys_used) - + if _HASH_FUNCTION is not None: break else: @@ -1864,7 +1866,7 @@ set_hash_format(None) def get_current_hash_algorithm_used(): """Returns the current hash algorithm name used. - + Where the python version >= 3.9, this is expected to return md5. If python's version is <= 3.8, this returns md5 on non-FIPS-mode platforms, and sha1 or sha256 on FIPS-mode Linux platforms. diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 5ed31cb..0cfb70f 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -899,7 +899,7 @@ class HashTestCase(unittest.TestCase): class FIPSHashTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): - super(FIPSHashTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) ############################### # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 diff --git a/SCons/cppTests.py b/SCons/cppTests.py index 99fa6cf..a9aef9d 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -853,7 +853,7 @@ class fileTestCase(unittest.TestCase): """) class MyPreProcessor(cpp.DumbPreProcessor): def __init__(self, *args, **kw): - cpp.DumbPreProcessor.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.files = [] def __call__(self, file): self.files.append(file) diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py index a465057..82e2c72 100644 --- a/bin/SConsDoc.py +++ b/bin/SConsDoc.py @@ -499,7 +499,7 @@ class Function(Item): class Tool(Item): def __init__(self, name): - Item.__init__(self, name) + super().__init__(name) self.entity = self.name.replace('+', 'X') diff --git a/doc/man/scons.xml b/doc/man/scons.xml index d7fe39d..733db78 100644 --- a/doc/man/scons.xml +++ b/doc/man/scons.xml @@ -145,7 +145,7 @@ to support additional input file types. <para>Information about files involved in the build, including a cryptographic hash of the contents, -is cached for later reuse, +is cached for later reuse, By default content hashes are used to determine if a file has changed since the last build, but this can be controlled by selecting an appropriate @@ -3983,7 +3983,7 @@ actually two bytes.</para> <para>Checks whether the C compiler (as defined by the &cv-link-CC; &consvar;) works, by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure.</para> @@ -4003,7 +4003,7 @@ be accepted or rejected by the compiler. <para>Checks whether the C++ compiler (as defined by the &cv-link-CXX; &consvar;) works, by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure.</para> @@ -4023,7 +4023,7 @@ be accepted or rejected by the compiler. <para>Checks whether the shared-object C compiler (as defined by the &cv-link-SHCC; &consvar;) works by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure.</para> @@ -4045,7 +4045,7 @@ be created. <para>Checks whether the shared-object C++ compiler (as defined by the &cv-link-SHCXX; &consvar;) works by trying to compile a small source file. -This provides a more rigorous check: +This provides a more rigorous check: by default, &SCons; itself only detects if there is a program with the correct name, not if it is a functioning compiler. Returns a boolean indicating success or failure.</para> @@ -4064,7 +4064,7 @@ be created. <varlistentry> <term><replaceable>context</replaceable>.<methodname>CheckProg</methodname>(<parameter>prog_name</parameter>)</term> <listitem> -<para>Checks if +<para>Checks if <parameter>prog_name</parameter> exists in the path &SCons; will use at build time. (<replaceable>context</replaceable>.<varname>env['ENV']['PATH']</varname>). @@ -4088,7 +4088,35 @@ Returns a boolean indicating success or failure.</para> </listitem> </varlistentry> - <varlistentry> + <varlistentry> + <term><replaceable>context</replaceable>.<methodname>CheckMember</methodname>(<parameter>aggregate_member, + [header, language]</parameter>) + </term> + <listitem> + <para>Checks for the existence of a member of the C/C++ struct or class. + <parameter>aggregate_member</parameter> + specifies the struct/class and member to check for. + <parameter>header</parameter> + is a string containing one or more + <literal>#include</literal> + lines that will be inserted into the program + that will be run to test for the existence of the member. + Example: + </para> + + <programlisting language="python"> +sconf.CheckMember('struct tm.tm_sec', '#include <time.h>') + </programlisting> + + <para> + Returns a boolean indicating success or failure. + </para> + + </listitem> + </varlistentry> + + + <varlistentry> <term><replaceable>context</replaceable>.<methodname>Define</methodname>(<parameter>symbol, [value, comment]</parameter>)</term> <listitem> <para>This method does not check for anything, but rather forces @@ -4100,7 +4128,7 @@ it will be be used as the macro replacement value. If <parameter>value</parameter> is a string and needs to display with quotes, the quotes need to be included, as in <literal>'"string"'</literal> -If the optional +If the optional <parameter>comment</parameter> is given, it is inserted as a comment above the macro definition (suitable comment marks will be added automatically). @@ -6410,27 +6438,200 @@ env.Command('marker', 'input_file', action=[MyBuildAction, Touch('$TARGET')]) <refsect3 id='variable_substitution'> <title>Variable Substitution</title> -<para>Before executing a command, -&scons; -performs variable substitution on the string that makes up -the action part of the builder. -Variables to be interpolated are indicated in the -string with a leading -<literal>$</literal>, to distinguish them from plain text -which is not to be substituted. -The name may be surrounded by curly braces -(<literal>${}</literal>) -to separate the name from surrounding characters if necessary. -Curly braces are required when you use -Python list subscripting/slicing notation on a variable -to select one or more items from a list, -or access a variable's special attributes, -or use Python expression substitution. +<para> +Before executing a command, &scons; +performs parameter expansion (<firstterm>substitution</firstterm>) +on the string that makes up the action part of the builder. +The format of a substitutable parameter is +<literal>${<replaceable>expression</replaceable>}</literal>. +If <replaceable>expression</replaceable> refers to a variable, +the braces in <literal>${<replaceable>expression</replaceable>}</literal> +can be omitted <emphasis>unless</emphasis> the variable name is +immediately followed by a character that could either be interpreted +as part of the name, or is &Python; syntax such as +<emphasis role="bold">[</emphasis> (for indexing/slicing) +or <emphasis role="bold">.</emphasis> (for attribute access - +see <link linkend="special_attributes">Special Attributes</link> below). </para> <para> +If <replaceable>expression</replaceable> refers to a &consvar;, +it is replaced with the value of that variable in the +&consenv; at the time of execution. +If <replaceable>expression</replaceable> looks like +a variable name but is not defined in the &consenv; +it is replaced with an empty string. +If <replaceable>expression</replaceable> refers to one of the +<link linkend="special_variables">Special Variables</link> +(see below) the corresponding value of the variable is substituted. +<replaceable>expression</replaceable> may also be +a &Python; expression to be evaluated. +See <link linkend='python_code_substitution'>Python Code Substitution</link> +below for a description. +</para> + +<para>&SCons; uses the following rules when converting &consvars; into +command line strings:</para> + +<itemizedlist> + <listitem> + <para>If the value is a string it is interpreted as space delimited + command line arguments.</para> + </listitem> + + <listitem> + <para>If the value is a list it is interpreted as a list of command + line arguments. Each element of the list is converted to a string.</para> + </listitem> + + <listitem> + <para>Anything that is not a list or string is converted to a string and + interpreted as a single command line argument.</para> + </listitem> + + <listitem> + <para>Newline characters (<literal>\n</literal>) delimit lines. + The newline parsing is done after + all other parsing, so it is not possible for arguments (e.g. file names) to + contain embedded newline characters.</para> + </listitem> + + <listitem> + <para>For a literal <emphasis role="bold">$</emphasis> + use <emphasis role="bold">$$</emphasis>. + For example, <literal>$$FOO</literal> will be left in the + final string as <literal>$FOO</literal>.</para> + </listitem> +</itemizedlist> + +<para> +When a build action is executed, a hash of the command line is +saved, together with other information about the target(s) built +by the action, for future use in rebuild determination. +This is called the <firstterm>&buildsig;</firstterm> +(or <firstterm>&build_action; signature</firstterm>). +The escape sequence +<emphasis role="bold">$(</emphasis> +<replaceable>subexpression</replaceable> +<emphasis role="bold">$)</emphasis> +may be used to indicate parts of a command line +that may change without +causing a rebuild--that is, +which are not to be included when calculating the &buildsig;. +All text from +<emphasis role="bold">$(</emphasis> +up to and including the matching +<emphasis role="bold">$)</emphasis> +will be removed from the command line +before it is added to the &buildsig; +while only the +<emphasis role="bold">$(</emphasis> +and +<emphasis role="bold">$)</emphasis> +will be removed before the command is executed. +For example, the command line string:</para> + +<programlisting language="python"> +"echo Last build occurred $( $TODAY $). > $TARGET" +</programlisting> + +<para>would execute the command:</para> + +<screen> +echo Last build occurred $TODAY. > $TARGET +</screen> + +<para>but the build signature added to any target files would be computed from:</para> + +<screen> +echo Last build occurred . > $TARGET +</screen> + +<para>While &consvars; are normally directly substituted, +if a &consvar; has a value which +is a callable &Python; object +(a function, or a class with a <literal>__call__</literal> method), +that object is called during substitution. +The callable must accept four arguments: +<parameter>target</parameter>, +<parameter>source</parameter>, +<parameter>env</parameter> and +<parameter>for_signature</parameter>. +<parameter>source</parameter> is a list of source nodes, +<parameter>target</parameter> is a list of target nodes, +<parameter>env</parameter> is the &consenv; to use for context, +and <parameter>for_signature</parameter> is +a boolean value that tells the callable +if it is being called for the purpose of generating a build signature. +Since the build signature is used for rebuild determination, +variable elements that do not affect whether +a rebuild should be triggered +should be omitted from the returned string +if <parameter>for_signature</parameter> is true. +See <emphasis role="bold">$(</emphasis> +and <emphasis role="bold">$)</emphasis> above +for the syntax. +</para> + +<para> +&SCons; will insert whatever +the callable returns +into the expanded string: +</para> + +<programlisting language="python"> +def foo(target, source, env, for_signature): + return "bar" + +# Will expand $BAR to "bar baz" +env = Environment(FOO=foo, BAR="$FOO baz") +</programlisting> + +<para>As a reminder, substitution happens when +<literal>$BAR</literal> is actually used in a +builder action. The value of <literal>env['BAR']</literal> +will be exactly as it was set: <literal>"$FOO baz"</literal>. +This can make debugging tricky, +as the substituted result is not available at the time +the SConscript files are being interpreted and +thus not available to <systemitem>print()</systemitem>. +However, you can perform the substitution on demand +by calling the &f-link-env-subst; method for this purpose. +</para> + +<para>You can use this feature to pass arguments to a +callable variable by creating a callable class +that stores passed arguments in the instance, +and then uses them +(in the <methodname>__call__</methodname> method) +when the instance is called. +Note that in this case, +the entire variable expansion must +be enclosed by curly braces +so that the arguments will +be associated with the +instantiation of the class:</para> + +<programlisting language="python"> +class foo: + def __init__(self, arg): + self.arg = arg + + def __call__(self, target, source, env, for_signature): + return self.arg + " bar" + +# Will expand $BAR to "my argument bar baz" +env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") +</programlisting> + +</refsect3> + +<refsect3 id='special_variables'> +<title>Substitution: Special Variables</title> + +<para> Besides regular &consvars;, scons provides the following -special variables for use in expanding commands:</para> +<firstterm>Special Variables</firstterm> for use in expanding commands:</para> <variablelist> <varlistentry> @@ -6502,8 +6703,12 @@ changed since the target was last built.</para> </varlistentry> </variablelist> -<para>These names are reserved -and may not be assigned to or used as &consvars;.</para> +<para> +These names are reserved +and may not be assigned to or used as &consvars;. +&SCons; computes them in a context-dependent manner +and they and are not retrieved from a &consenv;. +</para> <para>For example, the following builder call: </para> @@ -6529,6 +6734,11 @@ In the previous example, a string would expand to: <computeroutput>bar.c</computeroutput>. </para> +</refsect3> + +<refsect3 id='special_attributes'> +<title>Substitution: Special Attributes +</title> <para>A variable name may have the following modifiers appended within the enclosing curly braces @@ -6624,119 +6834,6 @@ Some modifiers can be combined, like <literal>${TARGET.file.suffix}</literal>, etc. </para> -<para>The curly brace notation may also be used -to enclose a Python expression to be evaluated. -See <xref linkend='python_code_substitution'/> below -for a description.</para> - -<para>The special escape sequences -<emphasis role="bold">$(</emphasis> -and -<emphasis role="bold">$)</emphasis> -may be used to surround parts of a command line -that may change -<emphasis>without</emphasis> -causing a rebuild--that is, -which are not included in the &buildsig; -of target files built with this command. -All text between -<emphasis role="bold">$(</emphasis> -and -<emphasis role="bold">$)</emphasis> -will be removed from the command line -before it is added to the &buildsig; -and the -<emphasis role="bold">$(</emphasis> -and -<emphasis role="bold">$)</emphasis> -will be removed before the command is executed. -For example, the command line:</para> - -<programlisting language="python"> -echo Last build occurred $( $TODAY $). > $TARGET -</programlisting> - -<para>would execute the command:</para> - -<screen> -echo Last build occurred $TODAY. > $TARGET -</screen> - -<para>but the command portion of the -the &buildsig; computed for any target files built -by this action would be:</para> - -<screen> -echo Last build occurred . > $TARGET -</screen> - -<para>While &consvars; are normally directly substituted, -if a variable refers to a &consvar; -whose value is a &Python; function, -that function is called during substitution. -Such a function must accept four arguments: -<parameter>target</parameter>, -<parameter>source</parameter>, -<parameter>env</parameter> and -<parameter>for_signature</parameter>. -<parameter>source</parameter> is a list of source nodes, -<parameter>target</parameter> is a list of target nodes, -<parameter>env</parameter> is the &consenv; to use for context, -and <parameter>for_signature</parameter> is -a Boolean value that tells the function -if it is being called for the purpose of generating a &buildsig;. -Since the &buildsig; is used for rebuild determination, -the function should omit variable elements -that do not affect whether a rebuild should be triggered -(see <emphasis role="bold">$(</emphasis> -and <emphasis role="bold">$)</emphasis> -above) if <parameter>for_signature</parameter> is true. -</para> - -<para> -&SCons; will insert whatever -the called function returns -into the expanded string: -</para> - -<programlisting language="python"> -def foo(target, source, env, for_signature): - return "bar" - -# Will expand $BAR to "bar baz" -env = Environment(FOO=foo, BAR="$FOO baz") -</programlisting> - -<para>As a reminder, substitution happens when -<literal>$BAR</literal> is actually used in a -builder action. The value of <literal>env['BAR']</literal> -will be exactly as it was set: <literal>"$FOO baz"</literal>. -</para> - -<para>You can use this feature to pass arguments to a -Python function by creating a callable class -that stores one or more arguments in an object, -and then uses them when the -<methodname>__call__()</methodname> -method is called. -Note that in this case, -the entire variable expansion must -be enclosed by curly braces -so that the arguments will -be associated with the -instantiation of the class:</para> - -<programlisting language="python"> -class foo: - def __init__(self, arg): - self.arg = arg - - def __call__(self, target, source, env, for_signature): - return self.arg + " bar" - -# Will expand $BAR to "my argument bar baz" -env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") -</programlisting> </refsect3> <refsect3 id='python_code_substitution'> @@ -6744,8 +6841,8 @@ env=Environment(FOO=foo, BAR="${FOO('my argument')} baz") <para> If a substitutable expression using the notation -<literal>${something}</literal> does not appear to match one of -the other substitution patterns, +<literal>${<replaceable>expression</replaceable>}</literal> +does not appear to match one of the other substitution patterns, it is evaluated as a Python expression. This uses Python's <function>eval</function> function, with the <parameter>globals</parameter> parameter set to @@ -6811,45 +6908,6 @@ which &SCons; passes to <function>eval</function> which returns the value. </para> -<para>&SCons; uses the following rules when converting &consvars; into -command lines:</para> - -<variablelist> - <varlistentry> - <term>string</term> - <listitem> -<para>When the value is a string it is interpreted as a space delimited list of -command line arguments.</para> - </listitem> - </varlistentry> - - <varlistentry> - <term>list</term> - <listitem> -<para>When the value is a list it is interpreted as a list of command line -arguments. Each element of the list is converted to a string.</para> - </listitem> - </varlistentry> - - <varlistentry> - <term>other</term> - <listitem> -<para>Anything that is not a list or string is converted to a string and -interpreted as a single command line argument.</para> - </listitem> - </varlistentry> - - <varlistentry> - <term>newline</term> - <listitem> -<para>Newline characters (<literal>\n</literal>) delimit lines. -The newline parsing is done after -all other parsing, so it is not possible for arguments (e.g. file names) to -contain embedded newline characters.</para> - </listitem> - </varlistentry> -</variablelist> - <note><para> Use of the Python <function>eval</function> function is considered to have security implications, since, @@ -6909,7 +6967,7 @@ is optional, the default is no <parameter>arg</parameter>. <para> The function can use use <function>str</function>(<parameter>node</parameter>) -to fetch the name of the file, +to fetch the name of the file, <replaceable>node</replaceable>.<function>dir</function> to fetch the directory the file is in, <replaceable>node</replaceable>.<function>get_contents</function>() @@ -6930,7 +6988,7 @@ in order to build Nodes with correct paths. Using &f-link-FindPathDirs; with an argument of <literal>CPPPATH</literal> as the <parameter>path_function</parameter> in the &f-Scanner; call means the scanner function will be called with the paths extracted -from &cv-CPPPATH; in the environment <parameter>env</parameter> +from &cv-CPPPATH; in the environment <parameter>env</parameter> passed as the <parameter>paths</parameter> parameter. </para> <para> diff --git a/doc/scons.mod b/doc/scons.mod index 5579d4d..8a6df17 100644 --- a/doc/scons.mod +++ b/doc/scons.mod @@ -440,15 +440,15 @@ <!ENTITY Dictionary "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>Dictionary</literal>"> -<!ENTITY Emitter "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>Emitter</literal>"> -<!ENTITY emitter "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>emitter</literal>"> +<!ENTITY Emitter "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>Emitter</phrase>"> +<!ENTITY emitter "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>emitter</phrase>"> -<!ENTITY factory "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>factory</literal>"> +<!ENTITY factory "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>factory</phrase>"> -<!ENTITY Generator "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>Generator</literal>"> -<!ENTITY generator "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>generator</literal>"> +<!ENTITY Generator "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>Generator</phrase>"> +<!ENTITY generator "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>generator</phrase>"> -<!ENTITY Nodes "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>Nodes</literal>"> +<!ENTITY Nodes "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>Nodes</phrase>"> <!ENTITY contentsig "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>content signature</phrase>"> <!ENTITY contentsigs "<phrase xmlns='http://www.scons.org/dbxsd/v1.0'>content signatures</phrase>"> @@ -462,12 +462,12 @@ <!-- Python keyword arguments --> -<!ENTITY action "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>action=</literal>"> -<!ENTITY batch_key "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>batch_key=</literal>"> -<!ENTITY cmdstr "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>cmdstr=</literal>"> -<!ENTITY exitstatfunc "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>exitstatfunc=</literal>"> -<!ENTITY strfunction "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>strfunction=</literal>"> -<!ENTITY varlist "<literal xmlns='http://www.scons.org/dbxsd/v1.0'>varlist=</literal>"> +<!ENTITY action "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>action=</parameter>"> +<!ENTITY batch_key "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>batch_key=</parameter>"> +<!ENTITY cmdstr "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>cmdstr=</parameter>"> +<!ENTITY exitstatfunc "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>exitstatfunc=</parameter>"> +<!ENTITY strfunction "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>strfunction=</parameter>"> +<!ENTITY varlist "<parameter xmlns='http://www.scons.org/dbxsd/v1.0'>varlist=</parameter>"> <!-- File and program names used in examples. --> diff --git a/doc/user/builders-writing.xml b/doc/user/builders-writing.xml index 5a8851d..a53e70e 100644 --- a/doc/user/builders-writing.xml +++ b/doc/user/builders-writing.xml @@ -2,7 +2,7 @@ <!DOCTYPE sconsdoc [ <!ENTITY % scons SYSTEM "../scons.mod"> %scons; - + <!ENTITY % builders-mod SYSTEM "../generated/builders.mod"> %builders-mod; <!ENTITY % functions-mod SYSTEM "../generated/functions.mod"> @@ -81,14 +81,14 @@ </para> <programlisting> -bld = Builder(action = 'foobuild < $SOURCE > $TARGET') +bld = Builder(action='foobuild < $SOURCE > $TARGET') </programlisting> <para> All the above line does is create a free-standing &Builder; object. - The next section will show us how to actually use it. + The next section will show how to actually use it. </para> @@ -105,7 +105,7 @@ bld = Builder(action = 'foobuild < $SOURCE > $TARGET') for files to be built. This is done through the &cv-link-BUILDERS; &consvar; in an environment. - The &cv-BUILDERS; variable is a Python dictionary + The &cv-link-BUILDERS; variable is a &Python; dictionary that maps the names by which you want to call various &Builder; objects to the objects themselves. For example, if we want to call the @@ -221,7 +221,7 @@ hello.c To be able to use both our own defined &Builder; objects and the default &Builder; objects in the same &consenv;, - you can either add to the &cv-BUILDERS; variable + you can either add to the &cv-link-BUILDERS; variable using the &Append; function: </para> @@ -296,8 +296,8 @@ env.Program('hello.c') suffixes to the target and/or the source file. For example, rather than having to specify explicitly that you want the <literal>Foo</literal> - &Builder; to build the <literal>file.foo</literal> - target file from the <literal>file.input</literal> source file, + &Builder; to build the <filename>file.foo</filename> + target file from the <filename>file.input</filename> source file, you can give the <literal>.foo</literal> and <literal>.input</literal> suffixes to the &Builder;, making for more compact and readable calls to @@ -361,7 +361,7 @@ env.Foo('file2') In &SCons;, you don't have to call an external command to build a file. - You can, instead, define a Python function + You can, instead, define a &Python; function that a &Builder; object can invoke to build your target file (or files). Such a &buildfunc; definition looks like: @@ -383,7 +383,7 @@ def build_function(target, source, env): <variablelist> <varlistentry> - <term>target</term> + <term><parameter>target</parameter></term> <listitem> <para> @@ -392,14 +392,14 @@ def build_function(target, source, env): the target or targets to be built by this function. The file names of these target(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. </para> </listitem> </varlistentry> <varlistentry> - <term>source</term> + <term><parameter>source</parameter></term> <listitem> <para> @@ -408,21 +408,21 @@ def build_function(target, source, env): the sources to be used by this function to build the targets. The file names of these source(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. </para> </listitem> </varlistentry> <varlistentry> - <term>env</term> + <term><parameter>env</parameter></term> <listitem> <para> The &consenv; used for building the target(s). The function may use any of the - environment's construction variables + environment's &consvars; in any way to affect how it builds the targets. </para> @@ -446,7 +446,7 @@ def build_function(target, source, env): <para> - Once you've defined the Python function + Once you've defined the &Python; function that will build your target file, defining a &Builder; object for it is as simple as specifying the name of the function, @@ -479,7 +479,7 @@ file.input <para> And notice that the output changes slightly, - reflecting the fact that a Python function, + reflecting the fact that a &Python; function, not an external command, is now called to build the target file: @@ -497,8 +497,8 @@ file.input <para> &SCons; Builder objects can create an action "on the fly" - by using a function called a &generator;. - (Note: this is not the same thing as a Python generator function + by using a function called a <firstterm>&Generator;</firstterm>. + (Note: this is not the same thing as a &Python; generator function described in <ulink url="https://www.python.org/dev/peps/pep-0255/">PEP 255</ulink>) This provides a great deal of flexibility to construct just the right list of commands @@ -521,7 +521,7 @@ def generate_actions(source, target, env, for_signature): <variablelist> <varlistentry> - <term>source</term> + <term><parameter>source</parameter></term> <listitem> <para> @@ -531,7 +531,7 @@ def generate_actions(source, target, env, for_signature): by the command or other action generated by this function. The file names of these source(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. </para> </listitem> @@ -539,7 +539,7 @@ def generate_actions(source, target, env, for_signature): </varlistentry> <varlistentry> - <term>target</term> + <term><parameter>target</parameter></term> <listitem> <para> @@ -549,7 +549,7 @@ def generate_actions(source, target, env, for_signature): by the command or other action generated by this function. The file names of these target(s) - may be extracted using the Python &str; function. + may be extracted using the &Python; &str; function. </para> </listitem> @@ -557,14 +557,14 @@ def generate_actions(source, target, env, for_signature): </varlistentry> <varlistentry> - <term>env</term> + <term><parameter>env</parameter></term> <listitem> <para> The &consenv; used for building the target(s). - The generator may use any of the - environment's construction variables + The &generator; may use any of the + environment's &consvars; in any way to determine what command or other action to return. @@ -574,13 +574,13 @@ def generate_actions(source, target, env, for_signature): </varlistentry> <varlistentry> - <term>for_signature</term> + <term><parameter>for_signature</parameter></term> <listitem> <para> A flag that specifies whether the - generator is being called to contribute to a build signature, + &generator; is being called to contribute to a &buildsig;, as opposed to actually executing the command. <!-- XXX NEED MORE HERE, describe generators use in signatures --> @@ -604,8 +604,8 @@ def generate_actions(source, target, env, for_signature): Once you've defined a &generator;, you create a &Builder; to use it - by specifying the generator keyword argument - instead of <literal>action</literal>. + by specifying the <parameter>generator</parameter> keyword argument + instead of <parameter>action</parameter>. </para> @@ -652,9 +652,9 @@ env.Foo('file') <para> Note that it's illegal to specify both an - <literal>action</literal> + <parameter>action</parameter> and a - <literal>generator</literal> + <parameter>generator</parameter> for a &Builder;. </para> @@ -672,7 +672,7 @@ env.Foo('file') that takes as its arguments the list of the targets passed to the builder, the list of the sources passed to the builder, - and the construction environment. + and the &consenv;. The emitter function should return the modified lists of targets that should be built and sources from which the targets will be built. @@ -739,7 +739,7 @@ env.Foo('file') </sconstruct> <para> - + And would yield the following output: </para> @@ -751,16 +751,15 @@ env.Foo('file') <para> One very flexible thing that you can do is - use a construction variable to specify - different emitter functions for different - construction variable. + use a &consvar; to specify + different emitter functions for different &consenvs;. To do this, specify a string - containing a construction variable + containing a &consvar; expansion as the emitter when you call the &f-link-Builder; function, - and set that construction variable to + and set that &consvar; to the desired emitter function - in different construction environments: + in different &consenvs;: </para> @@ -827,9 +826,9 @@ cat is a powerful concept, but sometimes all you really want is to be able to use an existing builder but change its concept of what targets are created. - In this case, + In this case, trying to recreate the logic of an existing Builder to - supply a special emitter can be a lot of work. + supply a special emitter can be a lot of work. The typical case for this is when you want to use a compiler flag that causes additional files to be generated. For example the GNU linker accepts an option @@ -844,12 +843,12 @@ cat <para> To help with this, &SCons; provides &consvars; which correspond - to a few standard builders: - &cv-link-PROGEMITTER; for &b-link-Program;; - &cv-link-LIBEMITTER; for &b-link-Library;; - &cv-link-SHLIBEMITTER; for &b-link-SharedLibrary; and + to a few standard builders: + &cv-link-PROGEMITTER; for &b-link-Program;; + &cv-link-LIBEMITTER; for &b-link-Library;; + &cv-link-SHLIBEMITTER; for &b-link-SharedLibrary; and &cv-link-LDMODULEEMITTER; for &b-link-LoadableModule;;. - Adding an emitter to one of these will cause it to be + Adding an emitter to one of these will cause it to be invoked in addition to any existing emitter for the corresponding builder. @@ -944,10 +943,10 @@ main() <para> The <filename>site_scons</filename> directories give you a place to - put Python modules and packages that you can import into your &SConscript; files - (<filename>site_scons</filename>), + put &Python; modules and packages that you can import into your + &SConscript; files (at the top level) add-on tools that can integrate into &SCons; - (<filename>site_scons/site_tools</filename>), + (in a <filename>site_tools</filename> subdirectory), and a <filename>site_scons/site_init.py</filename> file that gets read before any &SConstruct; or &SConscript; file, allowing you to change &SCons;'s default behavior. @@ -957,8 +956,10 @@ main() <para> Each system type (Windows, Mac, Linux, etc.) searches a canonical - set of directories for <filename>site_scons</filename>; see the man page for details. - The top-level SConstruct's <filename>site_scons</filename> dir is always searched last, + set of directories for <filename>site_scons</filename>; + see the man page for details. + The top-level SConstruct's <filename>site_scons</filename> dir + (that is, the one in the project) is always searched last, and its dir is placed first in the tool path so it overrides all others. @@ -969,8 +970,8 @@ main() If you get a tool from somewhere (the &SCons; wiki or a third party, for instance) and you'd like to use it in your project, a <filename>site_scons</filename> dir is the simplest place to put it. - Tools come in two flavors; either a Python function that operates on - an &Environment; or a Python module or package containing two functions, + Tools come in two flavors; either a &Python; function that operates on + an &Environment; or a &Python; module or package containing two functions, <function>exists()</function> and <function>generate()</function>. </para> @@ -1023,7 +1024,7 @@ env.AddHeader('tgt', 'src') </para> - <!-- + <!-- <scons_output example="builderswriting_site1" os="posix" suffix="1"> <scons_output_command>scons -Q</scons_output_command> </scons_output> @@ -1050,15 +1051,16 @@ env.AddHeader('tgt', 'src') </para> <para> - Many people have a library of utility Python functions they'd like - to include in &SConscript;s; just put that module in - <filename>site_scons/my_utils.py</filename> or any valid Python module name of your + Many people have a collection of utility &Python; functions they'd like + to include in their &SConscript; files: just put them in + <filename>site_scons/my_utils.py</filename> + or any valid &Python; module name of your choice. For instance you can do something like this in <filename>site_scons/my_utils.py</filename> to add <function>build_id</function> and <function>MakeWorkDir</function> functions: </para> - + <scons_example name="builderswriting_site2"> <file name="site_scons/my_utils.py" printme="1"> from SCons.Script import * # for Execute and Mkdir @@ -1073,16 +1075,15 @@ def MakeWorkDir(workdir): </file> <file name="SConscript"> import my_utils + MakeWorkDir('/tmp/work') print("build_id=" + my_utils.build_id()) </file> </scons_example> <para> - - And then in your &SConscript; or any sub-&SConscript; anywhere in - your build, you can import <filename>my_utils</filename> and use it: - + And then in your &SConscript; or any sub-&SConscript; anywhere in + your build, you can import <filename>my_utils</filename> and use it: </para> <sconstruct> @@ -1092,11 +1093,12 @@ my_utils.MakeWorkDir('/tmp/work') </sconstruct> <para> - Note that although you can put this library in + You can put this collection in its own module in a + <filename>site_scons</filename> and import it as in the example, + or you can include it in <filename>site_scons/site_init.py</filename>, - it is no better there than <filename>site_scons/my_utils.py</filename> - since you still have to import that module into your &SConscript;. - Also note that in order to refer to objects in the SCons namespace + which is automatically imported (unless you disable site directories). + Note that in order to refer to objects in the SCons namespace such as &Environment; or &Mkdir; or &Execute; in any file other than a &SConstruct; or &SConscript; you always need to do </para> @@ -1105,22 +1107,20 @@ from SCons.Script import * </sconstruct> <para> - This is true in modules in <filename>site_scons</filename> such as + This is true of modules in <filename>site_scons</filename> such as <filename>site_scons/site_init.py</filename> as well. </para> <para> - - You can use any of the user- or machine-wide site dirs such as + You can use any of the user- or machine-wide site directories such as <filename>~/.scons/site_scons</filename> instead of <filename>./site_scons</filename>, or use the - <option>--site-dir</option> option to point to your own dir. + <option>--site-dir</option> option to point to your own directory. <filename>site_init.py</filename> and - <filename>site_tools</filename> will be located under that dir. - To avoid using a <filename>site_scons</filename> dir at all, + <filename>site_tools</filename> will be located under that directory. + To avoid using a <filename>site_scons</filename> directory at all, even if it exists, use the <option>--no-site-dir</option> option. - </para> </section> diff --git a/src/test_setup.py b/src/test_setup.py index 8410be3..5db21a1 100644 --- a/src/test_setup.py +++ b/src/test_setup.py @@ -81,7 +81,7 @@ class MyTestSCons(TestSCons.TestSCons): ] def __init__(self): - TestSCons.TestSCons.__init__(self) + super().__init__() self.root = self.workpath('root') self.prefix = self.root + os.path.splitdrive(sys.prefix)[1] diff --git a/test/Configure/config-h.py b/test/Configure/config-h.py index f617d92..e5df625 100644 --- a/test/Configure/config-h.py +++ b/test/Configure/config-h.py @@ -1,6 +1,7 @@ #!/usr/bin/env python +# MIT License # -# __COPYRIGHT__ +# Copyright The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -25,7 +26,6 @@ Verify creation of a config.h file from a Configure context. """ -__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import re @@ -46,6 +46,10 @@ r2 = conf.CheckFunc('noFunctionCall') r3 = conf.CheckFunc('memmove') r4 = conf.CheckType('int') r5 = conf.CheckType('noType') + +m1 = conf.CheckMember('struct timespec.tv_sec', '#include <time.h>') +m2 = conf.CheckMember('struct timespec.tv_nanosec', '#include <time.h>') + r6 = conf.CheckCHeader('stdio.h', '<>') r7 = conf.CheckCHeader('hopefullynoc-header.h') r8 = conf.CheckCXXHeader('vector', '<>') @@ -66,6 +70,8 @@ Checking for C function noFunctionCall()... no Checking for C function memmove()... yes Checking for C type int... yes Checking for C type noType... no +Checking for C member struct timespec.tv_sec... yes +Checking for C member struct timespec.tv_nanosec... no Checking for C header file stdio.h... yes Checking for C header file hopefullynoc-header.h... no Checking for C++ header file vector... yes @@ -104,6 +110,12 @@ expected_config_h = ("""\ /* Define to 1 if the system has the type `noType'. */ /* #undef HAVE_NOTYPE */ +/* Define to 1 if the system has the member `struct timespec.tv_sec`. */ +#define HAVE_STRUCT_TIMESPEC_TV_SEC 1 + +/* Define to 1 if the system has the member `struct timespec.tv_nanosec`. */ +/* #undef HAVE_STRUCT_TIMESPEC_TV_NANOSEC */ + /* Define to 1 if you have the <stdio.h> header file. */ #define HAVE_STDIO_H 1 diff --git a/test/ninja/force_scons_callback.py b/test/ninja/force_scons_callback.py index 55c12ca..c99ed58 100644 --- a/test/ninja/force_scons_callback.py +++ b/test/ninja/force_scons_callback.py @@ -37,47 +37,50 @@ except ImportError: _python_ = TestSCons._python_ _exe = TestSCons._exe -ninja_bin = os.path.abspath(os.path.join( - ninja.__file__, - os.pardir, - 'data', - 'bin', - 'ninja' + _exe)) +ninja_bin = os.path.abspath( + os.path.join(ninja.__file__, os.pardir, "data", "bin", "ninja" + _exe) +) -test.dir_fixture('ninja-fixture') +test.dir_fixture("ninja-fixture") -test.file_fixture('ninja_test_sconscripts/sconstruct_force_scons_callback', 'SConstruct') +test.file_fixture( + "ninja_test_sconscripts/sconstruct_force_scons_callback", "SConstruct" +) # generate simple build test.run(stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_contain_all(test.stdout(), 'Executing:') -test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) -if test.stdout().count('scons: Building targets') != 2: +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_contain_all(test.stdout(), "Executing:") +test.must_contain_all(test.stdout(), "ninja%(_exe)s -f" % locals()) +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) # clean build and ninja files -test.run(arguments='-c', stdout=None) -test.must_contain_all_lines(test.stdout(), [ - 'Removed out.txt', - 'Removed out2.txt', - 'Removed build.ninja']) +test.run(arguments="-c", stdout=None) +test.must_contain_all_lines( + test.stdout(), ["Removed out.txt", "Removed out2.txt", "Removed build.ninja"] +) # only generate the ninja file -test.run(arguments='--disable-execute-ninja', stdout=None) -test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) -test.must_not_exist(test.workpath('out.txt')) -test.must_not_exist(test.workpath('out2.txt')) +test.run(arguments="--disable-execute-ninja", stdout=None) +test.must_contain_all_lines(test.stdout(), ["Generating: build.ninja"]) +test.must_not_exist(test.workpath("out.txt")) +test.must_not_exist(test.workpath("out2.txt")) # run ninja independently -program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +program = test.workpath("run_ninja_env.bat") if IS_WINDOWS else ninja_bin test.run(program=program, stdout=None) -if test.stdout().count('scons: Building targets') != 1: +if test.stdout().count("Defer to SCons to build") != 1: test.fail_test() -test.must_match('out.txt', 'foo.c' + os.linesep) -test.must_match('out2.txt', "test2.cpp" + os.linesep) +test.must_match("out.txt", "foo.c" + os.linesep) +test.must_match("out2.txt", "test2.cpp" + os.linesep) + +# only generate the ninja file with specific NINJA_SCONS_DAEMON_PORT +test.run(arguments="PORT=9999 --disable-execute-ninja", stdout=None) +# Verify that port # propagates to call to ninja_run_daemon.py +test.must_contain(test.workpath("build.ninja"), "ninja_run_daemon.py 9999") test.pass_test() diff --git a/test/ninja/iterative_speedup.py b/test/ninja/iterative_speedup.py index 05e372c..e5673b0 100644 --- a/test/ninja/iterative_speedup.py +++ b/test/ninja/iterative_speedup.py @@ -109,7 +109,7 @@ def generate_source(parent_source, current_source): test.write('source_{}.h'.format(current_source), """ #include <stdio.h> #include <stdlib.h> - + int print_function%(current_source)s(); """ % locals()) @@ -125,7 +125,7 @@ def mod_source_return(test_num): int print_function%(test_num)s() - { + { int test = 5 + 5; print_function%(parent_source)s(); return test; @@ -143,7 +143,7 @@ def mod_source_orig(test_num): int print_function%(test_num)s() - { + { return print_function%(parent_source)s(); } """ % locals()) @@ -190,13 +190,15 @@ jobs = '-j' + str(get_num_cpus()) ninja_program = [test.workpath('run_ninja_env.bat'), jobs] if IS_WINDOWS else [ninja_bin, jobs] -start = time.perf_counter() test.run(arguments='--disable-execute-ninja', stdout=None) +test.run(program=ninja_program, arguments='run-ninja-scons-daemon', stdout=None) +start = time.perf_counter() test.run(program=ninja_program, stdout=None) stop = time.perf_counter() ninja_times += [stop - start] test.run(program=test.workpath('print_bin'), stdout="main print") + for test_mod in tests_mods: mod_source_return(test_mod) start = time.perf_counter() diff --git a/test/ninja/mingw_command_generator_action.py b/test/ninja/mingw_command_generator_action.py new file mode 100644 index 0000000..58c5106 --- /dev/null +++ b/test/ninja/mingw_command_generator_action.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import os +import sys + +import TestSCons +from TestCmd import IS_WINDOWS +import SCons +from SCons.Platform.mingw import MINGW_DEFAULT_PATHS +from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS + +test = TestSCons.TestSCons() + +if sys.platform not in ('cygwin', 'win32',): + test.skip_test("Skipping mingw test on non-Windows platform %s." % sys.platform) + +dp = MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS +gcc = SCons.Tool.find_program_path(test.Environment(), 'gcc', default_paths=dp) +if not gcc: + test.skip_test("Skipping mingw test, no MinGW found.\n") + +# ninja must have the os environment setup to work properly +os.environ["PATH"] += os.pathsep + os.path.dirname(gcc) + +try: + import ninja +except ImportError: + test.skip_test("Could not find module in python") + +_python_ = TestSCons._python_ +_exe = TestSCons._exe + +ninja_bin = os.path.abspath(os.path.join( + ninja.BIN_DIR, + 'ninja' + _exe)) + +test.dir_fixture('ninja-fixture') + +test.file_fixture('ninja_test_sconscripts/sconstruct_mingw_command_generator_action', 'SConstruct') + +# generate simple build +test.run(stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_contain_all(test.stdout(), 'Executing:') +test.must_contain_all(test.stdout(), 'ninja%(_exe)s -f' % locals()) +test.run(program=test.workpath('test' + _exe), stdout="library_function") + +# clean build and ninja files +test.run(arguments='-c', stdout=None) + +# only generate the ninja file +test.run(arguments='--disable-execute-ninja', stdout=None) +test.must_contain_all_lines(test.stdout(), ['Generating: build.ninja']) +test.must_not_exist(test.workpath('test' + _exe)) + +# run ninja independently +program = test.workpath('run_ninja_env.bat') if IS_WINDOWS else ninja_bin +test.run(program=program, stdout=None) +test.run(program=test.workpath('test' + _exe), stdout="library_function") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/ninja/ninja_handle_control_c_rebuild.py b/test/ninja/ninja_handle_control_c_rebuild.py index c2c8c8d..9f6b413 100644 --- a/test/ninja/ninja_handle_control_c_rebuild.py +++ b/test/ninja/ninja_handle_control_c_rebuild.py @@ -22,7 +22,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # """ -This test ensures if ninja get's a control-c (or other interrupting signal) while +This test ensures if ninja gets a control-c (or other interrupting signal) while regenerating the build.ninja, it doesn't remove the build.ninja leaving it in an unworkable state. """ diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja index 0124576..34d7872 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja +++ b/test/ninja/ninja_test_sconscripts/sconstruct_control_c_ninja @@ -1,7 +1,7 @@ import os import signal -SetOption('experimental','ninja') +SetOption('experimental', 'ninja') DefaultEnvironment(tools=[]) env = Environment() @@ -9,4 +9,4 @@ env.Tool('ninja') env.Program(target='foo', source='foo.c') if ARGUMENTS.get('NINJA_DISABLE_AUTO_RUN', 0): - os.kill(os.getppid(),signal.SIGINT ) + os.kill(os.getppid(), signal.SIGINT) diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback index 55729a6..ef3562b 100644 --- a/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback +++ b/test/ninja/ninja_test_sconscripts/sconstruct_force_scons_callback @@ -1,8 +1,11 @@ -SetOption('experimental','ninja') +SetOption("experimental", "ninja") DefaultEnvironment(tools=[]) env = Environment(tools=[]) -env.Tool('ninja') +daemon_port = ARGUMENTS.get("PORT", False) +if daemon_port: + env["NINJA_SCONS_DAEMON_PORT"] = int(daemon_port) +env.Tool("ninja") -env.Command('out.txt', 'foo.c', 'echo $SOURCE> $TARGET', NINJA_FORCE_SCONS_BUILD=True) -env.Command('out2.txt', 'test2.cpp', 'echo $SOURCE> $TARGET')
\ No newline at end of file +env.Command("out.txt", "foo.c", "echo $SOURCE> $TARGET", NINJA_FORCE_SCONS_BUILD=True) +env.Command("out2.txt", "test2.cpp", "echo $SOURCE> $TARGET") diff --git a/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action new file mode 100644 index 0000000..e3fcfe2 --- /dev/null +++ b/test/ninja/ninja_test_sconscripts/sconstruct_mingw_command_generator_action @@ -0,0 +1,7 @@ +SetOption('experimental','ninja') +DefaultEnvironment(tools=[]) + +env = Environment(tools=['mingw']) +env.Tool('ninja') +dll = env.SharedLibrary(target='test_impl', source='test_impl.c') +env.Program(target='test', source='test1.c', LIBS=[dll])
\ No newline at end of file diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py index 2eeaad0..1999f74 100644 --- a/testing/framework/TestCommon.py +++ b/testing/framework/TestCommon.py @@ -261,7 +261,7 @@ class TestCommon(TestCmd): calling the base class initialization, and then changing directory to the workdir. """ - TestCmd.__init__(self, **kw) + super().__init__(**kw) os.chdir(self.workdir) def options_arguments(self, options, arguments): diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py index 18dcb94..9368953 100644 --- a/testing/framework/TestRuntest.py +++ b/testing/framework/TestRuntest.py @@ -146,7 +146,7 @@ class TestRuntest(TestCommon): del kw['things_to_copy'] orig_cwd = os.getcwd() - TestCommon.__init__(self, **kw) + super().__init__(**kw) dirs = [os.environ.get('SCONS_RUNTEST_DIR', orig_cwd)] diff --git a/testing/framework/TestSCons.py b/testing/framework/TestSCons.py index 0bf6abd..079e17d 100644 --- a/testing/framework/TestSCons.py +++ b/testing/framework/TestSCons.py @@ -321,7 +321,7 @@ class TestSCons(TestCommon): if kw.get('ignore_python_version', -1) != -1: del kw['ignore_python_version'] - TestCommon.__init__(self, **kw) + super().__init__(**kw) if not self.external: import SCons.Node.FS @@ -1755,7 +1755,7 @@ class TimeSCons(TestSCons): if 'verbose' not in kw and not self.calibrate: kw['verbose'] = True - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) # TODO(sgk): better way to get the script dir than sys.argv[0] self.test_dir = os.path.dirname(sys.argv[0]) diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py index a57ca88..e647fe2 100644 --- a/testing/framework/TestSCons_time.py +++ b/testing/framework/TestSCons_time.py @@ -199,7 +199,7 @@ class TestSCons_time(TestCommon): if 'workdir' not in kw: kw['workdir'] = '' - TestCommon.__init__(self, **kw) + super().__init__(**kw) def archive_split(self, path): if path[-7:] == '.tar.gz': diff --git a/testing/framework/TestSConsign.py b/testing/framework/TestSConsign.py index 699e929..b0562bf 100644 --- a/testing/framework/TestSConsign.py +++ b/testing/framework/TestSConsign.py @@ -64,7 +64,7 @@ class TestSConsign(TestSCons): os.chdir(script_dir) self.script_dir = os.getcwd() - TestSCons.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.my_kw = { 'interpreter' : python, # imported from TestSCons diff --git a/testing/framework/TestUnit/taprunner.py b/testing/framework/TestUnit/taprunner.py index 0dde327..001db5c 100644 --- a/testing/framework/TestUnit/taprunner.py +++ b/testing/framework/TestUnit/taprunner.py @@ -43,29 +43,29 @@ class TAPTestResult(TextTestResult): self.stream.flush() def addSuccess(self, test): - super(TextTestResult, self).addSuccess(test) + super().addSuccess(test) self._process(test, "ok") def addFailure(self, test, err): - super(TextTestResult, self).addFailure(test, err) + super().addFailure(test, err) self._process(test, "not ok", "FAIL") # [ ] add structured data about assertion def addError(self, test, err): - super(TextTestResult, self).addError(test, err) + super().addError(test, err) self._process(test, "not ok", "ERROR") # [ ] add structured data about exception def addSkip(self, test, reason): - super(TextTestResult, self).addSkip(test, reason) + super().addSkip(test, reason) self._process(test, "ok", directive=(" # SKIP %s" % reason)) def addExpectedFailure(self, test, err): - super(TextTestResult, self).addExpectedFailure(test, err) + super().addExpectedFailure(test, err) self._process(test, "not ok", directive=" # TODO") def addUnexpectedSuccess(self, test): - super(TextTestResult, self).addUnexpectedSuccess(test) + super().addUnexpectedSuccess(test) self._process(test, "not ok", "FAIL (unexpected success)") """ @@ -90,7 +90,7 @@ class TAPTestRunner(TextTestRunner): for case in test: case.suite = test - return super(TAPTestRunner, self).run(test) + return super().run(test) if __name__ == "__main__": |