diff options
Diffstat (limited to 'SCons')
32 files changed, 853 insertions, 193 deletions
diff --git a/SCons/Action.py b/SCons/Action.py index b7c6bb7..81dc033 100644 --- a/SCons/Action.py +++ b/SCons/Action.py @@ -848,7 +848,7 @@ class CommandAction(_ActionAction): # variables. if SCons.Debug.track_instances: logInstanceCreation(self, 'Action.CommandAction') - _ActionAction.__init__(self, **kw) + super().__init__(**kw) if is_List(cmd): if [c for c in cmd if is_List(c)]: raise TypeError("CommandAction should be given only " @@ -1231,7 +1231,7 @@ class FunctionAction(_ActionAction): # This is weird, just do the best we can. self.funccontents = _object_contents(execfunction) - _ActionAction.__init__(self, **kw) + super().__init__(**kw) def function_name(self): try: diff --git a/SCons/Builder.py b/SCons/Builder.py index 41475ac..ab51c32 100644 --- a/SCons/Builder.py +++ b/SCons/Builder.py @@ -130,8 +130,8 @@ class DictCmdGenerator(SCons.Util.Selector): to return the proper action based on the file suffix of the source file.""" - def __init__(self, dict=None, source_ext_match=1): - SCons.Util.Selector.__init__(self, dict) + def __init__(self, mapping=None, source_ext_match=True): + super().__init__(mapping) self.source_ext_match = source_ext_match def src_suffixes(self): @@ -222,10 +222,11 @@ class OverrideWarner(UserDict): can actually invoke multiple builders. This class only emits the warnings once, no matter how many Builders are invoked. """ - def __init__(self, dict): - UserDict.__init__(self, dict) + def __init__(self, mapping): + super().__init__(mapping) if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.OverrideWarner') self.already_warned = None + def warn(self): if self.already_warned: return @@ -245,7 +246,7 @@ def Builder(**kw): kw['action'] = SCons.Action.CommandGeneratorAction(kw['generator'], {}) del kw['generator'] elif 'action' in kw: - source_ext_match = kw.get('source_ext_match', 1) + source_ext_match = kw.get('source_ext_match', True) if 'source_ext_match' in kw: del kw['source_ext_match'] if SCons.Util.is_Dict(kw['action']): @@ -876,7 +877,7 @@ class CompositeBuilder(SCons.Util.Proxy): def __init__(self, builder, cmdgen): if SCons.Debug.track_instances: logInstanceCreation(self, 'Builder.CompositeBuilder') - SCons.Util.Proxy.__init__(self, builder) + super().__init__(builder) # cmdgen should always be an instance of DictCmdGenerator. self.cmdgen = cmdgen diff --git a/SCons/Environment.py b/SCons/Environment.py index d0a4baf..3507263 100644 --- a/SCons/Environment.py +++ b/SCons/Environment.py @@ -275,12 +275,12 @@ class BuilderDict(UserDict): the Builders. We need to do this because every time someone changes the Builders in the Environment's BUILDERS dictionary, we must update the Environment's attributes.""" - def __init__(self, dict, env): + def __init__(self, mapping, env): # Set self.env before calling the superclass initialization, # because it will end up calling our other methods, which will # need to point the values in this dictionary to self.env. self.env = env - UserDict.__init__(self, dict) + super().__init__(mapping) def __semi_deepcopy__(self): # These cannot be copied since they would both modify the same builder object, and indeed @@ -294,15 +294,15 @@ class BuilderDict(UserDict): pass else: self.env.RemoveMethod(method) - UserDict.__setitem__(self, item, val) + super().__setitem__(item, val) BuilderWrapper(self.env, val, item) def __delitem__(self, item): - UserDict.__delitem__(self, item) + super().__delitem__(item) delattr(self.env, item) - def update(self, dict): - for i, v in dict.items(): + def update(self, mapping): + for i, v in mapping.items(): self.__setitem__(i, v) @@ -637,7 +637,7 @@ class SubstitutionEnvironment: it is assumed to be a command and the rest of the string is executed; the result of that evaluation is then added to the dict. """ - dict = { + mapping = { 'ASFLAGS' : CLVar(''), 'CFLAGS' : CLVar(''), 'CCFLAGS' : CLVar(''), @@ -667,12 +667,12 @@ class SubstitutionEnvironment: arg = self.backtick(arg[1:]) # utility function to deal with -D option - def append_define(name, dict = dict): + def append_define(name, mapping=mapping): t = name.split('=') if len(t) == 1: - dict['CPPDEFINES'].append(name) + mapping['CPPDEFINES'].append(name) else: - dict['CPPDEFINES'].append([t[0], '='.join(t[1:])]) + mapping['CPPDEFINES'].append([t[0], '='.join(t[1:])]) # Loop through the flags and add them to the appropriate option. # This tries to strike a balance between checking for all possible @@ -702,67 +702,67 @@ class SubstitutionEnvironment: append_define(arg) elif append_next_arg_to == '-include': t = ('-include', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-imacros': t = ('-imacros', self.fs.File(arg)) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-isysroot': t = ('-isysroot', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '-isystem': t = ('-isystem', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-iquote': t = ('-iquote', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-idirafter': t = ('-idirafter', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) elif append_next_arg_to == '-arch': t = ('-arch', arg) - dict['CCFLAGS'].append(t) - dict['LINKFLAGS'].append(t) + mapping['CCFLAGS'].append(t) + mapping['LINKFLAGS'].append(t) elif append_next_arg_to == '--param': t = ('--param', arg) - dict['CCFLAGS'].append(t) + mapping['CCFLAGS'].append(t) else: - dict[append_next_arg_to].append(arg) + mapping[append_next_arg_to].append(arg) append_next_arg_to = None elif not arg[0] in ['-', '+']: - dict['LIBS'].append(self.fs.File(arg)) + mapping['LIBS'].append(self.fs.File(arg)) elif arg == '-dylib_file': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) append_next_arg_to = 'LINKFLAGS' elif arg[:2] == '-L': if arg[2:]: - dict['LIBPATH'].append(arg[2:]) + mapping['LIBPATH'].append(arg[2:]) else: append_next_arg_to = 'LIBPATH' elif arg[:2] == '-l': if arg[2:]: - dict['LIBS'].append(arg[2:]) + mapping['LIBS'].append(arg[2:]) else: append_next_arg_to = 'LIBS' elif arg[:2] == '-I': if arg[2:]: - dict['CPPPATH'].append(arg[2:]) + mapping['CPPPATH'].append(arg[2:]) else: append_next_arg_to = 'CPPPATH' elif arg[:4] == '-Wa,': - dict['ASFLAGS'].append(arg[4:]) - dict['CCFLAGS'].append(arg) + mapping['ASFLAGS'].append(arg[4:]) + mapping['CCFLAGS'].append(arg) elif arg[:4] == '-Wl,': if arg[:11] == '-Wl,-rpath=': - dict['RPATH'].append(arg[11:]) + mapping['RPATH'].append(arg[11:]) elif arg[:7] == '-Wl,-R,': - dict['RPATH'].append(arg[7:]) + mapping['RPATH'].append(arg[7:]) elif arg[:6] == '-Wl,-R': - dict['RPATH'].append(arg[6:]) + mapping['RPATH'].append(arg[6:]) else: - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:4] == '-Wp,': - dict['CPPFLAGS'].append(arg) + mapping['CPPFLAGS'].append(arg) elif arg[:2] == '-D': if arg[2:]: append_define(arg[2:]) @@ -771,32 +771,32 @@ class SubstitutionEnvironment: elif arg == '-framework': append_next_arg_to = 'FRAMEWORKS' elif arg[:14] == '-frameworkdir=': - dict['FRAMEWORKPATH'].append(arg[14:]) + mapping['FRAMEWORKPATH'].append(arg[14:]) elif arg[:2] == '-F': if arg[2:]: - dict['FRAMEWORKPATH'].append(arg[2:]) + mapping['FRAMEWORKPATH'].append(arg[2:]) else: append_next_arg_to = 'FRAMEWORKPATH' - elif arg in [ + elif arg in ( '-mno-cygwin', '-pthread', '-openmp', '-fmerge-all-constants', '-fopenmp', - ]: - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + ): + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg == '-mwindows': - dict['LINKFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg[:5] == '-std=': if '++' in arg[5:]: - key='CXXFLAGS' + key = 'CXXFLAGS' else: - key='CFLAGS' - dict[key].append(arg) + key = 'CFLAGS' + mapping[key].append(arg) elif arg[0] == '+': - dict['CCFLAGS'].append(arg) - dict['LINKFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) + mapping['LINKFLAGS'].append(arg) elif arg in [ '-include', '-imacros', @@ -809,11 +809,11 @@ class SubstitutionEnvironment: ]: append_next_arg_to = arg else: - dict['CCFLAGS'].append(arg) + mapping['CCFLAGS'].append(arg) for arg in flags: do_parse(arg) - return dict + return mapping def MergeFlags(self, args, unique=True): """Merge flags into construction variables. @@ -2375,7 +2375,7 @@ class OverrideEnvironment(Base): if SCons.Debug.track_instances: logInstanceCreation(self, 'Environment.OverrideEnvironment') self.__dict__['__subject'] = subject if overrides is None: - self.__dict__['overrides'] = dict() + self.__dict__['overrides'] = {} else: self.__dict__['overrides'] = overrides @@ -2472,6 +2472,11 @@ class OverrideEnvironment(Base): self.__dict__['overrides'].update(other) def _update_onlynew(self, other): + """Update a dict with new keys. + + Unlike the .update method, if the key is already present, + it is not replaced. + """ for k, v in other.items(): if k not in self.__dict__['overrides']: self.__dict__['overrides'][k] = v @@ -2516,28 +2521,35 @@ def NoSubstitutionProxy(subject): class _NoSubstitutionProxy(Environment): def __init__(self, subject): self.__dict__['__subject'] = subject + def __getattr__(self, name): return getattr(self.__dict__['__subject'], name) + def __setattr__(self, name, value): return setattr(self.__dict__['__subject'], name, value) + def executor_to_lvars(self, kwdict): if 'executor' in kwdict: kwdict['lvars'] = kwdict['executor'].get_lvars() del kwdict['executor'] else: kwdict['lvars'] = {} - def raw_to_mode(self, dict): + + def raw_to_mode(self, mapping): try: - raw = dict['raw'] + raw = mapping['raw'] except KeyError: pass else: - del dict['raw'] - dict['mode'] = raw + del mapping['raw'] + mapping['mode'] = raw + def subst(self, string, *args, **kwargs): return string + def subst_kw(self, kw, *args, **kwargs): return kw + def subst_list(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2545,6 +2557,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst_list(*nargs, **nkw) + def subst_target_source(self, string, *args, **kwargs): nargs = (string, self,) + args nkw = kwargs.copy() @@ -2552,6 +2565,7 @@ def NoSubstitutionProxy(subject): self.executor_to_lvars(nkw) self.raw_to_mode(nkw) return SCons.Subst.scons_subst(*nargs, **nkw) + return _NoSubstitutionProxy(subject) # Local Variables: diff --git a/SCons/EnvironmentTests.py b/SCons/EnvironmentTests.py index c505422..8359405 100644 --- a/SCons/EnvironmentTests.py +++ b/SCons/EnvironmentTests.py @@ -810,11 +810,13 @@ sys.exit(0) "-fmerge-all-constants " "-fopenmp " "-mno-cygwin -mwindows " - "-arch i386 -isysroot /tmp " + "-arch i386 " + "-isysroot /tmp " "-iquote /usr/include/foo1 " "-isystem /usr/include/foo2 " "-idirafter /usr/include/foo3 " "-imacros /usr/include/foo4 " + "-include /usr/include/foo5 " "--param l1-cache-size=32 --param l2-cache-size=6144 " "+DD64 " "-DFOO -DBAR=value -D BAZ " @@ -832,6 +834,7 @@ sys.exit(0) ('-isystem', '/usr/include/foo2'), ('-idirafter', '/usr/include/foo3'), ('-imacros', env.fs.File('/usr/include/foo4')), + ('-include', env.fs.File('/usr/include/foo5')), ('--param', 'l1-cache-size=32'), ('--param', 'l2-cache-size=6144'), '+DD64'], repr(d['CCFLAGS']) assert d['CXXFLAGS'] == ['-std=c++0x'], repr(d['CXXFLAGS']) diff --git a/SCons/Errors.py b/SCons/Errors.py index 42db072..04cea38 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -99,8 +99,8 @@ class BuildError(Exception): self.action = action self.command = command - Exception.__init__(self, node, errstr, status, exitstatus, filename, - executor, action, command, exc_info) + super().__init__(node, errstr, status, exitstatus, filename, + executor, action, command, exc_info) def __str__(self): if self.filename: @@ -128,7 +128,7 @@ class ExplicitExit(Exception): self.node = node self.status = status self.exitstatus = status - Exception.__init__(self, *args) + super().__init__(*args) def convert_to_BuildError(status, exc_info=None): """Convert a return code to a BuildError Exception. diff --git a/SCons/Job.py b/SCons/Job.py index bcdb88a..b398790 100644 --- a/SCons/Job.py +++ b/SCons/Job.py @@ -238,7 +238,7 @@ else: and a boolean indicating whether the task executed successfully. """ def __init__(self, requestQueue, resultsQueue, interrupted): - threading.Thread.__init__(self) + super().__init__() self.daemon = True self.requestQueue = requestQueue self.resultsQueue = resultsQueue diff --git a/SCons/JobTests.py b/SCons/JobTests.py index 5b5a590..54d7fa4 100644 --- a/SCons/JobTests.py +++ b/SCons/JobTests.py @@ -410,13 +410,13 @@ class DummyNodeInfo: class testnode (SCons.Node.Node): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.executed self.ninfo = DummyNodeInfo() class goodnode (testnode): def __init__(self): - SCons.Node.Node.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.up_to_date self.ninfo = DummyNodeInfo() @@ -430,7 +430,7 @@ class slowgoodnode (goodnode): class badnode (goodnode): def __init__(self): - goodnode.__init__(self) + super().__init__() self.expect_to_be = SCons.Node.failed def build(self, **kw): raise Exception('badnode exception') diff --git a/SCons/Memoize.py b/SCons/Memoize.py index 8c3303f..b02c144 100644 --- a/SCons/Memoize.py +++ b/SCons/Memoize.py @@ -159,7 +159,7 @@ class CountDict(Counter): def __init__(self, cls_name, method_name, keymaker): """ """ - Counter.__init__(self, cls_name, method_name) + super().__init__(cls_name, method_name) self.keymaker = keymaker def count(self, *args, **kw): """ Counts whether the computed key value is already present diff --git a/SCons/Node/Alias.py b/SCons/Node/Alias.py index b5e4eb4..1125c22 100644 --- a/SCons/Node/Alias.py +++ b/SCons/Node/Alias.py @@ -99,7 +99,7 @@ class Alias(SCons.Node.Node): BuildInfo = AliasBuildInfo def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.changed_since_last_build = 1 self.store_info = 0 diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 5f05a86..c1bfd6a 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -82,7 +82,7 @@ class EntryProxyAttributeError(AttributeError): of the underlying Entry involved in an AttributeError exception. """ def __init__(self, entry_proxy, attribute): - AttributeError.__init__(self) + super().__init__() self.entry_proxy = entry_proxy self.attribute = attribute def __str__(self): @@ -579,7 +579,7 @@ class Base(SCons.Node.Node): signatures.""" if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Base') - SCons.Node.Node.__init__(self) + super().__init__() # Filenames and paths are probably reused and are intern'ed to save some memory. # Filename with extension as it was specified when the object was @@ -982,7 +982,7 @@ class Entry(Base): 'contentsig'] def __init__(self, name, directory, fs): - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._func_exists = 3 self._func_get_contents = 1 @@ -1574,7 +1574,7 @@ class Dir(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.Dir') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def _morph(self): @@ -2563,7 +2563,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): if key != 'dependency_map' and hasattr(self, 'dependency_map'): del self.dependency_map - return super(FileBuildInfo, self).__setattr__(key, value) + return super().__setattr__(key, value) def convert_to_sconsign(self): """ @@ -2674,7 +2674,7 @@ class File(Base): def __init__(self, name, directory, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.File') - Base.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self._morph() def Entry(self, name): diff --git a/SCons/Node/FSTests.py b/SCons/Node/FSTests.py index d78940e..b93efc8 100644 --- a/SCons/Node/FSTests.py +++ b/SCons/Node/FSTests.py @@ -2568,7 +2568,7 @@ class FileTestCase(_tempdirTestCase): class ChangedNode(SCons.Node.FS.File): def __init__(self, name, directory=None, fs=None): - SCons.Node.FS.File.__init__(self, name, directory, fs) + super().__init__(name, directory, fs) self.name = name self.Tag('found_includes', []) self.stored_info = None @@ -2608,7 +2608,7 @@ class FileTestCase(_tempdirTestCase): class ChangedEnvironment(SCons.Environment.Base): def __init__(self): - SCons.Environment.Base.__init__(self) + super().__init__() self.decide_source = self._changed_timestamp_then_content class FakeNodeInfo: diff --git a/SCons/Node/NodeTests.py b/SCons/Node/NodeTests.py index b36c2e9..ee4d080 100644 --- a/SCons/Node/NodeTests.py +++ b/SCons/Node/NodeTests.py @@ -160,7 +160,7 @@ class NoneBuilder(Builder): class ListBuilder(Builder): def __init__(self, *nodes): - Builder.__init__(self) + super().__init__() self.nodes = nodes def execute(self, target, source, env): if hasattr(self, 'status'): @@ -200,7 +200,7 @@ class MyNode(SCons.Node.Node): simulate a real, functional Node subclass. """ def __init__(self, name): - SCons.Node.Node.__init__(self) + super().__init__() self.name = name self.Tag('found_includes', []) def __str__(self): @@ -1226,7 +1226,7 @@ class NodeTestCase(unittest.TestCase): """Test the get_string() method.""" class TestNode(MyNode): def __init__(self, name, sig): - MyNode.__init__(self, name) + super().__init__(name) self.sig = sig def for_signature(self): diff --git a/SCons/Node/Python.py b/SCons/Node/Python.py index c6850ab..80d2762 100644 --- a/SCons/Node/Python.py +++ b/SCons/Node/Python.py @@ -83,7 +83,7 @@ class Value(SCons.Node.Node): BuildInfo = ValueBuildInfo def __init__(self, value, built_value=None, name=None): - SCons.Node.Node.__init__(self) + super().__init__() self.value = value self.changed_since_last_build = 6 self.store_info = 0 diff --git a/SCons/SConf.py b/SCons/SConf.py index d8291d5..4e8d410 100644 --- a/SCons/SConf.py +++ b/SCons/SConf.py @@ -142,7 +142,7 @@ SCons.Warnings.enableWarningClass(SConfWarning) # some error definitions class SConfError(SCons.Errors.UserError): def __init__(self,msg): - SCons.Errors.UserError.__init__(self,msg) + super().__init__(msg) class ConfigureDryRunError(SConfError): """Raised when a file or directory needs to be updated during a Configure @@ -152,13 +152,13 @@ class ConfigureDryRunError(SConfError): msg = 'Cannot create configure directory "%s" within a dry-run.' % str(target) else: msg = 'Cannot update configure test "%s" within a dry-run.' % str(target) - SConfError.__init__(self,msg) + super().__init__(msg) class ConfigureCacheError(SConfError): """Raised when a use explicitely requested the cache feature, but the test is run the first time.""" def __init__(self,target): - SConfError.__init__(self, '"%s" is not yet built and cache is forced.' % str(target)) + super().__init__('"%s" is not yet built and cache is forced.' % str(target)) # define actions for building text files diff --git a/SCons/SConsign.py b/SCons/SConsign.py index 5b78855..ecca391 100644 --- a/SCons/SConsign.py +++ b/SCons/SConsign.py @@ -248,7 +248,7 @@ class DB(Base): determined by the database module. """ def __init__(self, dir): - Base.__init__(self) + super().__init__() self.dir = dir @@ -319,7 +319,7 @@ class Dir(Base): """ fp - file pointer to read entries from """ - Base.__init__(self) + super().__init__() if not fp: return @@ -352,7 +352,7 @@ class DirFile(Dir): fp = None try: - Dir.__init__(self, fp, dir) + super().__init__(fp, dir) except KeyboardInterrupt: raise except Exception: diff --git a/SCons/Script/Interactive.py b/SCons/Script/Interactive.py index 26a8bcd..d694740 100644 --- a/SCons/Script/Interactive.py +++ b/SCons/Script/Interactive.py @@ -29,7 +29,7 @@ # of its own, which might or might not be a good thing. Nevertheless, # here are some enhancements that will probably be requested some day # and are worth keeping in mind (assuming this takes off): -# +# # - A command to re-read / re-load the SConscript files. This may # involve allowing people to specify command-line options (e.g. -f, # -I, --no-site-dir) that affect how the SConscript files are read. @@ -257,7 +257,12 @@ version Prints SCons version information. # from SCons.Debug import Trace # Trace('node %s, ref_count %s !!!\n' % (node, node.ref_count)) - SCons.SConsign.Reset() + # TODO: REMOVE WPD DEBUG 02/14/2022 + # This call was clearing the list of sconsign files to be written, so it would + # only write the results of the first build command. All others wouldn't be written + # to .SConsign. + # Pretty sure commenting this out is the correct fix. + # SCons.SConsign.Reset() SCons.Script.Main.progress_display("scons: done clearing node information.") def do_clean(self, argv): diff --git a/SCons/Script/Main.py b/SCons/Script/Main.py index a340f5b..ab2dc0e 100644 --- a/SCons/Script/Main.py +++ b/SCons/Script/Main.py @@ -987,7 +987,6 @@ def _main(parser): # This would then cause subtle bugs, as already happened in #2971. if options.interactive: SCons.Node.interactive = True - # That should cover (most of) the options. # Next, set up the variables that hold command-line arguments, # so the SConscript files that we read and execute have access to them. diff --git a/SCons/Subst.py b/SCons/Subst.py index 298df38..7535772 100644 --- a/SCons/Subst.py +++ b/SCons/Subst.py @@ -132,7 +132,7 @@ class CmdStringHolder(collections.UserString): proper escape sequences inserted. """ def __init__(self, cmd, literal=None): - collections.UserString.__init__(self, cmd) + super().__init__(cmd) self.literal = literal def is_literal(self): @@ -490,7 +490,7 @@ class ListSubber(collections.UserList): internally. """ def __init__(self, env, mode, conv, gvars): - collections.UserList.__init__(self, []) + super().__init__([]) self.env = env self.mode = mode self.conv = conv diff --git a/SCons/SubstTests.py b/SCons/SubstTests.py index b956a25..1a203a0 100644 --- a/SCons/SubstTests.py +++ b/SCons/SubstTests.py @@ -121,7 +121,7 @@ class SubstTestCase(unittest.TestCase): class MyNode(DummyNode): """Simple node work-alike with some extra stuff for testing.""" def __init__(self, name): - DummyNode.__init__(self, name) + super().__init__(name) class Attribute: pass self.attribute = Attribute() diff --git a/SCons/Tool/GettextCommon.py b/SCons/Tool/GettextCommon.py index 16900e0..058b997 100644 --- a/SCons/Tool/GettextCommon.py +++ b/SCons/Tool/GettextCommon.py @@ -206,7 +206,7 @@ class _POFileBuilder(BuilderBase): del kw['target_alias'] if 'target_factory' not in kw: kw['target_factory'] = _POTargetFactory(env, alias=alias).File - BuilderBase.__init__(self, **kw) + super().__init__(**kw) def _execute(self, env, target, source, *args, **kw): """ Execute builder's actions. diff --git a/SCons/Tool/MSCommon/sdk.py b/SCons/Tool/MSCommon/sdk.py index d95df91..aa94f4d 100644 --- a/SCons/Tool/MSCommon/sdk.py +++ b/SCons/Tool/MSCommon/sdk.py @@ -131,7 +131,7 @@ class WindowsSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\Microsoft SDKs\Windows\v%s\InstallationFolder' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.version class PlatformSDK(SDKDefinition): @@ -140,7 +140,7 @@ class PlatformSDK(SDKDefinition): """ HKEY_FMT = r'Software\Microsoft\MicrosoftSDK\InstalledSDKS\%s\Install Dir' def __init__(self, *args, **kw): - SDKDefinition.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.hkey_data = self.uuid # diff --git a/SCons/Tool/msvs.py b/SCons/Tool/msvs.py index 7a827f4..887cb59 100644 --- a/SCons/Tool/msvs.py +++ b/SCons/Tool/msvs.py @@ -320,7 +320,7 @@ class _GenerateV7User(_UserGenerator): self.usrhead = V9UserHeader self.usrconf = V9UserConfiguration self.usrdebg = V9DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -395,7 +395,7 @@ class _GenerateV10User(_UserGenerator): self.usrhead = V10UserHeader self.usrconf = V10UserConfiguration self.usrdebg = V10DebugSettings - _UserGenerator.__init__(self, dspfile, source, env) + super().__init__(dspfile, source, env) def UserProject(self): confkeys = sorted(self.configs.keys()) @@ -1487,7 +1487,7 @@ class _DSWGenerator: class _GenerateV7DSW(_DSWGenerator): """Generates a Solution file for MSVS .NET""" def __init__(self, dswfile, source, env): - _DSWGenerator.__init__(self, dswfile, source, env) + super().__init__(dswfile, source, env) self.file = None self.version = self.env['MSVS_VERSION'] diff --git a/SCons/Tool/ninja/NinjaState.py b/SCons/Tool/ninja/NinjaState.py index d7c260e..fff60be 100644 --- a/SCons/Tool/ninja/NinjaState.py +++ b/SCons/Tool/ninja/NinjaState.py @@ -24,11 +24,15 @@ import io import os +import pathlib +import signal +import tempfile import shutil import sys from os.path import splitext from tempfile import NamedTemporaryFile import ninja +import hashlib import SCons from SCons.Script import COMMAND_LINE_TARGETS @@ -83,6 +87,7 @@ class NinjaState: # to make the SCONS_INVOCATION variable properly quoted for things # like CCFLAGS scons_escape = env.get("ESCAPE", lambda x: x) + scons_daemon_port = int(env.get('NINJA_SCONS_DAEMON_PORT',-1)) # if SCons was invoked from python, we expect the first arg to be the scons.py # script, otherwise scons was invoked from the scons script @@ -184,10 +189,10 @@ class NinjaState: "restat": 1, }, "TEMPLATE": { - "command": "$SCONS_INVOCATION $out", - "description": "Rendering $SCONS_INVOCATION $out", - "pool": "scons_pool", - "restat": 1, + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_daemon_build.py'} {scons_daemon_port} {get_path(env.get('NINJA_DIR'))} $out", + "description": "Defer to SCons to build $out", + "pool": "local_pool", + "restat": 1 }, "SCONS": { "command": "$SCONS_INVOCATION $out", @@ -211,6 +216,29 @@ class NinjaState: # output. "restat": 1, }, + + "SCONS_DAEMON": { + "command": f"{sys.executable} {pathlib.Path(__file__).parent / 'ninja_run_daemon.py'} {scons_daemon_port} {env.get('NINJA_DIR').abspath} {str(env.get('NINJA_SCONS_DAEMON_KEEP_ALIVE'))} $SCONS_INVOCATION", + "description": "Starting scons daemon...", + "pool": "local_pool", + # restat + # if present, causes Ninja to re-stat the command's outputs + # after execution of the command. Each output whose + # modification time the command did not change will be + # treated as though it had never needed to be built. This + # may cause the output's reverse dependencies to be removed + # from the list of pending build actions. + # + # We use restat any time we execute SCons because + # SCons calls in Ninja typically create multiple + # targets. But since SCons is doing it's own up to + # date-ness checks it may only update say one of + # them. Restat will find out which of the multiple + # build targets did actually change then only rebuild + # those targets which depend specifically on that + # output. + "restat": 1, + }, "REGENERATE": { "command": "$SCONS_INVOCATION_W_TARGETS", "description": "Regenerating $self", @@ -245,6 +273,9 @@ class NinjaState: if not node.has_builder(): return False + if isinstance(node, SCons.Node.Python.Value): + return False + if isinstance(node, SCons.Node.Alias.Alias): build = alias_to_ninja_build(node) else: @@ -256,7 +287,28 @@ class NinjaState: node_string = str(node) if node_string in self.builds: - raise InternalError("Node {} added to ninja build state more than once".format(node_string)) + # TODO: If we work out a way to handle Alias() with same name as file this logic can be removed + # This works around adding Alias with the same name as a Node. + # It's not great way to workaround because it force renames the alias, + # but the alternative is broken ninja support. + warn_msg = f"Alias {node_string} name the same as File node, ninja does not support this. Renaming Alias {node_string} to {node_string}_alias." + if isinstance(node, SCons.Node.Alias.Alias): + for i, output in enumerate(build["outputs"]): + if output == node_string: + build["outputs"][i] += "_alias" + node_string += "_alias" + print(warn_msg) + elif self.builds[node_string]["rule"] == "phony": + for i, output in enumerate(self.builds[node_string]["outputs"]): + if output == node_string: + self.builds[node_string]["outputs"][i] += "_alias" + tmp_build = self.builds[node_string].copy() + del self.builds[node_string] + node_string += "_alias" + self.builds[node_string] = tmp_build + print(warn_msg) + else: + raise InternalError("Node {} added to ninja build state more than once".format(node_string)) self.builds[node_string] = build self.built.update(build["outputs"]) return True @@ -330,8 +382,12 @@ class NinjaState: ) template_builders = [] + scons_compiledb = False for build in [self.builds[key] for key in sorted(self.builds.keys())]: + if "compile_commands.json" in build["outputs"]: + scons_compiledb = True + if build["rule"] == "TEMPLATE": template_builders.append(build) continue @@ -345,10 +401,10 @@ class NinjaState: # generated sources or else we will create a dependency # cycle. if ( - generated_source_files - and not build["rule"] == "INSTALL" - and set(build["outputs"]).isdisjoint(generated_source_files) - and set(build.get("implicit", [])).isdisjoint(generated_source_files) + generated_source_files + and not build["rule"] == "INSTALL" + and set(build["outputs"]).isdisjoint(generated_source_files) + and set(build.get("implicit", [])).isdisjoint(generated_source_files) ): # Make all non-generated source targets depend on # _generated_sources. We use order_only for generated @@ -422,41 +478,10 @@ class NinjaState: ninja.build(**build) - template_builds = dict() + scons_daemon_dirty = str(pathlib.Path(get_path(self.env.get("NINJA_DIR"))) / "scons_daemon_dirty") for template_builder in template_builders: - - # Special handling for outputs and implicit since we need to - # aggregate not replace for each builder. - for agg_key in ["outputs", "implicit", "inputs"]: - new_val = template_builds.get(agg_key, []) - - # Use pop so the key is removed and so the update - # below will not overwrite our aggregated values. - cur_val = template_builder.pop(agg_key, []) - if is_List(cur_val): - new_val += cur_val - else: - new_val.append(cur_val) - template_builds[agg_key] = new_val - - # Collect all other keys - template_builds.update(template_builder) - - if template_builds.get("outputs", []): - - # Try to clean up any dependency cycles. If we are passing an - # ouptut node to SCons, it will build any dependencys if ninja - # has not already. - for output in template_builds.get("outputs", []): - inputs = template_builds.get('inputs') - if inputs and output in inputs: - inputs.remove(output) - - implicits = template_builds.get('implicit') - if implicits and output in implicits: - implicits.remove(output) - - ninja.build(**template_builds) + template_builder["implicit"] += [scons_daemon_dirty] + ninja.build(**template_builder) # We have to glob the SCons files here to teach the ninja file # how to regenerate itself. We'll never see ourselves in the @@ -483,30 +508,56 @@ class NinjaState: } ) - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - ninja.build( - "compile_commands.json", - rule="CMD", - pool="console", - implicit=[str(self.ninja_file)], - variables={ - "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( - # NINJA_COMPDB_EXPAND - should only be true for ninja - # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) - # https://github.com/ninja-build/ninja/pull/1223 - # TODO: add check in generate to check version and enable this by default if it's available. - self.ninja_bin_path, str(self.ninja_file), - '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' - ) - }, - ) + if not scons_compiledb: + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[str(self.ninja_file)], + variables={ + "cmd": "{} -f {} -t compdb {}CC CXX > compile_commands.json".format( + # NINJA_COMPDB_EXPAND - should only be true for ninja + # This was added to ninja's compdb tool in version 1.9.0 (merged April 2018) + # https://github.com/ninja-build/ninja/pull/1223 + # TODO: add check in generate to check version and enable this by default if it's available. + self.ninja_bin_path, str(self.ninja_file), + '-x ' if self.env.get('NINJA_COMPDB_EXPAND', True) else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) ninja.build( - "compiledb", rule="phony", implicit=["compile_commands.json"], + ["run_ninja_scons_daemon_phony", scons_daemon_dirty], + rule="SCONS_DAEMON", ) + + daemon_dir = pathlib.Path(tempfile.gettempdir()) / ('scons_daemon_' + str(hashlib.md5(str(get_path(self.env["NINJA_DIR"])).encode()).hexdigest())) + pidfile = None + if os.path.exists(scons_daemon_dirty): + pidfile = scons_daemon_dirty + elif os.path.exists(daemon_dir / 'pidfile'): + pidfile = daemon_dir / 'pidfile' + + if pidfile: + with open(pidfile) as f: + pid = int(f.readline()) + try: + os.kill(pid, signal.SIGINT) + except OSError: + pass + + if os.path.exists(scons_daemon_dirty): + os.unlink(scons_daemon_dirty) + + # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -588,7 +639,13 @@ class SConsToNinjaTranslator: elif isinstance(action, COMMAND_TYPES): build = get_command(env, node, action) else: - raise Exception("Got an unbuildable ListAction for: {}".format(str(node))) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } if build is not None: build["order_only"] = get_order_only(node) @@ -657,7 +714,7 @@ class SConsToNinjaTranslator: return results[0] all_outputs = list({output for build in results for output in build["outputs"]}) - dependencies = list({dep for build in results for dep in build["implicit"]}) + dependencies = list({dep for build in results for dep in build.get("implicit", [])}) if results[0]["rule"] == "CMD" or results[0]["rule"] == "GENERATED_CMD": cmdline = "" @@ -669,6 +726,9 @@ class SConsToNinjaTranslator: # condition if not cmdstr. So here we strip preceding # and proceeding whitespace to make strings like the # above become empty strings and so will be skipped. + if not cmd.get("variables") or not cmd["variables"].get("cmd"): + continue + cmdstr = cmd["variables"]["cmd"].strip() if not cmdstr: continue @@ -717,4 +777,10 @@ class SConsToNinjaTranslator: "implicit": dependencies, } - raise Exception("Unhandled list action with rule: " + results[0]["rule"]) + return { + "rule": "TEMPLATE", + "order_only": get_order_only(node), + "outputs": get_outputs(node), + "inputs": get_inputs(node), + "implicit": get_dependencies(node, skip_sources=True), + } diff --git a/SCons/Tool/ninja/Utils.py b/SCons/Tool/ninja/Utils.py index 3adbb53..888218d 100644 --- a/SCons/Tool/ninja/Utils.py +++ b/SCons/Tool/ninja/Utils.py @@ -123,7 +123,7 @@ def get_dependencies(node, skip_sources=False): get_path(src_file(child)) for child in filter_ninja_nodes(node.children()) if child not in node.sources - ] + ] return [get_path(src_file(child)) for child in filter_ninja_nodes(node.children())] @@ -370,7 +370,7 @@ def ninja_contents(original): """Return a dummy content without doing IO""" def wrapper(self): - if isinstance(self, SCons.Node.Node) and self.is_sconscript(): + if isinstance(self, SCons.Node.Node) and (self.is_sconscript() or self.is_conftest()): return original(self) return bytes("dummy_ninja_contents", encoding="utf-8") diff --git a/SCons/Tool/ninja/__init__.py b/SCons/Tool/ninja/__init__.py index cc69554..3c72ee4 100644 --- a/SCons/Tool/ninja/__init__.py +++ b/SCons/Tool/ninja/__init__.py @@ -26,6 +26,7 @@ import importlib import os +import random import subprocess import sys @@ -187,6 +188,11 @@ def generate(env): env["NINJA_ALIAS_NAME"] = env.get("NINJA_ALIAS_NAME", "generate-ninja") env['NINJA_DIR'] = env.Dir(env.get("NINJA_DIR", '#/.ninja')) + env["NINJA_SCONS_DAEMON_KEEP_ALIVE"] = env.get("NINJA_SCONS_DAEMON_KEEP_ALIVE", 180000) + env["NINJA_SCONS_DAEMON_PORT"] = env.get('NINJA_SCONS_DAEMON_PORT', random.randint(10000, 60000)) + + if GetOption("disable_ninja"): + env.SConsignFile(os.path.join(str(env['NINJA_DIR']),'.ninja.sconsign')) # here we allow multiple environments to construct rules and builds # into the same ninja file @@ -200,6 +206,7 @@ def generate(env): SCons.Warnings.SConsWarning("Generating multiple ninja files not supported, set ninja file name before tool initialization.") ninja_file = [NINJA_STATE.ninja_file] + def ninja_generate_deps(env): """Return a list of SConscripts TODO: Should we also include files loaded from site_scons/*** @@ -456,3 +463,6 @@ def generate(env): env['TEMPFILEDIR'] = "$NINJA_DIR/.response_files" env["TEMPFILE"] = NinjaNoResponseFiles + + + env.Alias('run-ninja-scons-daemon', 'run_ninja_scons_daemon_phony') diff --git a/SCons/Tool/ninja/ninja.xml b/SCons/Tool/ninja/ninja.xml index bf58237..51ff435 100644 --- a/SCons/Tool/ninja/ninja.xml +++ b/SCons/Tool/ninja/ninja.xml @@ -73,6 +73,8 @@ See its __doc__ string for a discussion of the format. <item>_NINJA_REGENERATE_DEPS_FUNC</item> <!-- <item>__NINJA_NO</item>--> <item>IMPLICIT_COMMAND_DEPENDENCIES</item> + <item>NINJA_SCONS_DAEMON_KEEP_ALIVE</item> + <item>NINJA_SCONS_DAEMON_PORT</item> <!-- TODO: Document these --> @@ -350,5 +352,24 @@ python -m pip install ninja </summary> </cvar> + <cvar name="NINJA_SCONS_DAEMON_KEEP_ALIVE"> + <summary> + <para> + The number of seconds for the SCons deamon launched by ninja to stay alive. + (Default: 180000) + </para> + </summary> + </cvar> + + <cvar name="NINJA_SCONS_DAEMON_PORT"> + <summary> + <para> + The TCP/IP port for the SCons daemon to listen on. + <emphasis>NOTE: You cannot use a port already being listened to on your build machine.</emphasis> + (Default: random number between 10000,60000) + </para> + </summary> + </cvar> + </sconsdoc> diff --git a/SCons/Tool/ninja/ninja_daemon_build.py b/SCons/Tool/ninja/ninja_daemon_build.py new file mode 100644 index 0000000..f34935b --- /dev/null +++ b/SCons/Tool/ninja/ninja_daemon_build.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script is intended to execute a single build target. This script should be +called by ninja, passing the port, ninja dir, and build target via arguments. +The script then executes a simple get request to the scons daemon which is listening +on from localhost on the set port. +""" + +import http.client +import sys +import time +import os +import logging +import pathlib +import tempfile +import hashlib +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon_request.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +while True: + try: + logging.debug(f"Sending request: {sys.argv[3]}") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) + conn.request("GET", "/?build=" + sys.argv[3]) + response = None + + while not response: + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException: + logging.debug(f"Error: {traceback.format_exc()}") + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode("utf-8")) + exit(1) + logging.debug(f"Request Done: {sys.argv[3]}") + exit(0) + + except Exception: + logging.debug(f"Failed to send command: {traceback.format_exc()}") + exit(1) + + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_run_daemon.py b/SCons/Tool/ninja/ninja_run_daemon.py new file mode 100644 index 0000000..057537a --- /dev/null +++ b/SCons/Tool/ninja/ninja_run_daemon.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script is intended to be called by ninja to start up the scons daemon process. It will +launch the server and attempt to connect to it. This process needs to completely detach +from the spawned process so ninja can consider the build edge completed. It should be passed +the args which should be forwarded to the scons daemon process which could be any number of +# arguments. However the first few arguments are required to be port, ninja dir, and keep alive +timeout in seconds. + +The scons_daemon_dirty file acts as a pidfile marker letting this script quickly skip over +restarting the server if the server is running. The assumption here is the pidfile should only +exist if the server is running. +""" + +import subprocess +import sys +import os +import pathlib +import tempfile +import hashlib +import logging +import time +import http.client +import traceback + +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) + +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + +if not os.path.exists(ninja_builddir / "scons_daemon_dirty"): + cmd = [ + sys.executable, + str(pathlib.Path(__file__).parent / "ninja_scons_daemon.py"), + ] + sys.argv[1:] + logging.debug(f"Starting daemon with {' '.join(cmd)}") + + p = subprocess.Popen( + cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False + ) + + with open(daemon_dir / "pidfile", "w") as f: + f.write(str(p.pid)) + with open(ninja_builddir / "scons_daemon_dirty", "w") as f: + f.write(str(p.pid)) + + error_msg = f"ERROR: Failed to connect to scons daemon.\n Check {daemon_dir / 'scons_daemon.log'} for more info.\n" + + while True: + try: + logging.debug("Attempting to connect scons daemon") + conn = http.client.HTTPConnection( + "127.0.0.1", port=int(sys.argv[1]), timeout=60 + ) + conn.request("GET", "/?ready=true") + response = None + + try: + response = conn.getresponse() + except (http.client.RemoteDisconnected, http.client.ResponseNotReady): + time.sleep(0.01) + except http.client.HTTPException: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + else: + msg = response.read() + status = response.status + if status != 200: + print(msg.decode("utf-8")) + exit(1) + logging.debug("Server Responded it was ready!") + break + + except ConnectionRefusedError: + logging.debug(f"Server not ready, server PID: {p.pid}") + time.sleep(1) + if p.poll is not None: + logging.debug(f"Server process died, aborting: {p.returncode}") + sys.exit(p.returncode) + except ConnectionResetError: + logging.debug("Server ConnectionResetError") + sys.stderr.write(error_msg) + exit(1) + except Exception: + logging.debug(f"Error: {traceback.format_exc()}") + sys.stderr.write(error_msg) + exit(1) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/ninja/ninja_scons_daemon.py b/SCons/Tool/ninja/ninja_scons_daemon.py new file mode 100644 index 0000000..88f3b22 --- /dev/null +++ b/SCons/Tool/ninja/ninja_scons_daemon.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# Copyright The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +""" +This script primarily consists of two threads, the http server thread and the scons interactive +process thread. The http server thread will listen on the passed port for http get request +which should indicate some action for the scons interactive process to take. + +The daemon will keep log files in a tmp directory correlated to the hash of the absolute path +of the ninja build dir passed. The daemon will also use a keep alive time to know when to shut +itself down after the passed timeout of no activity. Any time the server receives a get request, +the keep alive time will be reset. +""" + +import http.server +import socketserver +from urllib.parse import urlparse, parse_qs +import time +from threading import Condition +from subprocess import PIPE, Popen +import sys +import os +import threading +import queue +import pathlib +import logging +from timeit import default_timer as timer +import traceback +import tempfile +import hashlib + +port = int(sys.argv[1]) +ninja_builddir = pathlib.Path(sys.argv[2]) +daemon_keep_alive = int(sys.argv[3]) +args = sys.argv[4:] + +daemon_dir = pathlib.Path(tempfile.gettempdir()) / ( + "scons_daemon_" + str(hashlib.md5(str(ninja_builddir).encode()).hexdigest()) +) +os.makedirs(daemon_dir, exist_ok=True) +logging.basicConfig( + filename=daemon_dir / "scons_daemon.log", + filemode="a", + format="%(asctime)s %(message)s", + level=logging.DEBUG, +) + + +def daemon_log(message): + logging.debug(message) + + +def custom_readlines(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + if buf.endswith("scons>>>"): + yield buf + buf = "" + + +def custom_readerr(handle, line_separator="\n", chunk_size=1): + buf = "" + while not handle.closed: + data = handle.read(chunk_size) + if not data: + break + buf += data.decode("utf-8") + if line_separator in buf: + chunks = buf.split(line_separator) + buf = chunks.pop() + for chunk in chunks: + yield chunk + line_separator + + +def enqueue_output(out, queue): + for line in iter(custom_readlines(out)): + queue.put(line) + out.close() + + +def enqueue_error(err, queue): + for line in iter(custom_readerr(err)): + queue.put(line) + err.close() + + +input_q = queue.Queue() +output_q = queue.Queue() +error_q = queue.Queue() + +finished_building = [] +error_nodes = [] + +building_cv = Condition() +error_cv = Condition() + +thread_error = False + + +def daemon_thread_func(): + global thread_error + global finished_building + global error_nodes + try: + args_list = args + ["--interactive"] + daemon_log(f"Starting daemon with args: {' '.join(args_list)}") + daemon_log(f"cwd: {os.getcwd()}") + + p = Popen(args_list, stdout=PIPE, stderr=PIPE, stdin=PIPE) + + t = threading.Thread(target=enqueue_output, args=(p.stdout, output_q)) + t.daemon = True + t.start() + + te = threading.Thread(target=enqueue_error, args=(p.stderr, error_q)) + te.daemon = True + te.start() + + daemon_ready = False + building_node = None + + while p.poll() is None: + + while True: + try: + line = output_q.get(block=False, timeout=0.01) + except queue.Empty: + break + else: + daemon_log("output: " + line.strip()) + + if "scons: building terminated because of errors." in line: + error_output = "" + while True: + try: + error_output += error_q.get(block=False, timeout=0.01) + except queue.Empty: + break + error_nodes += [{"node": building_node, "error": error_output}] + daemon_ready = True + building_node = None + with building_cv: + building_cv.notify() + + elif line == "scons>>>": + with error_q.mutex: + error_q.queue.clear() + daemon_ready = True + with building_cv: + building_cv.notify() + building_node = None + + while daemon_ready and not input_q.empty(): + + try: + building_node = input_q.get(block=False, timeout=0.01) + except queue.Empty: + break + if "exit" in building_node: + p.stdin.write("exit\n".encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + raise + + else: + input_command = "build " + building_node + "\n" + daemon_log("input: " + input_command.strip()) + + p.stdin.write(input_command.encode("utf-8")) + p.stdin.flush() + with building_cv: + finished_building += [building_node] + daemon_ready = False + + time.sleep(0.01) + except Exception: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + +daemon_thread = threading.Thread(target=daemon_thread_func) +daemon_thread.daemon = True +daemon_thread.start() + +logging.debug( + f"Starting request server on port {port}, keep alive: {daemon_keep_alive}" +) + +keep_alive_timer = timer() +httpd = None + + +def server_thread_func(): + class S(http.server.BaseHTTPRequestHandler): + def do_GET(self): + global thread_error + global keep_alive_timer + global error_nodes + + try: + gets = parse_qs(urlparse(self.path).query) + build = gets.get("build") + if build: + keep_alive_timer = timer() + + daemon_log(f"Got request: {build[0]}") + input_q.put(build[0]) + + def pred(): + return build[0] in finished_building + + with building_cv: + building_cv.wait_for(pred) + + for error_node in error_nodes: + if error_node["node"] == build[0]: + self.send_response(500) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write(error_node["error"].encode()) + return + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + return + + exitbuild = gets.get("exit") + if exitbuild: + input_q.put("exit") + + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + + except Exception: + thread_error = True + daemon_log("SERVER ERROR: " + traceback.format_exc()) + raise + + def log_message(self, format, *args): + return + + httpd = socketserver.TCPServer(("127.0.0.1", port), S) + httpd.serve_forever() + + +server_thread = threading.Thread(target=server_thread_func) +server_thread.daemon = True +server_thread.start() + +while timer() - keep_alive_timer < daemon_keep_alive and not thread_error: + time.sleep(1) + +if thread_error: + daemon_log(f"Shutting server on port {port} down because thread error.") +else: + daemon_log( + f"Shutting server on port {port} down because timed out: {daemon_keep_alive}" + ) + +# if there are errors, don't immediately shut down the daemon +# the process which started the server is attempt to connect to +# the daemon before allowing jobs to start being sent. If the daemon +# shuts down too fast, the launch script will think it has not +# started yet and sit and wait. If the launch script is able to connect +# and then the connection is dropped, it will immediately exit with fail. +time.sleep(5) + +if os.path.exists(ninja_builddir / "scons_daemon_dirty"): + os.unlink(ninja_builddir / "scons_daemon_dirty") +if os.path.exists(daemon_dir / "pidfile"): + os.unlink(daemon_dir / "pidfile") + +httpd.shutdown() +server_thread.join() + + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/SCons/Tool/tex.py b/SCons/Tool/tex.py index d8b694e..6d0a5fb 100644 --- a/SCons/Tool/tex.py +++ b/SCons/Tool/tex.py @@ -492,21 +492,23 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None return result -def LaTeXAuxAction(target = None, source= None, env=None): - result = InternalLaTeXAuxAction( LaTeXAction, target, source, env ) +def LaTeXAuxAction(target=None, source=None, env=None): + result = InternalLaTeXAuxAction(LaTeXAction, target, source, env) return result + LaTeX_re = re.compile("\\\\document(style|class)") -def is_LaTeX(flist,env,abspath): + +def is_LaTeX(flist, env, abspath) -> bool: """Scan a file list to decide if it's TeX- or LaTeX-flavored.""" # We need to scan files that are included in case the # \documentclass command is in them. # get path list from both env['TEXINPUTS'] and env['ENV']['TEXINPUTS'] - savedpath = modify_env_var(env, 'TEXINPUTS', abspath) - paths = env['ENV']['TEXINPUTS'] + savedpath = modify_env_var(env, "TEXINPUTS", abspath) + paths = env["ENV"]["TEXINPUTS"] if SCons.Util.is_List(paths): pass else: @@ -516,54 +518,58 @@ def is_LaTeX(flist,env,abspath): # now that we have the path list restore the env if savedpath is _null: try: - del env['ENV']['TEXINPUTS'] + del env["ENV"]["TEXINPUTS"] except KeyError: - pass # was never set + pass # was never set else: - env['ENV']['TEXINPUTS'] = savedpath + env["ENV"]["TEXINPUTS"] = savedpath if Verbose: - print("is_LaTeX search path ",paths) - print("files to search :",flist) + print("is_LaTeX search path ", paths) + print("files to search: ", flist) # Now that we have the search path and file list, check each one + file_test = False for f in flist: if Verbose: - print(" checking for Latex source ",str(f)) + print(f" checking for Latex source {f}") content = f.get_text_contents() if LaTeX_re.search(content): if Verbose: - print("file %s is a LaTeX file" % str(f)) - return 1 + print(f"file {f} is a LaTeX file") + return True if Verbose: - print("file %s is not a LaTeX file" % str(f)) + print(f"file {f} is not a LaTeX file") # now find included files - inc_files = [ ] - inc_files.extend( include_re.findall(content) ) + inc_files = [] + inc_files.extend(include_re.findall(content)) if Verbose: - print("files included by '%s': "%str(f),inc_files) + print(f"files included by '{f}': ", inc_files) # inc_files is list of file names as given. need to find them # using TEXINPUTS paths. # search the included files for src in inc_files: - srcNode = FindFile(src,['.tex','.ltx','.latex'],paths,env,requireExt=False) + srcNode = FindFile( + src, [".tex", ".ltx", ".latex"], paths, env, requireExt=False + ) # make this a list since is_LaTeX takes a list. - fileList = [srcNode,] + fileList = [srcNode] if Verbose: - print("FindFile found ",srcNode) + print("FindFile found ", srcNode) if srcNode is not None: file_test = is_LaTeX(fileList, env, abspath) # return on first file that finds latex is needed. if file_test: - return file_test + return True if Verbose: - print(" done scanning ",str(f)) + print(f" done scanning {f}") + + return False - return 0 def TeXLaTeXFunction(target = None, source= None, env=None): """A builder for TeX and LaTeX that scans the source file to diff --git a/SCons/UtilTests.py b/SCons/UtilTests.py index 5ed31cb..0cfb70f 100644 --- a/SCons/UtilTests.py +++ b/SCons/UtilTests.py @@ -899,7 +899,7 @@ class HashTestCase(unittest.TestCase): class FIPSHashTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): - super(FIPSHashTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) ############################### # algorithm mocks, can check if we called with usedforsecurity=False for python >= 3.9 diff --git a/SCons/cppTests.py b/SCons/cppTests.py index 99fa6cf..a9aef9d 100644 --- a/SCons/cppTests.py +++ b/SCons/cppTests.py @@ -853,7 +853,7 @@ class fileTestCase(unittest.TestCase): """) class MyPreProcessor(cpp.DumbPreProcessor): def __init__(self, *args, **kw): - cpp.DumbPreProcessor.__init__(self, *args, **kw) + super().__init__(*args, **kw) self.files = [] def __call__(self, file): self.files.append(file) |