From 50c9c963970a5b377b2202e884f7b0d8498f0a66 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 2 Apr 2019 12:49:00 -0700 Subject: Fix issue #2811 spurious rebuilds due to incorrect waiting_parents on nodes when builder has more than one target and source file generated --- src/engine/SCons/Node/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 131953b..e4c8e6c 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -784,6 +784,25 @@ class Node(object, with_metaclass(NoSlotsPyPy)): for parent in self.waiting_parents: parent.implicit = None + # Handle issue where builder emits more than one target and + # the source file for the builder is generated. + # in that case only the first target was getting it's .implicit + # cleared when the source file is built (second scan). + # leaving only partial implicits from scan before source file is generated + # typically the compiler only. Then scanned files are appended + # This is persisted to sconsign and rebuild causes false rebuilds + # because the ordering of the implicit list then changes to what it + # should have been. + # This is at least the following bugs + # https://github.com/SCons/scons/issues/2811 + # https://jira.mongodb.org/browse/SERVER-33111 + try: + for peer in parent.attributes.target_peers: + peer.implicit = None + except AttributeError as e: + pass + + self.clear() if self.pseudo: -- cgit v0.12 From d79290b03480e810acca5a61368d7b254d22fda5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 2 Apr 2019 14:06:25 -0700 Subject: update changes.txt --- src/CHANGES.txt | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 15e8b5f..c9a117e 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -7,14 +7,24 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER + From William Deegan: + - Fix spurious rebuilds on second build for cases where builder has > 1 target and the source file + is generated. This was causing the > 1th target to not have it's implicit list cleared when the source + file was actually built, leaving an implicit list similar to follows for 2nd and higher target + ['/usr/bin/python', 'xxx', 'yyy', 'zzz'] + This was getting persisted to SConsign and on rebuild it would be corrected to be similar to this + ['zzz', 'yyy', 'xxx', '/usr/bin/python'] + Which would trigger a rebuild because the order changed. + The fix involved added logic to mark all shared targets as peers and then ensure they're implicit + list is all cleared together. + + + From Mats Wichmann: - scons-time takes more care closing files and uses safer mkdtemp to avoid possible races on multi-job runs. - Use importlib to dynamically load tool and platform modules instead of imp module - From John Doe: - - - Whatever John Doe did. RELEASE 3.0.5 - Mon, 26 Mar 2019 15:04:42 -0700 -- cgit v0.12 From 0b9d8672365b01468fca5ba95594b92d307c94e9 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 2 Apr 2019 14:24:58 -0700 Subject: Move target_peers to slots from attributes --- src/engine/SCons/Builder.py | 7 +++++++ src/engine/SCons/Node/__init__.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/Builder.py b/src/engine/SCons/Builder.py index 010d5ff..bc243dd 100644 --- a/src/engine/SCons/Builder.py +++ b/src/engine/SCons/Builder.py @@ -563,6 +563,13 @@ class BuilderBase(object): tlist, slist = self._create_nodes(env, target, source) + # If there is more than one target ensure that if we need to reset + # the implicit list to new scan of dependency all targets implicit lists + # are cleared. (SCons GH Issue #2811 and MongoDB SERVER-33111) + if len(tlist) > 1: + for t in tlist: + t.target_peers = tlist + # Check for errors with the specified target/source lists. _node_errors(self, env, tlist, slist) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index e4c8e6c..8e7699b 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -529,6 +529,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): __slots__ = ['sources', 'sources_set', + 'target_peers', '_specific_sources', 'depends', 'depends_set', @@ -797,7 +798,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): # https://github.com/SCons/scons/issues/2811 # https://jira.mongodb.org/browse/SERVER-33111 try: - for peer in parent.attributes.target_peers: + for peer in parent.target_peers: peer.implicit = None except AttributeError as e: pass -- cgit v0.12 From 095970b093e2eb1c9d86ae55379d3bb0412cfd85 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 10:34:34 -0400 Subject: Initial testcase logic to fix Issue #2811 --- test/TaskMaster/bug_2811/fixture_dir/SConstruct | 62 +++++++++++++++++++++++++ test/TaskMaster/bug_2811/fixture_dir/a | 8 ++++ test/TaskMaster/bug_2811/fixture_dir/b.in | 2 + test/TaskMaster/bug_2811/fixture_dir/c | 0 test/TaskMaster/bug_2811/fixture_dir/f | 0 test/TaskMaster/bug_2811/fixture_dir/g | 0 6 files changed, 72 insertions(+) create mode 100644 test/TaskMaster/bug_2811/fixture_dir/SConstruct create mode 100644 test/TaskMaster/bug_2811/fixture_dir/a create mode 100644 test/TaskMaster/bug_2811/fixture_dir/b.in create mode 100644 test/TaskMaster/bug_2811/fixture_dir/c create mode 100644 test/TaskMaster/bug_2811/fixture_dir/f create mode 100644 test/TaskMaster/bug_2811/fixture_dir/g diff --git a/test/TaskMaster/bug_2811/fixture_dir/SConstruct b/test/TaskMaster/bug_2811/fixture_dir/SConstruct new file mode 100644 index 0000000..5fcc5f2 --- /dev/null +++ b/test/TaskMaster/bug_2811/fixture_dir/SConstruct @@ -0,0 +1,62 @@ +""" +This issue requires the following. +1. Generated source file which outputs 2 (or more) files +2. Action string gets scanned providing only compiler as part of implicit scan +3. Generated file gets built. Without the bugfix only the first target's .implicit list is cleared. +4. builder/executor/action gets tried again and implicits scanned. 2nd to Nth targets end up + with the compiler at the beginning of the implicit list and the rest of the scanned files added to that list. +5. That bimplicit gets saved into sconsign +6. Second run loads sconsign, now with generated file present a regular implicit scan occurs. This yields 2nd through + Nth target's implicit lists changing when compared to SConsign's which have been loaded. +7. This forces rebuild of source file and this propagates to massive recompile +""" +import SCons.Tool + + +def _dwo_emitter(target, source, env): + new_targets = [] + for t in target: + base, ext = SCons.Util.splitext(str(t)) + if not any(ext == env[osuffix] for osuffix in ['OBJSUFFIX', 'SHOBJSUFFIX']): + continue + # TODO: Move 'dwo' into DWOSUFFIX so it can be customized? For + # now, GCC doesn't let you control the output filename, so it + # doesn't matter. + dwotarget = (t.builder.target_factory or env.File)(base + ".dwo") + new_targets.append(dwotarget) + targets = target + new_targets + return (targets, source) + + +bld = Builder(action='cp $SOURCE $TARGET') + +env = Environment(BUILDERS={'Foo': bld}) + +env['SHCCCOM'] = 'cp $SOURCE $TARGET && cp $SOURCE ${TARGETS[1]}' +env['SHCCCOMSTR'] = env['SHCCCOM'] + + +suffixes = ['.c'] + +for object_builder in SCons.Tool.createObjBuilders(env): + emitterdict = object_builder.builder.emitter + for suffix in emitterdict.iterkeys(): + if not suffix in suffixes: + continue + base = emitterdict[suffix] + emitterdict[suffix] = SCons.Builder.ListEmitter([ + base, + _dwo_emitter, + ]) + +vs = ['a'] + +for v in vs: + env.Foo('%s.c' % v, v) + env.SharedObject('%s.c'%v) + +env.Foo('b','b.in') +# seems like processing is always alphabetical.. + + +# code: language=python insertSpaces=4 tabSize=4 \ No newline at end of file diff --git a/test/TaskMaster/bug_2811/fixture_dir/a b/test/TaskMaster/bug_2811/fixture_dir/a new file mode 100644 index 0000000..d3f62b8 --- /dev/null +++ b/test/TaskMaster/bug_2811/fixture_dir/a @@ -0,0 +1,8 @@ +#include "b" +// #include "c" +// #include "d" +// #include "e" + +int dummy() { + return 1; +} \ No newline at end of file diff --git a/test/TaskMaster/bug_2811/fixture_dir/b.in b/test/TaskMaster/bug_2811/fixture_dir/b.in new file mode 100644 index 0000000..867daaf --- /dev/null +++ b/test/TaskMaster/bug_2811/fixture_dir/b.in @@ -0,0 +1,2 @@ +#include "f" +#include "g" diff --git a/test/TaskMaster/bug_2811/fixture_dir/c b/test/TaskMaster/bug_2811/fixture_dir/c new file mode 100644 index 0000000..e69de29 diff --git a/test/TaskMaster/bug_2811/fixture_dir/f b/test/TaskMaster/bug_2811/fixture_dir/f new file mode 100644 index 0000000..e69de29 diff --git a/test/TaskMaster/bug_2811/fixture_dir/g b/test/TaskMaster/bug_2811/fixture_dir/g new file mode 100644 index 0000000..e69de29 -- cgit v0.12 From 3031ae8c0063bf49bafcb8a9f77a2822221f76d5 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 10:36:30 -0400 Subject: Force runtest to skip looking in this directory --- test/TaskMaster/bug_2811/fixture_dir/sconstest.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test/TaskMaster/bug_2811/fixture_dir/sconstest.skip diff --git a/test/TaskMaster/bug_2811/fixture_dir/sconstest.skip b/test/TaskMaster/bug_2811/fixture_dir/sconstest.skip new file mode 100644 index 0000000..e69de29 -- cgit v0.12 From 9cacbc061d39a46b792223d0b74b2dcaf5fa2aba Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 10:43:42 -0400 Subject: Change test logic to run once, and then run again expecting up to date. --- ...n_one_target_generated_source_false_rebuilds.py | 54 ++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/TaskMaster/bug_2811/issue_2811_emitter_more_than_one_target_generated_source_false_rebuilds.py diff --git a/test/TaskMaster/bug_2811/issue_2811_emitter_more_than_one_target_generated_source_false_rebuilds.py b/test/TaskMaster/bug_2811/issue_2811_emitter_more_than_one_target_generated_source_false_rebuilds.py new file mode 100644 index 0000000..8f2f328 --- /dev/null +++ b/test/TaskMaster/bug_2811/issue_2811_emitter_more_than_one_target_generated_source_false_rebuilds.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +""" +This issue requires the following. +1. Generated source file which outputs 2 (or more) files +2. Action string gets scanned providing only compiler as part of implicit scan +3. Generated file gets built. Without the bugfix only the first target's .implicit list is cleared. +4. builder/executor/action gets tried again and implicits scanned. 2nd to Nth targets end up + with the compiler at the beginning of the implicit list and the rest of the scanned files added to that list. +5. That bimplicit gets saved into sconsign +6. Second run loads sconsign, now with generated file present a regular implicit scan occurs. This yields 2nd through + Nth target's implicit lists changing when compared to SConsign's which have been loaded. +7. This forces rebuild of source file and this propagates to massive recompile +""" +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +import TestSCons + +test = TestSCons.TestSCons() +test.dir_fixture('fixture_dir') + +test.run() + +# Should not rebuild +test.up_to_date(arguments = '.') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 4af18c70308e70cf60fdb3843ce9234415857c4e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 11:29:05 -0400 Subject: Fix sider complain --- src/engine/SCons/Node/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 8e7699b..964e85d 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -800,7 +800,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): try: for peer in parent.target_peers: peer.implicit = None - except AttributeError as e: + except AttributeError: pass -- cgit v0.12 From c581c35b7bbc95329dcd48738715b40f331b56c0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 12:58:52 -0400 Subject: Fix test to work with py3.5+ --- test/TaskMaster/bug_2811/fixture_dir/SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TaskMaster/bug_2811/fixture_dir/SConstruct b/test/TaskMaster/bug_2811/fixture_dir/SConstruct index 5fcc5f2..c124902 100644 --- a/test/TaskMaster/bug_2811/fixture_dir/SConstruct +++ b/test/TaskMaster/bug_2811/fixture_dir/SConstruct @@ -40,7 +40,7 @@ suffixes = ['.c'] for object_builder in SCons.Tool.createObjBuilders(env): emitterdict = object_builder.builder.emitter - for suffix in emitterdict.iterkeys(): + for suffix in emitterdict.keys(): if not suffix in suffixes: continue base = emitterdict[suffix] -- cgit v0.12 From 1ada08832807e71c4a7660bde286fff9bf02e287 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 13:03:18 -0400 Subject: [skip ci] Add text to RELEASE.txt to cover this change --- src/RELEASE.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 06051ea..101eb21 100755 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -41,8 +41,18 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - - List modifications to existing features, where the previous behavior - wouldn't actually be considered a bug + - Fix spurious rebuilds on second build for cases where builder has > 1 target and the source file + is generated. This was causing the > 1th target to not have it's implicit list cleared when the source + file was actually built, leaving an implicit list similar to follows for 2nd and higher target + ['/usr/bin/python', 'xxx', 'yyy', 'zzz'] + This was getting persisted to SConsign and on rebuild it would be corrected to be similar to this + ['zzz', 'yyy', 'xxx', '/usr/bin/python'] + Which would trigger a rebuild because the order changed. + The fix involved added logic to mark all shared targets as peers and then ensure they're implicit + list is all cleared together. + - Fix Issue #3349 - SCons Exception EnvironmentError is conflicting with Python's EnvironmentError. + Renamed to SConsEnvironmentError + - Fix Issue #3350 - mslink failing when too many objects. This is resolved by adding TEMPFILEARGJOIN variable FIXES -- cgit v0.12 From 3fdeb198e8c487834045ad763fb3df6b07ec8b00 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 26 Apr 2019 13:07:33 -0400 Subject: [skip ci] remove text from RELEASE.txt --- src/RELEASE.txt | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/RELEASE.txt b/src/RELEASE.txt index 101eb21..06051ea 100755 --- a/src/RELEASE.txt +++ b/src/RELEASE.txt @@ -41,18 +41,8 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY - - Fix spurious rebuilds on second build for cases where builder has > 1 target and the source file - is generated. This was causing the > 1th target to not have it's implicit list cleared when the source - file was actually built, leaving an implicit list similar to follows for 2nd and higher target - ['/usr/bin/python', 'xxx', 'yyy', 'zzz'] - This was getting persisted to SConsign and on rebuild it would be corrected to be similar to this - ['zzz', 'yyy', 'xxx', '/usr/bin/python'] - Which would trigger a rebuild because the order changed. - The fix involved added logic to mark all shared targets as peers and then ensure they're implicit - list is all cleared together. - - Fix Issue #3349 - SCons Exception EnvironmentError is conflicting with Python's EnvironmentError. - Renamed to SConsEnvironmentError - - Fix Issue #3350 - mslink failing when too many objects. This is resolved by adding TEMPFILEARGJOIN variable + - List modifications to existing features, where the previous behavior + wouldn't actually be considered a bug FIXES -- cgit v0.12