From 3b0556d4f8758edf433a44bf9909a800d0177ddc Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 15:14:39 -0400 Subject: Fix some bad code formatting. --- src/engine/SCons/Node/FS.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 5dceaa0..f7319b1 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -690,10 +690,15 @@ class Base(SCons.Node.Node): @SCons.Memoize.CountMethodCall def stat(self): - try: return self._memo['stat'] - except KeyError: pass - try: result = self.fs.stat(self.get_abspath()) - except os.error: result = None + try: + return self._memo['stat'] + except KeyError: + pass + try: + result = self.fs.stat(self.get_abspath()) + except os.error: + result = None + self._memo['stat'] = result return result @@ -705,13 +710,17 @@ class Base(SCons.Node.Node): def getmtime(self): st = self.stat() - if st: return st[stat.ST_MTIME] - else: return None + if st: + return st[stat.ST_MTIME] + else: + return None def getsize(self): st = self.stat() - if st: return st[stat.ST_SIZE] - else: return None + if st: + return st[stat.ST_SIZE] + else: + return None def isdir(self): st = self.stat() @@ -3246,6 +3255,9 @@ class File(Base): def changed_timestamp_then_content(self, target, prev_ni): if not self.changed_timestamp_match(target, prev_ni): try: + if str(self) == 'beta.h' and prev_ni.csig == '2ff783593a2224d0574181661ab5f1b7': + print("in problem code") + print('Setting csig [%s]:%s'%(str(self),prev_ni.csig)) self.get_ninfo().csig = prev_ni.csig except AttributeError: pass -- cgit v0.12 From c8b4d3bc32476914aefa7cc50c6376d401478a01 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 15:15:51 -0400 Subject: Apply patch from wblevins for issue #2980 This fixes that issue, but as discussion indicates breaks other tests. WIP --- src/engine/SCons/Node/__init__.py | 80 +++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index c18a954..22f1be5 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1406,6 +1406,60 @@ class Node(object, with_metaclass(NoSlotsPyPy)): return None return self._tags.get(key, None) + + def _build_dependency_map(self, binfo): + """ + Build mapping from file -> signiture + + Args: + self - self + binfo - buildinfo from node being considered + + Returns: + dictionary of file->signiture mappings + """ + + # For an "empty" binfo properties like bsources + # do not exist: check this to avoid exception. + if (len(binfo.bsourcesigs) + len(binfo.bdependsigs) + \ + len(binfo.bimplicitsigs)) == 0: + return{} + + pairs = [ + (binfo.bsources, binfo.bsourcesigs), + (binfo.bdepends, binfo.bdependsigs), + (binfo.bimplicit, binfo.bimplicitsigs) + ] + + m = {} + for children, signatures in pairs: + for child, signature in zip(children, signatures): + schild = str(child) + m[schild] = signature + return m + + def _get_previous_signatures(self, dmap, children): + """ + Return a list of corresponding csigs from previous + build in order of the node/files in children. + + Args: + self - self + dmap - Dictionary of file -> csig + children - List of children + + Returns: + List of csigs for provided list of children + """ + prev = [] + for c in map(str, children): + # If there is no previous signature, + # we place None in the corresponding position. + prev.append(dmap.get(c)) + return prev + + + def changed(self, node=None, allowcache=False): """ Returns if the node is up-to-date with respect to the BuildInfo @@ -1437,27 +1491,37 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = False bi = node.get_stored_info().binfo - then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + # then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + # TODO: Can we move this into the if diff below? + dmap = self._build_dependency_map(bi) children = self.children() - diff = len(children) - len(then) + # diff = len(children) - len(then) + diff = len(children) - len(dmap) if diff: # The old and new dependency lists are different lengths. # This always indicates that the Node must be rebuilt. - # We also extend the old dependency list with enough None - # entries to equal the new dependency list, for the benefit - # of the loop below that updates node information. - then.extend([None] * diff) - if t: Trace(': old %s new %s' % (len(then), len(children))) + + # TODO: Below is from new logic + # # We also extend the old dependency list with enough None + # # entries to equal the new dependency list, for the benefit + # # of the loop below that updates node information. + # then.extend([None] * diff) + # if t: Trace(': old %s new %s' % (len(then), len(children))) + result = True + # Now build new then based on map built above. + then = self._get_previous_signatures(dmap, children) + for child, prev_ni in zip(children, then): if _decider_map[child.changed_since_last_build](child, self, prev_ni): if t: Trace(': %s changed' % child) result = True - contents = self.get_executor().get_contents() if self.has_builder(): + contents = self.get_executor().get_contents() + import SCons.Util newsig = SCons.Util.MD5signature(contents) if bi.bactsig != newsig: -- cgit v0.12 From 57aefe3ca22c13c9f8cfc093f0fbcde97a61ea54 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 15:30:48 -0400 Subject: Modified fix for issue #2980 where the more complicated logic is only applied when the number of children in the current build doesn't match the number in the previous build as retrieved from the sconsign file. --- src/CHANGES.txt | 6 ++++-- src/engine/SCons/Node/__init__.py | 12 ++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index bd1e56a..e2fb598 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -15,7 +15,7 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Fix the PATH created by scons.bat (and other .bat files) to provide a normalized PATH. Some pythons in the 3.6 series are no longer able to handle paths which have ".." in them and end up crashing. This is done by cd'ing into the directory - we want to add to the path and then useing %CD% to give us the normalized directory + we want to add to the path and then using %CD% to give us the normalized directory See bug filed under Python 3.6: https://bugs.python.org/issue32457. Note: On Win32 PATH's which have not been normalized may cause undefined behavior by other executables being run by SCons (or any subprocesses of executables being run by SCons). @@ -55,7 +55,9 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Fix new logic which populates JAVAINCLUDES to handle the case where javac is not found. - Fix GH Issue #3225 SCons.Util.Flatten() doesn't handle MappingView's produced by dictionary as return values from dict().{items(), keys(), values()}. - + - Fix issue #2980 with credit to William Blevins. This is an issue where using TimeStamp-MD5 Decider + and CacheDir can yield incorrect md5's being written into the .sconsign. + From Andrew Featherstone - Removed unused --warn options from the man page and source code. diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 22f1be5..cdaccca 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1491,13 +1491,11 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = False bi = node.get_stored_info().binfo - # then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs # TODO: Can we move this into the if diff below? - dmap = self._build_dependency_map(bi) children = self.children() - # diff = len(children) - len(then) - diff = len(children) - len(dmap) + diff = len(children) - len(then) if diff: # The old and new dependency lists are different lengths. # This always indicates that the Node must be rebuilt. @@ -1508,11 +1506,13 @@ class Node(object, with_metaclass(NoSlotsPyPy)): # # of the loop below that updates node information. # then.extend([None] * diff) # if t: Trace(': old %s new %s' % (len(then), len(children))) + dmap = self._build_dependency_map(bi) + # Now build new then based on map built above. + then = self._get_previous_signatures(dmap, children) + result = True - # Now build new then based on map built above. - then = self._get_previous_signatures(dmap, children) for child, prev_ni in zip(children, then): if _decider_map[child.changed_since_last_build](child, self, prev_ni): -- cgit v0.12 From d5ebc3fa8dc67635600eb7b46f5350b13d3f2caf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 15:52:28 -0400 Subject: Added docstring to File.changed_timestamp_then_content()include a note and reference to issue #2980 --- src/engine/SCons/Node/FS.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f7319b1..bba5cbb 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3253,11 +3253,24 @@ class File(Base): return self.state != SCons.Node.up_to_date def changed_timestamp_then_content(self, target, prev_ni): + """ + Used when decider for file is Timestamp-MD5 + + NOTE: If the timestamp hasn't change this will skip md5'ing the + file and just copy the prev_ni provided. If the prev_ni + is wrong. It will propogate it. + See: https://github.com/SCons/scons/issues/2980 + + Args: + self - self + target - + prev_ni - The NodeInfo object loaded from previous builds .sconsign + + Returns: + Boolean - Indicates if node(File) has changed. + """ if not self.changed_timestamp_match(target, prev_ni): try: - if str(self) == 'beta.h' and prev_ni.csig == '2ff783593a2224d0574181661ab5f1b7': - print("in problem code") - print('Setting csig [%s]:%s'%(str(self),prev_ni.csig)) self.get_ninfo().csig = prev_ni.csig except AttributeError: pass -- cgit v0.12 From 5fc244b2e43172df4c888f7c31942ee489b01a8f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 15:54:09 -0400 Subject: Updated changelog info for this patch with more details. --- src/CHANGES.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index e2fb598..a36a1e7 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -56,7 +56,10 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Fix GH Issue #3225 SCons.Util.Flatten() doesn't handle MappingView's produced by dictionary as return values from dict().{items(), keys(), values()}. - Fix issue #2980 with credit to William Blevins. This is an issue where using TimeStamp-MD5 Decider - and CacheDir can yield incorrect md5's being written into the .sconsign. + and CacheDir can yield incorrect md5's being written into the .sconsign. The difference between + William Blevins patch and the current code is that the more complicated creation of file to csig + map is only done when the count of children for the current node doesn't match the previous count + which is loaded from the sconsign. From Andrew Featherstone - Removed unused --warn options from the man page and source code. -- cgit v0.12 From 66fb81887a0ad6d2745d6ceded97fadbd2a22c09 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 17:06:58 -0400 Subject: rename variable then to previous_children to make the code a bit easier to understand. --- src/engine/SCons/Node/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index cdaccca..b02c84d 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1491,11 +1491,11 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = False bi = node.get_stored_info().binfo - then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + previous_children = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs # TODO: Can we move this into the if diff below? children = self.children() - diff = len(children) - len(then) + diff = len(children) - len(previous_children) if diff: # The old and new dependency lists are different lengths. # This always indicates that the Node must be rebuilt. @@ -1505,16 +1505,18 @@ class Node(object, with_metaclass(NoSlotsPyPy)): # # entries to equal the new dependency list, for the benefit # # of the loop below that updates node information. # then.extend([None] * diff) - # if t: Trace(': old %s new %s' % (len(then), len(children))) + + if t: Trace(': old %s new %s' % (len(previous_children), len(children))) + dmap = self._build_dependency_map(bi) # Now build new then based on map built above. - then = self._get_previous_signatures(dmap, children) + previous_children = self._get_previous_signatures(dmap, children) result = True - for child, prev_ni in zip(children, then): + for child, prev_ni in zip(children, previous_children): if _decider_map[child.changed_since_last_build](child, self, prev_ni): if t: Trace(': %s changed' % child) result = True -- cgit v0.12 From 5c0dbb5fd3b61dfb6d974048dfdf3d150b34f2bf Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 17:07:45 -0400 Subject: remove unittest.TestSuite() no longer needed as unittest.main() is available now. (And seems actually a little faster) --- src/engine/SCons/Node/FSTests.py | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 698f574..63a1ba4 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -3752,38 +3752,7 @@ class AbsolutePathTestCase(unittest.TestCase): if __name__ == "__main__": - suite = unittest.TestSuite() - suite.addTest(VariantDirTestCase()) - suite.addTest(find_fileTestCase()) - suite.addTest(StringDirTestCase()) - suite.addTest(stored_infoTestCase()) - suite.addTest(has_src_builderTestCase()) - suite.addTest(prepareTestCase()) - suite.addTest(SConstruct_dirTestCase()) - suite.addTest(clearTestCase()) - suite.addTest(disambiguateTestCase()) - suite.addTest(postprocessTestCase()) - suite.addTest(SpecialAttrTestCase()) - suite.addTest(SaveStringsTestCase()) - tclasses = [ - AbsolutePathTestCase, - BaseTestCase, - CacheDirTestCase, - DirTestCase, - DirBuildInfoTestCase, - DirNodeInfoTestCase, - EntryTestCase, - FileTestCase, - FileBuildInfoTestCase, - FileNodeInfoTestCase, - FSTestCase, - GlobTestCase, - RepositoryTestCase, - ] - for tclass in tclasses: - names = unittest.getTestCaseNames(tclass, 'test_') - suite.addTests(list(map(tclass, names))) - TestUnit.run(suite) + unittest.main() # Local Variables: # tab-width:4 -- cgit v0.12 From 8e5547b0be903028e171ca42ea3e9c9f154c9238 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 29 Mar 2018 20:51:53 -0400 Subject: Attribute patch correctly to Piotr Bartosik per email from William Blevins --- src/CHANGES.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index a36a1e7..8793a19 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -60,6 +60,11 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE William Blevins patch and the current code is that the more complicated creation of file to csig map is only done when the count of children for the current node doesn't match the previous count which is loaded from the sconsign. + - Fix issue #2980 with credit to Piotr Bartosik (and William Blevins). This is an issue where using + TimeStamp-MD5 Decider and CacheDir can yield incorrect md5's being written into the .sconsign. + The difference between Piotr Bartosik's patch and the current code is that the more complicated + creation of file to csig map is only done when the count of children for the current node doesn't + match the previous count which is loaded from the sconsign. From Andrew Featherstone - Removed unused --warn options from the man page and source code. -- cgit v0.12 From bf1a0e2f502bae80f7a5e4be5eff79888ed297b6 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 5 Apr 2018 07:23:21 -0700 Subject: Switch to use unittest.main() instead of building up a TestSuite which is never used --- src/engine/SCons/Node/NodeTests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py index ca6c883..7dc5f5d 100644 --- a/src/engine/SCons/Node/NodeTests.py +++ b/src/engine/SCons/Node/NodeTests.py @@ -1349,6 +1349,7 @@ class NodeListTestCase(unittest.TestCase): if __name__ == "__main__": unittest.main() + # Local Variables: # tab-width:4 # indent-tabs-mode:nil -- cgit v0.12 From 140cea983b9d9e02cc76b1ebcb47c0fb4a88b367 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 10 Apr 2018 18:55:11 -0700 Subject: Fix comment spelling --- src/engine/SCons/Node/FS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index bba5cbb..31c7441 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3256,7 +3256,7 @@ class File(Base): """ Used when decider for file is Timestamp-MD5 - NOTE: If the timestamp hasn't change this will skip md5'ing the + NOTE: If the timestamp hasn't changed this will skip md5'ing the file and just copy the prev_ni provided. If the prev_ni is wrong. It will propogate it. See: https://github.com/SCons/scons/issues/2980 -- cgit v0.12 From da31f2954ae37e1c07a408bccbbf39a9c14b2446 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 22 Apr 2018 14:30:21 -0700 Subject: Create test to verify fix for issue #2980 There are still possible errors due to timestamp-MD5 + cachedir + changed implicit or regular dependencies (but the same # of such as the previous build). These are not yet handled as the fix being used for changed number of such breaks a number of tests. This fix doe reduce the number of possible issues. --- src/engine/SCons/Node/FSTests.py | 137 ++++++++++++++++++++++++++++++++++++++ src/engine/SCons/Node/__init__.py | 7 +- 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 63a1ba4..0b57e2a 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -41,6 +41,7 @@ import SCons.Errors import SCons.Node.FS import SCons.Util import SCons.Warnings +import SCons.Environment built_it = None @@ -2460,6 +2461,142 @@ class FileTestCase(_tempdirTestCase): assert not build_f1.exists(), "%s did not realize that %s disappeared" % (build_f1, src_f1) assert not os.path.exists(build_f1.get_abspath()), "%s did not get removed after %s was removed" % (build_f1, src_f1) + def test_changed(self): + """ + Verify that changes between BuildInfo's list of souces, depends, and implicit + dependencies do not corrupt content signiture values written to .SConsign + when using CacheDir and Timestamp-MD5 decider. + This is for issue #2980 + """ + # node should have + # 1 source (for example main.cpp) + # 0 depends + # N implicits (for example ['alpha.h', 'beta.h', 'gamma.h', '/usr/bin/g++']) + + class ChangedNode(SCons.Node.FS.File): + def __init__(self, name, directory=None, fs=None): + SCons.Node.FS.File.__init__(self, name, directory, fs) + self.name = name + self.Tag('found_includes', []) + self.stored_info = None + self.build_env = None + self.changed_since_last_build = 4 + self.timestamp = 1 + + def get_stored_info(self): + return self.stored_info + + def get_build_env(self): + return self.build_env + + def get_timestamp(self): + """ Fake timestamp so they always match""" + return self.timestamp + + def get_contents(self): + return self.name + + def get_ninfo(self): + """ mocked to ensure csig will equal the filename""" + try: + return self.ninfo + except AttributeError: + self.ninfo = FakeNodeInfo(self.name, self.timestamp) + return self.ninfo + + def get_csig(self): + ninfo = self.get_ninfo() + try: + return ninfo.csig + except AttributeError: + pass + + return "Should Never Happen" + + class ChangedEnvironment(SCons.Environment.Base): + + def __init__(self): + SCons.Environment.Base.__init__(self) + self.decide_source = self._changed_timestamp_then_content + + class FakeNodeInfo(object): + def __init__(self, csig, timestamp): + self.csig = csig + self.timestamp = timestamp + + + #Create nodes + fs = SCons.Node.FS.FS() + d = self.fs.Dir('.') + + node = ChangedNode('main.o',d,fs) # main.o + s1 = ChangedNode('main.cpp',d,fs) # main.cpp + s1.timestamp = 2 # this changed + i1 = ChangedNode('alpha.h',d,fs) # alpha.h - The bug is caused because the second build adds this file + i1.timestamp = 2 # This is the new file. + i2 = ChangedNode('beta.h',d,fs) # beta.h + i3 = ChangedNode('gamma.h',d,fs) # gamma.h - In the bug beta.h's csig from binfo overwrites this ones + i4 = ChangedNode('/usr/bin/g++',d,fs) # /usr/bin/g++ + + + + node.add_source([s1]) + node.add_dependency([]) + node.implicit = [i1, i2, i3, i4] + node.implicit_set = set() + # node._add_child(node.implicit, node.implicit_set, [n7, n8, n9]) + # node._add_child(node.implicit, node.implicit_set, [n10, n11, n12]) + + # Mock out node's scan method + # node.scan = lambda *args: None + + # Mock out nodes' children() ? + # Should return Node's. + # All those nodes should have changed_since_last_build set to match Timestamp-MD5's + # decider method... + + # Generate sconsign entry needed + sconsign_entry = SCons.SConsign.SConsignEntry() + sconsign_entry.binfo = node.new_binfo() + sconsign_entry.ninfo = node.new_ninfo() + + # mock out loading info from sconsign + # This will cause node.get_stored_info() to return our freshly created sconsign_entry + node.stored_info = sconsign_entry + + # binfo = information from previous build (from sconsign) + # We'll set the following attributes (which are lists): "bsources", "bsourcesigs", + # "bdepends","bdependsigs", "bimplicit", "bimplicitsigs" + bi = sconsign_entry.binfo + bi.bsources = ['main.cpp'] + bi.bsourcesigs=[FakeNodeInfo('main.cpp',1),] + + bi.bdepends = [] + bi.bdependsigs = [] + + bi.bimplicit = ['beta.h','gamma.h'] + bi.bimplicitsigs = [FakeNodeInfo('beta.h',1), FakeNodeInfo('gamma.h',1)] + + ni = sconsign_entry.ninfo + # We'll set the following attributes (which are lists): sources, depends, implicit lists + + #Set timestamp-md5 + #Call changed + #Check results + node.build_env = ChangedEnvironment() + + changed = node.changed() + + # change to true to debug + if False: + print("Changed:%s"%changed) + print("%15s -> csig:%s"%(s1.name, s1.ninfo.csig)) + print("%15s -> csig:%s"%(i1.name, i1.ninfo.csig)) + print("%15s -> csig:%s"%(i2.name, i2.ninfo.csig)) + print("%15s -> csig:%s"%(i3.name, i3.ninfo.csig)) + print("%15s -> csig:%s"%(i4.name, i4.ninfo.csig)) + + self.assertEqual(i2.name,i2.ninfo.csig, "gamma.h's fake csig should equal gamma.h but equals:%s"%i2.ninfo.csig) class GlobTestCase(_tempdirTestCase): diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index b02c84d..ce2a4e3 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1458,8 +1458,6 @@ class Node(object, with_metaclass(NoSlotsPyPy)): prev.append(dmap.get(c)) return prev - - def changed(self, node=None, allowcache=False): """ Returns if the node is up-to-date with respect to the BuildInfo @@ -1492,7 +1490,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): bi = node.get_stored_info().binfo previous_children = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs - # TODO: Can we move this into the if diff below? + children = self.children() diff = len(children) - len(previous_children) @@ -1515,6 +1513,8 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = True + # TODO: There are still possible errors due to timestamp-MD5 + cachedir + changed implicit or regular dependencies (but the same # of such as the previous build). + for child, prev_ni in zip(children, previous_children): if _decider_map[child.changed_since_last_build](child, self, prev_ni): @@ -1524,7 +1524,6 @@ class Node(object, with_metaclass(NoSlotsPyPy)): if self.has_builder(): contents = self.get_executor().get_contents() - import SCons.Util newsig = SCons.Util.MD5signature(contents) if bi.bactsig != newsig: if t: Trace(': bactsig %s != newsig %s' % (bi.bactsig, newsig)) -- cgit v0.12 From 8c3455de4b02bd064938af54c9e0ef3249853372 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 24 May 2018 14:50:59 -0700 Subject: Remove extraneous parens in if statements --- src/engine/SCons/dblite.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/dblite.py b/src/engine/SCons/dblite.py index 87a1763..628182b 100644 --- a/src/engine/SCons/dblite.py +++ b/src/engine/SCons/dblite.py @@ -75,7 +75,8 @@ class dblite(object): def __init__(self, file_base_name, flag, mode): assert flag in (None, "r", "w", "c", "n") - if (flag is None): flag = "r" + if flag is None: + flag = "r" base, ext = os.path.splitext(file_base_name) if ext == dblite_suffix: @@ -106,13 +107,13 @@ class dblite(object): self._chown_to = -1 # don't chown self._chgrp_to = -1 # don't chgrp - if (self._flag == "n"): + if self._flag == "n": self._open(self._file_name, "wb", self._mode) else: try: f = self._open(self._file_name, "rb") except IOError as e: - if (self._flag != "c"): + if self._flag != "c": raise e self._open(self._file_name, "wb", self._mode) else: @@ -132,7 +133,7 @@ class dblite(object): corruption_warning(self._file_name) def close(self): - if (self._needs_sync): + if self._needs_sync: self.sync() def __del__(self): -- cgit v0.12 From 20680e3f72ef140474bb17fbbd3f61f3716a496e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 24 May 2018 16:11:36 -0700 Subject: add method find_repo_file() which finds the file in it's known repositories. Minor reformat in rfile(). --- src/engine/SCons/Node/FS.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 31c7441..f0576ca 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3326,9 +3326,12 @@ class File(Base): result = self if not self.exists(): norm_name = _my_normcase(self.name) - for dir in self.dir.get_all_rdirs(): - try: node = dir.entries[norm_name] - except KeyError: node = dir.file_on_disk(self.name) + for repo_dir in self.dir.get_all_rdirs(): + try: + node = repo_dir.entries[norm_name] + except KeyError: + node = repo_dir.file_on_disk(self.name) + if node and node.exists() and \ (isinstance(node, File) or isinstance(node, Entry) or not node.is_derived()): @@ -3350,6 +3353,28 @@ class File(Base): self._memo['rfile'] = result return result + def find_repo_file(self): + """ + For this node, find if there exists a corresponding file in one or more repositories + :return: list of corresponding files in repositories + """ + retvals = [] + + norm_name = _my_normcase(self.name) + for repo_dir in self.dir.get_all_rdirs(): + try: + node = repo_dir.entries[norm_name] + except KeyError: + node = repo_dir.file_on_disk(self.name) + + if node and node.exists() and \ + (isinstance(node, File) or isinstance(node, Entry) \ + or not node.is_derived()): + retvals.append(node) + + return retvals + + def rstr(self): return str(self.rfile()) -- cgit v0.12 From 56c26b38b9a4baea4270395304e08eb9e7151bab Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 24 May 2018 16:12:59 -0700 Subject: change exists_file() method to skip adding files AND sigs for files in the ignore set. previously was only skipping signitures which left the sconsign in an inconsistant state. (More file names than sigs) --- src/engine/SCons/Node/__init__.py | 83 +++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index ce2a4e3..1f98a30 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -139,6 +139,7 @@ def exists_entry(node): node.disambiguate() return _exists_map[node._func_exists](node) + def exists_file(node): # Duplicate from source path if we are set up to do this. if node.duplicate and not node.is_derived() and not node.linked: @@ -1136,7 +1137,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bactsig = SCons.Util.MD5signature(executor.get_contents()) if self._specific_sources: - sources = [ s for s in self.sources if not s in ignore_set] + sources = [s for s in self.sources if not s in ignore_set] else: sources = executor.get_unignored_sources(self, self.ignore) @@ -1146,11 +1147,13 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bsourcesigs = [s.get_ninfo() for s in binfo.bsources] - binfo.bdepends = self.depends - binfo.bdependsigs = [d.get_ninfo() for d in self.depends if d not in ignore_set] + binfo.bdepends = [d for d in self.depends if d not in ignore_set] + binfo.bdependsigs = [d.get_ninfo() for d in self.depends] - binfo.bimplicit = self.implicit or [] - binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit if i not in ignore_set] + # TODO: If anything matches the ignore_set: bimplicit and bimplicitsigs will have different lengths and potentially the items in each list will not represent the same file. (any item but the last one matches any item in ignore_set) + # binfo.bimplicit = self.implicit or [] + binfo.bimplicit = [i for i in self.implicit or [] if i not in ignore_set] + binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit] return binfo @@ -1406,24 +1409,23 @@ class Node(object, with_metaclass(NoSlotsPyPy)): return None return self._tags.get(key, None) - def _build_dependency_map(self, binfo): """ - Build mapping from file -> signiture + Build mapping from file -> signature Args: self - self binfo - buildinfo from node being considered Returns: - dictionary of file->signiture mappings + dictionary of file->signature mappings """ # For an "empty" binfo properties like bsources # do not exist: check this to avoid exception. if (len(binfo.bsourcesigs) + len(binfo.bdependsigs) + \ len(binfo.bimplicitsigs)) == 0: - return{} + return {} pairs = [ (binfo.bsources, binfo.bsourcesigs), @@ -1452,10 +1454,33 @@ class Node(object, with_metaclass(NoSlotsPyPy)): List of csigs for provided list of children """ prev = [] - for c in map(str, children): - # If there is no previous signature, - # we place None in the corresponding position. - prev.append(dmap.get(c)) + + for c in children: + c_str = str(c).replace('\\','/') + df = dmap.get(c_str) + if df: + prev.append(df) + continue + + try: + # We're not finding the file as listed in the + # current children list in the list loaded from + # SConsign. So let's see if it was previously + # retrieved from the repo. + # Also since we only have find_repo_file() on File objects + # Handle if we have any other Node type not having that method + for rf in c.find_repo_file(): + rfs = str(rf) + df = dmap.get(rfs) + if df: + prev.append(df) + break + else: + prev.append(None) + except AttributeError as e: + prev.append(None) + + # prev = [dmap.get(str(c), dmap.get(str(c.find_repo_file()[0]))) for c in children] return prev def changed(self, node=None, allowcache=False): @@ -1482,40 +1507,31 @@ class Node(object, with_metaclass(NoSlotsPyPy)): @see: FS.File.changed(), FS.File.release_target_info() """ t = 0 - if t: Trace('changed(%s [%s], %s)' % (self, classname(self), node)) + if t: + Trace('changed(%s [%s], %s)' % (self, classname(self), node)) if node is None: node = self result = False + children = self.children() + bi = node.get_stored_info().binfo - previous_children = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + # previous_children = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + previous_map_by_file_name = self._build_dependency_map(bi) - children = self.children() + # Now build new then based on map built above. + previous_children = self._get_previous_signatures(previous_map_by_file_name, children) - diff = len(children) - len(previous_children) + diff = len(children) - len(previous_map_by_file_name) if diff: # The old and new dependency lists are different lengths. # This always indicates that the Node must be rebuilt. - - # TODO: Below is from new logic - # # We also extend the old dependency list with enough None - # # entries to equal the new dependency list, for the benefit - # # of the loop below that updates node information. - # then.extend([None] * diff) - if t: Trace(': old %s new %s' % (len(previous_children), len(children))) - - dmap = self._build_dependency_map(bi) - - # Now build new then based on map built above. - previous_children = self._get_previous_signatures(dmap, children) + if t: Trace(': old %s new %s' % (len(v), len(children))) result = True - # TODO: There are still possible errors due to timestamp-MD5 + cachedir + changed implicit or regular dependencies (but the same # of such as the previous build). - - for child, prev_ni in zip(children, previous_children): if _decider_map[child.changed_since_last_build](child, self, prev_ni): if t: Trace(': %s changed' % child) @@ -1530,7 +1546,8 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = True if not result: - if t: Trace(': up to date') + if t: + Trace(': up to date') if t: Trace('\n') -- cgit v0.12 From 002f2a16b97b14123244389f30b03c5b3c0c1e9b Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 16 Jun 2018 18:12:44 -0400 Subject: try travis ci mac builds as well --- .travis.yml | 6 ++++- .travis/install.sh | 79 +++++++++++++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index ea63e57..daf635f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,10 @@ addons: apt: update: true +os: + - linux + - osx + install: - ./.travis/install.sh @@ -15,7 +19,7 @@ install: matrix: allow_failures: - python: pypy - + - os: osx jobs: include: diff --git a/.travis/install.sh b/.travis/install.sh index 69e833a..17fd79f 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -1,44 +1,49 @@ #!/usr/bin/env bash set -x -# setup clang for clang tests using local clang installation - - -if [ ! -f /usr/local/clang-5.0.0/bin/clang ]; then - echo "No Clang 5.0.0 trying 7.0.0" - sudo ln -s /usr/local/clang-7.0.0/bin/clang /usr/bin/clang - sudo ln -s /usr/local/clang-7.0.0/bin/clang++ /usr/bin/clang++ +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + echo "OSX" else - echo "Clang 5.0.0" - sudo ln -s /usr/local/clang-5.0.0/bin/clang /usr/bin/clang - sudo ln -s /usr/local/clang-5.0.0/bin/clang++ /usr/bin/clang++ -fi + # dependencies for clang tests + sudo apt-get -y install clang -# dependencies for gdc tests -sudo apt-get -y install gdc -# dependencies for docbook tests -sudo apt-get -y install docbook-xml xsltproc libxml2-dev libxslt-dev fop docbook-xsl-doc-pdf -# dependencies for latex tests -sudo apt-get -y install texlive texlive-latex3 biber texmaker ghostscript -# need some things for building dependencies for other tests -sudo apt-get -y install python-pip python-dev build-essential libpcre3-dev autoconf automake libtool bison subversion git -# dependencies for docbook tests continued -sudo pip install lxml -# dependencies for D tests -sudo wget http://master.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list -wget -qO - https://dlang.org/d-keyring.gpg | sudo apt-key add - -sudo apt-get update && sudo apt-get -y --allow-unauthenticated install dmd-bin -# dependencies for ldc tests -wget https://github.com/ldc-developers/ldc/releases/download/v1.4.0/ldc2-1.4.0-linux-x86_64.tar.xz -tar xf ldc2-1.4.0-linux-x86_64.tar.xz -sudo cp -rf ldc2-1.4.0-linux-x86_64/* / + # setup clang for clang tests using local clang installation + if [ ! -f /usr/local/clang-5.0.0/bin/clang ]; then + echo "No Clang 5.0.0 trying 7.0.0" + sudo ln -s /usr/local/clang-7.0.0/bin/clang /usr/bin/clang + sudo ln -s /usr/local/clang-7.0.0/bin/clang++ /usr/bin/clang++ + else + echo "Clang 5.0.0" + sudo ln -s /usr/local/clang-5.0.0/bin/clang /usr/bin/clang + sudo ln -s /usr/local/clang-5.0.0/bin/clang++ /usr/bin/clang++ + fi -ls -l /usr/lib/*python*{so,a}* + # dependencies for gdc tests + sudo apt-get -y install gdc + # dependencies for docbook tests + sudo apt-get -y install docbook-xml xsltproc libxml2-dev libxslt-dev fop docbook-xsl-doc-pdf + # dependencies for latex tests + sudo apt-get -y install texlive-full biber texmaker + # need some things for building dependencies for other tests + sudo apt-get -y install python-pip python-dev build-essential libpcre3-dev autoconf automake libtool bison subversion git + # dependencies for docbook tests continued + sudo pip install lxml + # dependencies for D tests + sudo wget http://master.dl.sourceforge.net/project/d-apt/files/d-apt.list -O /etc/apt/sources.list.d/d-apt.list + wget -qO - https://dlang.org/d-keyring.gpg | sudo apt-key add - + sudo apt-get update && sudo apt-get -y --allow-unauthenticated install dmd-bin + # dependencies for ldc tests + wget https://github.com/ldc-developers/ldc/releases/download/v1.4.0/ldc2-1.4.0-linux-x86_64.tar.xz + tar xf ldc2-1.4.0-linux-x86_64.tar.xz + sudo cp -rf ldc2-1.4.0-linux-x86_64/* / -# For now skip swig if py27 -if [[ "$PYVER" == 27 ]]; then - # dependencies for swig tests - wget https://github.com/swig/swig/archive/rel-3.0.12.tar.gz - tar xzf rel-3.0.12.tar.gz - cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. -fi \ No newline at end of file + ls -l /usr/lib/*python*{so,a}* + + # For now skip swig if py27 + if [[ "$PYVER" == 27 ]]; then + # dependencies for swig tests + wget https://github.com/swig/swig/archive/rel-3.0.12.tar.gz + tar xzf rel-3.0.12.tar.gz + cd swig-rel-3.0.12 && ./autogen.sh && ./configure --prefix=/usr && make && sudo make install && cd .. + fi +fi -- cgit v0.12 From 0536ab52936a409b1a7207141c094782e7647b30 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 16 Jun 2018 18:15:57 -0400 Subject: add sudo: required --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index daf635f..d65ace0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ os: - linux - osx +sudo: required + + install: - ./.travis/install.sh -- cgit v0.12 From f42a5a0660d782de6d54d56485e9dac417277124 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 16 Jun 2018 18:21:45 -0400 Subject: add brew install python --- .travis/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis/install.sh b/.travis/install.sh index 17fd79f..15a36a3 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -3,6 +3,7 @@ set -x if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then echo "OSX" + brew install python --framework --universal else # dependencies for clang tests sudo apt-get -y install clang -- cgit v0.12 From 2d11749c15d1725a99e887ad0242beaed0d326c2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 16 Jun 2018 20:26:59 -0400 Subject: set osx_image: xcode9.4 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d65ace0..cce5e71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ os: sudo: required +osx_image: xcode9.4 + install: - ./.travis/install.sh -- cgit v0.12 From f5da82812631222e0c5a23c51c9b4cf25b5952d7 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 22 Jun 2018 20:28:39 -0700 Subject: Update CHANGES.txt with info on bug fixed. --- src/CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 8793a19..0e50ab6 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -65,6 +65,10 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE The difference between Piotr Bartosik's patch and the current code is that the more complicated creation of file to csig map is only done when the count of children for the current node doesn't match the previous count which is loaded from the sconsign. + - Move SCons test framework files to testing/framework and remove all references to QMtest. + QMTest has not been used by SCons for some time now. + - Fixed issue causing stack trace when python Action function contains a unicode string when being + run with Python 2.7 From Andrew Featherstone - Removed unused --warn options from the man page and source code. -- cgit v0.12 From 5c6fa923c7d18f5cdf8656ab79e5266541e9097f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 27 Jun 2018 13:15:29 -0700 Subject: Clear up some comments and convert comment to docstring where it makes sense --- src/engine/SCons/Node/FS.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f0576ca..8eefc5d 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -1065,21 +1065,22 @@ _classEntry = Entry class LocalFS(object): - - # This class implements an abstraction layer for operations involving - # a local file system. Essentially, this wraps any function in - # the os, os.path or shutil modules that we use to actually go do - # anything with or to the local file system. - # - # Note that there's a very good chance we'll refactor this part of - # the architecture in some way as we really implement the interface(s) - # for remote file system Nodes. For example, the right architecture - # might be to have this be a subclass instead of a base class. - # Nevertheless, we're using this as a first step in that direction. - # - # We're not using chdir() yet because the calling subclass method - # needs to use os.chdir() directly to avoid recursion. Will we - # really need this one? + """ + This class implements an abstraction layer for operations involving + a local file system. Essentially, this wraps any function in + the os, os.path or shutil modules that we use to actually go do + anything with or to the local file system. + + Note that there's a very good chance we'll refactor this part of + the architecture in some way as we really implement the interface(s) + for remote file system Nodes. For example, the right architecture + might be to have this be a subclass instead of a base class. + Nevertheless, we're using this as a first step in that direction. + + We're not using chdir() yet because the calling subclass method + needs to use os.chdir() directly to avoid recursion. Will we + really need this one? + """ #def chdir(self, path): # return os.chdir(path) def chmod(self, path, mode): @@ -3258,7 +3259,7 @@ class File(Base): NOTE: If the timestamp hasn't changed this will skip md5'ing the file and just copy the prev_ni provided. If the prev_ni - is wrong. It will propogate it. + is wrong. It will propagate it. See: https://github.com/SCons/scons/issues/2980 Args: @@ -3271,6 +3272,7 @@ class File(Base): """ if not self.changed_timestamp_match(target, prev_ni): try: + # NOTE: We're modifying the current node's csig in a query. self.get_ninfo().csig = prev_ni.csig except AttributeError: pass @@ -3284,6 +3286,12 @@ class File(Base): return 1 def changed_timestamp_match(self, target, prev_ni): + """ + Return True if the timestamps don't match or if there is no previous timestamp + :param target: + :param prev_ni: Information about the node from the previous build + :return: + """ try: return self.get_timestamp() != prev_ni.timestamp except AttributeError: -- cgit v0.12 From 6d1a481b4276d81e7162f131bb18e90f2c2c6ed0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 29 Jun 2018 15:53:15 -0700 Subject: pull changes for this issue from WIP branch on mongo tree --- .gitignore | 1 + src/engine/SCons/Node/__init__.py | 83 ++++++++++++++++++++++++++++----------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index db2cd3a..8abc1a7 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ htmlcov *.bkp *.bak *~ +!/test/Decider/switch-rebuild.py diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 1f98a30..76f5139 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1150,8 +1150,6 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bdepends = [d for d in self.depends if d not in ignore_set] binfo.bdependsigs = [d.get_ninfo() for d in self.depends] - # TODO: If anything matches the ignore_set: bimplicit and bimplicitsigs will have different lengths and potentially the items in each list will not represent the same file. (any item but the last one matches any item in ignore_set) - # binfo.bimplicit = self.implicit or [] binfo.bimplicit = [i for i in self.implicit or [] if i not in ignore_set] binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit] @@ -1449,36 +1447,72 @@ class Node(object, with_metaclass(NoSlotsPyPy)): self - self dmap - Dictionary of file -> csig children - List of children - + Returns: List of csigs for provided list of children """ prev = [] for c in children: - c_str = str(c).replace('\\','/') - df = dmap.get(c_str) + + # First try the simple name for node + try: + # Check if we have something derived from Node.FS.Base + c.fs + c_str = str(c) + if os.altsep: + c_str = c_str.replace(os.sep, os.altsep) + df = dmap.get(c_str) + if not df: + try: + # this should yield a path which matches what's in the sconsign + c_str = c.get_path() + if os.altsep: + c_str = c_str.replace(os.sep, os.altsep) + + df = dmap.get(c_str) + except AttributeError as e: + import pdb; + pdb.set_trace() + except AttributeError as e: + # We have a non file node, likely Value + c_str = str(c) + df = dmap.get(c_str) + if df: prev.append(df) continue - try: - # We're not finding the file as listed in the - # current children list in the list loaded from - # SConsign. So let's see if it was previously - # retrieved from the repo. - # Also since we only have find_repo_file() on File objects - # Handle if we have any other Node type not having that method - for rf in c.find_repo_file(): - rfs = str(rf) - df = dmap.get(rfs) - if df: - prev.append(df) - break - else: - prev.append(None) - except AttributeError as e: - prev.append(None) + prev.append(None) + continue + + # TODO: This may not be necessary at all.. + # try: + # # We're not finding the file as listed in the + # # current children list in the list loaded from + # # SConsign. So let's see if it was previously + # # retrieved from the repo. + # # Also since we only have find_repo_file() on File objects + # # Handle if we have any other Node type not having that method + # for rf in c.find_repo_file(): + # rfs = str(rf) + # df = dmap.get(rfs) + # if df: + # prev.append(df) + # break + # else: + # print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) + + # # TODO: may want to use c.fs.File(...,create=0). Though that doesn't resolve + # # test/Repository/JavaH.py failure while below does. + # possibles = [(f,v) for f,v in dmap.items() if c.Entry('#/%s'%f).rfile() == c] + # if len(possibles) == 1: + # prev.append(possibles[0][1]) + # else: + # prev.append(None) + # except AttributeError as e: + # print("CHANGE_DEBUG (Exception): file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) + # prev.append(None) # prev = [dmap.get(str(c), dmap.get(str(c.find_repo_file()[0]))) for c in children] return prev @@ -1506,6 +1540,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): @see: FS.File.changed(), FS.File.release_target_info() """ + t = 0 if t: Trace('changed(%s [%s], %s)' % (self, classname(self), node)) @@ -1536,6 +1571,8 @@ class Node(object, with_metaclass(NoSlotsPyPy)): if _decider_map[child.changed_since_last_build](child, self, prev_ni): if t: Trace(': %s changed' % child) result = True + # TODO: What if we remove matched items here. Then we can compare csigs + # for the unmatched vs current child? if self.has_builder(): contents = self.get_executor().get_contents() @@ -1698,7 +1735,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): removed = [x for x in old_bkids if not x in new_bkids] if removed: - removed = list(map(stringify, removed)) + removed = [stringify(r) for r in removed] fmt = "`%s' is no longer a dependency\n" lines.extend([fmt % s for s in removed]) -- cgit v0.12 From 303ff960140143a00d5870732c01061d7baba325 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 11 Jul 2018 12:25:06 -0400 Subject: clarify docstring on decider function --- src/engine/SCons/Node/FS.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 8eefc5d..1ad860a 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3263,8 +3263,8 @@ class File(Base): See: https://github.com/SCons/scons/issues/2980 Args: - self - self - target - + self - dependency + target - target prev_ni - The NodeInfo object loaded from previous builds .sconsign Returns: -- cgit v0.12 From 883770b811061663c9e5fd6dd6f583c443aa1c7c Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 18 Jul 2018 18:55:31 -0400 Subject: move comment to docstring for LinkFunc() --- src/engine/SCons/Node/FS.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 1ad860a..d315b59 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -285,11 +285,13 @@ def set_duplicate(duplicate): Link_Funcs.append(link_dict[func]) def LinkFunc(target, source, env): - # Relative paths cause problems with symbolic links, so - # we use absolute paths, which may be a problem for people - # who want to move their soft-linked src-trees around. Those - # people should use the 'hard-copy' mode, softlinks cannot be - # used for that; at least I have no idea how ... + """ + Relative paths cause problems with symbolic links, so + we use absolute paths, which may be a problem for people + who want to move their soft-linked src-trees around. Those + people should use the 'hard-copy' mode, softlinks cannot be + used for that; at least I have no idea how ... + """ src = source[0].get_abspath() dest = target[0].get_abspath() dir, file = os.path.split(dest) -- cgit v0.12 From f27ff8636b0db7cc025ca343b3eeac5dd11ba6b0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 18 Jul 2018 19:00:19 -0400 Subject: Check in before migrating logic from Node() -> File(). Since the Timestamp-MD5 decider issue we're trying to resolve only affects File() nodes. Additionally creating the map of file names -> csigs for info loaded from SConsign would only be used when using Timestamp-MD5 --- src/engine/SCons/Node/__init__.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 76f5139..823f505 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -45,6 +45,7 @@ from __future__ import print_function __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" +import os import collections import copy from itertools import chain @@ -381,6 +382,7 @@ class NodeInfoBase(object): """ state = other.__getstate__() self.__setstate__(state) + def format(self, field_list=None, names=0): if field_list is None: try: @@ -1214,7 +1216,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): return _exists_map[self._func_exists](self) def rexists(self): - """Does this node exist locally or in a repositiory?""" + """Does this node exist locally or in a repository?""" # There are no repositories by default: return _rexists_map[self._func_rexists](self) @@ -1464,6 +1466,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str) if not df: + print("No Luck1:%s"%c_str) try: # this should yield a path which matches what's in the sconsign c_str = c.get_path() @@ -1471,6 +1474,10 @@ class Node(object, with_metaclass(NoSlotsPyPy)): c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str) + if not df: + print("No Luck:%s"%[str(s) for s in c.find_repo_file()]) + print(" :%s"%c.rfile()) + except AttributeError as e: import pdb; pdb.set_trace() @@ -1482,10 +1489,14 @@ class Node(object, with_metaclass(NoSlotsPyPy)): if df: prev.append(df) continue + else: + print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) prev.append(None) continue + c.find_repo_file() + # TODO: This may not be necessary at all.. # try: # # We're not finding the file as listed in the @@ -1550,9 +1561,12 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = False children = self.children() - bi = node.get_stored_info().binfo - # previous_children = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + + # then_names = bi.bsources + bi.bdepends + bi.bimplicit + # print("CHANGED : %s"%[str(s) for s in then_names]) + # print("CHILDREN: %s"%[str(s) for s in children]) + previous_map_by_file_name = self._build_dependency_map(bi) # Now build new then based on map built above. -- cgit v0.12 From 8c588f8baf76c0cd491e0589e9d2122787ef2512 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Wed, 18 Jul 2018 23:41:27 -0400 Subject: Moved logic to handle Timestamp-MD5 decider issues into File() node. One minor change in Node.Changed() it now has to handle the decider called indirectly throwing DeciderNeedsNode exception which has a property of decider it should call. Also had to update the explain logic to handle this exception. --- src/engine/SCons/Node/FS.py | 95 +++++++++++++++++- src/engine/SCons/Node/__init__.py | 202 ++++++++++---------------------------- 2 files changed, 145 insertions(+), 152 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index d315b59..e39d5d6 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -56,6 +56,7 @@ import SCons.Util import SCons.Warnings from SCons.Debug import Trace +from . import DeciderNeedsNode print_duplicate = 0 @@ -3255,7 +3256,87 @@ class File(Base): def changed_state(self, target, prev_ni): return self.state != SCons.Node.up_to_date - def changed_timestamp_then_content(self, target, prev_ni): + + def _build_dependency_map(self, binfo): + """ + Build mapping from file -> signature + + Args: + self - self + binfo - buildinfo from node being considered + + Returns: + dictionary of file->signature mappings + """ + + # For an "empty" binfo properties like bsources + # do not exist: check this to avoid exception. + if (len(binfo.bsourcesigs) + len(binfo.bdependsigs) + \ + len(binfo.bimplicitsigs)) == 0: + return {} + + pairs = [ + (binfo.bsources, binfo.bsourcesigs), + (binfo.bdepends, binfo.bdependsigs), + (binfo.bimplicit, binfo.bimplicitsigs) + ] + + m = {} + for children, signatures in pairs: + for child, signature in zip(children, signatures): + schild = str(child) + m[schild] = signature + return m + + def _get_previous_signatures(self, dmap): + """ + Return a list of corresponding csigs from previous + build in order of the node/files in children. + + Args: + self - self + dmap - Dictionary of file -> csig + children - List of children + + Returns: + List of csigs for provided list of children + """ + prev = [] + + + + # First try the simple name for node + c_str = str(self) + if os.altsep: + c_str = c_str.replace(os.sep, os.altsep) + df = dmap.get(c_str) + if not df: + print("No Luck1:%s"%c_str) + try: + # this should yield a path which matches what's in the sconsign + c_str = self.get_path() + if os.altsep: + c_str = c_str.replace(os.sep, os.altsep) + + df = dmap.get(c_str) + if not df: + print("No Luck:%s"%[str(s) for s in self.find_repo_file()]) + print(" :%s"%self.rfile()) + + except AttributeError as e: + import pdb; + pdb.set_trace() + + + if df: + return df + else: + print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) + pass + + return None + + def changed_timestamp_then_content(self, target, prev_ni, node=None): """ Used when decider for file is Timestamp-MD5 @@ -3272,6 +3353,14 @@ class File(Base): Returns: Boolean - Indicates if node(File) has changed. """ + if node is None: + raise DeciderNeedsNode(self.changed_timestamp_then_content) + + # Now get sconsign name -> csig map and then get proper prev_ni if possible + bi = node.get_stored_info().binfo + dmap = self._build_dependency_map(bi) + prev_ni = self._get_previous_signatures(dmap) + if not self.changed_timestamp_match(target, prev_ni): try: # NOTE: We're modifying the current node's csig in a query. @@ -3315,7 +3404,9 @@ class File(Base): # ...and they'd like a local copy. e = LocalCopy(self, r, None) if isinstance(e, SCons.Errors.BuildError): - raise + # Likely this should be re-raising exception e + # (which would be BuildError) + raise e SCons.Node.store_info_map[self.store_info](self) if T: Trace(' 1\n') return 1 diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index 823f505..ecf2cd5 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -248,6 +248,21 @@ _target_from_source_map = {0 : target_from_source_none, # used by it. # + +class DeciderNeedsNode(Exception): + """ + Indicate that the decider needs the node as well as the target and the dependency. + Normally the node and the target are the same, but in the case of repository + They may be different. Also the NodeInfo is retrieved from the node + """ + def __init__(self, call_this_decider): + """ + :param call_this_decider: to return the decider to call directly since deciders + are called through several levels of indirection + """ + self.decider = call_this_decider + + # # First, the single decider functions # @@ -271,6 +286,7 @@ def changed_since_last_build_node(node, target, prev_ni): """ raise NotImplementedError + def changed_since_last_build_alias(node, target, prev_ni): cur_csig = node.get_csig() try: @@ -278,19 +294,24 @@ def changed_since_last_build_alias(node, target, prev_ni): except AttributeError: return 1 + def changed_since_last_build_entry(node, target, prev_ni): node.disambiguate() return _decider_map[node.changed_since_last_build](node, target, prev_ni) + def changed_since_last_build_state_changed(node, target, prev_ni): - return (node.state != SCons.Node.up_to_date) + return node.state != SCons.Node.up_to_date + def decide_source(node, target, prev_ni): return target.get_build_env().decide_source(node, target, prev_ni) + def decide_target(node, target, prev_ni): return target.get_build_env().decide_target(node, target, prev_ni) + def changed_since_last_build_python(node, target, prev_ni): cur_csig = node.get_csig() try: @@ -1409,125 +1430,6 @@ class Node(object, with_metaclass(NoSlotsPyPy)): return None return self._tags.get(key, None) - def _build_dependency_map(self, binfo): - """ - Build mapping from file -> signature - - Args: - self - self - binfo - buildinfo from node being considered - - Returns: - dictionary of file->signature mappings - """ - - # For an "empty" binfo properties like bsources - # do not exist: check this to avoid exception. - if (len(binfo.bsourcesigs) + len(binfo.bdependsigs) + \ - len(binfo.bimplicitsigs)) == 0: - return {} - - pairs = [ - (binfo.bsources, binfo.bsourcesigs), - (binfo.bdepends, binfo.bdependsigs), - (binfo.bimplicit, binfo.bimplicitsigs) - ] - - m = {} - for children, signatures in pairs: - for child, signature in zip(children, signatures): - schild = str(child) - m[schild] = signature - return m - - def _get_previous_signatures(self, dmap, children): - """ - Return a list of corresponding csigs from previous - build in order of the node/files in children. - - Args: - self - self - dmap - Dictionary of file -> csig - children - List of children - - Returns: - List of csigs for provided list of children - """ - prev = [] - - for c in children: - - # First try the simple name for node - try: - # Check if we have something derived from Node.FS.Base - c.fs - c_str = str(c) - if os.altsep: - c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str) - if not df: - print("No Luck1:%s"%c_str) - try: - # this should yield a path which matches what's in the sconsign - c_str = c.get_path() - if os.altsep: - c_str = c_str.replace(os.sep, os.altsep) - - df = dmap.get(c_str) - if not df: - print("No Luck:%s"%[str(s) for s in c.find_repo_file()]) - print(" :%s"%c.rfile()) - - except AttributeError as e: - import pdb; - pdb.set_trace() - except AttributeError as e: - # We have a non file node, likely Value - c_str = str(c) - df = dmap.get(c_str) - - if df: - prev.append(df) - continue - else: - print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) - - prev.append(None) - continue - - c.find_repo_file() - - # TODO: This may not be necessary at all.. - # try: - # # We're not finding the file as listed in the - # # current children list in the list loaded from - # # SConsign. So let's see if it was previously - # # retrieved from the repo. - # # Also since we only have find_repo_file() on File objects - # # Handle if we have any other Node type not having that method - # for rf in c.find_repo_file(): - # rfs = str(rf) - # df = dmap.get(rfs) - # if df: - # prev.append(df) - # break - # else: - # print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) - - # # TODO: may want to use c.fs.File(...,create=0). Though that doesn't resolve - # # test/Repository/JavaH.py failure while below does. - # possibles = [(f,v) for f,v in dmap.items() if c.Entry('#/%s'%f).rfile() == c] - # if len(possibles) == 1: - # prev.append(possibles[0][1]) - # else: - # prev.append(None) - # except AttributeError as e: - # print("CHANGE_DEBUG (Exception): file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) - # prev.append(None) - - # prev = [dmap.get(str(c), dmap.get(str(c.find_repo_file()[0]))) for c in children] - return prev - def changed(self, node=None, allowcache=False): """ Returns if the node is up-to-date with respect to the BuildInfo @@ -1551,54 +1453,48 @@ class Node(object, with_metaclass(NoSlotsPyPy)): @see: FS.File.changed(), FS.File.release_target_info() """ - t = 0 - if t: - Trace('changed(%s [%s], %s)' % (self, classname(self), node)) + if t: Trace('changed(%s [%s], %s)' % (self, classname(self), node)) if node is None: node = self result = False - children = self.children() bi = node.get_stored_info().binfo + then = bi.bsourcesigs + bi.bdependsigs + bi.bimplicitsigs + children = self.children() - # then_names = bi.bsources + bi.bdepends + bi.bimplicit - # print("CHANGED : %s"%[str(s) for s in then_names]) - # print("CHILDREN: %s"%[str(s) for s in children]) - - previous_map_by_file_name = self._build_dependency_map(bi) - - # Now build new then based on map built above. - previous_children = self._get_previous_signatures(previous_map_by_file_name, children) - - diff = len(children) - len(previous_map_by_file_name) + diff = len(children) - len(then) if diff: # The old and new dependency lists are different lengths. # This always indicates that the Node must be rebuilt. - - if t: Trace(': old %s new %s' % (len(v), len(children))) - + # We also extend the old dependency list with enough None + # entries to equal the new dependency list, for the benefit + # of the loop below that updates node information. + then.extend([None] * diff) + if t: Trace(': old %s new %s' % (len(then), len(children))) result = True - for child, prev_ni in zip(children, previous_children): - if _decider_map[child.changed_since_last_build](child, self, prev_ni): - if t: Trace(': %s changed' % child) - result = True - # TODO: What if we remove matched items here. Then we can compare csigs - # for the unmatched vs current child? - + for child, prev_ni in zip(children, then): + try: + if _decider_map[child.changed_since_last_build](child, self, prev_ni): + if t: Trace(': %s changed' % child) + result = True + except DeciderNeedsNode as e: + if e.decider(self, prev_ni, node=node): + if t: Trace(': %s changed' % child) + result = True + + contents = self.get_executor().get_contents() if self.has_builder(): - contents = self.get_executor().get_contents() - + import SCons.Util newsig = SCons.Util.MD5signature(contents) if bi.bactsig != newsig: if t: Trace(': bactsig %s != newsig %s' % (bi.bactsig, newsig)) result = True if not result: - if t: - Trace(': up to date') + if t: Trace(': up to date') if t: Trace('\n') @@ -1756,8 +1652,14 @@ class Node(object, with_metaclass(NoSlotsPyPy)): for k in new_bkids: if not k in old_bkids: lines.append("`%s' is a new dependency\n" % stringify(k)) - elif _decider_map[k.changed_since_last_build](k, self, osig[k]): - lines.append("`%s' changed\n" % stringify(k)) + else: + try: + changed = _decider_map[k.changed_since_last_build](k, self, osig[k]) + except DeciderNeedsNode as e: + changed = e.decider(self, osig[k], node=self) + + if changed: + lines.append("`%s' changed\n" % stringify(k)) if len(lines) == 0 and old_bkids != new_bkids: lines.append("the dependency order changed:\n" + -- cgit v0.12 From 5b8630a68bcf38a7375904be2d4415151ea7e950 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 3 Aug 2018 12:48:05 -0700 Subject: Change logic to ensure we only build the dependency map once per target file. --- src/engine/SCons/Node/FS.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index e39d5d6..7d8191c 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2485,7 +2485,7 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): class FileBuildInfo(SCons.Node.BuildInfoBase): - __slots__ = () + __slots__ = ('dependency_map') current_version_id = 2 def convert_to_sconsign(self): @@ -3261,6 +3261,9 @@ class File(Base): """ Build mapping from file -> signature + This method also updates binfo.dependency_map to avoid rebuilding + the map for every dependency of the node we're workingon. + Args: self - self binfo - buildinfo from node being considered @@ -3286,6 +3289,9 @@ class File(Base): for child, signature in zip(children, signatures): schild = str(child) m[schild] = signature + + # store this info so we can avoid regenerating it. + binfo.dependency_map = m return m def _get_previous_signatures(self, dmap): @@ -3358,8 +3364,13 @@ class File(Base): # Now get sconsign name -> csig map and then get proper prev_ni if possible bi = node.get_stored_info().binfo - dmap = self._build_dependency_map(bi) - prev_ni = self._get_previous_signatures(dmap) + try: + dependency_map = bi.dependency_map + except AttributeError as e: + dependency_map = self._build_dependency_map(bi) + + + prev_ni = self._get_previous_signatures(dependency_map) if not self.changed_timestamp_match(target, prev_ni): try: -- cgit v0.12 From 73f88ffd1ea1fa12004407502749bf015011fe66 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 23 Sep 2018 18:52:39 -0400 Subject: Add docstring --- src/engine/SCons/Node/FS.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 7d8191c..68cabb9 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2485,6 +2485,12 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): class FileBuildInfo(SCons.Node.BuildInfoBase): + """ + This is info loaded from sconsign. + We're adding dependency_map to cache file->csig mapping + for all dependencies. Currently this is only used when using + MD5-timestamp decider. (It's needed because + """ __slots__ = ('dependency_map') current_version_id = 2 @@ -3290,6 +3296,8 @@ class File(Base): schild = str(child) m[schild] = signature + + # store this info so we can avoid regenerating it. binfo.dependency_map = m return m -- cgit v0.12 From 1cd374a5d56698a669f544cf3226edb2b7ac353d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 27 Sep 2018 11:35:08 -0700 Subject: Revisit caching of filename -> csig map and invalidate when reasonable --- src/engine/SCons/Node/FS.py | 65 +++++++++++++++++---------------------- src/engine/SCons/Node/__init__.py | 2 +- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 68cabb9..605e80b 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -43,6 +43,7 @@ import stat import sys import time import codecs +from itertools import izip, chain import SCons.Action import SCons.Debug @@ -2483,6 +2484,12 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): if key not in ('__weakref__',): setattr(self, key, value) + def __eq__(self, other): + return self.csig == other.csig and self.timestamp == other.timestamp and self.size == other.size + + def __ne__(self, other): + return not self.__eq__(other) + class FileBuildInfo(SCons.Node.BuildInfoBase): """ @@ -2494,6 +2501,12 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): __slots__ = ('dependency_map') current_version_id = 2 + def __setattr__(self, key, value): + if key != 'dependency_map' and hasattr(self, 'dependency_map'): + del self.dependency_map + + return super(FileBuildInfo, self).__setattr__(key, value) + def convert_to_sconsign(self): """ Converts this FileBuildInfo object for writing to a .sconsign file @@ -3263,13 +3276,15 @@ class File(Base): return self.state != SCons.Node.up_to_date + # Caching node -> string mapping for the below method + __dmap_cache = {} + __dmap_sig_cache = {} + + def _build_dependency_map(self, binfo): """ Build mapping from file -> signature - This method also updates binfo.dependency_map to avoid rebuilding - the map for every dependency of the node we're workingon. - Args: self - self binfo - buildinfo from node being considered @@ -3281,26 +3296,15 @@ class File(Base): # For an "empty" binfo properties like bsources # do not exist: check this to avoid exception. if (len(binfo.bsourcesigs) + len(binfo.bdependsigs) + \ - len(binfo.bimplicitsigs)) == 0: + len(binfo.bimplicitsigs)) == 0: return {} - pairs = [ - (binfo.bsources, binfo.bsourcesigs), - (binfo.bdepends, binfo.bdependsigs), - (binfo.bimplicit, binfo.bimplicitsigs) - ] - - m = {} - for children, signatures in pairs: - for child, signature in zip(children, signatures): - schild = str(child) - m[schild] = signature - - # store this info so we can avoid regenerating it. - binfo.dependency_map = m - return m + binfo.dependency_map = { str(child):signature for child, signature in izip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit), + chain(binfo.bsourcesigs, binfo.bdependsigs, binfo.bimplicitsigs))} + + return binfo.dependency_map def _get_previous_signatures(self, dmap): """ @@ -3310,45 +3314,31 @@ class File(Base): Args: self - self dmap - Dictionary of file -> csig - children - List of children Returns: List of csigs for provided list of children """ prev = [] - - # First try the simple name for node c_str = str(self) if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str) + df = dmap.get(c_str, None) if not df: - print("No Luck1:%s"%c_str) try: # this should yield a path which matches what's in the sconsign c_str = self.get_path() if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str) - if not df: - print("No Luck:%s"%[str(s) for s in self.find_repo_file()]) - print(" :%s"%self.rfile()) + df = dmap.get(c_str, None) except AttributeError as e: import pdb; pdb.set_trace() - - if df: - return df - else: - print("CHANGE_DEBUG: file:%s PREV_BUILD_FILES:%s" % (c_str, ",".join(dmap.keys()))) - pass - - return None + return df def changed_timestamp_then_content(self, target, prev_ni, node=None): """ @@ -3372,11 +3362,12 @@ class File(Base): # Now get sconsign name -> csig map and then get proper prev_ni if possible bi = node.get_stored_info().binfo + rebuilt = False try: dependency_map = bi.dependency_map except AttributeError as e: dependency_map = self._build_dependency_map(bi) - + rebuilt = True prev_ni = self._get_previous_signatures(dependency_map) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index ecf2cd5..f3f142f 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1485,9 +1485,9 @@ class Node(object, with_metaclass(NoSlotsPyPy)): if t: Trace(': %s changed' % child) result = True - contents = self.get_executor().get_contents() if self.has_builder(): import SCons.Util + contents = self.get_executor().get_contents() newsig = SCons.Util.MD5signature(contents) if bi.bactsig != newsig: if t: Trace(': bactsig %s != newsig %s' % (bi.bactsig, newsig)) -- cgit v0.12 From c118af22cd8336c487da11aa2890a8807e712d7f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 27 Sep 2018 12:54:42 -0700 Subject: switch from izip to zip. no izip in py3 --- src/engine/SCons/Node/FS.py | 4 ++-- src/engine/SCons/Node/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 605e80b..06f6a6a 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -43,7 +43,7 @@ import stat import sys import time import codecs -from itertools import izip, chain +from itertools import chain import SCons.Action import SCons.Debug @@ -3301,7 +3301,7 @@ class File(Base): # store this info so we can avoid regenerating it. - binfo.dependency_map = { str(child):signature for child, signature in izip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit), + binfo.dependency_map = { str(child):signature for child, signature in zip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit), chain(binfo.bsourcesigs, binfo.bdependsigs, binfo.bimplicitsigs))} return binfo.dependency_map diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index f3f142f..af98891 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1636,7 +1636,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): # so we only print them after running them through this lambda # to turn them into the right relative Node and then return # its string. - def stringify( s, E=self.dir.Entry ) : + def stringify( s, E=self.dir.Entry): if hasattr( s, 'dir' ) : return str(E(s)) return str(s) -- cgit v0.12 From 835ef3b09d38cb39f4705e65c498de90ec2f2707 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 27 Sep 2018 13:04:25 -0700 Subject: add note about saxon-xslt version 5.5 needing xsl and source file argument order swapped --- src/engine/SCons/Tool/docbook/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/engine/SCons/Tool/docbook/__init__.py b/src/engine/SCons/Tool/docbook/__init__.py index d60789d..5b98db2 100644 --- a/src/engine/SCons/Tool/docbook/__init__.py +++ b/src/engine/SCons/Tool/docbook/__init__.py @@ -166,6 +166,8 @@ xsltproc_com_priority = ['xsltproc', 'saxon', 'saxon-xslt', 'xalan'] # see: http://saxon.sourceforge.net/ xsltproc_com = {'xsltproc' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE', 'saxon' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE $DOCBOOK_XSLTPROCPARAMS', + # Note if saxon-xslt is version 5.5 the proper arguments are: (swap order of docbook_xsl and source) + # 'saxon-xslt' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $SOURCE $DOCBOOK_XSL $DOCBOOK_XSLTPROCPARAMS', 'saxon-xslt' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -o $TARGET $DOCBOOK_XSL $SOURCE $DOCBOOK_XSLTPROCPARAMS', 'xalan' : '$DOCBOOK_XSLTPROC $DOCBOOK_XSLTPROCFLAGS -q -out $TARGET -xsl $DOCBOOK_XSL -in $SOURCE'} xmllint_com = {'xmllint' : '$DOCBOOK_XMLLINT $DOCBOOK_XMLLINTFLAGS --xinclude $SOURCE > $TARGET'} -- cgit v0.12 From 96d6e45186d84e9bb21bca8fd2a25b15e380e2d2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Thu, 27 Sep 2018 14:31:15 -0700 Subject: cleanup travis.xml file --- .travis.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index cce5e71..76f6db0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,13 +10,9 @@ addons: os: - linux - - osx sudo: required -osx_image: xcode9.4 - - install: - ./.travis/install.sh @@ -24,7 +20,6 @@ install: matrix: allow_failures: - python: pypy - - os: osx jobs: include: -- cgit v0.12 From 91171125c73be3814c4db1a833b81551d3f036b0 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 28 Sep 2018 13:10:35 -0700 Subject: fix whitespace issues --- src/engine/SCons/Node/FSTests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index 0b57e2a..f239889 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -2524,7 +2524,6 @@ class FileTestCase(_tempdirTestCase): self.csig = csig self.timestamp = timestamp - #Create nodes fs = SCons.Node.FS.FS() d = self.fs.Dir('.') @@ -2538,8 +2537,6 @@ class FileTestCase(_tempdirTestCase): i3 = ChangedNode('gamma.h',d,fs) # gamma.h - In the bug beta.h's csig from binfo overwrites this ones i4 = ChangedNode('/usr/bin/g++',d,fs) # /usr/bin/g++ - - node.add_source([s1]) node.add_dependency([]) node.implicit = [i1, i2, i3, i4] -- cgit v0.12 From e66888271c34ab081edc64a9245a435290db295d Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 9 Oct 2018 15:12:48 -0400 Subject: Add test with MD5-timestamp decider and Repository usage --- .gitignore | 6 +-- test/Decider/MD5-timestamp-Repository.py | 91 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) create mode 100644 test/Decider/MD5-timestamp-Repository.py diff --git a/.gitignore b/.gitignore index 8abc1a7..546c893 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,8 @@ __pycache__/ # Distribution / packaging .Python -build/** -bootstrap/** +/build/** +/bootstrap/** .idea/ src/build/** @@ -28,8 +28,6 @@ ENV/ env.bak/ venv.bak/ - - # mypy .mypy_cache/ diff --git a/test/Decider/MD5-timestamp-Repository.py b/test/Decider/MD5-timestamp-Repository.py new file mode 100644 index 0000000..1826f68 --- /dev/null +++ b/test/Decider/MD5-timestamp-Repository.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Verify behavior of the MD5-timestamp Decider() setting when combined with Repository() usage +""" + +import os +import stat + +import TestSCons + +test = TestSCons.TestSCons() + +test.subdir('Repository', 'work') +repository = test.workpath('Repository') + + +test.write(['Repository','content1.in'], "content1.in 1\n") +test.write(['Repository','content2.in'], "content2.in 1\n") +test.write(['Repository','content3.in'], "content3.in 1\n") +# test.writable('Repository', 0) + + +test.write(['work','SConstruct'], """\ +Repository(r'%s') +DefaultEnvironment(tools=[]) +m = Environment(tools=[]) +m.Decider('MD5-timestamp') +m.Command('content1.out', 'content1.in', Copy('$TARGET', '$SOURCE')) +m.Command('content2.out', 'content2.in', Copy('$TARGET', '$SOURCE')) +m.Command('content3.out', 'content3.in', Copy('$TARGET', '$SOURCE')) +"""%repository) + +test.run(chdir='work',arguments='.') + +test.up_to_date(chdir='work',arguments='.') + +test.sleep() + +test.write(['Repository','content1.in'], "content1.in 2\n") + +test.touch(['Repository','content2.in']) + +time_content = os.stat(os.path.join(repository,'content3.in'))[stat.ST_MTIME] +test.write(['Repository','content3.in'], "content3.in 2\n") +test.touch(['Repository','content3.in'], time_content) + +# We should only see content1.out rebuilt. The timestamp of content2.in +# has changed, but its content hasn't, so the follow-on content check says +# to not rebuild it. The content of content3.in has changed, but that's +# masked by the fact that its timestamp is the same as the last run. + +expect = test.wrap_stdout("""\ +Copy("content1.out", "%s") +"""%os.path.join(repository,'content1.in')) + +test.run(chdir='work', arguments='.', stdout=expect) + +test.up_to_date(chdir='work', arguments='.') + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v0.12 From 4f8b2d05b6e321e5ad8b33af5ea4164cf3c5a061 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 9 Oct 2018 15:19:30 -0400 Subject: Fix docstring on FileBuildInfo per comment from @dirkbaechle --- src/engine/SCons/Node/FS.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 06f6a6a..2f18706 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -2494,9 +2494,17 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): class FileBuildInfo(SCons.Node.BuildInfoBase): """ This is info loaded from sconsign. - We're adding dependency_map to cache file->csig mapping - for all dependencies. Currently this is only used when using - MD5-timestamp decider. (It's needed because + + Attributes unique to FileBuildInfo: + dependency_map : Caches file->csig mapping + for all dependencies. Currently this is only used when using + MD5-timestamp decider. + It's used to ensure that we copy the correct + csig from previous build to be written to .sconsign when current build + is done. Previously the matching of csig to file was strictly by order + they appeared in bdepends, bsources, or bimplicit, and so a change in order + or count of any of these could yield writing wrong csig, and then false positive + rebuilds """ __slots__ = ('dependency_map') current_version_id = 2 -- cgit v0.12 From b7161dc0fc1284111d8f7540995b4b3c3d591e3a Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 9 Oct 2018 16:10:09 -0400 Subject: Fix typos --- src/engine/SCons/Node/FSTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index f239889..b39d1e3 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -2464,7 +2464,7 @@ class FileTestCase(_tempdirTestCase): def test_changed(self): """ Verify that changes between BuildInfo's list of souces, depends, and implicit - dependencies do not corrupt content signiture values written to .SConsign + dependencies do not corrupt content signature values written to .SConsign when using CacheDir and Timestamp-MD5 decider. This is for issue #2980 """ -- cgit v0.12 From ea7b78b5eb7b3d299def7688bf7a36eaa5a14f73 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 9 Oct 2018 16:10:49 -0400 Subject: Clarify and simplify logic in Node.get_binfo() --- src/engine/SCons/Node/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Node/__init__.py b/src/engine/SCons/Node/__init__.py index af98891..131953b 100644 --- a/src/engine/SCons/Node/__init__.py +++ b/src/engine/SCons/Node/__init__.py @@ -1169,13 +1169,17 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bsources = [s for s in sources if s not in seen and not seen.add(s)] binfo.bsourcesigs = [s.get_ninfo() for s in binfo.bsources] - binfo.bdepends = [d for d in self.depends if d not in ignore_set] binfo.bdependsigs = [d.get_ninfo() for d in self.depends] - binfo.bimplicit = [i for i in self.implicit or [] if i not in ignore_set] - binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit] - + # Because self.implicit is initialized to None (and not empty list []) + # we have to handle this case + if not self.implicit: + binfo.bimplicit = [] + binfo.bimplicitsigs = [] + else: + binfo.bimplicit = [i for i in self.implicit if i not in ignore_set] + binfo.bimplicitsigs = [i.get_ninfo() for i in binfo.bimplicit] return binfo -- cgit v0.12 From 0978c790762c47834bd816b9f4b56bf7978f6b19 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 22 Oct 2018 08:29:30 -0700 Subject: Resolve comments from @GaryO to clarify new code. --- src/engine/SCons/Node/FS.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 2f18706..77c340f 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -76,6 +76,9 @@ def sconsign_dir(node): _sconsign_map = {0 : sconsign_none, 1 : sconsign_dir} +class FileBuildInfoFileToCsigMappingError(Exception): + pass + class EntryProxyAttributeError(AttributeError): """ An AttributeError subclass for recording and displaying the name @@ -2510,6 +2513,11 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): current_version_id = 2 def __setattr__(self, key, value): + + # If any attributes are changed in FileBuildInfo, we need to + # invalidate the cached map of file name to content signature + # heald in dependency_map. Currently only used with + # MD5-timestamp decider if key != 'dependency_map' and hasattr(self, 'dependency_map'): del self.dependency_map @@ -3343,8 +3351,7 @@ class File(Base): df = dmap.get(c_str, None) except AttributeError as e: - import pdb; - pdb.set_trace() + raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str) return df @@ -3361,11 +3368,15 @@ class File(Base): self - dependency target - target prev_ni - The NodeInfo object loaded from previous builds .sconsign + node - Node instance. This is the only changed* function which requires + node to function. So if we detect that it's not passed. + we throw DeciderNeedsNode, and caller should handle this and pass node. Returns: Boolean - Indicates if node(File) has changed. """ if node is None: + # We need required node argument to get BuildInfo to function raise DeciderNeedsNode(self.changed_timestamp_then_content) # Now get sconsign name -> csig map and then get proper prev_ni if possible -- cgit v0.12 From e5940adedbb88877eb0a52f9dd9fe26e032fc9ba Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Nov 2018 08:37:19 -0800 Subject: Change test for str(node1) is str(node2) to use ==. Expecting that the strings would have the same id() is not reasonable. Expecting their values are equal is. --- src/engine/SCons/Node/FSTests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/Node/FSTests.py b/src/engine/SCons/Node/FSTests.py index b39d1e3..23ec48e 100644 --- a/src/engine/SCons/Node/FSTests.py +++ b/src/engine/SCons/Node/FSTests.py @@ -1134,7 +1134,10 @@ class FSTestCase(_tempdirTestCase): e1 = fs.Entry(p) e2 = fs.Entry(path) assert e1 is e2, (e1, e2) - assert str(e1) is str(e2), (str(e1), str(e2)) + a=str(e1) + b=str(e2) + assert a == b, ("Strings should match for same file/node\n%s\n%s"%(a,b)) + # Test for a bug in 0.04 that did not like looking up # dirs with a trailing slash on Windows. -- cgit v0.12 From aec04a49172af53b27f12bfba398766bede3f8bb Mon Sep 17 00:00:00 2001 From: William Deegan Date: Mon, 12 Nov 2018 09:22:14 -0800 Subject: Fix duplicated items in src/CHANGES.txt --- src/CHANGES.txt | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 0e50ab6..964e5b8 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -24,8 +24,6 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE From William Deegan: - Remove long deprecated SCons.Options code and tests. This removes BoolOption,EnumOption, ListOption,PackageOption, and PathOption which have been replaced by *Variable() many years ago. - - Fix issue # 3106 MSVC if using MSVC_BATCH and target dir had a space would fail due to quirk in - MSVC's handling of escaped targetdirs when batch compiling. - Re-Enable parallel SCons (-j) when running via Pypy - Move SCons test framework files to testing/framework and remove all references to QMtest. QMTest has not been used by SCons for some time now. @@ -35,40 +33,28 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Default path for clang/clangxx : C:\Program Files\LLVM\bin - Default path for mingw : C:\MinGW\bin and/or C:\mingw-w64\*\mingw64\bin - Key program to locate mingw : mingw32-make (as the gcc with mingw prefix has no fixed name) - - Fix GH Issue #3141 unicode string in a TryAction() with python 2.7 crashes. - Fixed issue causing stack trace when python Action function contains a unicode string when being run with Python 2.7 - Add alternate path to QT install for Centos in qt tool: /usr/lib64/qt-3.3/bin - - Fix GH Issue #2580 - # in FRAMEWORKPATH doesn't get properly expanded. The # is left in the - command line. - - Fix GH Issue #3212 - Use of Py3 and CacheDir + Configure's TryCompile (or likely and Python Value Nodes) - yielded trying to combine strings and bytes which threw exception. - - Updated logic for mingw and clang on win32 to search default tool install paths if not - found in normal SCons PATH. If the user specifies PATH or tool specific paths they - will be used and the default paths below will be ignored. - - Default path for clang/clangxx : C:\Program Files\LLVM\bin - - Default path for mingw : c:\MinGW\bin - Fix Java tools to search reasonable default paths for Win32, Linux, macOS. Add required paths for swig and java native interface to JAVAINCLUDES. You should add these to your CPPPATH if you need to compile with them. This handles spaces in paths in default Java paths on windows. - Added more java paths to match install for Centos 7 of openjdk - Fix new logic which populates JAVAINCLUDES to handle the case where javac is not found. - - Fix GH Issue #3225 SCons.Util.Flatten() doesn't handle MappingView's produced by dictionary as return - values from dict().{items(), keys(), values()}. - - Fix issue #2980 with credit to William Blevins. This is an issue where using TimeStamp-MD5 Decider - and CacheDir can yield incorrect md5's being written into the .sconsign. The difference between - William Blevins patch and the current code is that the more complicated creation of file to csig - map is only done when the count of children for the current node doesn't match the previous count - which is loaded from the sconsign. + - Fix GH Issue #2580 - # in FRAMEWORKPATH doesn't get properly expanded. The # is left in the + command line. - Fix issue #2980 with credit to Piotr Bartosik (and William Blevins). This is an issue where using TimeStamp-MD5 Decider and CacheDir can yield incorrect md5's being written into the .sconsign. The difference between Piotr Bartosik's patch and the current code is that the more complicated creation of file to csig map is only done when the count of children for the current node doesn't match the previous count which is loaded from the sconsign. - - Move SCons test framework files to testing/framework and remove all references to QMtest. - QMTest has not been used by SCons for some time now. - - Fixed issue causing stack trace when python Action function contains a unicode string when being - run with Python 2.7 + - Fix issue # 3106 MSVC if using MSVC_BATCH and target dir had a space would fail due to quirk in + MSVC's handling of escaped targetdirs when batch compiling. + - Fix GH Issue #3141 unicode string in a TryAction() with python 2.7 crashes. + - Fix GH Issue #3212 - Use of Py3 and CacheDir + Configure's TryCompile (or likely and Python Value Nodes) + yielded trying to combine strings and bytes which threw exception. + - Fix GH Issue #3225 SCons.Util.Flatten() doesn't handle MappingView's produced by dictionary as return + values from dict().{items(), keys(), values()}. From Andrew Featherstone - Removed unused --warn options from the man page and source code. -- cgit v0.12