From 28d90499867ace9e16c117b1cbf960aca8e67c8e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 26 Feb 2019 21:52:46 -0800 Subject: On windows first try with native file paths with \\ then swap path to normalized path string with / separators. On a fresh windows build the node string will have windows dirsep and not normalizd. This yielded broken builds for the Meta project' --- src/engine/SCons/Node/FS.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 77c340f..f2ba7be 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3338,17 +3338,29 @@ class File(Base): # First try the simple name for node c_str = str(self) + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df + if not df: try: # this should yield a path which matches what's in the sconsign c_str = self.get_path() + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df except AttributeError as e: raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str) @@ -3388,16 +3400,23 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True - prev_ni = self._get_previous_signatures(dependency_map) + new_prev_ni = self._get_previous_signatures(dependency_map) + new = self.changed_timestamp_match(target, new_prev_ni) + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) + - if not self.changed_timestamp_match(target, prev_ni): + if not new: try: # NOTE: We're modifying the current node's csig in a query. - self.get_ninfo().csig = prev_ni.csig + self.get_ninfo().csig = new_prev_ni.csig except AttributeError: pass return False - return self.changed_content(target, prev_ni) + return self.changed_content(target, new_prev_ni) def changed_timestamp_newer(self, target, prev_ni): try: -- cgit v0.12 From 7ec2458832f757d138a396c0bb1412df5f391c9e Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 1 Mar 2019 12:26:21 -0800 Subject: Added logic to shortcut comparing prev_ni if there is no dependency map from previous build. This should speed up md5-timestamp builds for clean builds. Also added debug logic to dump and check aagainst previous implementation at top of FS.PY MD5_TIMESTAMP_DEBUG flag. currently set to False --- src/engine/SCons/Node/FS.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f2ba7be..f520af1 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -61,6 +61,8 @@ from . import DeciderNeedsNode print_duplicate = 0 +MD5_TIMESTAMP_DEBUG = False + def sconsign_none(node): raise NotImplementedError @@ -3335,9 +3337,16 @@ class File(Base): List of csigs for provided list of children """ prev = [] + # MD5_TIMESTAMP_DEBUG = False + + if len(dmap) == 0: + if MD5_TIMESTAMP_DEBUG: print("Nothing dmap shortcutting") + return None + if MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap)) # First try the simple name for node c_str = str(self) + if MD5_TIMESTAMP_DEBUG: print("Checking :%s"%c_str) df = dmap.get(c_str, None) if df: return df @@ -3345,6 +3354,7 @@ class File(Base): if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3353,12 +3363,14 @@ class File(Base): # this should yield a path which matches what's in the sconsign c_str = self.get_path() df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3400,13 +3412,21 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True + if len(dependency_map) == 0: + # If there's no dependency map, there's no need to find the + # prev_ni as there aren't any + # shortcut the rest of the logic + if MD5_TIMESTAMP_DEBUG: print("Skipping checks len(dmap)=0") + return True new_prev_ni = self._get_previous_signatures(dependency_map) new = self.changed_timestamp_match(target, new_prev_ni) - old = self.changed_timestamp_match(target, prev_ni) - if old != new: - print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) - new_prev_ni = self._get_previous_signatures(dependency_map) + if MD5_TIMESTAMP_DEBUG: + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) if not new: -- cgit v0.12 From bbb5d2ccc6a0b37611b4877d8bb1d6b969da7bf2 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 2 Mar 2019 14:10:59 -0500 Subject: Forgot to add current file csig when shortcutting calling changed_content(). Directly calling get_csig() when there is no previous build history --- src/engine/SCons/Node/FS.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f520af1..61054f3 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3417,7 +3417,12 @@ class File(Base): # prev_ni as there aren't any # shortcut the rest of the logic if MD5_TIMESTAMP_DEBUG: print("Skipping checks len(dmap)=0") + + # We still need to get the current file's csig + # This should be slightly faster than calling self.changed_content(target, new_prev_ni) + self.get_csig() return True + new_prev_ni = self._get_previous_signatures(dependency_map) new = self.changed_timestamp_match(target, new_prev_ni) -- cgit v0.12 From 9bbb976181f4cd5b6261993bae9fb5535bf07587 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Tue, 26 Feb 2019 21:52:46 -0800 Subject: On windows first try with native file paths with \\ then swap path to normalized path string with / separators. On a fresh windows build the node string will have windows dirsep and not normalizd. This yielded broken builds for the Meta project' --- src/engine/SCons/Node/FS.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index 77c340f..f2ba7be 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -3338,17 +3338,29 @@ class File(Base): # First try the simple name for node c_str = str(self) + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df + if not df: try: # this should yield a path which matches what's in the sconsign c_str = self.get_path() + df = dmap.get(c_str, None) + if df: + return df + if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - - df = dmap.get(c_str, None) + df = dmap.get(c_str, None) + if df: + return df except AttributeError as e: raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str) @@ -3388,16 +3400,23 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True - prev_ni = self._get_previous_signatures(dependency_map) + new_prev_ni = self._get_previous_signatures(dependency_map) + new = self.changed_timestamp_match(target, new_prev_ni) + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) + - if not self.changed_timestamp_match(target, prev_ni): + if not new: try: # NOTE: We're modifying the current node's csig in a query. - self.get_ninfo().csig = prev_ni.csig + self.get_ninfo().csig = new_prev_ni.csig except AttributeError: pass return False - return self.changed_content(target, prev_ni) + return self.changed_content(target, new_prev_ni) def changed_timestamp_newer(self, target, prev_ni): try: -- cgit v0.12 From 9c6608bc1069d8bfd199e30801b2d1118c0137af Mon Sep 17 00:00:00 2001 From: William Deegan Date: Fri, 1 Mar 2019 12:26:21 -0800 Subject: Added logic to shortcut comparing prev_ni if there is no dependency map from previous build. This should speed up md5-timestamp builds for clean builds. Also added debug logic to dump and check aagainst previous implementation at top of FS.PY MD5_TIMESTAMP_DEBUG flag. currently set to False --- src/engine/SCons/Node/FS.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Node/FS.py b/src/engine/SCons/Node/FS.py index f2ba7be..f520af1 100644 --- a/src/engine/SCons/Node/FS.py +++ b/src/engine/SCons/Node/FS.py @@ -61,6 +61,8 @@ from . import DeciderNeedsNode print_duplicate = 0 +MD5_TIMESTAMP_DEBUG = False + def sconsign_none(node): raise NotImplementedError @@ -3335,9 +3337,16 @@ class File(Base): List of csigs for provided list of children """ prev = [] + # MD5_TIMESTAMP_DEBUG = False + + if len(dmap) == 0: + if MD5_TIMESTAMP_DEBUG: print("Nothing dmap shortcutting") + return None + if MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap)) # First try the simple name for node c_str = str(self) + if MD5_TIMESTAMP_DEBUG: print("Checking :%s"%c_str) df = dmap.get(c_str, None) if df: return df @@ -3345,6 +3354,7 @@ class File(Base): if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3353,12 +3363,14 @@ class File(Base): # this should yield a path which matches what's in the sconsign c_str = self.get_path() df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df if os.altsep: c_str = c_str.replace(os.sep, os.altsep) df = dmap.get(c_str, None) + if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) if df: return df @@ -3400,13 +3412,21 @@ class File(Base): dependency_map = self._build_dependency_map(bi) rebuilt = True + if len(dependency_map) == 0: + # If there's no dependency map, there's no need to find the + # prev_ni as there aren't any + # shortcut the rest of the logic + if MD5_TIMESTAMP_DEBUG: print("Skipping checks len(dmap)=0") + return True new_prev_ni = self._get_previous_signatures(dependency_map) new = self.changed_timestamp_match(target, new_prev_ni) - old = self.changed_timestamp_match(target, prev_ni) - if old != new: - print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) - new_prev_ni = self._get_previous_signatures(dependency_map) + if MD5_TIMESTAMP_DEBUG: + old = self.changed_timestamp_match(target, prev_ni) + + if old != new: + print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) + new_prev_ni = self._get_previous_signatures(dependency_map) if not new: -- cgit v0.12 From 3a107dcd6c93520f41c54b25a322317bfd1b8460 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sat, 2 Mar 2019 21:56:53 -0500 Subject: Add test to cover this issue with MD5-timestamp decider breaking dependencies only on windows when one generated souce implicitly depends on another generated source. --- test/Decider/MD5-winonly-firstbuild.py | 60 +++++++++++++++++++++++++ test/Decider/MD5-winonly-fixture/SConstruct | 9 ++++ test/Decider/MD5-winonly-fixture/sconstest.skip | 0 test/Decider/MD5-winonly-fixture/test_lex.l | 10 +++++ test/Decider/MD5-winonly-fixture/test_parse.y | 43 ++++++++++++++++++ 5 files changed, 122 insertions(+) create mode 100644 test/Decider/MD5-winonly-firstbuild.py create mode 100644 test/Decider/MD5-winonly-fixture/SConstruct create mode 100644 test/Decider/MD5-winonly-fixture/sconstest.skip create mode 100644 test/Decider/MD5-winonly-fixture/test_lex.l create mode 100644 test/Decider/MD5-winonly-fixture/test_parse.y diff --git a/test/Decider/MD5-winonly-firstbuild.py b/test/Decider/MD5-winonly-firstbuild.py new file mode 100644 index 0000000..4047ea8 --- /dev/null +++ b/test/Decider/MD5-winonly-firstbuild.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# +# __COPYRIGHT__ +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + +""" +Test but which only shows on windows when one generated file depends on another generated file +In this case flex and yacc are an example. +""" + +import os +import stat + +import TestSCons +from TestCmd import IS_WINDOWS + + +test = TestSCons.TestSCons() + +if IS_WINDOWS: + yacc = test.where_is('yacc') or test.where_is('bison') or test.where_is('win_bison') + lex = test.where_is('flex') or test.where_is('lex') or test.where_is('win_flex') + # print("Lex:%s yacc:%s"%(lex,yacc)) + if not yacc or not lex: + test.skip("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test") +else: + test.skip("Windows only test") + + +test.dir_fixture('MD5-winonly-fixture') +test.run() + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/test/Decider/MD5-winonly-fixture/SConstruct b/test/Decider/MD5-winonly-fixture/SConstruct new file mode 100644 index 0000000..c194c47 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/SConstruct @@ -0,0 +1,9 @@ +DefaultEnvironment(tools=[]) +env=Environment() +env.Decider('MD5-timestamp') +env.AppendENVPath('PATH', r'd:\mesa\flexbison') +env.Tool('lex') +env.Tool('yacc') +env['YACCFLAGS'] = '-d' +env.Append(LEXFLAGS = ['--wincompat']) +env.SharedLibrary('dummy',['test_lex.l','test_parse.y']) diff --git a/test/Decider/MD5-winonly-fixture/sconstest.skip b/test/Decider/MD5-winonly-fixture/sconstest.skip new file mode 100644 index 0000000..e69de29 diff --git a/test/Decider/MD5-winonly-fixture/test_lex.l b/test/Decider/MD5-winonly-fixture/test_lex.l new file mode 100644 index 0000000..269b984 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/test_lex.l @@ -0,0 +1,10 @@ +%{ + +#include +#include "test_parse.h" +int c; +%} +%% + +%% + diff --git a/test/Decider/MD5-winonly-fixture/test_parse.y b/test/Decider/MD5-winonly-fixture/test_parse.y new file mode 100644 index 0000000..3ead9d2 --- /dev/null +++ b/test/Decider/MD5-winonly-fixture/test_parse.y @@ -0,0 +1,43 @@ +%{ +#include + +int regs[26]; +int base; + +%} + +%start digit + +%union { int a; } + + +%token DIGIT LETTER + +%left '|' +%left '&' +%left '+' '-' +%left '*' '/' '%' +%left UMINUS /*supplies precedence for unary minus */ + +%% /* beginning of rules section */ + +digit: DIGIT; + + +%% +main() +{ + return(yyparse()); +} + +yyerror(s) +char *s; +{ + fprintf(stderr, "%s\n",s); + return(0); +} + +yywrap() +{ + return(1); +} \ No newline at end of file -- cgit v0.12 From a6849f5629cbcc5a1c8693f398b9641cbc92208f Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 3 Mar 2019 10:29:52 -0500 Subject: fix typo in skip_test --- test/Decider/MD5-winonly-firstbuild.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Decider/MD5-winonly-firstbuild.py b/test/Decider/MD5-winonly-firstbuild.py index 4047ea8..1e1b51e 100644 --- a/test/Decider/MD5-winonly-firstbuild.py +++ b/test/Decider/MD5-winonly-firstbuild.py @@ -43,9 +43,9 @@ if IS_WINDOWS: lex = test.where_is('flex') or test.where_is('lex') or test.where_is('win_flex') # print("Lex:%s yacc:%s"%(lex,yacc)) if not yacc or not lex: - test.skip("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test") + test.skip_test("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test") else: - test.skip("Windows only test") + test.skip_test("Windows only test") test.dir_fixture('MD5-winonly-fixture') -- cgit v0.12 From 9fe21847c4c987792326d515a851b673d96af930 Mon Sep 17 00:00:00 2001 From: William Deegan Date: Sun, 3 Mar 2019 12:56:55 -0500 Subject: Fix logic in test to only use --wincompat if win_flex. Improve test skip messaging --- test/Decider/MD5-winonly-firstbuild.py | 4 ++-- test/Decider/MD5-winonly-fixture/SConstruct | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/Decider/MD5-winonly-firstbuild.py b/test/Decider/MD5-winonly-firstbuild.py index 1e1b51e..87f7999 100644 --- a/test/Decider/MD5-winonly-firstbuild.py +++ b/test/Decider/MD5-winonly-firstbuild.py @@ -43,9 +43,9 @@ if IS_WINDOWS: lex = test.where_is('flex') or test.where_is('lex') or test.where_is('win_flex') # print("Lex:%s yacc:%s"%(lex,yacc)) if not yacc or not lex: - test.skip_test("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test") + test.skip_test("On windows but no flex/lex/win_flex and yacc/bison/win_bison required for test\n") else: - test.skip_test("Windows only test") + test.skip_test("Windows only test\n") test.dir_fixture('MD5-winonly-fixture') diff --git a/test/Decider/MD5-winonly-fixture/SConstruct b/test/Decider/MD5-winonly-fixture/SConstruct index c194c47..9775a43 100644 --- a/test/Decider/MD5-winonly-fixture/SConstruct +++ b/test/Decider/MD5-winonly-fixture/SConstruct @@ -5,5 +5,6 @@ env.AppendENVPath('PATH', r'd:\mesa\flexbison') env.Tool('lex') env.Tool('yacc') env['YACCFLAGS'] = '-d' -env.Append(LEXFLAGS = ['--wincompat']) +if env['YACC'] == 'win_flex': + env.Append(LEXFLAGS = ['--wincompat']) env.SharedLibrary('dummy',['test_lex.l','test_parse.y']) -- cgit v0.12