Merge branch 'fix_win_md5_decider_malfunction_on_fresh_build' of github.com:bdbaddog/scons into fix_win_md5_decider_malfunction_on_fresh_build

9 files changed, 159 insertions, 31 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 0d02ec0..0a80095 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -14,17 +14,28 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       The Configure logic directly calls the decider when using --config=force but wasn't handling
       that exception.  This would yield minimally configure tests using TryLink() not running and
       leaving TypeError Nonetype exception in config.log
+    - Fix Issue #3303 - Handle --config=force overwriting the Environment passed into Configure()'s
+      Decider and not clearing it when the configure context is completed.
+    - Add default paths for yacc tool on windows to include cygwin, mingw, and chocolatey
 
   From Daniel Moody:
     - Change the default for AppendENVPath to delete_existing=0, so path
       order will not be changed, unless explicitly set (Issue #3276)
     - Fixed bug which threw error when running SCons on windows system with no MSVC installed.
+    - Update link tool to convert target to node before accessing node member
+    - Update mingw tool to remove MSVC like nologo CCFLAG
+    - Add default paths for lex tool on windows to include cygwin, mingw, and chocolatey
+    - Add lex construction variable LEXUNISTD for turning off unix headers on windows
+    - Update lex tool to use win_flex on windows if available
 
   From Mats Wichmann:
     - Quiet open file ResourceWarnings on Python >= 3.6 caused by
       not using a context manager around Popen.stdout
     - Add the textfile tool to the default tool list
-    - Fix syntax on is/is not cluases: should not use with a literal
+    - Fix syntax on is/is not clauses: should not use with a literal
+
+  From Bernhard M. Wiedemann:
+    - Do not store build host+user name if reproducible builds are wanted
 
 
 RELEASE 3.0.4 - Mon, 20 Jan 2019 22:49:27 +0000
@@ -42,8 +53,6 @@ RELEASE 3.0.4 - Mon, 20 Jan 2019 22:49:27 +0000
       Issues #3268 & Issue #3222
     - Initial support for ARM targets with Visual Studio 2017 - Issue #3182 (You must set TARGET_ARCH for this to work)
     - Update TempFileMunge class to use PRINT_CMD_LINE_FUNC
-    - Update link tool to convert target to node before accessing node member
-    - Update mingw tool to remove MSVC like nologo CCFLAG
 
   From Tobias Herzog
     - Enhance cpp scanner regex logic to detect if/elif expressions without whitespaces but
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index 152f0bc..eea8aed 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -2372,6 +2372,7 @@ class OverrideEnvironment(Base):
         kw = copy_non_reserved_keywords(kw)
         self.__dict__['overrides'].update(semi_deepcopy(kw))
 
+
 # The entry point that will be used by the external world
 # to refer to a construction environment.  This allows the wrapper
 # interface to extend a construction environment for its own purposes
diff --git a/src/engine/SCons/Platform/win32.py b/src/engine/SCons/Platform/win32.py
index 2d40fb8..be30546 100644
--- a/src/engine/SCons/Platform/win32.py
+++ b/src/engine/SCons/Platform/win32.py
@@ -43,6 +43,10 @@ from SCons.Platform.virtualenv import ImportVirtualenv
 from SCons.Platform.virtualenv import ignore_virtualenv, enable_virtualenv
 import SCons.Util
 
+CHOCO_DEFAULT_PATH = [
+    r'C:\ProgramData\chocolatey\bin'
+]
+
 try:
     import msvcrt
     import win32api
diff --git a/src/engine/SCons/SConf.py b/src/engine/SCons/SConf.py
index f23c401..b123c11 100644
--- a/src/engine/SCons/SConf.py
+++ b/src/engine/SCons/SConf.py
@@ -324,24 +324,6 @@ class SConfBuildTask(SCons.Taskmaster.AlwaysTask):
             s = sys.stdout = sys.stderr = Streamer(sys.stdout)
             try:
                 env = self.targets[0].get_build_env()
-                if cache_mode == FORCE:
-                    # Set up the Decider() to force rebuilds by saying
-                    # that every source has changed.  Note that we still
-                    # call the environment's underlying source decider so
-                    # that the correct .sconsign info will get calculated
-                    # and keep the build state consistent.
-                    def force_build(dependency, target, prev_ni,
-                                    env_decider=env.decide_source,
-                                    node=None):
-                        try:
-                            env_decider(dependency, target, prev_ni)
-                        except DeciderNeedsNode as e:
-                            e.decider(target, prev_ni, node=target)
-                        except Exception as e:
-                            raise e
-                        return True
-                    if env.decide_source.__code__ is not force_build.__code__:
-                        env.Decider(force_build)
                 env['PSTDOUT'] = env['PSTDERR'] = s
                 try:
                     sconf.cached = 0
@@ -412,12 +394,42 @@ class SConfBase(object):
         build tests in the VariantDir, not in the SourceDir)
         """
         global SConfFS
+
+        # Now create isolated override so setting source_decider doesn't affect parent Environment
+        if cache_mode == FORCE:
+            self.original_env = env
+            self.env = env.Clone()
+
+            # Set up the Decider() to force rebuilds by saying
+            # that every source has changed.  Note that we still
+            # call the environment's underlying source decider so
+            # that the correct .sconsign info will get calculated
+            # and keep the build state consistent.
+            def force_build(dependency, target, prev_ni,
+                            env_decider=env.decide_source,
+                            node=None):
+                try:
+                    env_decider(dependency, target, prev_ni)
+                except DeciderNeedsNode as e:
+                    e.decider(target, prev_ni, node=target)
+                except Exception as e:
+                    raise e
+                return True
+
+            if self.env.decide_source.__code__ is not force_build.__code__:
+                self.env.Decider(force_build)
+
+        else:
+            self.env = env
+
+        # print("Override env:%s"%env)
+
         if not SConfFS:
             SConfFS = SCons.Node.FS.default_fs or \
                       SCons.Node.FS.FS(env.fs.pathTop)
         if sconf_global is not None:
             raise SCons.Errors.UserError
-        self.env = env
+
         if log_file is not None:
             log_file = SConfFS.File(env.subst(log_file))
         self.logfile = log_file
@@ -456,6 +468,7 @@ class SConfBase(object):
                 env = sconf.Finish()
         """
         self._shutdown()
+
         return self.env
 
     def Define(self, name, value = None, comment = None):
@@ -510,6 +523,20 @@ class SConfBase(object):
                 n.attributes = SCons.Node.Node.Attrs()
             n.attributes.keep_targetinfo = 1
 
+            if True:
+                # Some checkers have intermediate files (for example anything that compiles a c file into a program to run
+                # Those files need to be set to not release their target info, otherwise taskmaster will throw a
+                # Nonetype not callable
+                for c in n.children(scan=False):
+                    # Keep debug code here.
+                    # print("Checking [%s] for builders and then setting keep_targetinfo"%c)
+                    if  c.has_builder():
+                        n.store_info = 0
+                        if not hasattr(c, 'attributes'):
+                            c.attributes = SCons.Node.Node.Attrs()
+                        c.attributes.keep_targetinfo = 1
+                    # pass
+
         ret = 1
 
         try:
@@ -746,10 +773,18 @@ class SConfBase(object):
             self.logstream.write("\n")
             self.logstream.close()
             self.logstream = None
-        # remove the SConfSourceBuilder from the environment
-        blds = self.env['BUILDERS']
-        del blds['SConfSourceBuilder']
-        self.env.Replace( BUILDERS=blds )
+
+        # Now reset the decider if we changed it due to --config=force
+        # We saved original Environment passed in and cloned it to isolate
+        # it from being changed.
+        if cache_mode == FORCE:
+            self.env.Decider(self.original_env.decide_source)
+
+            # remove the SConfSourceBuilder from the environment
+            blds = self.env['BUILDERS']
+            del blds['SConfSourceBuilder']
+            self.env.Replace( BUILDERS=blds )
+
         self.active = 0
         sconf_global = None
         if not self.config_h is None:
diff --git a/src/engine/SCons/SConfTests.py b/src/engine/SCons/SConfTests.py
index 73fcf8d..f770450 100644
--- a/src/engine/SCons/SConfTests.py
+++ b/src/engine/SCons/SConfTests.py
@@ -194,7 +194,7 @@ class SConfTestCase(unittest.TestCase):
                         pass
                     def add_post_action(self, *actions):
                         pass
-                    def children(self):
+                    def children(self, scan = 1):
                         return []
                     def get_state(self):
                         return self.state
diff --git a/src/engine/SCons/Tool/lex.py b/src/engine/SCons/Tool/lex.py
index 280c768..a63ddc9 100644
--- a/src/engine/SCons/Tool/lex.py
+++ b/src/engine/SCons/Tool/lex.py
@@ -34,10 +34,14 @@ selection method.
 __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"
 
 import os.path
+import sys
 
 import SCons.Action
 import SCons.Tool
 import SCons.Util
+from SCons.Platform.mingw import MINGW_DEFAULT_PATHS
+from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS
+from SCons.Platform.win32 import CHOCO_DEFAULT_PATH
 
 LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR")
 
@@ -64,6 +68,29 @@ def lexEmitter(target, source, env):
                 target.append(fileName)
     return (target, source)
 
+def get_lex_path(env, append_paths=False):
+    """
+    Find the a path containing the lex or flex binaries. If a construction 
+    environment is passed in then append the path to the ENV PATH.
+    """
+    # save existing path to reset if we don't want to append any paths
+    envPath = env['ENV']['PATH']
+    bins = ['flex', 'lex', 'win_flex']
+
+    for prog in bins:
+        bin_path = SCons.Tool.find_program_path(
+            env, 
+            prog, 
+            default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
+        if bin_path:
+            if not append_paths:
+                env['ENV']['PATH'] = envPath
+            else:
+                env.AppendENVPath('PATH', os.path.dirname(bin_path))
+            return bin_path
+    SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH')
+
+
 def generate(env):
     """Add Builders and construction variables for lex to an Environment."""
     c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
@@ -83,12 +110,23 @@ def generate(env):
     cxx_file.add_action(".ll", LexAction)
     cxx_file.add_emitter(".ll", lexEmitter)
 
-    env["LEX"]      = env.Detect("flex") or "lex"
     env["LEXFLAGS"] = SCons.Util.CLVar("")
-    env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
+
+    if sys.platform == 'win32':
+        get_lex_path(env, append_paths=True)
+        env["LEX"] = env.Detect(['flex', 'lex', 'win_flex'])
+        if not env.get("LEXUNISTD"):
+            env["LEXUNISTD"] = SCons.Util.CLVar("")
+        env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET"
+    else:
+        env["LEX"] = env.Detect(["flex", "lex"])
+        env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
 
 def exists(env):
-    return env.Detect(["flex", "lex"])
+    if sys.platform == 'win32':
+        return get_lex_path(env)
+    else:
+        return env.Detect(["flex", "lex"])
 
 # Local Variables:
 # tab-width:4
diff --git a/src/engine/SCons/Tool/lex.xml b/src/engine/SCons/Tool/lex.xml
index 0388ee3..f933451 100644
--- a/src/engine/SCons/Tool/lex.xml
+++ b/src/engine/SCons/Tool/lex.xml
@@ -33,6 +33,7 @@ Sets construction variables for the &lex; lexical analyser.
 <item>LEX</item>
 <item>LEXFLAGS</item>
 <item>LEXCOM</item>
+<item>LEXUNISTD</item>
 </sets>
 <uses>
 <item>LEXCOMSTR</item>
@@ -78,4 +79,12 @@ General options passed to the lexical analyzer generator.
 </summary>
 </cvar>
 
+<cvar name="LEXUNISTD">
+<summary>
+<para>
+Used only on windows environments to set a lex flag to prevent 'unistd.h' from being included. The default value is '--nounistd'.
+</para>
+</summary>
+</cvar>
+
 </sconsdoc>
diff --git a/src/engine/SCons/Tool/msvc.py b/src/engine/SCons/Tool/msvc.py
index 9f3c1fa..2352088 100644
--- a/src/engine/SCons/Tool/msvc.py
+++ b/src/engine/SCons/Tool/msvc.py
@@ -271,6 +271,10 @@ def generate(env):
     env['SHOBJPREFIX']    = '$OBJPREFIX'
     env['SHOBJSUFFIX']    = '$OBJSUFFIX'
 
+    # MSVC probably wont support unistd.h so default
+    # without it for lex generation
+    env["LEXUNISTD"] = SCons.Util.CLVar("--nounistd")
+
     # Set-up ms tools paths
     msvc_setup_env_once(env)
 
diff --git a/src/engine/SCons/Tool/yacc.py b/src/engine/SCons/Tool/yacc.py
index cd9b9a8..5db49d3 100644
--- a/src/engine/SCons/Tool/yacc.py
+++ b/src/engine/SCons/Tool/yacc.py
@@ -41,6 +41,7 @@ import SCons.Tool
 import SCons.Util
 from SCons.Platform.mingw import MINGW_DEFAULT_PATHS
 from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS
+from SCons.Platform.win32 import CHOCO_DEFAULT_PATH
 
 YaccAction = SCons.Action.Action("$YACCCOM", "$YACCCOMSTR")
 
@@ -97,6 +98,28 @@ def ymEmitter(target, source, env):
 def yyEmitter(target, source, env):
     return _yaccEmitter(target, source, env, ['.yy'], '$YACCHXXFILESUFFIX')
 
+def get_yacc_path(env, append_paths=False):
+    """
+    Find the a path containing the lex or flex binaries. If a construction 
+    environment is passed in then append the path to the ENV PATH.
+    """
+    # save existing path to reset if we don't want to append any paths
+    envPath = env['ENV']['PATH']
+    bins = ['bison', 'yacc', 'win_bison']
+
+    for prog in bins:
+        bin_path = SCons.Tool.find_program_path(
+            env, 
+            prog, 
+            default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
+        if bin_path:
+            if not append_paths:
+                env['ENV']['PATH'] = envPath
+            else:
+                env.AppendENVPath('PATH', os.path.dirname(bin_path))
+            return bin_path
+    SCons.Warnings.Warning('lex tool requested, but lex or flex binary not found in ENV PATH')
+
 def generate(env):
     """Add Builders and construction variables for yacc to an Environment."""
     c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
@@ -124,7 +147,12 @@ def generate(env):
         else:
             SCons.Warnings.Warning('yacc tool requested, but bison binary not found in ENV PATH')
 
-    env['YACC']      = env.Detect('bison') or 'yacc'
+    if sys.platform == 'win32':
+        get_yacc_path(env, append_paths=True)
+        env["YACC"] = env.Detect(['bison', 'yacc', 'win_bison'])
+    else:
+        env["YACC"] = env.Detect(["bison", "yacc"])
+    
     env['YACCFLAGS'] = SCons.Util.CLVar('')
     env['YACCCOM']   = '$YACC $YACCFLAGS -o $TARGET $SOURCES'
     env['YACCHFILESUFFIX'] = '.h'