Improvements to lex and yacc tools

The mocked tools mylex.py and myyacc.py now understand the file-generation
options, and generate a dummy file with predictable contents, for
checking.  This allows more testing of the path through the SCons support
for these two without needing live commands.

New tests added which invoke the file-generation options, and make
sure the extra files are created, and that SCons detects and tracks
the added targets.  Work is done in a subdirectory, which exposes some
existing known inconsistent behavior (the regular generated file goes
in the subdir per the LEXCOM and YACCOM generated line, while the ones
generated from commandline options go in the topdir) - but we're going
to allow that behavior to continue for backwards compat.

Same fix applied to yacc tool that PR #4168 did for lex - do subst_list()
instead of subst() to preserve spaces in paths.  That fix left the lex
tool unable to pass the new test, as it could not see the individual
arguments in the FLAGS variable, which was solved by indexing into the
subst'd list so we can iterate over the args again.

Test and tool cleanup; add DefaultEnvironment calls, etc.

Note this mentions, but does not address the problem described in issue 4154.

Signed-off-by: Mats Wichmann <mats@linux.com>

1 files changed, 35 insertions, 23 deletions

diff --git a/SCons/Tool/lex.py b/SCons/Tool/lex.py
index 96f9bcb..262fe25 100644
--- a/SCons/Tool/lex.py
+++ b/SCons/Tool/lex.py
@@ -23,6 +23,9 @@
 
 """Tool-specific initialization for lex.
 
+This tool should support multiple lex implementations,
+but is in actuality biased towards GNU Flex.
+
 There normally shouldn't be any need to import this module directly.
 It will usually be imported through the generic SCons.Tool.Tool()
 selection method.
@@ -30,14 +33,17 @@ selection method.
 
 import os.path
 import sys
+from typing import Optional
 
 import SCons.Action
 import SCons.Tool
-import SCons.Util
 import SCons.Warnings
 from SCons.Platform.mingw import MINGW_DEFAULT_PATHS
 from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS
 from SCons.Platform.win32 import CHOCO_DEFAULT_PATH
+from SCons.Util import CLVar, to_String
+
+DEFAULT_PATHS = CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS
 
 LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR")
 
@@ -47,20 +53,25 @@ else:
     BINS = ["flex", "lex"]
 
 
-def lexEmitter(target, source, env):
-    sourceBase, sourceExt = os.path.splitext(SCons.Util.to_String(source[0]))
+def lexEmitter(target, source, env) -> tuple:
+    """Adds extra files generated by lex program to target list."""
 
+    sourceBase, sourceExt = os.path.splitext(to_String(source[0]))
     if sourceExt == ".lm":           # If using Objective-C
         target = [sourceBase + ".m"] # the extension is ".m".
 
-    # This emitter essentially tries to add to the target all extra
-    # files generated by flex.
-
-    # Different options that are used to trigger the creation of extra files.
+    # With --header-file and ----tables-file, the file to write is defined
+    # by the option argument. Extract this and include in the list of targets.
+    # NOTE: a filename passed to the command this way is not modified by SCons,
+    # and so will be interpreted relative to the project top directory at
+    # execution time, while the name added to the target list will be
+    # interpreted relative to the SConscript directory - a possibile mismatch.
+    #
+    # These are GNU flex-only options.
+    # TODO: recognize --outfile also?
     file_gen_options = ["--header-file=", "--tables-file="]
-
     lexflags = env.subst_list("$LEXFLAGS", target=target, source=source)
-    for option in SCons.Util.CLVar(lexflags):
+    for option in lexflags[0]:
         for fileGenOption in file_gen_options:
             l = len(fileGenOption)
             if option[:l] == fileGenOption:
@@ -68,28 +79,29 @@ def lexEmitter(target, source, env):
                 # file name to the target list.
                 file_name = option[l:].strip()
                 target.append(file_name)
+
     return target, source
 
 
-def get_lex_path(env, append_paths=False):
+def get_lex_path(env, append_paths=False) -> Optional[str]:
     """
-    Find the path to the lex tool, searching several possible names
+    Returns the path to the lex tool, searching several possible names.
 
-    Only called in the Windows case, so the default_path
-    can be Windows-specific
+    Only called in the Windows case, so the `default_path` argument to
+    :func:`find_program_path` can be Windows-specific.
 
-    :param env: current construction environment
-    :param append_paths: if set, add the path to the tool to PATH
-    :return: path to lex tool, if found
+    Args:
+        env: current construction environment
+        append_paths: if set, add the path to the tool to PATH
     """
     for prog in BINS:
         bin_path = SCons.Tool.find_program_path(
             env,
             prog,
-            default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
+            default_paths=DEFAULT_PATHS,
+            add_path=append_paths,
+        )
         if bin_path:
-            if append_paths:
-                env.AppendENVPath('PATH', os.path.dirname(bin_path))
             return bin_path
 
     SCons.Warnings.warn(
@@ -98,7 +110,7 @@ def get_lex_path(env, append_paths=False):
     )
 
 
-def generate(env):
+def generate(env) -> None:
     """Add Builders and construction variables for lex to an Environment."""
     c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
 
@@ -117,21 +129,21 @@ def generate(env):
     cxx_file.add_action(".ll", LexAction)
     cxx_file.add_emitter(".ll", lexEmitter)
 
-    env["LEXFLAGS"] = SCons.Util.CLVar("")
+    env["LEXFLAGS"] = CLVar("")
 
     if sys.platform == 'win32':
         # ignore the return - we do not need the full path here
         _ = get_lex_path(env, append_paths=True)
         env["LEX"] = env.Detect(BINS)
         if not env.get("LEXUNISTD"):
-            env["LEXUNISTD"] = SCons.Util.CLVar("")
+            env["LEXUNISTD"] = CLVar("")
         env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET"
     else:
         env["LEX"] = env.Detect(BINS)
         env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
 
 
-def exists(env):
+def exists(env) -> Optional[str]:
     if sys.platform == 'win32':
         return get_lex_path(env)
     else: