From 395e09a4d28d25ae030c50cf82a0729f6c4c3bee Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Fri, 26 Jan 2018 19:20:14 +0200 Subject: New conditional C Scanner; Improvements for virtual C Pre-Processor --- src/CHANGES.txt | 11 +++++ src/engine/SCons/Scanner/C.py | 42 ++++++++++++------- src/engine/SCons/Scanner/CTests.py | 68 +++++++++++++++++++++++++++--- src/engine/SCons/cpp.py | 85 ++++++++++++++++++++++++++------------ src/engine/SCons/cppTests.py | 37 +++++++++++++---- 5 files changed, 186 insertions(+), 57 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index c02bb36..c13e3de 100644 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -33,6 +33,17 @@ RELEASE 3.1.0.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE - Updated manpage scons.xml to fix a nested list problem - Updated doc terminionly: use prepend instead of append as appropriate + From Ivan Kravets, PlatformIO + - New conditional C Scanner (`SCons.Scanner.C.CConditionalScanner()`) + which interprets C/C Preprocessor conditional syntax (#ifdef, #if, #else, + #elif, #define, etc.) + - Improvements for virtual C Pre-Processor: + * Handle UNSIGNED LONG and LONG numeric constants in DEC (keep support for HEX) + * Skip unrecognized directives, such as `#if( defined ...)` + * Ignore `#include DYNAMIC_INCLUDE` directive that depends on a dynamic + macro which is not located in a state TABLE. + * Cleanup CPP expressions before evaluating (strip comments, carriage returns) + RELEASE 3.0.1 - Mon, 12 Nov 2017 15:31:33 -0700 From Daniel Moody: diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index c2e50a6..db7463f 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -1,6 +1,6 @@ """SCons.Scanner.C -This module implements the dependency scanner for C/C++ code. +This module implements the dependency scanner for C/C++ code. """ @@ -72,7 +72,11 @@ def dictify_CPPDEFINES(env): result = {} for c in cppdefines: if SCons.Util.is_Sequence(c): - result[c[0]] = c[1] + # handle tuple with 1 item (e.g. tuple("DEFINE", )) + if len(c) > 1: + result[c[0]] = c[1] + else: + result[c[0]] = None else: result[c] = None return result @@ -108,21 +112,27 @@ class SConsCPPScannerWrapper(object): def select(self, node): return self + +def CConditionalScanner(): + """ + Return an advanced conditional Scanner instance for scanning source files + + Interprets C/C++ Preprocessor conditional syntax + (#ifdef, #if, defined, #else, #elif, etc.). + """ + return SConsCPPScannerWrapper("CScanner", "CPPPATH") + + def CScanner(): - """Return a prototype Scanner instance for scanning source files - that use the C pre-processor""" - - # Here's how we would (or might) use the CPP scanner code above that - # knows how to evaluate #if/#ifdef/#else/#elif lines when searching - # for #includes. This is commented out for now until we add the - # right configurability to let users pick between the scanners. - #return SConsCPPScannerWrapper("CScanner", "CPPPATH") - - cs = SCons.Scanner.ClassicCPP("CScanner", - "$CPPSUFFIXES", - "CPPPATH", - '^[ \t]*#[ \t]*(?:include|import)[ \t]*(<|")([^>"]+)(>|")') - return cs + """Return a simplified classic Scanner instance for scanning source files + + Takes into account the type of bracketing used to include the file, and + uses classic CPP rules for searching for the files based on the bracketing. + """ + return SCons.Scanner.ClassicCPP( + "CScanner", "$CPPSUFFIXES", "CPPPATH", + '^[ \t]*#[ \t]*(?:include|import)[ \t]*(<|")([^>"]+)(>|")') + # Local Variables: # tab-width:4 diff --git a/src/engine/SCons/Scanner/CTests.py b/src/engine/SCons/Scanner/CTests.py index 9c7df12..94d3dbf 100644 --- a/src/engine/SCons/Scanner/CTests.py +++ b/src/engine/SCons/Scanner/CTests.py @@ -27,7 +27,6 @@ import SCons.compat import collections import os -import sys import unittest import TestCmd @@ -45,8 +44,11 @@ os.chdir(test.workpath('')) # create some source files and headers: test.write('f1.cpp',""" -#include \"f1.h\" +#ifdef INCLUDE_F2 /* multi-line comment */ #include +#else +#include \"f1.h\" +#endif int main() { @@ -56,8 +58,15 @@ int main() test.write('f2.cpp',""" #include \"d1/f1.h\" -#include + +#if 5UL < 10 && !defined(DUMMY_MACRO) // some comment + #if NESTED_CONDITION + #include + #endif +#else #include \"f1.h\" +#endif + #import int main() @@ -261,7 +270,7 @@ class CScannerTestCase4(unittest.TestCase): deps = s(env.File('f2.cpp'), env, path) headers = ['d1/f1.h', 'f1.h', 'd1/d2/f1.h', 'd1/d2/f4.h'] deps_match(self, deps, headers) - + class CScannerTestCase5(unittest.TestCase): def runTest(self): """Make sure files in repositories will get scanned""" @@ -282,7 +291,7 @@ class CScannerTestCase5(unittest.TestCase): # Make sure rexists() got called on the file node being # scanned, essential for cooperation with VariantDir functionality. assert n.GetTag('rexists_called') - + headers = ['f1.h', 'f2.h', 'f3-test.h', 'd1/f1.h', 'd1/f2.h', 'd1/f3-test.h'] deps_match(self, deps, headers) @@ -340,7 +349,7 @@ class CScannerTestCase9(unittest.TestCase): # Did we catch the warning associated with not finding fb.h? assert to.out - + deps_match(self, deps, [ 'fa.h' ]) test.unlink('fa.h') @@ -440,6 +449,50 @@ class CScannerTestCase15(unittest.TestCase): assert suffix in s.get_skeys(env), "%s not in skeys" % suffix +class CConditionalScannerTestCase1(unittest.TestCase): + def runTest(self): + """Find local files with no CPPPATH""" + env = DummyEnvironment(CPPPATH=[]) + s = SCons.Scanner.C.CConditionalScanner() + path = s.path(env) + deps = s(env.File('f1.cpp'), env, path) + headers = ['f1.h'] + deps_match(self, deps, headers) + + +class CConditionalScannerTestCase2(unittest.TestCase): + def runTest(self): + """Find local files with no CPPPATH based on #ifdef""" + env = DummyEnvironment(CPPPATH=[], CPPDEFINES=["INCLUDE_F2"]) + s = SCons.Scanner.C.CConditionalScanner() + path = s.path(env) + deps = s(env.File('f1.cpp'), env, path) + headers = ['f2.h', 'fi.h'] + deps_match(self, deps, headers) + + +class CConditionalScannerTestCase3(unittest.TestCase): + def runTest(self): + """Find files in explicit subdirectories, ignore missing file""" + env = DummyEnvironment( + CPPPATH=[test.workpath("d1")], + CPPDEFINES=[("NESTED_CONDITION", 1)] + ) + s = SCons.Scanner.C.CConditionalScanner() + deps = s(env.File('f2.cpp'), env, s.path(env)) + headers = ['d1/f1.h', 'd1/d2/f1.h'] + deps_match(self, deps, headers) + + # disable nested conditions + env = DummyEnvironment( + CPPPATH=[test.workpath("d1")], + CPPDEFINES=[("NESTED_CONDITION", 0)] + ) + s = SCons.Scanner.C.CConditionalScanner() + deps = s(env.File('f2.cpp'), env, s.path(env)) + headers = ['d1/f1.h'] + deps_match(self, deps, headers) + def suite(): suite = unittest.TestSuite() @@ -457,6 +510,9 @@ def suite(): suite.addTest(CScannerTestCase13()) suite.addTest(CScannerTestCase14()) suite.addTest(CScannerTestCase15()) + suite.addTest(CConditionalScannerTestCase1()) + suite.addTest(CConditionalScannerTestCase2()) + suite.addTest(CConditionalScannerTestCase3()) return suite if __name__ == "__main__": diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 161f070..42693b9 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -99,8 +99,19 @@ e = '^\s*#\s*(' + '|'.join(l) + ')(.*)$' # And last but not least, compile the expression. CPP_Expression = re.compile(e, re.M) +# A list with RE to cleanup CPP Expressions (tuples) +# We should remove all comments and carriage returns (\r) before evaluating +CPP_Expression_Cleaner_List = [ + "/\*.*\*/", + "/\*.*", + "//.*", + "\r" +] +CPP_Expression_Cleaner_RE = re.compile( + "\s*(" + "|".join(CPP_Expression_Cleaner_List) + ")") - +def Cleanup_CPP_Expressions(ts): + return [(t[0], CPP_Expression_Cleaner_RE.sub("", t[1])) for t in ts] # # Second "subsystem" of regular expressions that we set up: @@ -118,7 +129,6 @@ CPP_to_Python_Ops_Dict = { '||' : ' or ', '?' : ' and ', ':' : ' or ', - '\r' : '', } CPP_to_Python_Ops_Sub = lambda m: CPP_to_Python_Ops_Dict[m.group(0)] @@ -141,12 +151,10 @@ CPP_to_Python_Ops_Expression = re.compile(expr) # A separate list of expressions to be evaluated and substituted # sequentially, not all at once. CPP_to_Python_Eval_List = [ - ['defined\s+(\w+)', '"\\1" in __dict__'], - ['defined\s*\((\w+)\)', '"\\1" in __dict__'], - ['/\*.*\*/', ''], - ['/\*.*', ''], - ['//.*', ''], - ['(0x[0-9A-Fa-f]*)[UL]+', '\\1'], + ['defined\s+(\w+)', '"\\1" in __dict__'], + ['defined\s*\((\w+)\)', '"\\1" in __dict__'], + ['(0x[0-9A-Fa-f]+)(?:L|UL)?', '\\1'], + ['(\d+)(?:L|UL)?', '\\1'], ] # Replace the string representations of the regular expressions in the @@ -284,7 +292,13 @@ class PreProcessor(object): global CPP_Expression, Table contents = line_continuations.sub('', contents) cpp_tuples = CPP_Expression.findall(contents) - return [(m[0],) + Table[m[0]].match(m[1]).groups() for m in cpp_tuples] + cpp_tuples = Cleanup_CPP_Expressions(cpp_tuples) + result = [] + for t in cpp_tuples: + m = Table[t[0]].match(t[1]) + if m: + result.append((t[0],) + m.groups()) + return result def __call__(self, file): """ @@ -354,8 +368,10 @@ class PreProcessor(object): track #define values. """ t = CPP_to_Python(' '.join(t[1:])) - try: return eval(t, self.cpp_namespace) - except (NameError, TypeError): return 0 + try: + return eval(t, self.cpp_namespace) + except (NameError, TypeError, SyntaxError): + return 0 def initialize_result(self, fname): self.result = [fname] @@ -482,7 +498,11 @@ class PreProcessor(object): try: expansion = int(expansion) except (TypeError, ValueError): - pass + # handle "defined" chain "! (defined (A) || defined (B)" ... + if "defined " in expansion: + self.cpp_namespace[name] = self.eval_expression(t[2:]) + return + if args: evaluator = FunctionEvaluator(name, args[1:-1], expansion) self.cpp_namespace[name] = evaluator @@ -508,15 +528,21 @@ class PreProcessor(object): Default handling of a #include line. """ t = self.resolve_include(t) + if not t: + return include_file = self.find_include_file(t) - if include_file: - #print("include_file =", include_file) - self.result.append(include_file) - contents = self.read_file(include_file) - new_tuples = [('scons_current_file', include_file)] + \ - self.tupleize(contents) + \ - [('scons_current_file', self.current_file)] - self.tuples[:] = new_tuples + self.tuples + if not include_file: + return + # print("include_file =", include_file) + self.result.append(include_file) + # handle infinite recursion + for t in self.tuples: + if t[0] == 'scons_current_file' and t[1] == include_file: + return + new_tuples = [('scons_current_file', include_file)] + \ + self.tupleize(self.read_file(include_file)) + \ + [('scons_current_file', self.current_file)] + self.tuples[:] = new_tuples + self.tuples # Date: Tue, 22 Nov 2005 20:26:09 -0500 # From: Stefan Seefeld @@ -542,18 +568,25 @@ class PreProcessor(object): This handles recursive expansion of values without "" or <> surrounding the name until an initial " or < is found, to handle - #include FILE - - where FILE is a #define somewhere else.""" - - s = t[1] + where FILE is a #define somewhere else. + """ + s = t[1].strip() while not s[0] in '<"': - #print("s =", s) try: s = self.cpp_namespace[s] except KeyError: m = function_name.search(s) + + # Date: Mon, 28 Nov 2016 17:47:13 UTC + # From: Ivan Kravets + # + # Ignore `#include` directive that depends on dynamic macro + # which is not located in state TABLE + # For example, `#include MYCONFIG_FILE` + if not m: + return None + s = self.cpp_namespace[m.group(1)] if callable(s): args = function_arg_separator.split(m.group(2)) diff --git a/src/engine/SCons/cppTests.py b/src/engine/SCons/cppTests.py index 40236e4..1447a7b 100644 --- a/src/engine/SCons/cppTests.py +++ b/src/engine/SCons/cppTests.py @@ -26,7 +26,6 @@ from __future__ import absolute_import __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import atexit -import sys import unittest import TestUnit @@ -67,7 +66,7 @@ substitution_input = """ ifdef_input = """ #define DEFINED 0 -#ifdef DEFINED +#ifdef DEFINED /* multi-line comment */ #include "file7-yes" #else #include "file7-no" @@ -82,7 +81,7 @@ ifdef_input = """ if_boolean_input = """ -#define ZERO 0 +#define ZERO 0 // single-line comment #define ONE 1 #if ZERO @@ -132,27 +131,35 @@ if_boolean_input = """ if_defined_input = """ -#define DEFINED 0 +#define DEFINED_A 0 +#define DEFINED_B 0 -#if defined(DEFINED) +#if defined(DEFINED_A) #include "file15-yes" #endif -#if ! defined(DEFINED) +#if ! defined(DEFINED_A) #include #else #include #endif -#if defined DEFINED +#if defined DEFINED_A #include "file17-yes" #endif -#if ! defined DEFINED +#if ! defined DEFINED_A #include #else #include #endif + +#if ! (defined (DEFINED_A) || defined (DEFINED_B) +#include +#else +#include +#endif + """ @@ -226,11 +233,17 @@ expression_input = """ #include "file29-yes" #endif -#if ! (ONE != ONE) +#if ! (ONE != ONE) #include #else #include #endif + +#if 123456789UL || 0x13L +#include +#else +#include +#endif """ @@ -466,6 +479,7 @@ class PreProcessorTestCase(cppAllTestCase): ('include', '<', 'file16-yes'), ('include', '"', 'file17-yes'), ('include', '<', 'file18-yes'), + ('include', '<', 'file19-yes'), ] expression_expect = [ @@ -481,6 +495,7 @@ class PreProcessorTestCase(cppAllTestCase): ('include', '<', 'file28-yes'), ('include', '"', 'file29-yes'), ('include', '<', 'file30-yes'), + ('include', '<', 'file301-yes'), ] undef_expect = [ @@ -561,6 +576,8 @@ class DumbPreProcessorTestCase(cppAllTestCase): ('include', '"', 'file17-yes'), ('include', '<', 'file18-no'), ('include', '<', 'file18-yes'), + ('include', '<', 'file19-no'), + ('include', '<', 'file19-yes'), ] expression_expect = [ @@ -588,6 +605,8 @@ class DumbPreProcessorTestCase(cppAllTestCase): ('include', '"', 'file29-yes'), ('include', '<', 'file30-yes'), ('include', '<', 'file30-no'), + ('include', '<', 'file301-yes'), + ('include', '<', 'file301-no'), ] undef_expect = [ -- cgit v0.12 From 5fcdb300cbe1a64488c7686a2a71cd20aafd146d Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Mar 2018 12:40:17 +0200 Subject: Avoid infinite recursion for conditional PreProcessor --- src/engine/SCons/cpp.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 42693b9..35b8799 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -531,14 +531,10 @@ class PreProcessor(object): if not t: return include_file = self.find_include_file(t) - if not include_file: + # avoid infinite recursion + if not include_file or include_file in self.result: return - # print("include_file =", include_file) self.result.append(include_file) - # handle infinite recursion - for t in self.tuples: - if t[0] == 'scons_current_file' and t[1] == include_file: - return new_tuples = [('scons_current_file', include_file)] + \ self.tupleize(self.read_file(include_file)) + \ [('scons_current_file', self.current_file)] -- cgit v0.12 From 080d987d57bb3d0fe5a6a5f879cdb8b35cd8c576 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Mar 2018 14:03:33 +0200 Subject: Implement CPP tuples caching per a file --- src/engine/SCons/cpp.py | 88 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 35b8799..72018c5 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -239,6 +239,9 @@ function_arg_separator = re.compile(',\s*') class PreProcessor(object): + + FILE_TUPLES_CACHE = None + """ The main workhorse class for handling C pre-processing. """ @@ -277,57 +280,81 @@ class PreProcessor(object): d[op] = getattr(self, 'do_' + op) self.default_table = d - # Controlling methods. - - def tupleize(self, contents): + def __call__(self, file): """ - Turns the contents of a file into a list of easily-processed - tuples describing the CPP lines in the file. + Pre-processes a file. - The first element of each tuple is the line's preprocessor - directive (#if, #include, #define, etc., minus the initial '#'). - The remaining elements are specific to the type of directive, as - pulled apart by the regular expression. + This is the main public entry point. """ - global CPP_Expression, Table - contents = line_continuations.sub('', contents) - cpp_tuples = CPP_Expression.findall(contents) - cpp_tuples = Cleanup_CPP_Expressions(cpp_tuples) - result = [] - for t in cpp_tuples: - m = Table[t[0]].match(t[1]) - if m: - result.append((t[0],) + m.groups()) - return result + self.current_file = file + return self.process_file(file) - def __call__(self, file): + def process_file(self, file): """ Pre-processes a file. - This is the main public entry point. + This is the main internal entry point. """ - self.current_file = file - return self.process_contents(self.read_file(file), file) + return self._process_tuples(self.tupleize_file(file), file) - def process_contents(self, contents, fname=None): + def process_contents(self, contents): """ Pre-processes a file contents. - This is the main internal entry point. + Is used by tests """ + return self._process_tuples(self.tupleize(contents)) + + def _process_tuples(self, tuples, file=None): self.stack = [] self.dispatch_table = self.default_table.copy() - self.current_file = fname - self.tuples = self.tupleize(contents) + self.current_file = file + self.tuples = tuples - self.initialize_result(fname) + self.initialize_result(file) while self.tuples: t = self.tuples.pop(0) # Uncomment to see the list of tuples being processed (e.g., # to validate the CPP lines are being translated correctly). - #print(t) + # print(t) self.dispatch_table[t[0]](t) - return self.finalize_result(fname) + return self.finalize_result(file) + + def tupleize_file(self, file): + if not PreProcessor.FILE_TUPLES_CACHE: + PreProcessor.FILE_TUPLES_CACHE = {} + file_tuples = PreProcessor.FILE_TUPLES_CACHE.get(file) + if not file_tuples: + file_tuples = self._parse_tuples(self.read_file(file)) + PreProcessor.FILE_TUPLES_CACHE[file] = file_tuples + return self._match_tuples(file_tuples) + + def tupleize(self, contents): + """ + Turns the contents of a file into a list of easily-processed + tuples describing the CPP lines in the file. + + The first element of each tuple is the line's preprocessor + directive (#if, #include, #define, etc., minus the initial '#'). + The remaining elements are specific to the type of directive, as + pulled apart by the regular expression. + """ + return self._match_tuples(self._parse_tuples(contents)) + + def _parse_tuples(self, contents): + global CPP_Expression + contents = line_continuations.sub('', contents) + tuples = CPP_Expression.findall(contents) + return Cleanup_CPP_Expressions(tuples) + + def _match_tuples(self, tuples): + global Table + result = [] + for t in tuples: + m = Table[t[0]].match(t[1]) + if m: + result.append((t[0],) + m.groups()) + return result # Dispatch table stack manipulation methods. @@ -535,6 +562,7 @@ class PreProcessor(object): if not include_file or include_file in self.result: return self.result.append(include_file) + # print include_file, len(self.tuples) new_tuples = [('scons_current_file', include_file)] + \ self.tupleize(self.read_file(include_file)) + \ [('scons_current_file', self.current_file)] -- cgit v0.12 From 80e62ede63890d52fb7395c7588758bc07f8fba6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Mar 2018 14:28:51 +0200 Subject: Restore support for "all" option for Conditional PreProcessor --- src/engine/SCons/Scanner/C.py | 5 +++-- src/engine/SCons/cpp.py | 14 +++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index db7463f..0d18879 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -96,10 +96,11 @@ class SConsCPPScannerWrapper(object): def __init__(self, name, variable): self.name = name self.path = SCons.Scanner.FindPathDirs(variable) - def __call__(self, node, env, path = ()): + def __call__(self, node, env, path = (), all=False): cpp = SConsCPPScanner(current = node.get_dir(), cpppath = path, - dict = dictify_CPPDEFINES(env)) + dict = dictify_CPPDEFINES(env), + all = all) result = cpp(node) for included, includer in cpp.missing: fmt = "No dependency generated for file: %s (included from: %s) -- file not found" diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 72018c5..22b7a7b 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -245,7 +245,7 @@ class PreProcessor(object): """ The main workhorse class for handling C pre-processing. """ - def __init__(self, current=os.curdir, cpppath=(), dict={}, all=0): + def __init__(self, current=os.curdir, cpppath=(), dict={}, all=False): global Table cpppath = tuple(cpppath) @@ -263,8 +263,8 @@ class PreProcessor(object): self.cpp_namespace = dict.copy() self.cpp_namespace['__dict__'] = self.cpp_namespace - if all: - self.do_include = self.all_include + # IF all=True, return all found includes without nested parsing + self.all = all # For efficiency, a dispatch table maps each C preprocessor # directive (#if, #define, etc.) to the method that should be @@ -563,6 +563,10 @@ class PreProcessor(object): return self.result.append(include_file) # print include_file, len(self.tuples) + + if self.all: + return + new_tuples = [('scons_current_file', include_file)] + \ self.tupleize(self.read_file(include_file)) + \ [('scons_current_file', self.current_file)] @@ -619,10 +623,6 @@ class PreProcessor(object): return None return (t[0], s[0], s[1:-1]) - def all_include(self, t): - """ - """ - self.result.append(self.resolve_include(t)) class DumbPreProcessor(PreProcessor): """A preprocessor that ignores all #if/#elif/#else/#endif directives -- cgit v0.12 From e2312b8dfb4b993e8abbb1cf1043906d4bf31b14 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 8 Mar 2018 16:25:09 +0200 Subject: Allow to control maximum depth of nested includes for conditional PreProcessor --- src/engine/SCons/Scanner/C.py | 4 ++-- src/engine/SCons/cpp.py | 28 +++++++++++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index 0d18879..bd75bc9 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -96,11 +96,11 @@ class SConsCPPScannerWrapper(object): def __init__(self, name, variable): self.name = name self.path = SCons.Scanner.FindPathDirs(variable) - def __call__(self, node, env, path = (), all=False): + def __call__(self, node, env, path = (), depth=-1): cpp = SConsCPPScanner(current = node.get_dir(), cpppath = path, dict = dictify_CPPDEFINES(env), - all = all) + depth = depth) result = cpp(node) for included, includer in cpp.missing: fmt = "No dependency generated for file: %s (included from: %s) -- file not found" diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 22b7a7b..cfff053 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -245,7 +245,7 @@ class PreProcessor(object): """ The main workhorse class for handling C pre-processing. """ - def __init__(self, current=os.curdir, cpppath=(), dict={}, all=False): + def __init__(self, current=os.curdir, cpppath=(), dict={}, all=0, depth=-1): global Table cpppath = tuple(cpppath) @@ -263,8 +263,15 @@ class PreProcessor(object): self.cpp_namespace = dict.copy() self.cpp_namespace['__dict__'] = self.cpp_namespace - # IF all=True, return all found includes without nested parsing - self.all = all + # Return all includes without resolving + if all: + self.do_include = self.all_include + + # Max depth of nested includes: + # -1 = unlimited + # 0 - disabled nesting + # >0 - number of allowed nested includes + self.depth = depth # For efficiency, a dispatch table maps each C preprocessor # directive (#if, #define, etc.) to the method that should be @@ -564,8 +571,14 @@ class PreProcessor(object): self.result.append(include_file) # print include_file, len(self.tuples) - if self.all: - return + # Handle maximum depth of nested includes + if self.depth != -1: + current_depth = 0 + for t in self.tuples: + if t[0] != "scons_current_file": + current_depth += 1 + if current_depth >= self.depth: + return new_tuples = [('scons_current_file', include_file)] + \ self.tupleize(self.read_file(include_file)) + \ @@ -623,6 +636,11 @@ class PreProcessor(object): return None return (t[0], s[0], s[1:-1]) + def all_include(self, t): + """ + """ + self.result.append(self.resolve_include(t)) + class DumbPreProcessor(PreProcessor): """A preprocessor that ignores all #if/#elif/#else/#endif directives -- cgit v0.12 From 1ec74bcc6f5a17217cfa7067a18d172ed224b876 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 19 Nov 2018 21:55:31 +0200 Subject: Remember known CPPPATH for dependencies --- src/engine/SCons/Scanner/C.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index bd75bc9..ede74b9 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -29,6 +29,7 @@ This module implements the dependency scanner for C/C++ code. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" + import SCons.Node.FS import SCons.Scanner import SCons.Util @@ -46,16 +47,25 @@ class SConsCPPScanner(SCons.cpp.PreProcessor): def __init__(self, *args, **kw): SCons.cpp.PreProcessor.__init__(self, *args, **kw) self.missing = [] + self._known_paths = tuple() + def initialize_result(self, fname): self.result = SCons.Util.UniqueList([fname]) - def finalize_result(self, fname): - return self.result[1:] + def find_include_file(self, t): keyword, quote, fname = t - result = SCons.Node.FS.find_file(fname, self.searchpath[quote]) - if not result: + result = SCons.Node.FS.find_file( + fname, self._known_paths + self.searchpath[quote]) + if result: + result_path = result.get_abspath() + for p in self.searchpath[quote]: + if result_path.startswith(p.get_abspath()): + self._known_paths.add(p) + break + else: self.missing.append((fname, self.current_file)) return result + def read_file(self, file): try: with open(str(file.rfile())) as fp: -- cgit v0.12 From 037c8733cf74d9404a73d8ef6c5549a1d7684f25 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Tue, 20 Nov 2018 01:21:45 +0200 Subject: Handle CWD when searching for a file --- src/engine/SCons/Scanner/C.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index ede74b9..37bf244 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -29,7 +29,6 @@ This module implements the dependency scanner for C/C++ code. __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" - import SCons.Node.FS import SCons.Scanner import SCons.Util @@ -47,20 +46,22 @@ class SConsCPPScanner(SCons.cpp.PreProcessor): def __init__(self, *args, **kw): SCons.cpp.PreProcessor.__init__(self, *args, **kw) self.missing = [] - self._known_paths = tuple() + self._known_paths = [] def initialize_result(self, fname): self.result = SCons.Util.UniqueList([fname]) def find_include_file(self, t): keyword, quote, fname = t - result = SCons.Node.FS.find_file( - fname, self._known_paths + self.searchpath[quote]) + paths = tuple(self._known_paths) + self.searchpath[quote] + if quote == '"': + paths = (self.current_file.dir, ) + paths + result = SCons.Node.FS.find_file(fname, paths) if result: result_path = result.get_abspath() for p in self.searchpath[quote]: if result_path.startswith(p.get_abspath()): - self._known_paths.add(p) + self._known_paths.append(p) break else: self.missing.append((fname, self.current_file)) -- cgit v0.12 From 04bf778cb8749845cf1f21b6f06a742dac01b6ad Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 21 Nov 2018 00:43:40 +0200 Subject: Stop handling "define" and "undef" when condition fails --- src/engine/SCons/cpp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index e5f7345..4b921e4 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -447,7 +447,7 @@ class PreProcessor(object): d = self.dispatch_table p = self.stack[-1] if self.stack else self.default_table - for k in ('import', 'include', 'include_next'): + for k in ('import', 'include', 'include_next', 'define', 'undef'): d[k] = p[k] def stop_handling_includes(self, t=None): @@ -463,6 +463,8 @@ class PreProcessor(object): d['import'] = self.do_nothing d['include'] = self.do_nothing d['include_next'] = self.do_nothing + d['define'] = self.do_nothing + d['undef'] = self.do_nothing # Default methods for handling all of the preprocessor directives. # (Note that what actually gets called for a given directive at any -- cgit v0.12 From fa2df78952debfb465515235ff9d0227aa140316 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Aug 2019 21:09:15 +0300 Subject: Fix incorrect calculation of includes depth --- src/engine/SCons/cpp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 40d4a0c..7535dbb 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -218,7 +218,7 @@ class FunctionEvaluator(object): parts = [] for s in self.expansion: - if not s in self.args: + if s not in self.args: s = repr(s) parts.append(s) statement = ' + '.join(parts) @@ -580,13 +580,13 @@ class PreProcessor(object): if self.depth != -1: current_depth = 0 for t in self.tuples: - if t[0] != "scons_current_file": + if t[0] == "scons_current_file": current_depth += 1 if current_depth >= self.depth: return new_tuples = [('scons_current_file', include_file)] + \ - self.tupleize(self.read_file(include_file)) + \ + self.tupleize_file(include_file) + \ [('scons_current_file', self.current_file)] self.tuples[:] = new_tuples + self.tuples -- cgit v0.12 From b9b4a8f887868aeb4242816d1a88dc07f9435c20 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Mon, 26 Aug 2019 21:46:09 +0300 Subject: Remove tuples caching, save memory --- src/engine/SCons/cpp.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py index 7535dbb..f55ea21 100644 --- a/src/engine/SCons/cpp.py +++ b/src/engine/SCons/cpp.py @@ -243,8 +243,6 @@ function_arg_separator = re.compile(r',\s*') class PreProcessor(object): - FILE_TUPLES_CACHE = None - """ The main workhorse class for handling C pre-processing. """ @@ -305,7 +303,7 @@ class PreProcessor(object): This is the main internal entry point. """ - return self._process_tuples(self.tupleize_file(file), file) + return self._process_tuples(self.tupleize(self.read_file(file)), file) def process_contents(self, contents): """ @@ -330,15 +328,6 @@ class PreProcessor(object): self.dispatch_table[t[0]](t) return self.finalize_result(file) - def tupleize_file(self, file): - if not PreProcessor.FILE_TUPLES_CACHE: - PreProcessor.FILE_TUPLES_CACHE = {} - file_tuples = PreProcessor.FILE_TUPLES_CACHE.get(file) - if not file_tuples: - file_tuples = self._parse_tuples(self.read_file(file)) - PreProcessor.FILE_TUPLES_CACHE[file] = file_tuples - return self._match_tuples(file_tuples) - def tupleize(self, contents): """ Turns the contents of a file into a list of easily-processed @@ -586,7 +575,7 @@ class PreProcessor(object): return new_tuples = [('scons_current_file', include_file)] + \ - self.tupleize_file(include_file) + \ + self.tupleize(self.read_file(include_file)) + \ [('scons_current_file', self.current_file)] self.tuples[:] = new_tuples + self.tuples -- cgit v0.12 From 8bc187941d5ea8a11200432c099802eefa7a2356 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 12 Feb 2020 22:01:05 +0200 Subject: Update changes with CConditionalScanner and improved virtual C Pre-Processor --- src/CHANGES.txt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index b3a7422..6b6b7fd 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -11,17 +11,17 @@ NOTE: Please include a reference to any Issues resolved by your changes in the b RELEASE VERSION/DATE TO BE FILLED IN LATER From William Deegan: - - Fix broken clang + MSVC 2019 combination by using MSVC configuration logic to + - Fix broken clang + MSVC 2019 combination by using MSVC configuration logic to propagate'VCINSTALLDIR' and 'VCToolsInstallDir' which clang tools use to locate header files and libraries from MSVC install. (Fixes GH Issue #3480) - - Added C:\msys64\mingw64\bin to default mingw and clang windows PATH's. This + - Added C:\msys64\mingw64\bin to default mingw and clang windows PATH's. This is a reasonable default and also aligns with changes in Appveyor's VS2019 image. - Drop support for Python 2.7. SCons will be Python 3.5+ going forward. - Change SCons.Node.ValueWithMemo to consider any name passed when memoizing Value() nodes - Fix Github Issue #3550 - When using Substfile() with a value like Z:\mongo\build\install\bin the implementation using re.sub() would end up interpreting the string and finding regex escape characters where it should have been simply replacing existing text. Switched to use string.replace(). - + From Jeremy Elson: - Updated design doc to use the correct syntax for Depends() @@ -32,6 +32,17 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Added support for explicitly passing a name when creating Value() nodes. This may be useful when the value can't be converted to a string or if having a name is otherwise desirable. + From Ivan Kravets, PlatformIO + - New conditional C Scanner (`SCons.Scanner.C.CConditionalScanner()`) + which interprets C/C Preprocessor conditional syntax (#ifdef, #if, #else, + #elif, #define, etc.) + - Improvements for virtual C Pre-Processor: + * Handle UNSIGNED LONG and LONG numeric constants in DEC (keep support for HEX) + * Skip unrecognized directives, such as `#if( defined ...)` + * Ignore `#include DYNAMIC_INCLUDE` directive that depends on a dynamic + macro which is not located in a state TABLE. + * Cleanup CPP expressions before evaluating (strip comments, carriage returns) + From Andrew Morrow: - Fix Issue #3469 - Fixed improper reuse of temporary and compiled files by Configure when changing the order and/or number of tests. This is done by using the hash of the generated temporary files @@ -102,7 +113,7 @@ RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 (e.g. memmove) were incorrectly recognized as not available. From Jakub Kulik - - Fix stacktrace when using SCons with Python 3.5+ and SunOS/Solaris related tools. + - Fix stacktrace when using SCons with Python 3.5+ and SunOS/Solaris related tools. From Philipp Maierhöfer: - Avoid crash with UnicodeDecodeError on Python 3 when a Latex log file in -- cgit v0.12 From fe2680ddd07a4bc7c6064e7193f4979060e178ba Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Thu, 13 Feb 2020 17:33:47 +0200 Subject: Isolate CConditionalScanner from CScanner --- src/engine/SCons/Scanner/C.py | 142 +++++++++++++++++++++++++++++++----------- 1 file changed, 107 insertions(+), 35 deletions(-) diff --git a/src/engine/SCons/Scanner/C.py b/src/engine/SCons/Scanner/C.py index 37bf244..29cc396 100644 --- a/src/engine/SCons/Scanner/C.py +++ b/src/engine/SCons/Scanner/C.py @@ -46,27 +46,16 @@ class SConsCPPScanner(SCons.cpp.PreProcessor): def __init__(self, *args, **kw): SCons.cpp.PreProcessor.__init__(self, *args, **kw) self.missing = [] - self._known_paths = [] - def initialize_result(self, fname): self.result = SCons.Util.UniqueList([fname]) - + def finalize_result(self, fname): + return self.result[1:] def find_include_file(self, t): keyword, quote, fname = t - paths = tuple(self._known_paths) + self.searchpath[quote] - if quote == '"': - paths = (self.current_file.dir, ) + paths - result = SCons.Node.FS.find_file(fname, paths) - if result: - result_path = result.get_abspath() - for p in self.searchpath[quote]: - if result_path.startswith(p.get_abspath()): - self._known_paths.append(p) - break - else: + result = SCons.Node.FS.find_file(fname, self.searchpath[quote]) + if not result: self.missing.append((fname, self.current_file)) return result - def read_file(self, file): try: with open(str(file.rfile())) as fp: @@ -83,11 +72,7 @@ def dictify_CPPDEFINES(env): result = {} for c in cppdefines: if SCons.Util.is_Sequence(c): - # handle tuple with 1 item (e.g. tuple("DEFINE", )) - if len(c) > 1: - result[c[0]] = c[1] - else: - result[c[0]] = None + result[c[0]] = c[1] else: result[c] = None return result @@ -107,11 +92,10 @@ class SConsCPPScannerWrapper(object): def __init__(self, name, variable): self.name = name self.path = SCons.Scanner.FindPathDirs(variable) - def __call__(self, node, env, path = (), depth=-1): + def __call__(self, node, env, path = ()): cpp = SConsCPPScanner(current = node.get_dir(), cpppath = path, - dict = dictify_CPPDEFINES(env), - depth = depth) + dict = dictify_CPPDEFINES(env)) result = cpp(node) for included, includer in cpp.missing: fmt = "No dependency generated for file: %s (included from: %s) -- file not found" @@ -124,26 +108,114 @@ class SConsCPPScannerWrapper(object): def select(self, node): return self +def CScanner(): + """Return a prototype Scanner instance for scanning source files + that use the C pre-processor""" + + # Here's how we would (or might) use the CPP scanner code above that + # knows how to evaluate #if/#ifdef/#else/#elif lines when searching + # for #includes. This is commented out for now until we add the + # right configurability to let users pick between the scanners. + #return SConsCPPScannerWrapper("CScanner", "CPPPATH") -def CConditionalScanner(): + cs = SCons.Scanner.ClassicCPP("CScanner", + "$CPPSUFFIXES", + "CPPPATH", + '^[ \t]*#[ \t]*(?:include|import)[ \t]*(<|")([^>"]+)(>|")') + return cs + + +# +# ConditionalScanner +# + + +class SConsCPPConditionalScanner(SCons.cpp.PreProcessor): """ - Return an advanced conditional Scanner instance for scanning source files + SCons-specific subclass of the cpp.py module's processing. - Interprets C/C++ Preprocessor conditional syntax - (#ifdef, #if, defined, #else, #elif, etc.). + We subclass this so that: 1) we can deal with files represented + by Nodes, not strings; 2) we can keep track of the files that are + missing. """ - return SConsCPPScannerWrapper("CScanner", "CPPPATH") + def __init__(self, *args, **kw): + SCons.cpp.PreProcessor.__init__(self, *args, **kw) + self.missing = [] + self._known_paths = [] + + def initialize_result(self, fname): + self.result = SCons.Util.UniqueList([fname]) -def CScanner(): - """Return a simplified classic Scanner instance for scanning source files + def find_include_file(self, t): + keyword, quote, fname = t + paths = tuple(self._known_paths) + self.searchpath[quote] + if quote == '"': + paths = (self.current_file.dir,) + paths + result = SCons.Node.FS.find_file(fname, paths) + if result: + result_path = result.get_abspath() + for p in self.searchpath[quote]: + if result_path.startswith(p.get_abspath()): + self._known_paths.append(p) + break + else: + self.missing.append((fname, self.current_file)) + return result + + def read_file(self, file): + try: + with open(str(file.rfile())) as fp: + return fp.read() + except EnvironmentError: + self.missing.append((file, self.current_file)) + return "" + + +class SConsCPPConditionalScannerWrapper(object): + """ + The SCons wrapper around a cpp.py scanner. + + This is the actual glue between the calling conventions of generic + SCons scanners, and the (subclass of) cpp.py class that knows how + to look for #include lines with reasonably real C-preprocessor-like + evaluation of #if/#ifdef/#else/#elif lines. + """ - Takes into account the type of bracketing used to include the file, and - uses classic CPP rules for searching for the files based on the bracketing. + def __init__(self, name, variable): + self.name = name + self.path = SCons.Scanner.FindPathDirs(variable) + + def __call__(self, node, env, path=(), depth=-1): + cpp = SConsCPPConditionalScanner( + current=node.get_dir(), + cpppath=path, + dict=dictify_CPPDEFINES(env), + depth=depth, + ) + result = cpp(node) + for included, includer in cpp.missing: + fmt = "No dependency generated for file: %s (included from: %s) -- file not found" + SCons.Warnings.warn( + SCons.Warnings.DependencyWarning, fmt % (included, includer) + ) + return result + + def recurse_nodes(self, nodes): + return nodes + + def select(self, node): + return self + + +def CConditionalScanner(): + """ + Return an advanced conditional Scanner instance for scanning source files + + Interprets C/C++ Preprocessor conditional syntax + (#ifdef, #if, defined, #else, #elif, etc.). """ - return SCons.Scanner.ClassicCPP( - "CScanner", "$CPPSUFFIXES", "CPPPATH", - '^[ \t]*#[ \t]*(?:include|import)[ \t]*(<|")([^>"]+)(>|")') + return SConsCPPConditionalScannerWrapper("CConditionalScanner", "CPPPATH") # Local Variables: -- cgit v0.12