Merge pull request #4629 from jcbrill/jbrill-gh4623-int

Fix Issue with SConsCPPConditionalScanner misinterpreting ifdefs suffixed with L or UL [Alternate Implementation]

4 files changed, 370 insertions, 17 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 098bba3..b75f856 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -59,6 +59,22 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       and nodes from the MSVSProject return values.  Resolves #4613.
     - MSVS: Remove the platform specification (i.e., platform = 'win32') from select
       test script environments.  The platform specification appears superfluous.
+    - SCons C preprocessor changes:
+      - Preserve literals that contain valid integer substring specifications.
+        Previously, the integer suffix could be stripped from a symbol that contained
+        an integer and suffix substring.
+      - Update the optional integer suffixes to include the z|Z and wb|WB suffixes.
+      - Update the optional integer suffixes to include support for alternate
+        orderings of unsigned with long or long long as defined in the c/cpp
+        grammar.
+      - Update the optional integer suffixes for case insensitive specifications as
+        defined in the c/cpp grammar.
+      - Add support for binary integer constants.
+      - Add support for octal integer constants.  Previously, octal integers were
+        evaluated as decimal integers.  A literal zero (0) is treated as an octal
+        number.
+      - Change the attempted conversion of a define expansion from using int() to
+        a constant expression evaluation.
 
   From Alex James:
     - On Darwin, PermissionErrors are now handled while trying to access
diff --git a/RELEASE.txt b/RELEASE.txt
index 6ba7fc0..2cb6638 100644
--- a/RELEASE.txt
+++ b/RELEASE.txt
@@ -56,12 +56,23 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
   in the projects argument list rather than manually removing solution file
   names and nodes from the MSVSProject return values.
 
+- SCons C preprocessor:
+  - Update the optional integer suffixes to include the z|Z and wb|WB
+    suffixes.
+  - Add support for binary integer constants.
+  - Add support for octal integer constants.  Previously, octal integers
+    were evaluated as decimal integers.  A literal zero (0) is treated as an
+    octal number.
+  - Change the method for attempted conversion of a define expansion value
+    to an integer from a literal to a constant expression evaluation.
+
 - Add a tag to each CacheDir to let systems ignore backing it up
   (per https://bford.info/cachedir/). Update the way a CacheDir
   is created, since it now has to create two files.
 
 FIXES
 -----
+
 - PackageVariable now does what the documentation always said it does
   if the variable is used on the command line with one of the enabling
   string as the value: the variable's default value is produced (previously
@@ -104,6 +115,17 @@ FIXES
   solution file.  User-specified project GUIDs should now be correctly
   written to the solution file.
 
+- SCons C preprocessor: Preserve literals that contain valid integer
+  substring specifications.  Previously, the integer suffix could be
+  stripped from a symbol that contained an integer and suffix substring.
+
+- SCons C preprocessor: Update the optional integer suffixes to include
+  support for the alternate orderings of unsigned with long or long long as
+  defined in the c/cpp grammar.
+
+- SCons C preprocessor: Update the optional integer suffixes for case
+  insensitive specifications as defined in the c/cpp grammar.
+
 - Fix nasm test for missing include file, cleanup.
 
 - Skip running a few validation tests if the user is root and the test is
diff --git a/SCons/cpp.py b/SCons/cpp.py
index 1093ae2..63e49ae 100644
--- a/SCons/cpp.py
+++ b/SCons/cpp.py
@@ -146,13 +146,27 @@ expr = '|'.join(map(re.escape, l))
 # ...and compile the expression.
 CPP_to_Python_Ops_Expression = re.compile(expr)
 
+# integer specifications
+int_suffix_opt = r'(?:[uU](?:l{1,2}|L{1,2}|[zZ]|wb|WB)?|(?:l{1,2}|L{1,2}|[zZ]|wb|WB)[uU]?)?'
+
+hex_integer = fr'(0[xX][0-9A-Fa-f]+){int_suffix_opt}'
+bin_integer = fr'(0[bB][01]+){int_suffix_opt}'
+oct_integer = fr'(0[0-7]*){int_suffix_opt}'
+dec_integer = fr'([1-9][0-9]*){int_suffix_opt}'
+
+int_boundary = r'[a-zA-Z0-9_]'
+int_neg_lookbehind = fr'(?<!{int_boundary})'
+int_neg_lookahead = fr'(?!{int_boundary})'
+
 # A separate list of expressions to be evaluated and substituted
 # sequentially, not all at once.
 CPP_to_Python_Eval_List = [
-    [r'defined\s+(\w+)',                 '"\\1" in __dict__'],
-    [r'defined\s*\((\w+)\)',             '"\\1" in __dict__'],
-    [r'(0x[0-9A-Fa-f]+)(?:L|UL)?',  '\\1'],
-    [r'(\d+)(?:L|UL)?',  '\\1'],
+    [r'defined\s+(\w+)', '"\\1" in __dict__'],
+    [r'defined\s*\((\w+)\)', '"\\1" in __dict__'],
+    [fr'{int_neg_lookbehind}{hex_integer}{int_neg_lookahead}', '\\1'],
+    [fr'{int_neg_lookbehind}{bin_integer}{int_neg_lookahead}', '\\1'],
+    [fr'{int_neg_lookbehind}{oct_integer}{int_neg_lookahead}', '0o\\1'],
+    [fr'{int_neg_lookbehind}{dec_integer}{int_neg_lookahead}', '\\1'],
 ]
 
 # Replace the string representations of the regular expressions in the
@@ -250,6 +264,9 @@ class PreProcessor:
         self.cpp_namespace = dict.copy()
         self.cpp_namespace['__dict__'] = self.cpp_namespace
 
+        # Namespace for constant expression evaluation (literals only)
+        self.constant_expression_namespace = {}
+
         # Return all includes without resolving
         if all:
             self.do_include = self.all_include
@@ -369,6 +386,25 @@ class PreProcessor:
     def scons_current_file(self, t) -> None:
         self.current_file = t[1]
 
+    def eval_constant_expression(self, s):
+        """
+        Evaluates a C preprocessor expression.
+
+        This is done by converting it to a Python equivalent and
+        eval()ing it in the C preprocessor namespace we use to
+        track #define values.
+
+        Returns None if the eval() result is not an integer.
+        """
+        s = CPP_to_Python(s)
+        try:
+            rval = eval(s, self.constant_expression_namespace)
+        except (NameError, TypeError, SyntaxError) as e:
+            rval = None
+        if not isinstance(rval, int):
+            rval = None
+        return rval
+
     def eval_expression(self, t):
         """
         Evaluates a C preprocessor expression.
@@ -507,9 +543,11 @@ class PreProcessor:
         Default handling of a #define line.
         """
         _, name, args, expansion = t
-        try:
-            expansion = int(expansion)
-        except (TypeError, ValueError):
+
+        rval = self.eval_constant_expression(expansion)
+        if rval is not None:
+            expansion = rval
+        else:
             # handle "defined" chain "! (defined (A) || defined (B)" ...
             if "defined " in expansion:
                 self.cpp_namespace[name] = self.eval_expression(t[2:])
diff --git a/SCons/cppTests.py b/SCons/cppTests.py
index 85f01b7..8b079ec 100644
--- a/SCons/cppTests.py
+++ b/SCons/cppTests.py
@@ -151,7 +151,7 @@ if_defined_input = """
 #include <file18-yes>
 #endif
 
-#if ! (defined (DEFINED_A) || defined (DEFINED_B)
+#if ! (defined (DEFINED_A) || defined (DEFINED_B))
 #include <file19-no>
 #else
 #include <file19-yes>
@@ -236,11 +236,189 @@ expression_input = """
 #include <file30-no>
 #endif
 
-#if	123456789UL || 0x13L
-#include <file301-yes>
+#if	(123456789UL || 0x13L)
+#include <file301or-yes>
 #else
-#include <file301-no>
+#include <file301or-no>
 #endif
+
+#if	(123456789UL && 0x0LU)
+#include <file301and-yes>
+#else
+#include <file301and-no>
+#endif
+
+#if	123456789UL
+#include <file301ul-yes>
+#else
+#include <file301ul-no>
+#endif
+
+#if 1234U
+#include <file301u-yes>
+#else
+#include <file301u-no>
+#endif
+
+#if 1234L
+#include <file301l-yes>
+#else
+#include <file301l-no>
+#endif
+
+#if 1234ULL
+#include <file301ull-yes>
+#else
+#include <file301ull-no>
+#endif
+
+#define X1234UL 1
+#if X1234UL
+#include <file302-yes>
+#else
+#include <file302-no>
+#endif
+
+#define X1234U 1
+#if X1234U
+#include <file303-yes>
+#else
+#include <file303-no>
+#endif
+
+#define X1234L 1
+#if X1234L
+#include <file304-yes>
+#else
+#include <file304-no>
+#endif
+
+#define X1234ULL 1
+#if X1234ULL
+#include <file305-yes>
+#else
+#include <file305-no>
+#endif
+
+#define DEC0 0
+#define HEX0 0x0
+#define HEXF 0xF
+
+#if DEC0
+#include <file401-yes>
+#else
+#include <file401-no>
+#endif
+
+#if ! DEC0
+#include <file402-yes>
+#else
+#include <file402-no>
+#endif
+
+#if (DEC0)
+#include <file403-yes>
+#else
+#include <file403-no>
+#endif
+
+#if !(DEC0)
+#include <file404-yes>
+#else
+#include <file404-no>
+#endif
+
+#if HEX0
+#include <file411-yes>
+#else
+#include <file411-no>
+#endif
+
+#if ! HEX0
+#include <file412-yes>
+#else
+#include <file412-no>
+#endif
+
+#if (HEX0)
+#include <file413-yes>
+#else
+#include <file413-no>
+#endif
+
+#if !(HEX0)
+#include <file414-yes>
+#else
+#include <file414-no>
+#endif
+
+#if HEXF
+#include <file421-yes>
+#else
+#include <file421-no>
+#endif
+
+#if ! HEXF
+#include <file422-yes>
+#else
+#include <file422-no>
+#endif
+
+#if (HEXF)
+#include <file423-yes>
+#else
+#include <file423-no>
+#endif
+
+#if !(HEXF)
+#include <file424-yes>
+#else
+#include <file424-no>
+#endif
+
+#if defined(DEC0) && (DEC0 & 0x1)
+#include <file431-yes>
+#else
+#include <file431-no>
+#endif
+
+#if !(defined(HEXF) && (HEXF & 0x1))
+#include <file432-yes>
+#else
+#include <file432-no>
+#endif
+
+#define X2345ULL 1
+#if !(X2345ULL > 4567ull)
+#include <file501-yes>
+#else
+#include <file501-no>
+#endif
+
+#if !0ull
+#include <file502-yes>
+#else
+#include <file502-no>
+#endif
+
+#define X0U 0U
+#if X0U
+#include <file503-yes>
+#else
+#include <file503-no>
+#endif
+
+#define XF1 (0x0U & 0x1U)
+#if XF1
+#include <file504-yes>
+#else
+#include <file504-no>
+#endif
+
+#define ABC00 0U
+#define ABC01 1U
+#define ABC_(a, b) ABC##a##b
+#define ABC ABC_(ZERO, ZERO)
 """
 
 
@@ -303,8 +481,8 @@ macro_function_input = """
 #define	FUNC39a(x0, y0)	FILE39
 #define	FUNC40a(x0, y0)	FILE40
 
-#define	FUNC39b(x1, y2)	FUNC39a(x1, y1)
-#define	FUNC40b(x1, y2)	FUNC40a(x1, y1)
+#define	FUNC39b(x1, y1)	FUNC39a(x1, y1)
+#define	FUNC40b(x1, y1)	FUNC40a(x1, y1)
 
 #define	FUNC39c(x2, y2)	FUNC39b(x2, y2)
 #define	FUNC40c(x2, y2)	FUNC40b(x2, y2)
@@ -480,7 +658,12 @@ class cppTestCase(unittest.TestCase):
         """Test #if with arithmetic expressions"""
         expect = self.expression_expect
         result = self.cpp.process_contents(expression_input)
-        assert expect == result, (expect, result)
+        if expect != result:
+            for e,r in zip(expect, result):
+                if e != r:
+                    print("ERROR->",end="")
+                print(f"{e}: {r}")
+        assert expect == result, f"\nexpect:{expect}\nresult:{result}"
 
     def test_undef(self) -> None:
         """Test #undef handling"""
@@ -588,7 +771,41 @@ class PreProcessorTestCase(cppAllTestCase):
         ('include', '<', 'file28-yes'),
         ('include', '"', 'file29-yes'),
         ('include', '<', 'file30-yes'),
-        ('include', '<', 'file301-yes'),
+
+        ('include', '<', 'file301or-yes'),
+        ('include', '<', 'file301and-no'),
+        ('include', '<', 'file301ul-yes'),
+        ('include', '<', 'file301u-yes'),
+        ('include', '<', 'file301l-yes'),
+        ('include', '<', 'file301ull-yes'),
+
+        ('include', '<', 'file302-yes'),
+        ('include', '<', 'file303-yes'),
+        ('include', '<', 'file304-yes'),
+        ('include', '<', 'file305-yes'),
+
+        ('include', '<', 'file401-no'),
+        ('include', '<', 'file402-yes'),
+        ('include', '<', 'file403-no'),
+        ('include', '<', 'file404-yes'),
+
+        ('include', '<', 'file411-no'),
+        ('include', '<', 'file412-yes'),
+        ('include', '<', 'file413-no'),
+        ('include', '<', 'file414-yes'),
+
+        ('include', '<', 'file421-yes'),
+        ('include', '<', 'file422-no'),
+        ('include', '<', 'file423-yes'),
+        ('include', '<', 'file424-no'),
+
+        ('include', '<', 'file431-no'),
+        ('include', '<', 'file432-no'),
+
+        ('include', '<', 'file501-yes'),
+        ('include', '<', 'file502-yes'),
+        ('include', '<', 'file503-no'),
+        ('include', '<', 'file504-no'),
     ]
 
     undef_expect = [
@@ -717,8 +934,68 @@ class DumbPreProcessorTestCase(cppAllTestCase):
         ('include', '"', 'file29-yes'),
         ('include', '<', 'file30-yes'),
         ('include', '<', 'file30-no'),
-        ('include', '<', 'file301-yes'),
-        ('include', '<', 'file301-no'),
+
+        ('include', '<', 'file301or-yes'),
+        ('include', '<', 'file301or-no'),
+        ('include', '<', 'file301and-yes'),
+        ('include', '<', 'file301and-no'),
+        ('include', '<', 'file301ul-yes'),
+        ('include', '<', 'file301ul-no'),
+        ('include', '<', 'file301u-yes'),
+        ('include', '<', 'file301u-no'),
+        ('include', '<', 'file301l-yes'),
+        ('include', '<', 'file301l-no'),
+        ('include', '<', 'file301ull-yes'),
+        ('include', '<', 'file301ull-no'),
+
+        ('include', '<', 'file302-yes'),
+        ('include', '<', 'file302-no'),
+        ('include', '<', 'file303-yes'),
+        ('include', '<', 'file303-no'),
+        ('include', '<', 'file304-yes'),
+        ('include', '<', 'file304-no'),
+        ('include', '<', 'file305-yes'),
+        ('include', '<', 'file305-no'),
+
+        ('include', '<', 'file401-yes'),
+        ('include', '<', 'file401-no'),
+        ('include', '<', 'file402-yes'),
+        ('include', '<', 'file402-no'),
+        ('include', '<', 'file403-yes'),
+        ('include', '<', 'file403-no'),
+        ('include', '<', 'file404-yes'),
+        ('include', '<', 'file404-no'),
+
+        ('include', '<', 'file411-yes'),
+        ('include', '<', 'file411-no'),
+        ('include', '<', 'file412-yes'),
+        ('include', '<', 'file412-no'),
+        ('include', '<', 'file413-yes'),
+        ('include', '<', 'file413-no'),
+        ('include', '<', 'file414-yes'),
+        ('include', '<', 'file414-no'),
+
+        ('include', '<', 'file421-yes'),
+        ('include', '<', 'file421-no'),
+        ('include', '<', 'file422-yes'),
+        ('include', '<', 'file422-no'),
+        ('include', '<', 'file423-yes'),
+        ('include', '<', 'file423-no'),
+        ('include', '<', 'file424-yes'),
+        ('include', '<', 'file424-no'),
+        ('include', '<', 'file431-yes'),
+        ('include', '<', 'file431-no'),
+        ('include', '<', 'file432-yes'),
+        ('include', '<', 'file432-no'),
+
+        ('include', '<', 'file501-yes'),
+        ('include', '<', 'file501-no'),
+        ('include', '<', 'file502-yes'),
+        ('include', '<', 'file502-no'),
+        ('include', '<', 'file503-yes'),
+        ('include', '<', 'file503-no'),
+        ('include', '<', 'file504-yes'),
+        ('include', '<', 'file504-no'),
     ]
 
     undef_expect = [