bpo-46521: Fix codeop to use a new partial-input mode of the parser (GH-31010)

1 files changed, 11 insertions, 43 deletions

diff --git a/Lib/codeop.py b/Lib/codeop.py
index 6b56be4..568e9bb 100644
--- a/Lib/codeop.py
+++ b/Lib/codeop.py
@@ -10,30 +10,6 @@ and:
   syntax error (OverflowError and ValueError can be produced by
   malformed literals).
 
-Approach:
-
-First, check if the source consists entirely of blank lines and
-comments; if so, replace it with 'pass', because the built-in
-parser doesn't always do the right thing for these.
-
-Compile three times: as is, with \n, and with \n\n appended.  If it
-compiles as is, it's complete.  If it compiles with one \n appended,
-we expect more.  If it doesn't compile either way, we compare the
-error we get when compiling with \n or \n\n appended.  If the errors
-are the same, the code is broken.  But if the errors are different, we
-expect more.  Not intuitive; not even guaranteed to hold in future
-releases; but this matches the compiler's behavior from Python 1.4
-through 2.2, at least.
-
-Caveat:
-
-It is possible (but not likely) that the parser stops parsing with a
-successful outcome before reaching the end of the source; in this
-case, trailing symbols may be ignored instead of causing an error.
-For example, a backslash followed by two newlines may be followed by
-arbitrary garbage.  This will be fixed once the API for the parser is
-better.
-
 The two interfaces are:
 
 compile_command(source, filename, symbol):
@@ -64,7 +40,11 @@ _features = [getattr(__future__, fname)
 
 __all__ = ["compile_command", "Compile", "CommandCompiler"]
 
-PyCF_DONT_IMPLY_DEDENT = 0x200          # Matches pythonrun.h.
+# The following flags match the values from Include/cpython/compile.h
+# Caveat emptor: These flags are undocumented on purpose and depending
+# on their effect outside the standard library is **unsupported**.
+PyCF_DONT_IMPLY_DEDENT = 0x200          
+PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
 
 def _maybe_compile(compiler, source, filename, symbol):
     # Check for source consisting of only blank lines and comments.
@@ -86,24 +66,12 @@ def _maybe_compile(compiler, source, filename, symbol):
     with warnings.catch_warnings():
         warnings.simplefilter("error")
 
-        code1 = err1 = err2 = None
-        try:
-            code1 = compiler(source + "\n", filename, symbol)
-        except SyntaxError as e:
-            err1 = e
-
         try:
-            code2 = compiler(source + "\n\n", filename, symbol)
+            compiler(source + "\n", filename, symbol)
         except SyntaxError as e:
-            err2 = e
-
-    try:
-        if not code1 and _is_syntax_error(err1, err2):
-            raise err1
-        else:
-            return None
-    finally:
-        err1 = err2 = None
+            if "incomplete input" in str(e):
+                return None
+            raise
 
 def _is_syntax_error(err1, err2):
     rep1 = repr(err1)
@@ -115,7 +83,7 @@ def _is_syntax_error(err1, err2):
     return False
 
 def _compile(source, filename, symbol):
-    return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT)
+    return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT)
 
 def compile_command(source, filename="<input>", symbol="single"):
     r"""Compile a command and determine whether it is incomplete.
@@ -144,7 +112,7 @@ class Compile:
     statement, it "remembers" and compiles all subsequent program texts
     with the statement in force."""
     def __init__(self):
-        self.flags = PyCF_DONT_IMPLY_DEDENT
+        self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
 
     def __call__(self, source, filename, symbol):
         codeob = compile(source, filename, symbol, self.flags, True)