bpo-40612: Fix SyntaxError edge cases in traceback formatting (GH-20072)

This fixes both the traceback.py module and the C code for formatting syntax errors (in Python/pythonrun.c). They now both consistently do the following:

- Suppress caret if it points left of text
- Allow caret pointing just past end of line
- If caret points past end of line, clip to *just* past end of line

The syntax error formatting code in traceback.py was mostly rewritten; small, subtle changes were applied to the C code in pythonrun.c.

There's still a difference when the text contains embedded newlines. Neither handles these very well, and I don't think the case occurs in practice.

Automerge-Triggered-By: @gvanrossum

1 files changed, 26 insertions, 8 deletions

diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index 7361d09..f9a5f2f 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -58,13 +58,13 @@ class TracebackCases(unittest.TestCase):
                                         SyntaxError)
         self.assertIn("^", err[2]) # third line has caret
         self.assertEqual(err[2].count('\n'), 1)   # and no additional newline
-        self.assertEqual(err[1].find("+"), err[2].find("^"))  # in the right place
+        self.assertEqual(err[1].find("+") + 1, err[2].find("^"))  # in the right place
 
         err = self.get_exception_format(self.syntax_error_with_caret_non_ascii,
                                         SyntaxError)
         self.assertIn("^", err[2]) # third line has caret
         self.assertEqual(err[2].count('\n'), 1)   # and no additional newline
-        self.assertEqual(err[1].find("+"), err[2].find("^"))  # in the right place
+        self.assertEqual(err[1].find("+") + 1, err[2].find("^"))  # in the right place
 
     def test_nocaret(self):
         exc = SyntaxError("error", ("x.py", 23, None, "bad syntax"))
@@ -78,14 +78,13 @@ class TracebackCases(unittest.TestCase):
         self.assertEqual(len(err), 4)
         self.assertEqual(err[1].strip(), "print(2)")
         self.assertIn("^", err[2])
-        self.assertEqual(err[1].find(")"), err[2].find("^"))
+        self.assertEqual(err[1].find(")") + 1, err[2].find("^"))
 
+        # No caret for "unexpected indent"
         err = self.get_exception_format(self.syntax_error_bad_indentation2,
                                         IndentationError)
-        self.assertEqual(len(err), 4)
+        self.assertEqual(len(err), 3)
         self.assertEqual(err[1].strip(), "print(2)")
-        self.assertIn("^", err[2])
-        self.assertEqual(err[1].find("p"), err[2].find("^"))
 
     def test_base_exception(self):
         # Test that exceptions derived from BaseException are formatted right
@@ -656,7 +655,7 @@ class BaseExceptionReportingTests:
         self.assertIn('inner_raise() # Marker', blocks[2])
         self.check_zero_div(blocks[2])
 
-    @support.skip_if_new_parser("Pegen is arguably better here, so no need to fix this")
+    @unittest.skipIf(support.use_old_parser(), "Pegen is arguably better here, so no need to fix this")
     def test_syntax_error_offset_at_eol(self):
         # See #10186.
         def e():
@@ -666,7 +665,7 @@ class BaseExceptionReportingTests:
         def e():
             exec("x = 5 | 4 |")
         msg = self.get_report(e).splitlines()
-        self.assertEqual(msg[-2], '              ^')
+        self.assertEqual(msg[-2], '               ^')
 
     def test_message_none(self):
         # A message that looks like "None" should not be treated specially
@@ -679,6 +678,25 @@ class BaseExceptionReportingTests:
         err = self.get_report(Exception(''))
         self.assertIn('Exception\n', err)
 
+    def test_syntax_error_various_offsets(self):
+        for offset in range(-5, 10):
+            for add in [0, 2]:
+                text = " "*add + "text%d" % offset
+                expected = ['  File "file.py", line 1']
+                if offset < 1:
+                    expected.append("    %s" % text.lstrip())
+                elif offset <= 6:
+                    expected.append("    %s" % text.lstrip())
+                    expected.append("    %s^" % (" "*(offset-1)))
+                else:
+                    expected.append("    %s" % text.lstrip())
+                    expected.append("    %s^" % (" "*5))
+                expected.append("SyntaxError: msg")
+                expected.append("")
+                err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text)))
+                exp = "\n".join(expected)
+                self.assertEqual(exp, err)
+
 
 class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase):
     #