bpo-40334: Catch E_EOF error, when the tokenizer returns ERRORTOKEN (GH-19743)

An E_EOF error was only being caught after the parser exited before this commit. There are some cases though, where the tokenizer returns ERRORTOKEN *and* has set an E_EOF error (like when EOF directly follows a line continuation character) which weren't correctly handled before.

1 files changed, 0 insertions, 2 deletions

diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
index f806578..9ef8eb1 100644
--- a/Lib/test/test_eof.py
+++ b/Lib/test/test_eof.py
@@ -26,7 +26,6 @@ class EOFTestCase(unittest.TestCase):
         else:
             raise support.TestFailed
 
-    @support.skip_if_new_parser("TODO for PEG -- fails with new parser")
     def test_line_continuation_EOF(self):
         """A continuation at the end of input must be an error; bpo2180."""
         expect = 'unexpected EOF while parsing (<string>, line 1)'
@@ -37,7 +36,6 @@ class EOFTestCase(unittest.TestCase):
             exec('\\')
         self.assertEqual(str(excinfo.exception), expect)
 
-    @unittest.skip("TODO for PEG -- fails even with old parser now")
     @unittest.skipIf(not sys.executable, "sys.executable required")
     def test_line_continuation_EOF_from_file_bpo2180(self):
         """Ensure tok_nextc() does not add too many ending newlines."""