- Issue #11250: Back port fix from 3.3 branch, so that 2to3 can handle files

  with line feeds.  This was ported from the sandbox to the 3.3 branch, but
  didn't make it into 3.2.

- Re-enable lib2to3's test_parser.py tests, though with an expected failure
  (see issue 13125).

1 files changed, 26 insertions, 12 deletions

diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index ce39e41..f32404c 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -14,10 +14,21 @@ from .support import driver, test_dir
 
 # Python imports
 import os
+import unittest
 
 # Local imports
 from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+
+
+class TestDriver(support.TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
 
 
 class GrammarTest(support.TestCase):
@@ -147,19 +158,22 @@ class TestParserIdempotency(support.TestCase):
 
     """A cut-down version of pytree_idempotency.py."""
 
+    # Issue 13125
+    @unittest.expectedFailure
     def test_all_project_files(self):
         for filepath in support.all_project_files():
             with open(filepath, "rb") as fp:
                 encoding = tokenize.detect_encoding(fp.readline)[0]
             self.assertTrue(encoding is not None,
                             "can't detect encoding for %s" % filepath)
-            with open(filepath, "r") as fp:
+            with open(filepath, "r", encoding=encoding) as fp:
                 source = fp.read()
-                source = source.decode(encoding)
-            tree = driver.parse_string(source)
+            try:
+                tree = driver.parse_string(source)
+            except ParseError as err:
+                print('ParseError on file', filepath, err)
+                continue
             new = str(tree)
-            if encoding:
-                new = new.encode(encoding)
             if diff(filepath, new):
                 self.fail("Idempotency failed: %s" % filepath)
 
@@ -202,14 +216,14 @@ class TestLiterals(GrammarTest):
         self.validate(s)
 
 
-def diff(fn, result, encoding):
-    f = open("@", "w")
-    try:
-        f.write(result.encode(encoding))
-    finally:
-        f.close()
+def diff(fn, result):
     try:
+        with open('@', 'w') as f:
+            f.write(str(result))
         fn = fn.replace('"', '\\"')
         return os.system('diff -u "%s" @' % fn)
     finally:
-        os.remove("@")
+        try:
+            os.remove("@")
+        except OSError:
+            pass