From 78f89d8c38e2c71c5e6a89a32022626a1d8d8922 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 7 Oct 2011 14:44:49 -0400 Subject: - Issue #11250: Back port fix from 3.3 branch, so that 2to3 can handle files with line feeds. This was ported from the sandbox to the 3.3 branch, but didn't make it into 3.2. - Re-enable lib2to3's test_parser.py tests, though with an expected failure (see issue 13125). --- Lib/lib2to3/patcomp.py | 3 ++- Lib/lib2to3/pgen2/driver.py | 11 ++--------- Lib/lib2to3/tests/test_parser.py | 38 ++++++++++++++++++++++++++------------ Lib/test/test_lib2to3.py | 3 ++- Misc/NEWS | 7 +++++++ 5 files changed, 39 insertions(+), 23 deletions(-) diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py index bb538d5..0a259e9 100644 --- a/Lib/lib2to3/patcomp.py +++ b/Lib/lib2to3/patcomp.py @@ -11,6 +11,7 @@ The compiler compiles a pattern to a pytree.*Pattern instance. __author__ = "Guido van Rossum " # Python imports +import io import os # Fairly local imports @@ -32,7 +33,7 @@ class PatternSyntaxError(Exception): def tokenize_wrapper(input): """Tokenizes a string suppressing significant whitespace.""" skip = set((token.NEWLINE, token.INDENT, token.DEDENT)) - tokens = tokenize.generate_tokens(driver.generate_lines(input).__next__) + tokens = tokenize.generate_tokens(io.StringIO(input).readline) for quintuple in tokens: type, value, start, end, line_text = quintuple if type not in skip: diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py index ee77a13..e7828ff 100644 --- a/Lib/lib2to3/pgen2/driver.py +++ b/Lib/lib2to3/pgen2/driver.py @@ -17,6 +17,7 @@ __all__ = ["Driver", "load_grammar"] # Python imports import codecs +import io import os import logging import sys @@ -101,18 +102,10 @@ class Driver(object): def parse_string(self, text, debug=False): """Parse a string and return the syntax tree.""" - tokens = tokenize.generate_tokens(generate_lines(text).__next__) + tokens = tokenize.generate_tokens(io.StringIO(text).readline) return self.parse_tokens(tokens, debug) -def generate_lines(text): - """Generator that behaves like readline without using StringIO.""" - for line in text.splitlines(True): - yield line - while True: - yield "" - - def load_grammar(gt="Grammar.txt", gp=None, save=True, force=False, logger=None): """Load the grammar (maybe from a pickle).""" diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index ce39e41..f32404c 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -14,10 +14,21 @@ from .support import driver, test_dir # Python imports import os +import unittest # Local imports from lib2to3.pgen2 import tokenize from ..pgen2.parse import ParseError +from lib2to3.pygram import python_symbols as syms + + +class TestDriver(support.TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = driver.parse_string(s) + self.assertEqual(t.children[0].children[0].type, syms.print_stmt) + self.assertEqual(t.children[1].children[0].type, syms.print_stmt) class GrammarTest(support.TestCase): @@ -147,19 +158,22 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" + # Issue 13125 + @unittest.expectedFailure def test_all_project_files(self): for filepath in support.all_project_files(): with open(filepath, "rb") as fp: encoding = tokenize.detect_encoding(fp.readline)[0] self.assertTrue(encoding is not None, "can't detect encoding for %s" % filepath) - with open(filepath, "r") as fp: + with open(filepath, "r", encoding=encoding) as fp: source = fp.read() - source = source.decode(encoding) - tree = driver.parse_string(source) + try: + tree = driver.parse_string(source) + except ParseError as err: + print('ParseError on file', filepath, err) + continue new = str(tree) - if encoding: - new = new.encode(encoding) if diff(filepath, new): self.fail("Idempotency failed: %s" % filepath) @@ -202,14 +216,14 @@ class TestLiterals(GrammarTest): self.validate(s) -def diff(fn, result, encoding): - f = open("@", "w") - try: - f.write(result.encode(encoding)) - finally: - f.close() +def diff(fn, result): try: + with open('@', 'w') as f: + f.write(str(result)) fn = fn.replace('"', '\\"') return os.system('diff -u "%s" @' % fn) finally: - os.remove("@") + try: + os.remove("@") + except OSError: + pass diff --git a/Lib/test/test_lib2to3.py b/Lib/test/test_lib2to3.py index 0d6f9a3..1afaf70 100644 --- a/Lib/test/test_lib2to3.py +++ b/Lib/test/test_lib2to3.py @@ -1,6 +1,7 @@ # Skipping test_parser and test_all_fixers # because of running from lib2to3.tests import (test_fixers, test_pytree, test_util, test_refactor, + test_parser, test_main as test_main_) import unittest from test.support import run_unittest @@ -9,7 +10,7 @@ def suite(): tests = unittest.TestSuite() loader = unittest.TestLoader() for m in (test_fixers, test_pytree,test_util, test_refactor, - test_main_): + test_parser, test_main_): tests.addTests(loader.loadTestsFromModule(m)) return tests diff --git a/Misc/NEWS b/Misc/NEWS index 44435fe..f49097d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -39,6 +39,10 @@ Core and Builtins Library ------- +- Issue #11250: Back port fix from 3.3 branch, so that 2to3 can handle files + with line feeds. This was ported from the sandbox to the 3.3 branch, but + didn't make it into 3.2. + - Issue #7367: Fix pkgutil.walk_paths to skip directories whose contents cannot be read. @@ -101,6 +105,9 @@ Tests - Issue #12821: Fix test_fcntl failures on OpenBSD 5. +- Re-enable lib2to3's test_parser.py tests, though with an expected failure + (see issue 13125). + Extension Modules ----------------- -- cgit v0.12