diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/lib2to3/fixes/fix_imports.py | 2 | ||||
| -rw-r--r-- | Lib/lib2to3/fixes/fix_next.py | 2 | ||||
| -rw-r--r-- | Lib/lib2to3/fixes/fix_renames.py | 2 | ||||
| -rw-r--r-- | Lib/lib2to3/fixes/fix_urllib.py | 10 | ||||
| -rw-r--r-- | Lib/lib2to3/main.py | 11 | ||||
| -rw-r--r-- | Lib/lib2to3/pgen2/tokenize.py | 10 | ||||
| -rw-r--r-- | Lib/lib2to3/refactor.py | 43 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/data/bom.py | 3 | ||||
| -rwxr-xr-x | Lib/lib2to3/tests/test_fixers.py | 15 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/test_refactor.py | 20 |
10 files changed, 68 insertions, 50 deletions
diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py index 6c41c71..5c053c0 100644 --- a/Lib/lib2to3/fixes/fix_imports.py +++ b/Lib/lib2to3/fixes/fix_imports.py @@ -108,7 +108,7 @@ class FixImports(fixer_base.BaseFix): # Module usage could be in the trailer of an attribute lookup, so we # might have nested matches when "bare_with_attr" is present. if "bare_with_attr" not in results and \ - any([match(obj) for obj in attr_chain(node, "parent")]): + any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py index c999d6f..66a00c2 100644 --- a/Lib/lib2to3/fixes/fix_next.py +++ b/Lib/lib2to3/fixes/fix_next.py @@ -99,4 +99,4 @@ def find_assign(node): def is_subtree(root, node): if root == node: return True - return any([is_subtree(c, node) for c in root.children]) + return any(is_subtree(c, node) for c in root.children) diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py index 275e23f..c9f68d5 100644 --- a/Lib/lib2to3/fixes/fix_renames.py +++ b/Lib/lib2to3/fixes/fix_renames.py @@ -49,7 +49,7 @@ class FixRenames(fixer_base.BaseFix): match = super(FixRenames, self).match results = match(node) if results: - if any([match(obj) for obj in attr_chain(node, "parent")]): + if any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index d11220c..db18ca8 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -63,7 +63,8 @@ def build_pattern(): yield """import_name< 'import' dotted_as_name< module_as=%r 'as' any > > """ % old_module - yield """power< module_dot=%r trailer< '.' member=%s > any* > + # bare_with_attr has a special significance for FixImports.match(). + yield """power< bare_with_attr=%r trailer< '.' member=%s > any* > """ % (old_module, members) @@ -150,12 +151,11 @@ class FixUrllib(FixImports): def transform_dot(self, node, results): """Transform for calls to module members in code.""" - module_dot = results.get('module_dot') + module_dot = results.get('bare_with_attr') member = results.get('member') - # this may be a list of length one, or just a node + new_name = None if isinstance(member, list): member = member[0] - new_name = None for change in MAPPING[module_dot.value]: if member.value in change[1]: new_name = change[0] @@ -171,7 +171,7 @@ class FixUrllib(FixImports): self.transform_import(node, results) elif results.get('mod_member'): self.transform_member(node, results) - elif results.get('module_dot'): + elif results.get('bare_with_attr'): self.transform_dot(node, results) # Renaming and star imports are not supported for these modules. elif results.get('module_star'): diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py index 7c26cdb..6e09693 100644 --- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -90,8 +90,7 @@ def main(fixer_pkg, args=None): parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations (fixes/fix_*.py)") parser.add_option("-p", "--print-function", action="store_true", - help="DEPRECATED Modify the grammar so that print() is " - "a function") + help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", @@ -103,12 +102,10 @@ def main(fixer_pkg, args=None): # Parse command line arguments refactor_stdin = False + flags = {} options, args = parser.parse_args(args) if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") - if options.print_function: - warn("-p is deprecated; " - "detection of from __future__ import print_function is automatic") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if options.list_fixes: @@ -126,6 +123,8 @@ def main(fixer_pkg, args=None): if options.write: print("Can't write to stdin.", file=sys.stderr) return 2 + if options.print_function: + flags["print_function"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO @@ -146,7 +145,7 @@ def main(fixer_pkg, args=None): else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit), + rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 4585ca3..7ae0280 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -283,9 +283,13 @@ def detect_encoding(readline): # This behaviour mimics the Python interpreter raise SyntaxError("unknown encoding: " + encoding) - if bom_found and codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') + if bom_found: + if codec.name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + else: + # Allow it to be properly encoded and decoded. + encoding = 'utf-8-sig' return encoding first = read_or_stop() diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 5edf584..8bd61ad 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -18,7 +18,6 @@ import logging import operator import collections import io -import warnings from itertools import chain # Local imports @@ -139,26 +138,23 @@ def _detect_future_print(source): if have_docstring: break have_docstring = True - elif tp == token.NAME: - if value == "from": + elif tp == token.NAME and value == "from": + tp, value = advance() + if tp != token.NAME and value != "__future__": + break + tp, value = advance() + if tp != token.NAME and value != "import": + break + tp, value = advance() + if tp == token.OP and value == "(": tp, value = advance() - if tp != token.NAME and value != "__future__": - break + while tp == token.NAME: + if value == "print_function": + return True tp, value = advance() - if tp != token.NAME and value != "import": + if tp != token.OP and value != ",": break tp, value = advance() - if tp == token.OP and value == "(": - tp, value = advance() - while tp == token.NAME: - if value == "print_function": - return True - tp, value = advance() - if tp != token.OP and value != ",": - break - tp, value = advance() - else: - break else: break except StopIteration: @@ -172,7 +168,7 @@ class FixerError(Exception): class RefactoringTool(object): - _default_options = {} + _default_options = {"print_function" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -189,15 +185,16 @@ class RefactoringTool(object): self.explicit = explicit or [] self.options = self._default_options.copy() if options is not None: - if "print_function" in options: - warnings.warn("the 'print_function' option is deprecated", - DeprecationWarning) self.options.update(options) + if self.options["print_function"]: + self.grammar = pygram.python_grammar_no_print_statement + else: + self.grammar = pygram.python_grammar self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] self.wrote = False - self.driver = driver.Driver(pygram.python_grammar, + self.driver = driver.Driver(self.grammar, convert=pytree.convert, logger=self.logger) self.pre_order, self.post_order = self.get_fixers() @@ -353,7 +350,7 @@ class RefactoringTool(object): name, err.__class__.__name__, err) return finally: - self.driver.grammar = pygram.python_grammar + self.driver.grammar = self.grammar self.log_debug("Refactoring %s", name) self.refactor_tree(tree, name) return tree diff --git a/Lib/lib2to3/tests/data/bom.py b/Lib/lib2to3/tests/data/bom.py new file mode 100644 index 0000000..ecb782a --- /dev/null +++ b/Lib/lib2to3/tests/data/bom.py @@ -0,0 +1,3 @@ +# coding: utf-8 +print "BOM BOOM!" + diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 2a39359..ea361de 100755 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -1753,6 +1753,8 @@ class Test_urllib(FixerTestCase): for old, changes in self.modules.items(): for new, members in changes: for member in members: + new_import = ", ".join([n for (n, mems) + in self.modules[old]]) b = """ import %s foo(%s.%s) @@ -1760,9 +1762,16 @@ class Test_urllib(FixerTestCase): a = """ import %s foo(%s.%s) - """ % (", ".join([n for (n, mems) - in self.modules[old]]), - new, member) + """ % (new_import, new, member) + self.check(b, a) + b = """ + import %s + %s.%s(%s.%s) + """ % (old, old, member, old, member) + a = """ + import %s + %s.%s(%s.%s) + """ % (new_import, new, member, new, member) self.check(b, a) diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 5ba23a5..2263f50 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -4,6 +4,7 @@ Unit tests for refactor.py. import sys import os +import codecs import operator import io import tempfile @@ -45,12 +46,10 @@ class TestRefactoringTool(unittest.TestCase): return refactor.RefactoringTool(fixers, options, explicit) def test_print_function_option(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True}) - self.assertEqual(len(w), 1) - msg, = w - self.assertTrue(msg.category is DeprecationWarning) + rt = self.rt({"print_function" : True}) + self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement) + self.assertTrue(rt.driver.grammar is + pygram.python_grammar_no_print_statement) def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] @@ -179,10 +178,12 @@ from __future__ import print_function""" try: rt.refactor_file(test_file, True) - self.assertNotEqual(old_contents, read_file()) + new_contents = read_file() + self.assertNotEqual(old_contents, new_contents) finally: with open(test_file, "wb") as fp: fp.write(old_contents) + return new_contents def test_refactor_file(self): test_file = os.path.join(FIXER_DIR, "parrot_example.py") @@ -223,6 +224,11 @@ from __future__ import print_function""" fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_bom(self): + fn = os.path.join(TEST_DATA_DIR, "bom.py") + data = self.check_file_refactoring(fn) + self.assertTrue(data.startswith(codecs.BOM_UTF8)) + def test_crlf_newlines(self): old_sep = os.linesep os.linesep = "\r\n" |