diff options
Diffstat (limited to 'Lib/lib2to3')
-rw-r--r-- | Lib/lib2to3/fixer_base.py | 6 | ||||
-rw-r--r-- | Lib/lib2to3/fixer_util.py | 24 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_dict.py | 2 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_except.py | 2 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_imports.py | 64 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_imports2.py | 1 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_next.py | 2 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_numliterals.py | 2 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_renames.py | 2 | ||||
-rw-r--r-- | Lib/lib2to3/fixes/fix_urllib.py | 6 | ||||
-rw-r--r-- | Lib/lib2to3/pgen2/parse.py | 6 | ||||
-rw-r--r-- | Lib/lib2to3/pytree.py | 10 | ||||
-rwxr-xr-x | Lib/lib2to3/refactor.py | 11 | ||||
-rwxr-xr-x | Lib/lib2to3/tests/test_fixers.py | 50 | ||||
-rw-r--r-- | Lib/lib2to3/tests/test_refactor.py | 2 |
15 files changed, 104 insertions, 86 deletions
diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py index 5246b08..34d0728 100644 --- a/Lib/lib2to3/fixer_base.py +++ b/Lib/lib2to3/fixer_base.py @@ -7,12 +7,6 @@ import logging import itertools -# Get a usable 'set' constructor -try: - set -except NameError: - from sets import Set as set - # Local imports from .patcomp import PatternCompiler from . import pygram diff --git a/Lib/lib2to3/fixer_util.py b/Lib/lib2to3/fixer_util.py index b694d27..ea394e8 100644 --- a/Lib/lib2to3/fixer_util.py +++ b/Lib/lib2to3/fixer_util.py @@ -153,30 +153,6 @@ def is_list(node): and node.children[0].value == "[" and node.children[-1].value == "]") -########################################################### -### Common portability code. This allows fixers to do, eg, -### "from .util import set" and forget about it. -########################################################### - -try: - any = any -except NameError: - def any(l): - for o in l: - if o: - return True - return False - -try: - set = set -except NameError: - from sets import Set as set - -try: - reversed = reversed -except NameError: - def reversed(l): - return l[::-1] ########################################################### ### Misc diff --git a/Lib/lib2to3/fixes/fix_dict.py b/Lib/lib2to3/fixes/fix_dict.py index 485a08c..dec94c5 100644 --- a/Lib/lib2to3/fixes/fix_dict.py +++ b/Lib/lib2to3/fixes/fix_dict.py @@ -28,7 +28,7 @@ from .. import pytree from .. import patcomp from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot, set +from ..fixer_util import Name, Call, LParen, RParen, ArgList, Dot from .. import fixer_util diff --git a/Lib/lib2to3/fixes/fix_except.py b/Lib/lib2to3/fixes/fix_except.py index 8387913..dc2a5b1 100644 --- a/Lib/lib2to3/fixes/fix_except.py +++ b/Lib/lib2to3/fixes/fix_except.py @@ -25,7 +25,7 @@ The following cases will be converted: from .. import pytree from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Assign, Attr, Name, is_tuple, is_list, reversed +from ..fixer_util import Assign, Attr, Name, is_tuple, is_list def find_excepts(nodes): for i, n in enumerate(nodes): diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py index 38e868b..e48c4f0 100644 --- a/Lib/lib2to3/fixes/fix_imports.py +++ b/Lib/lib2to3/fixes/fix_imports.py @@ -1,9 +1,9 @@ """Fix incompatible imports and module references.""" -# Author: Collin Winter +# Authors: Collin Winter, Nick Edds # Local imports from .. import fixer_base -from ..fixer_util import Name, attr_chain, any, set +from ..fixer_util import Name, attr_chain MAPPING = {'StringIO': 'io', 'cStringIO': 'io', @@ -61,36 +61,49 @@ def alternates(members): def build_pattern(mapping=MAPPING): - mod_list = ' | '.join(["module='" + key + "'" for key in mapping.keys()]) - mod_name_list = ' | '.join(["module_name='" + key + "'" for key in mapping.keys()]) - yield """import_name< 'import' ((%s) + mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) + bare_names = alternates(mapping.keys()) + + yield """name_import=import_name< 'import' ((%s) | dotted_as_names< any* (%s) any* >) > """ % (mod_list, mod_list) yield """import_from< 'from' (%s) 'import' ['('] ( any | import_as_name< any 'as' any > | import_as_names< any* >) [')'] > - """ % mod_name_list + """ % mod_list yield """import_name< 'import' dotted_as_name< (%s) 'as' any > > - """ % mod_name_list - # Find usages of module members in code e.g. urllib.foo(bar) - yield """power< (%s) - trailer<'.' any > any* > - """ % mod_name_list - yield """bare_name=%s""" % alternates(mapping.keys()) + """ % mod_list + + # Find usages of module members in code e.g. thread.foo(bar) + yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names + class FixImports(fixer_base.BaseFix): - PATTERN = "|".join(build_pattern()) + order = "pre" # Pre-order tree traversal + # This is overridden in fix_imports2. mapping = MAPPING - # Don't match the node if it's within another match + def build_pattern(self): + return "|".join(build_pattern(self.mapping)) + + def compile_pattern(self): + # We override this, so MAPPING can be pragmatically altered and the + # changes will be reflected in PATTERN. + self.PATTERN = self.build_pattern() + super(FixImports, self).compile_pattern() + + # Don't match the node if it's within another match. def match(self, node): match = super(FixImports, self).match results = match(node) if results: - if any([match(obj) for obj in attr_chain(node, "parent")]): + # Module usage could be in the trailier of an attribute lookup, so + # we might have nested matches when "bare_with_attr" is present. + if "bare_with_attr" not in results and \ + any([match(obj) for obj in attr_chain(node, "parent")]): return False return results return False @@ -100,20 +113,17 @@ class FixImports(fixer_base.BaseFix): self.replace = {} def transform(self, node, results): - import_mod = results.get("module") - mod_name = results.get("module_name") - bare_name = results.get("bare_name") - - if import_mod or mod_name: - new_name = self.mapping[(import_mod or mod_name).value] - + import_mod = results.get("module_name") if import_mod: - self.replace[import_mod.value] = new_name + new_name = self.mapping[(import_mod or mod_name).value] + if "name_import" in results: + # If it's not a "from x import x, y" or "import x as y" import, + # marked its usage to be replaced. + self.replace[import_mod.value] = new_name import_mod.replace(Name(new_name, prefix=import_mod.get_prefix())) - elif mod_name: - mod_name.replace(Name(new_name, prefix=mod_name.get_prefix())) - elif bare_name: - bare_name = bare_name[0] + else: + # Replace usage of the module. + bare_name = results["bare_with_attr"][0] new_name = self.replace.get(bare_name.value) if new_name: bare_name.replace(Name(new_name, prefix=bare_name.get_prefix())) diff --git a/Lib/lib2to3/fixes/fix_imports2.py b/Lib/lib2to3/fixes/fix_imports2.py index e82f7e4..bcd7aa6 100644 --- a/Lib/lib2to3/fixes/fix_imports2.py +++ b/Lib/lib2to3/fixes/fix_imports2.py @@ -10,7 +10,6 @@ MAPPING = { class FixImports2(fix_imports.FixImports): - PATTERN = "|".join((fix_imports.build_pattern(MAPPING))) order = "post" diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py index 9f1861e..492b515 100644 --- a/Lib/lib2to3/fixes/fix_next.py +++ b/Lib/lib2to3/fixes/fix_next.py @@ -9,7 +9,7 @@ from ..pgen2 import token from ..pygram import python_symbols as syms from .. import fixer_base -from ..fixer_util import Name, Call, find_binding, any +from ..fixer_util import Name, Call, find_binding bind_warning = "Calls to builtin next() possibly shadowed by global binding" diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py index 682aac5..d821e39 100644 --- a/Lib/lib2to3/fixes/fix_numliterals.py +++ b/Lib/lib2to3/fixes/fix_numliterals.py @@ -6,7 +6,7 @@ # Local imports from ..pgen2 import token from .. import fixer_base -from ..fixer_util import Number, set +from ..fixer_util import Number class FixNumliterals(fixer_base.BaseFix): diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py index ea50b47..33d4746 100644 --- a/Lib/lib2to3/fixes/fix_renames.py +++ b/Lib/lib2to3/fixes/fix_renames.py @@ -8,7 +8,7 @@ Fixes: # Local imports from .. import fixer_base -from ..fixer_util import Name, attr_chain, any, set +from ..fixer_util import Name, attr_chain MAPPING = {"sys": {"maxint" : "maxsize"}, } diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py index 78bf7ab..ea7e9ca 100644 --- a/Lib/lib2to3/fixes/fix_urllib.py +++ b/Lib/lib2to3/fixes/fix_urllib.py @@ -7,7 +7,7 @@ # Local imports from .fix_imports import alternates, FixImports from .. import fixer_base -from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain, any, set +from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain MAPPING = {'urllib': [ ('urllib.request', @@ -65,7 +65,9 @@ def build_pattern(): class FixUrllib(FixImports): - PATTERN = "|".join(build_pattern()) + + def build_pattern(self): + return "|".join(build_pattern()) def transform_import(self, node, results): """Transform for the basic import case. Replaces the old diff --git a/Lib/lib2to3/pgen2/parse.py b/Lib/lib2to3/pgen2/parse.py index 9b1ac2a..6bebdbb 100644 --- a/Lib/lib2to3/pgen2/parse.py +++ b/Lib/lib2to3/pgen2/parse.py @@ -10,12 +10,6 @@ how this parsing engine works. """ -# Get a usable 'set' constructor -try: - set -except NameError: - from sets import Set as set - # Local imports from . import token diff --git a/Lib/lib2to3/pytree.py b/Lib/lib2to3/pytree.py index 08198fd..d345733 100644 --- a/Lib/lib2to3/pytree.py +++ b/Lib/lib2to3/pytree.py @@ -11,6 +11,9 @@ There's also a pattern matching implementation here. __author__ = "Guido van Rossum <guido@python.org>" +import sys +from StringIO import StringIO + HUGE = 0x7FFFFFFF # maximum repeat count, default max @@ -655,6 +658,11 @@ class WildcardPattern(BasePattern): elif self.name == "bare_name": yield self._bare_name_matches(nodes) else: + # The reason for this is that hitting the recursion limit usually + # results in some ugly messages about how RuntimeErrors are being + # ignored. + save_stderr = sys.stderr + sys.stderr = StringIO() try: for count, r in self._recursive_matches(nodes, 0): if self.name: @@ -667,6 +675,8 @@ class WildcardPattern(BasePattern): if self.name: r[self.name] = nodes[:count] yield count, r + finally: + sys.stderr = save_stderr def _iterative_matches(self, nodes): """Helper to iteratively yield the matches.""" diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 4fe18ba..e7dd66e 100755 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -123,8 +123,8 @@ class RefactoringTool(object): logger=self.logger) self.pre_order, self.post_order = self.get_fixers() - self.pre_order = get_headnode_dict(self.pre_order) - self.post_order = get_headnode_dict(self.post_order) + self.pre_order_mapping = get_headnode_dict(self.pre_order) + self.post_order_mapping = get_headnode_dict(self.post_order) self.files = [] # List of files that were or should be modified @@ -290,13 +290,12 @@ class RefactoringTool(object): # Two calls to chain are required because pre_order.values() # will be a list of lists of fixers: # [[<fixer ...>, <fixer ...>], [<fixer ...>]] - all_fixers = chain(chain(*self.pre_order.values()),\ - chain(*self.post_order.values())) + all_fixers = chain(self.pre_order, self.post_order) for fixer in all_fixers: fixer.start_tree(tree, name) - self.traverse_by(self.pre_order, tree.pre_order()) - self.traverse_by(self.post_order, tree.post_order()) + self.traverse_by(self.pre_order_mapping, tree.pre_order()) + self.traverse_by(self.post_order_mapping, tree.post_order()) for fixer in all_fixers: fixer.finish_tree(tree, name) diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 1de7e1b..1098d4c 100755 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -15,10 +15,7 @@ from itertools import chain from operator import itemgetter # Local imports -from .. import pygram -from .. import pytree -from .. import refactor -from .. import fixer_util +from lib2to3 import pygram, pytree, refactor, fixer_util class FixerTestCase(support.TestCase): @@ -30,10 +27,9 @@ class FixerTestCase(support.TestCase): self.fixer_log = [] self.filename = "<string>" - for order in (self.refactor.pre_order.values(),\ - self.refactor.post_order.values()): - for fixer in chain(*order): - fixer.log = self.fixer_log + for fixer in chain(self.refactor.pre_order, + self.refactor.post_order): + fixer.log = self.fixer_log def _check(self, before, after): before = support.reformat(before) @@ -1488,6 +1484,44 @@ class Test_imports(FixerTestCase): """ % (new, new) self.check(b, a) + b = """ + from %s import x + %s = 23 + """ % (old, old) + a = """ + from %s import x + %s = 23 + """ % (new, old) + self.check(b, a) + + s = """ + def f(): + %s.method() + """ % (old,) + self.unchanged(s) + + # test nested usage + b = """ + import %s + %s.bar(%s.foo) + """ % (old, old, old) + a = """ + import %s + %s.bar(%s.foo) + """ % (new, new, new) + self.check(b, a) + + b = """ + import %s + x.%s + """ % (old, old) + a = """ + import %s + x.%s + """ % (new, old) + self.check(b, a) + + class Test_imports2(Test_imports): fixer = "imports2" diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 1e15a1a..8d0c1da 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -161,7 +161,7 @@ class TestRefactoringTool(unittest.TestCase): self.assertEqual(len(rt.post_order), 0) rt = self.rt(explicit=["myfixes.fix_explicit"]) - for fix in rt.post_order[None]: + for fix in rt.post_order: if isinstance(fix, FixExplicit): break else: |