diff options
Diffstat (limited to 'Lib/lib2to3/refactor.py')
| -rw-r--r-- | Lib/lib2to3/refactor.py | 97 |
1 files changed, 58 insertions, 39 deletions
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index 55fd60f..8a40deb 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -8,23 +8,26 @@ recursively descend down directories. Imported as a module, this provides infrastructure to write your own refactoring tool. """ +from __future__ import with_statement + __author__ = "Guido van Rossum <guido@python.org>" # Python imports -import io import os import pkgutil import sys import logging import operator import collections +import StringIO from itertools import chain # Local imports from .pgen2 import driver, tokenize, token from .fixer_util import find_root from . import pytree, pygram +from . import btm_utils as bu from . import btm_matcher as bm @@ -54,7 +57,7 @@ def _get_head_types(pat): # Always return leafs if pat.type is None: raise _EveryNode - return {pat.type} + return set([pat.type]) if isinstance(pat, pytree.NegatedPattern): if pat.content: @@ -91,7 +94,7 @@ def _get_headnode_dict(fixer_list): head_nodes[fixer._accept_type].append(fixer) else: every.append(fixer) - for node_type in chain(pygram.python_grammar.symbol2number.values(), + for node_type in chain(pygram.python_grammar.symbol2number.itervalues(), pygram.python_grammar.tokens): head_nodes[node_type].extend(every) return dict(head_nodes) @@ -107,14 +110,30 @@ def get_fixers_from_package(pkg_name): def _identity(obj): return obj +if sys.version_info < (3, 0): + import codecs + _open_with_encoding = codecs.open + # codecs.open doesn't translate newlines sadly. + def _from_system_newlines(input): + return input.replace(u"\r\n", u"\n") + def _to_system_newlines(input): + if os.linesep != "\n": + return input.replace(u"\n", os.linesep) + else: + return input +else: + _open_with_encoding = open + _from_system_newlines = _identity + _to_system_newlines = _identity + def _detect_future_features(source): have_docstring = False - gen = tokenize.generate_tokens(io.StringIO(source).readline) + gen = tokenize.generate_tokens(StringIO.StringIO(source).readline) def advance(): - tok = next(gen) + tok = gen.next() return tok[0], tok[1] - ignore = frozenset({token.NEWLINE, tokenize.NL, token.COMMENT}) + ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT)) features = set() try: while True: @@ -125,20 +144,20 @@ def _detect_future_features(source): if have_docstring: break have_docstring = True - elif tp == token.NAME and value == "from": + elif tp == token.NAME and value == u"from": tp, value = advance() - if tp != token.NAME or value != "__future__": + if tp != token.NAME or value != u"__future__": break tp, value = advance() - if tp != token.NAME or value != "import": + if tp != token.NAME or value != u"import": break tp, value = advance() - if tp == token.OP and value == "(": + if tp == token.OP and value == u"(": tp, value = advance() while tp == token.NAME: features.add(value) tp, value = advance() - if tp != token.OP or value != ",": + if tp != token.OP or value != u",": break tp, value = advance() else: @@ -232,7 +251,7 @@ class RefactoringTool(object): try: fix_class = getattr(mod, class_name) except AttributeError: - raise FixerError("Can't find %s.%s" % (fix_name, class_name)) from None + raise FixerError("Can't find %s.%s" % (fix_name, class_name)) fixer = fix_class(self.options, self.fixer_log) if fixer.explicit and self.explicit is not True and \ fix_mod_path not in self.explicit: @@ -307,15 +326,15 @@ class RefactoringTool(object): """ try: f = open(filename, "rb") - except OSError as err: + except IOError as err: self.log_error("Can't open %s: %s", filename, err) return None, None try: encoding = tokenize.detect_encoding(f.readline)[0] finally: f.close() - with io.open(filename, "r", encoding=encoding, newline='') as f: - return f.read(), encoding + with _open_with_encoding(filename, "r", encoding=encoding) as f: + return _from_system_newlines(f.read()), encoding def refactor_file(self, filename, write=False, doctests_only=False): """Refactors a file.""" @@ -323,7 +342,7 @@ class RefactoringTool(object): if input is None: # Reading the file failed. return - input += "\n" # Silence certain parse errors + input += u"\n" # Silence certain parse errors if doctests_only: self.log_debug("Refactoring doctests in %s", filename) output = self.refactor_docstring(input, filename) @@ -335,7 +354,7 @@ class RefactoringTool(object): tree = self.refactor_string(input, filename) if self.write_unchanged_files or (tree and tree.was_changed): # The [:-1] is to take off the \n we added earlier - self.processed_file(str(tree)[:-1], filename, + self.processed_file(unicode(tree)[:-1], filename, write=write, encoding=encoding) else: self.log_debug("No changes in %s", filename) @@ -379,7 +398,7 @@ class RefactoringTool(object): else: tree = self.refactor_string(input, "<stdin>") if self.write_unchanged_files or (tree and tree.was_changed): - self.processed_file(str(tree), "<stdin>", input) + self.processed_file(unicode(tree), "<stdin>", input) else: self.log_debug("No changes in stdin") @@ -514,16 +533,16 @@ class RefactoringTool(object): set. """ try: - fp = io.open(filename, "w", encoding=encoding, newline='') - except OSError as err: + f = _open_with_encoding(filename, "w", encoding=encoding) + except os.error as err: self.log_error("Can't create %s: %s", filename, err) return - - with fp: - try: - fp.write(new_text) - except OSError as err: - self.log_error("Can't write %s: %s", filename, err) + try: + f.write(_to_system_newlines(new_text)) + except os.error as err: + self.log_error("Can't write %s: %s", filename, err) + finally: + f.close() self.log_debug("Wrote changes to %s", filename) self.wrote = True @@ -547,7 +566,7 @@ class RefactoringTool(object): block_lineno = None indent = None lineno = 0 - for line in input.splitlines(keepends=True): + for line in input.splitlines(True): lineno += 1 if line.lstrip().startswith(self.PS1): if block is not None: @@ -559,7 +578,7 @@ class RefactoringTool(object): indent = line[:i] elif (indent is not None and (line.startswith(indent + self.PS2) or - line == indent + self.PS2.rstrip() + "\n")): + line == indent + self.PS2.rstrip() + u"\n")): block.append(line) else: if block is not None: @@ -571,7 +590,7 @@ class RefactoringTool(object): if block is not None: result.extend(self.refactor_doctest(block, block_lineno, indent, filename)) - return "".join(result) + return u"".join(result) def refactor_doctest(self, block, lineno, indent, filename): """Refactors one doctest. @@ -586,17 +605,17 @@ class RefactoringTool(object): except Exception as err: if self.logger.isEnabledFor(logging.DEBUG): for line in block: - self.log_debug("Source: %s", line.rstrip("\n")) + self.log_debug("Source: %s", line.rstrip(u"\n")) self.log_error("Can't parse docstring in %s line %s: %s: %s", filename, lineno, err.__class__.__name__, err) return block if self.refactor_tree(tree, filename): - new = str(tree).splitlines(keepends=True) + new = unicode(tree).splitlines(True) # Undo the adjustment of the line numbers in wrap_toks() below. clipped, new = new[:lineno-1], new[lineno-1:] - assert clipped == ["\n"] * (lineno-1), clipped - if not new[-1].endswith("\n"): - new[-1] += "\n" + assert clipped == [u"\n"] * (lineno-1), clipped + if not new[-1].endswith(u"\n"): + new[-1] += u"\n" block = [indent + self.PS1 + new.pop(0)] if new: block += [indent + self.PS2 + line for line in new] @@ -637,7 +656,7 @@ class RefactoringTool(object): def wrap_toks(self, block, lineno, indent): """Wraps a tokenize stream to systematically modify start/end.""" - tokens = tokenize.generate_tokens(self.gen_lines(block, indent).__next__) + tokens = tokenize.generate_tokens(self.gen_lines(block, indent).next) for type, value, (line0, col0), (line1, col1), line_text in tokens: line0 += lineno - 1 line1 += lineno - 1 @@ -660,8 +679,8 @@ class RefactoringTool(object): for line in block: if line.startswith(prefix): yield line[len(prefix):] - elif line == prefix.rstrip() + "\n": - yield "\n" + elif line == prefix.rstrip() + u"\n": + yield u"\n" else: raise AssertionError("line=%r, prefix=%r" % (line, prefix)) prefix = prefix2 @@ -694,7 +713,7 @@ class MultiprocessRefactoringTool(RefactoringTool): self.queue = multiprocessing.JoinableQueue() self.output_lock = multiprocessing.Lock() processes = [multiprocessing.Process(target=self._child) - for i in range(num_processes)] + for i in xrange(num_processes)] try: for p in processes: p.start() @@ -702,7 +721,7 @@ class MultiprocessRefactoringTool(RefactoringTool): doctests_only) finally: self.queue.join() - for i in range(num_processes): + for i in xrange(num_processes): self.queue.put(None) for p in processes: if p.is_alive(): |