merge 2to3 improvments

18 files changed, 405 insertions, 140 deletions

diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py
index c361e61..ae99566 100644
--- a/Lib/lib2to3/fixer_base.py
+++ b/Lib/lib2to3/fixer_base.py
@@ -33,6 +33,8 @@ class BaseFix(object):
     explicit = False # Is this ignored by refactor.py -f all?
     run_order = 5   # Fixers will be sorted by run order before execution
                     # Lower numbers will be run first.
+    _accept_type = None # [Advanced and not public] This tells RefactoringTool
+                        # which node type to accept when there's not a pattern.
 
     # Shortcut for access to Python grammar symbols
     syms = pygram.python_symbols
diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py
index 103c3ef..998fe61 100644
--- a/Lib/lib2to3/fixes/fix_import.py
+++ b/Lib/lib2to3/fixes/fix_import.py
@@ -12,7 +12,7 @@ Becomes:
 
 # Local imports
 from .. import fixer_base
-from os.path import dirname, join, exists, pathsep
+from os.path import dirname, join, exists, sep
 from ..fixer_util import FromImport, syms, token
 
 
@@ -84,7 +84,7 @@ class FixImport(fixer_base.BaseFix):
         # so can't be a relative import.
         if not exists(join(dirname(base_path), '__init__.py')):
             return False
-        for ext in ['.py', pathsep, '.pyc', '.so', '.sl', '.pyd']:
+        for ext in ['.py', sep, '.pyc', '.so', '.sl', '.pyd']:
             if exists(base_path + ext):
                 return True
         return False
diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py
index 9aeca38..6c41c71 100644
--- a/Lib/lib2to3/fixes/fix_imports.py
+++ b/Lib/lib2to3/fixes/fix_imports.py
@@ -84,8 +84,6 @@ def build_pattern(mapping=MAPPING):
 
 class FixImports(fixer_base.BaseFix):
 
-    order = "pre" # Pre-order tree traversal
-
     # This is overridden in fix_imports2.
     mapping = MAPPING
 
diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py
index b5e0bdb..9d25b74 100644
--- a/Lib/lib2to3/fixes/fix_long.py
+++ b/Lib/lib2to3/fixes/fix_long.py
@@ -5,18 +5,15 @@
 """
 
 # Local imports
-from .. import fixer_base
-from ..fixer_util import Name, Number, is_probably_builtin
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import is_probably_builtin
 
 
 class FixLong(fixer_base.BaseFix):
 
     PATTERN = "'long'"
 
-    static_int = Name("int")
-
     def transform(self, node, results):
         if is_probably_builtin(node):
-            new = self.static_int.clone()
-            new.prefix = node.prefix
-            return new
+            node.value = "int"
+            node.changed()
diff --git a/Lib/lib2to3/fixes/fix_ne.py b/Lib/lib2to3/fixes/fix_ne.py
index 1472036..e3ee10f 100644
--- a/Lib/lib2to3/fixes/fix_ne.py
+++ b/Lib/lib2to3/fixes/fix_ne.py
@@ -12,9 +12,11 @@ from .. import fixer_base
 class FixNe(fixer_base.BaseFix):
     # This is so simple that we don't need the pattern compiler.
 
+    _accept_type = token.NOTEQUAL
+
     def match(self, node):
         # Override
-        return node.type == token.NOTEQUAL and node.value == "<>"
+        return node.value == "<>"
 
     def transform(self, node, results):
         new = pytree.Leaf(token.NOTEQUAL, "!=", prefix=node.prefix)
diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py
index 6552740..79207d4 100644
--- a/Lib/lib2to3/fixes/fix_numliterals.py
+++ b/Lib/lib2to3/fixes/fix_numliterals.py
@@ -12,10 +12,11 @@ from ..fixer_util import Number
 class FixNumliterals(fixer_base.BaseFix):
     # This is so simple that we don't need the pattern compiler.
 
+    _accept_type = token.NUMBER
+
     def match(self, node):
         # Override
-        return (node.type == token.NUMBER and
-                (node.value.startswith("0") or node.value[-1] in "Ll"))
+        return (node.value.startswith("0") or node.value[-1] in "Ll")
 
     def transform(self, node, results):
         val = node.value
diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py
new file mode 100644
index 0000000..9b1089c
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_operator.py
@@ -0,0 +1,40 @@
+"""Fixer for operator.{isCallable,sequenceIncludes}
+
+operator.isCallable(obj) -> hasattr(obj, '__call__')
+operator.sequenceIncludes(obj) -> operator.contains(obj)
+"""
+
+# Local imports
+from .. import fixer_base
+from ..fixer_util import Call, Name, String
+
+class FixOperator(fixer_base.BaseFix):
+
+    methods = "method=('isCallable'|'sequenceIncludes')"
+    func = "'(' func=any ')'"
+    PATTERN = """
+              power< module='operator'
+                trailer< '.' {methods} > trailer< {func} > >
+              |
+              power< {methods} trailer< {func} > >
+              """.format(methods=methods, func=func)
+
+    def transform(self, node, results):
+        method = results["method"][0]
+
+        if method.value == "sequenceIncludes":
+            if "module" not in results:
+                # operator may not be in scope, so we can't make a change.
+                self.warning(node, "You should use operator.contains here.")
+            else:
+                method.value = "contains"
+                method.changed()
+        elif method.value == "isCallable":
+            if "module" not in results:
+                self.warning(node,
+                             "You should use hasattr(%s, '__call__') here." %
+                             results["func"].value)
+            else:
+                func = results["func"]
+                args = [func.clone(), String(", "), String("'__call__'")]
+                return Call(Name("hasattr"), args, prefix=node.prefix)
diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py
index abce251..cc7934a 100644
--- a/Lib/lib2to3/fixes/fix_print.py
+++ b/Lib/lib2to3/fixes/fix_print.py
@@ -26,20 +26,15 @@ parend_expr = patcomp.compile_pattern(
               )
 
 
-class FixPrint(fixer_base.ConditionalFix):
+class FixPrint(fixer_base.BaseFix):
 
     PATTERN = """
               simple_stmt< any* bare='print' any* > | print_stmt
               """
 
-    skip_on = '__future__.print_function'
-
     def transform(self, node, results):
         assert results
 
-        if self.should_skip(node):
-            return
-
         bare_print = results.get("bare")
 
         if bare_print:
diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py
index a89266e..d11220c 100644
--- a/Lib/lib2to3/fixes/fix_urllib.py
+++ b/Lib/lib2to3/fixes/fix_urllib.py
@@ -12,13 +12,13 @@ from ..fixer_util import Name, Comma, FromImport, Newline, attr_chain
 MAPPING = {'urllib':  [
                 ('urllib.request',
                     ['URLOpener', 'FancyURLOpener', 'urlretrieve',
-                     '_urlopener', 'urlopen', 'urlcleanup']),
+                     '_urlopener', 'urlopen', 'urlcleanup',
+                     'pathname2url', 'url2pathname']),
                 ('urllib.parse',
                     ['quote', 'quote_plus', 'unquote', 'unquote_plus',
-                     'urlencode', 'pathname2url', 'url2pathname', 'splitattr',
-                     'splithost', 'splitnport', 'splitpasswd', 'splitport',
-                     'splitquery', 'splittag', 'splittype', 'splituser',
-                     'splitvalue', ]),
+                     'urlencode', 'splitattr', 'splithost', 'splitnport',
+                     'splitpasswd', 'splitport', 'splitquery', 'splittag',
+                     'splittype', 'splituser', 'splitvalue', ]),
                 ('urllib.error',
                     ['ContentTooShortError'])],
            'urllib2' : [
diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
index e1adc88..828b02f 100644
--- a/Lib/lib2to3/main.py
+++ b/Lib/lib2to3/main.py
@@ -4,19 +4,31 @@ Main program for 2to3.
 
 import sys
 import os
+import difflib
 import logging
 import shutil
 import optparse
 
 from . import refactor
 
+
+def diff_texts(a, b, filename):
+    """Return a unified diff of two strings."""
+    a = a.splitlines()
+    b = b.splitlines()
+    return difflib.unified_diff(a, b, filename, filename,
+                                "(original)", "(refactored)",
+                                lineterm="")
+
+
 class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
     """
     Prints output to stdout.
     """
 
-    def __init__(self, fixers, options, explicit, nobackups):
+    def __init__(self, fixers, options, explicit, nobackups, show_diffs):
         self.nobackups = nobackups
+        self.show_diffs = show_diffs
         super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
 
     def log_error(self, msg, *args, **kwargs):
@@ -42,9 +54,17 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
         if not self.nobackups:
             shutil.copymode(backup, filename)
 
-    def print_output(self, lines):
-        for line in lines:
-            print(line)
+    def print_output(self, old, new, filename, equal):
+        if equal:
+            self.log_message("No changes to %s", filename)
+        else:
+            self.log_message("Refactored %s", filename)
+            if self.show_diffs:
+                for line in diff_texts(old, new, filename):
+                    print(line)
+
+def warn(msg):
+    print >> sys.stderr, "WARNING: %s" % (msg,)
 
 
 def main(fixer_pkg, args=None):
@@ -70,9 +90,12 @@ def main(fixer_pkg, args=None):
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations (fixes/fix_*.py)")
     parser.add_option("-p", "--print-function", action="store_true",
-                      help="Modify the grammar so that print() is a function")
+                      help="DEPRECATED Modify the grammar so that print() is "
+                          "a function")
     parser.add_option("-v", "--verbose", action="store_true",
                       help="More verbose logging")
+    parser.add_option("--no-diffs", action="store_true",
+                      help="Don't show diffs of the refactoring")
     parser.add_option("-w", "--write", action="store_true",
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
@@ -81,6 +104,11 @@ def main(fixer_pkg, args=None):
     # Parse command line arguments
     refactor_stdin = False
     options, args = parser.parse_args(args)
+    if not options.write and options.no_diffs:
+        warn("not writing files and not printing diffs; that's not very useful")
+    if options.print_function:
+        warn("-p is deprecated; "
+             "detection of from __future__ import print_function is automatic")
     if not options.write and options.nobackups:
         parser.error("Can't use -n without -w")
     if options.list_fixes:
@@ -104,7 +132,6 @@ def main(fixer_pkg, args=None):
     logging.basicConfig(format='%(name)s: %(message)s', level=level)
 
     # Initialize the refactoring tool
-    rt_opts = {"print_function" : options.print_function}
     avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
     unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)
     explicit = set()
@@ -119,8 +146,8 @@ def main(fixer_pkg, args=None):
     else:
         requested = avail_fixes.union(explicit)
     fixer_names = requested.difference(unwanted_fixes)
-    rt = StdoutRefactoringTool(sorted(fixer_names), rt_opts, sorted(explicit),
-                               options.nobackups)
+    rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit),
+                               options.nobackups, not options.no_diffs)
 
     # Refactor all files and directories passed as arguments
     if not rt.errors:
diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py
index 24251cb..3831622 100644
--- a/Lib/lib2to3/patcomp.py
+++ b/Lib/lib2to3/patcomp.py
@@ -14,7 +14,7 @@ __author__ = "Guido van Rossum <guido@python.org>"
 import os
 
 # Fairly local imports
-from .pgen2 import driver, literals, token, tokenize, parse
+from .pgen2 import driver, literals, token, tokenize, parse, grammar
 
 # Really local imports
 from . import pytree
@@ -138,7 +138,7 @@ class PatternCompiler(object):
         node = nodes[0]
         if node.type == token.STRING:
             value = str(literals.evalString(node.value))
-            return pytree.LeafPattern(content=value)
+            return pytree.LeafPattern(_type_of_literal(value), value)
         elif node.type == token.NAME:
             value = node.value
             if value.isupper():
@@ -179,6 +179,15 @@ TOKEN_MAP = {"NAME": token.NAME,
              "TOKEN": None}
 
 
+def _type_of_literal(value):
+    if value[0].isalpha():
+        return token.NAME
+    elif value in grammar.opmap:
+        return grammar.opmap[value]
+    else:
+        return None
+
+
 def pattern_convert(grammar, raw_node_info):
     """Converts raw node information to a Node or Leaf instance."""
     type, value, context, children = raw_node_info
diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
index b7f867f..26caeb4 100644
--- a/Lib/lib2to3/pgen2/grammar.py
+++ b/Lib/lib2to3/pgen2/grammar.py
@@ -97,6 +97,19 @@ class Grammar(object):
         f.close()
         self.__dict__.update(d)
 
+    def copy(self):
+        """
+        Copy the grammar.
+        """
+        new = self.__class__()
+        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
+                          "tokens", "symbol2label"):
+            setattr(new, dict_attr, getattr(self, dict_attr).copy())
+        new.labels = self.labels[:]
+        new.states = self.states[:]
+        new.start = self.start
+        return new
+
     def report(self):
         """Dump the grammar tables to standard output, for debugging."""
         from pprint import pprint
diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py
index 49d5cef..d598317 100644
--- a/Lib/lib2to3/pygram.py
+++ b/Lib/lib2to3/pygram.py
@@ -28,4 +28,8 @@ class Symbols(object):
 
 
 python_grammar = driver.load_grammar(_GRAMMAR_FILE)
+
 python_symbols = Symbols(python_grammar)
+
+python_grammar_no_print_statement = python_grammar.copy()
+del python_grammar_no_print_statement.keywords["print"]
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index 339b94f..5edf584 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -14,14 +14,15 @@ __author__ = "Guido van Rossum <guido@python.org>"
 # Python imports
 import os
 import sys
-import difflib
 import logging
 import operator
-from collections import defaultdict
+import collections
+import io
+import warnings
 from itertools import chain
 
 # Local imports
-from .pgen2 import driver, tokenize
+from .pgen2 import driver, tokenize, token
 from . import pytree, pygram
 
 
@@ -37,7 +38,12 @@ def get_all_fix_names(fixer_pkg, remove_prefix=True):
             fix_names.append(name[:-3])
     return fix_names
 
-def get_head_types(pat):
+
+class _EveryNode(Exception):
+    pass
+
+
+def _get_head_types(pat):
     """ Accepts a pytree Pattern Node and returns a set
         of the pattern types which will match first. """
 
@@ -45,34 +51,50 @@ def get_head_types(pat):
         # NodePatters must either have no type and no content
         #   or a type and content -- so they don't get any farther
         # Always return leafs
+        if pat.type is None:
+            raise _EveryNode
         return set([pat.type])
 
     if isinstance(pat, pytree.NegatedPattern):
         if pat.content:
-            return get_head_types(pat.content)
-        return set([None]) # Negated Patterns don't have a type
+            return _get_head_types(pat.content)
+        raise _EveryNode # Negated Patterns don't have a type
 
     if isinstance(pat, pytree.WildcardPattern):
         # Recurse on each node in content
         r = set()
         for p in pat.content:
             for x in p:
-                r.update(get_head_types(x))
+                r.update(_get_head_types(x))
         return r
 
     raise Exception("Oh no! I don't understand pattern %s" %(pat))
 
-def get_headnode_dict(fixer_list):
+
+def _get_headnode_dict(fixer_list):
     """ Accepts a list of fixers and returns a dictionary
         of head node type --> fixer list.  """
-    head_nodes = defaultdict(list)
+    head_nodes = collections.defaultdict(list)
+    every = []
     for fixer in fixer_list:
-        if not fixer.pattern:
-            head_nodes[None].append(fixer)
-            continue
-        for t in get_head_types(fixer.pattern):
-            head_nodes[t].append(fixer)
-    return head_nodes
+        if fixer.pattern:
+            try:
+                heads = _get_head_types(fixer.pattern)
+            except _EveryNode:
+                every.append(fixer)
+            else:
+                for node_type in heads:
+                    head_nodes[node_type].append(fixer)
+        else:
+            if fixer._accept_type is not None:
+                head_nodes[fixer._accept_type].append(fixer)
+            else:
+                every.append(fixer)
+    for node_type in chain(pygram.python_grammar.symbol2number.values(),
+                           pygram.python_grammar.tokens):
+        head_nodes[node_type].extend(every)
+    return dict(head_nodes)
+
 
 def get_fixers_from_package(pkg_name):
     """
@@ -101,13 +123,56 @@ else:
     _to_system_newlines = _identity
 
 
+def _detect_future_print(source):
+    have_docstring = False
+    gen = tokenize.generate_tokens(io.StringIO(source).readline)
+    def advance():
+        tok = next(gen)
+        return tok[0], tok[1]
+    ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+    try:
+        while True:
+            tp, value = advance()
+            if tp in ignore:
+                continue
+            elif tp == token.STRING:
+                if have_docstring:
+                    break
+                have_docstring = True
+            elif tp == token.NAME:
+                if value == "from":
+                    tp, value = advance()
+                    if tp != token.NAME and value != "__future__":
+                        break
+                    tp, value = advance()
+                    if tp != token.NAME and value != "import":
+                        break
+                    tp, value = advance()
+                    if tp == token.OP and value == "(":
+                        tp, value = advance()
+                    while tp == token.NAME:
+                        if value == "print_function":
+                            return True
+                        tp, value = advance()
+                        if tp != token.OP and value != ",":
+                            break
+                        tp, value = advance()
+                else:
+                    break
+            else:
+                break
+    except StopIteration:
+        pass
+    return False
+
+
 class FixerError(Exception):
     """A fixer could not be loaded."""
 
 
 class RefactoringTool(object):
 
-    _default_options = {"print_function": False}
+    _default_options = {}
 
     CLASS_PREFIX = "Fix" # The prefix for fixer classes
     FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -124,20 +189,21 @@ class RefactoringTool(object):
         self.explicit = explicit or []
         self.options = self._default_options.copy()
         if options is not None:
+            if "print_function" in options:
+                warnings.warn("the 'print_function' option is deprecated",
+                              DeprecationWarning)
             self.options.update(options)
         self.errors = []
         self.logger = logging.getLogger("RefactoringTool")
         self.fixer_log = []
         self.wrote = False
-        if self.options["print_function"]:
-            del pygram.python_grammar.keywords["print"]
         self.driver = driver.Driver(pygram.python_grammar,
                                     convert=pytree.convert,
                                     logger=self.logger)
         self.pre_order, self.post_order = self.get_fixers()
 
-        self.pre_order_heads = get_headnode_dict(self.pre_order)
-        self.post_order_heads = get_headnode_dict(self.post_order)
+        self.pre_order_heads = _get_headnode_dict(self.pre_order)
+        self.post_order_heads = _get_headnode_dict(self.post_order)
 
         self.files = []  # List of files that were or should be modified
 
@@ -196,8 +262,9 @@ class RefactoringTool(object):
             msg = msg % args
         self.logger.debug(msg)
 
-    def print_output(self, lines):
-        """Called with lines of output to give to the user."""
+    def print_output(self, old_text, new_text, filename, equal):
+        """Called with the old version, new version, and filename of a
+        refactored file."""
         pass
 
     def refactor(self, items, write=False, doctests_only=False):
@@ -220,7 +287,8 @@ class RefactoringTool(object):
             dirnames.sort()
             filenames.sort()
             for name in filenames:
-                if not name.startswith(".") and name.endswith("py"):
+                if not name.startswith(".") and \
+                        os.path.splitext(name)[1].endswith("py"):
                     fullname = os.path.join(dirpath, name)
                     self.refactor_file(fullname, write, doctests_only)
             # Modify dirnames in-place to remove subdirs with leading dots
@@ -276,12 +344,16 @@ class RefactoringTool(object):
             An AST corresponding to the refactored input stream; None if
             there were errors during the parse.
         """
+        if _detect_future_print(data):
+            self.driver.grammar = pygram.python_grammar_no_print_statement
         try:
             tree = self.driver.parse_string(data)
         except Exception as err:
             self.log_error("Can't parse %s: %s: %s",
                            name, err.__class__.__name__, err)
             return
+        finally:
+            self.driver.grammar = pygram.python_grammar
         self.log_debug("Refactoring %s", name)
         self.refactor_tree(tree, name)
         return tree
@@ -338,12 +410,11 @@ class RefactoringTool(object):
         if not fixers:
             return
         for node in traversal:
-            for fixer in fixers[node.type] + fixers[None]:
+            for fixer in fixers[node.type]:
                 results = fixer.match(node)
                 if results:
                     new = fixer.transform(node, results)
-                    if new is not None and (new != node or
-                                            str(new) != str(node)):
+                    if new is not None:
                         node.replace(new)
                         node = new
 
@@ -357,10 +428,11 @@ class RefactoringTool(object):
             old_text = self._read_python_source(filename)[0]
             if old_text is None:
                 return
-        if old_text == new_text:
+        equal = old_text == new_text
+        self.print_output(old_text, new_text, filename, equal)
+        if equal:
             self.log_debug("No changes to %s", filename)
             return
-        self.print_output(diff_texts(old_text, new_text, filename))
         if write:
             self.write_file(new_text, filename, old_text, encoding)
         else:
@@ -582,12 +654,3 @@ class MultiprocessRefactoringTool(RefactoringTool):
         else:
             return super(MultiprocessRefactoringTool, self).refactor_file(
                 *args, **kwargs)
-
-
-def diff_texts(a, b, filename):
-    """Return a unified diff of two strings."""
-    a = a.splitlines()
-    b = b.splitlines()
-    return difflib.unified_diff(a, b, filename, filename,
-                                "(original)", "(refactored)",
-                                lineterm="")
diff --git a/Lib/lib2to3/tests/data/different_encoding.py b/Lib/lib2to3/tests/data/different_encoding.py
index 888f51f..9f32bd0 100644
--- a/Lib/lib2to3/tests/data/different_encoding.py
+++ b/Lib/lib2to3/tests/data/different_encoding.py
@@ -1,3 +1,6 @@
 #!/usr/bin/env python
-# -*- coding: iso-8859-1 -*-
-print u'��������������������������������������������������������������'
+# -*- coding: utf-8 -*-
+print u'ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ'
+
+def f(x):
+    print '%s\t->  α(%2i):%s  β(%s)'
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index c0f1e37..a25f5a8 100755
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -18,8 +18,6 @@ class FixerTestCase(support.TestCase):
     def setUp(self, fix_list=None, fixer_pkg="lib2to3", options=None):
         if fix_list is None:
             fix_list = [self.fixer]
-        if options is None:
-            options = {"print_function" : False}
         self.refactor = support.get_refactorer(fixer_pkg, fix_list, options)
         self.fixer_log = []
         self.filename = "<string>"
@@ -58,8 +56,7 @@ class FixerTestCase(support.TestCase):
     def assert_runs_after(self, *names):
         fixes = [self.fixer]
         fixes.extend(names)
-        options = {"print_function" : False}
-        r = support.get_refactorer("lib2to3", fixes, options)
+        r = support.get_refactorer("lib2to3", fixes)
         (pre, post) = r.get_fixers()
         n = "fix_" + self.fixer
         if post and post[-1].__class__.__module__.endswith(n):
@@ -379,18 +376,15 @@ class Test_print(FixerTestCase):
         self.unchanged(s)
 
     def test_idempotency_print_as_function(self):
-        print_stmt = pygram.python_grammar.keywords.pop("print")
-        try:
-            s = """print(1, 1+1, 1+1+1)"""
-            self.unchanged(s)
+        self.refactor.driver.grammar = pygram.python_grammar_no_print_statement
+        s = """print(1, 1+1, 1+1+1)"""
+        self.unchanged(s)
 
-            s = """print()"""
-            self.unchanged(s)
+        s = """print()"""
+        self.unchanged(s)
 
-            s = """print('')"""
-            self.unchanged(s)
-        finally:
-            pygram.python_grammar.keywords["print"] = print_stmt
+        s = """print('')"""
+        self.unchanged(s)
 
     def test_1(self):
         b = """print 1, 1+1, 1+1+1"""
@@ -462,31 +456,15 @@ class Test_print(FixerTestCase):
         a = """print(file=sys.stderr)"""
         self.check(b, a)
 
-    # With from __future__ import print_function
     def test_with_future_print_function(self):
-        # XXX: These tests won't actually do anything until the parser
-        #      is fixed so it won't crash when it sees print(x=y).
-        #      When #2412 is fixed, the try/except block can be taken
-        #      out and the tests can be run like normal.
-        # MvL: disable entirely for now, so that it doesn't print to stdout
-        return
-        try:
-            s = "from __future__ import print_function\n"\
-                "print('Hai!', end=' ')"
-            self.unchanged(s)
+        s = "from __future__ import print_function\n" \
+            "print('Hai!', end=' ')"
+        self.unchanged(s)
 
-            b = "print 'Hello, world!'"
-            a = "print('Hello, world!')"
-            self.check(b, a)
+        b = "print 'Hello, world!'"
+        a = "print('Hello, world!')"
+        self.check(b, a)
 
-            s = "from __future__ import *\n"\
-                "print('Hai!', end=' ')"
-            self.unchanged(s)
-        except:
-            return
-        else:
-            self.assertFalse(True, "#2421 has been fixed -- printing tests "\
-                                   "need to be updated!")
 
 class Test_exec(FixerTestCase):
     fixer = "exec"
@@ -1705,6 +1683,11 @@ class Test_imports_fixer_order(FixerTestCase, ImportsFixerTests):
         for key in ('dbhash', 'dumbdbm', 'dbm', 'gdbm'):
             self.modules[key] = mapping1[key]
 
+    def test_after_local_imports_refactoring(self):
+        for fix in ("imports", "imports2"):
+            self.fixer = fix
+            self.assert_runs_after("import")
+
 
 class Test_urllib(FixerTestCase):
     fixer = "urllib"
@@ -3504,6 +3487,7 @@ class Test_itertools_imports(FixerTestCase):
         s = "from itertools import foo"
         self.unchanged(s)
 
+
 class Test_import(FixerTestCase):
     fixer = "import"
 
@@ -3538,8 +3522,7 @@ class Test_import(FixerTestCase):
 
         self.always_exists = False
         self.present_files = set(['__init__.py'])
-        expected_extensions = ('.py', os.path.pathsep, '.pyc', '.so',
-                               '.sl', '.pyd')
+        expected_extensions = ('.py', os.path.sep, '.pyc', '.so', '.sl', '.pyd')
         names_to_test = (p("/spam/eggs.py"), "ni.py", p("../../shrubbery.py"))
 
         for name in names_to_test:
@@ -3569,6 +3552,13 @@ class Test_import(FixerTestCase):
         self.present_files = set(["__init__.py", "bar.py"])
         self.check(b, a)
 
+    def test_import_from_package(self):
+        b = "import bar"
+        a = "from . import bar"
+        self.always_exists = False
+        self.present_files = set(["__init__.py", "bar/"])
+        self.check(b, a)
+
     def test_comments_and_indent(self):
         b = "import bar # Foo"
         a = "from . import bar # Foo"
@@ -4095,3 +4085,26 @@ class Test_getcwdu(FixerTestCase):
         b = """os.getcwdu (  )"""
         a = """os.getcwd (  )"""
         self.check(b, a)
+
+
+class Test_operator(FixerTestCase):
+
+    fixer = "operator"
+
+    def test_operator_isCallable(self):
+        b = "operator.isCallable(x)"
+        a = "hasattr(x, '__call__')"
+        self.check(b, a)
+
+    def test_operator_sequenceIncludes(self):
+        b = "operator.sequenceIncludes(x, y)"
+        a = "operator.contains(x, y)"
+        self.check(b, a)
+
+    def test_bare_isCallable(self):
+        s = "isCallable(x)"
+        self.warns_unchanged(s, "You should use hasattr(x, '__call__') here.")
+
+    def test_bare_sequenceIncludes(self):
+        s = "sequenceIncludes(x, y)"
+        self.warns_unchanged(s, "You should use operator.contains here.")
diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py
index e55f555..b282cba 100644
--- a/Lib/lib2to3/tests/test_refactor.py
+++ b/Lib/lib2to3/tests/test_refactor.py
@@ -7,9 +7,12 @@ import os
 import operator
 import io
 import tempfile
+import shutil
 import unittest
+import warnings
 
 from lib2to3 import refactor, pygram, fixer_base
+from lib2to3.pgen2 import token
 
 from . import support
 
@@ -42,14 +45,11 @@ class TestRefactoringTool(unittest.TestCase):
         return refactor.RefactoringTool(fixers, options, explicit)
 
     def test_print_function_option(self):
-        gram = pygram.python_grammar
-        save = gram.keywords["print"]
-        try:
-            rt = self.rt({"print_function" : True})
-            self.assertRaises(KeyError, operator.itemgetter("print"),
-                              gram.keywords)
-        finally:
-            gram.keywords["print"] = save
+        with warnings.catch_warnings(record=True) as w:
+            refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True})
+        self.assertEqual(len(w), 1)
+        msg, = w
+        self.assertTrue(msg.category is DeprecationWarning)
 
     def test_fixer_loading_helpers(self):
         contents = ["explicit", "first", "last", "parrot", "preorder"]
@@ -61,19 +61,63 @@ class TestRefactoringTool(unittest.TestCase):
         self.assertEqual(full_names,
                          ["myfixes.fix_" + name for name in contents])
 
+    def test_detect_future_print(self):
+        run = refactor._detect_future_print
+        self.assertFalse(run(""))
+        self.assertTrue(run("from __future__ import print_function"))
+        self.assertFalse(run("from __future__ import generators"))
+        self.assertFalse(run("from __future__ import generators, feature"))
+        input = "from __future__ import generators, print_function"
+        self.assertTrue(run(input))
+        input ="from __future__ import print_function, generators"
+        self.assertTrue(run(input))
+        input = "from __future__ import (print_function,)"
+        self.assertTrue(run(input))
+        input = "from __future__ import (generators, print_function)"
+        self.assertTrue(run(input))
+        input = "from __future__ import (generators, nested_scopes)"
+        self.assertFalse(run(input))
+        input = """from __future__ import generators
+from __future__ import print_function"""
+        self.assertTrue(run(input))
+        self.assertFalse(run("from"))
+        self.assertFalse(run("from 4"))
+        self.assertFalse(run("from x"))
+        self.assertFalse(run("from x 5"))
+        self.assertFalse(run("from x im"))
+        self.assertFalse(run("from x import"))
+        self.assertFalse(run("from x import 4"))
+        input = "'docstring'\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "'docstring'\n'somng'\nfrom __future__ import print_function"
+        self.assertFalse(run(input))
+        input = "# comment\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "# comment\n'doc'\nfrom __future__ import print_function"
+        self.assertTrue(run(input))
+        input = "class x: pass\nfrom __future__ import print_function"
+        self.assertFalse(run(input))
+
     def test_get_headnode_dict(self):
         class NoneFix(fixer_base.BaseFix):
-            PATTERN = None
+            pass
 
         class FileInputFix(fixer_base.BaseFix):
             PATTERN = "file_input< any * >"
 
+        class SimpleFix(fixer_base.BaseFix):
+            PATTERN = "'name'"
+
         no_head = NoneFix({}, [])
         with_head = FileInputFix({}, [])
-        d = refactor.get_headnode_dict([no_head, with_head])
-        expected = {None: [no_head],
-                    pygram.python_symbols.file_input : [with_head]}
-        self.assertEqual(d, expected)
+        simple = SimpleFix({}, [])
+        d = refactor._get_headnode_dict([no_head, with_head, simple])
+        top_fixes = d.pop(pygram.python_symbols.file_input)
+        self.assertEqual(top_fixes, [with_head, no_head])
+        name_fixes = d.pop(token.NAME)
+        self.assertEqual(name_fixes, [simple, no_head])
+        for fixes in d.values():
+            self.assertEqual(fixes, [no_head])
 
     def test_fixer_loading(self):
         from myfixes.fix_first import FixFirst
@@ -106,10 +150,10 @@ class TestRefactoringTool(unittest.TestCase):
 
         class MyRT(refactor.RefactoringTool):
 
-            def print_output(self, lines):
-                diff_lines.extend(lines)
+            def print_output(self, old_text, new_text, filename, equal):
+                results.extend([old_text, new_text, filename, equal])
 
-        diff_lines = []
+        results = []
         rt = MyRT(_DEFAULT_FIXERS)
         save = sys.stdin
         sys.stdin = io.StringIO("def parrot(): pass\n\n")
@@ -117,12 +161,10 @@ class TestRefactoringTool(unittest.TestCase):
             rt.refactor_stdin()
         finally:
             sys.stdin = save
-        expected = """--- <stdin> (original)
-+++ <stdin> (refactored)
-@@ -1,2 +1,2 @@
--def parrot(): pass
-+def cheese(): pass""".splitlines()
-        self.assertEqual(diff_lines[:-1], expected)
+        expected = ["def parrot(): pass\n\n",
+                    "def cheese(): pass\n\n",
+                    "<stdin>", False]
+        self.assertEqual(results, expected)
 
     def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):
         def read_file():
@@ -145,6 +187,37 @@ class TestRefactoringTool(unittest.TestCase):
         test_file = os.path.join(FIXER_DIR, "parrot_example.py")
         self.check_file_refactoring(test_file, _DEFAULT_FIXERS)
 
+    def test_refactor_dir(self):
+        def check(structure, expected):
+            def mock_refactor_file(self, f, *args):
+                got.append(f)
+            save_func = refactor.RefactoringTool.refactor_file
+            refactor.RefactoringTool.refactor_file = mock_refactor_file
+            rt = self.rt()
+            got = []
+            dir = tempfile.mkdtemp(prefix="2to3-test_refactor")
+            try:
+                os.mkdir(os.path.join(dir, "a_dir"))
+                for fn in structure:
+                    open(os.path.join(dir, fn), "wb").close()
+                rt.refactor_dir(dir)
+            finally:
+                refactor.RefactoringTool.refactor_file = save_func
+                shutil.rmtree(dir)
+            self.assertEqual(got,
+                             [os.path.join(dir, path) for path in expected])
+        check([], [])
+        tree = ["nothing",
+                "hi.py",
+                ".dumb",
+                ".after.py",
+                "sappy"]
+        expected = ["hi.py"]
+        check(tree, expected)
+        tree = ["hi.py",
+                "a_dir/stuff.py"]
+        check(tree, tree)
+
     def test_file_encoding(self):
         fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
         self.check_file_refactoring(fn)
diff --git a/Lib/lib2to3/tests/test_util.py b/Lib/lib2to3/tests/test_util.py
index ef3256d..6186b4f 100644
--- a/Lib/lib2to3/tests/test_util.py
+++ b/Lib/lib2to3/tests/test_util.py
@@ -1,4 +1,4 @@
-""" Test suite for the code in fixes.util """
+""" Test suite for the code in fixer_util """
 
 # Testing imports
 from . import support
@@ -7,10 +7,10 @@ from . import support
 import os.path
 
 # Local imports
-from .. import pytree
-from .. import fixer_util
-from ..fixer_util import Attr, Name
-
+from lib2to3.pytree import Node, Leaf
+from lib2to3 import fixer_util
+from lib2to3.fixer_util import Attr, Name, Call, Comma
+from lib2to3.pgen2 import token
 
 def parse(code, strip_levels=0):
     # The topmost node is file_input, which we don't care about.
@@ -24,7 +24,7 @@ def parse(code, strip_levels=0):
 class MacroTestCase(support.TestCase):
     def assertStr(self, node, string):
         if isinstance(node, (tuple, list)):
-            node = pytree.Node(fixer_util.syms.simple_stmt, node)
+            node = Node(fixer_util.syms.simple_stmt, node)
         self.assertEqual(str(node), string)
 
 
@@ -78,6 +78,31 @@ class Test_Name(MacroTestCase):
         self.assertStr(Name("a", prefix="b"), "ba")
 
 
+class Test_Call(MacroTestCase):
+    def _Call(self, name, args=None, prefix=None):
+        """Help the next test"""
+        children = []
+        if isinstance(args, list):
+            for arg in args:
+                children.append(arg)
+                children.append(Comma())
+            children.pop()
+        return Call(Name(name), children, prefix)
+
+    def test(self):
+        kids = [None,
+                [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 2),
+                 Leaf(token.NUMBER, 3)],
+                [Leaf(token.NUMBER, 1), Leaf(token.NUMBER, 3),
+                 Leaf(token.NUMBER, 2), Leaf(token.NUMBER, 4)],
+                [Leaf(token.STRING, "b"), Leaf(token.STRING, "j", prefix=" ")]
+                ]
+        self.assertStr(self._Call("A"), "A()")
+        self.assertStr(self._Call("b", kids[1]), "b(1,2,3)")
+        self.assertStr(self._Call("a.b().c", kids[2]), "a.b().c(1,3,2,4)")
+        self.assertStr(self._Call("d", kids[3], prefix=" "), " d(b, j)")
+
+
 class Test_does_tree_import(support.TestCase):
     def _find_bind_rec(self, name, node):
         # Search a tree for a binding -- used to find the starting