Merged revisions 67427,67431,67433,67435,67630,67652,67656-67657,67674-67675,67678-67679,67705-67706,67716,67723,67765-67771,67774,67776,67778 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r67427 | benjamin.peterson | 2008-11-28 16:07:41 -0600 (Fri, 28 Nov 2008) | 1 line

  fix spelling in comment
........
  r67431 | benjamin.peterson | 2008-11-28 17:14:08 -0600 (Fri, 28 Nov 2008) | 1 line

  add a scripts directory; move things to it
........
  r67433 | benjamin.peterson | 2008-11-28 17:18:48 -0600 (Fri, 28 Nov 2008) | 1 line

  run svneol.py
........
  r67435 | benjamin.peterson | 2008-11-28 17:25:03 -0600 (Fri, 28 Nov 2008) | 1 line

  rename pre/post_order_mapping to pre/post_order_heads
........
  r67630 | alexandre.vassalotti | 2008-12-06 21:51:56 -0600 (Sat, 06 Dec 2008) | 2 lines

  Fix typo in the urllib2.HTTPDigestAuthHandler fixer.
........
  r67652 | armin.ronacher | 2008-12-07 15:39:43 -0600 (Sun, 07 Dec 2008) | 5 lines

  Added a fixer that cleans up a tuple argument to isinstance after the tokens
  in it were fixed.  This is mainly used to remove double occurrences of
  tokens as a leftover of the long -> int / unicode -> str conversion.
........
  r67656 | armin.ronacher | 2008-12-07 16:54:16 -0600 (Sun, 07 Dec 2008) | 3 lines

  Added missing copyright fo 2to3 fix_isinstance.
........
  r67657 | armin.ronacher | 2008-12-07 18:29:35 -0600 (Sun, 07 Dec 2008) | 3 lines

  2to3: intern and reduce fixes now add the imports if missing.  Because that is a common task the fixer_util module now has a function "touch_import" that adds imports if missing.
........
  r67674 | benjamin.peterson | 2008-12-08 19:58:11 -0600 (Mon, 08 Dec 2008) | 1 line

  copy permission bits when making backup files #4602
........
  r67675 | benjamin.peterson | 2008-12-08 19:59:11 -0600 (Mon, 08 Dec 2008) | 1 line

  add forgotten import
........
  r67678 | benjamin.peterson | 2008-12-08 20:08:30 -0600 (Mon, 08 Dec 2008) | 1 line

  fix #4602 for real
........
  r67679 | armin.ronacher | 2008-12-09 00:54:03 -0600 (Tue, 09 Dec 2008) | 3 lines

  Removed redudant code from the 2to3 long fixer.  This fixes #4590.
........
  r67705 | benjamin.peterson | 2008-12-11 13:04:08 -0600 (Thu, 11 Dec 2008) | 1 line

  put trailers after a range call after the list()
........
  r67706 | benjamin.peterson | 2008-12-11 13:17:57 -0600 (Thu, 11 Dec 2008) | 1 line

  add html related modules to the fix_imports mapping
........
  r67716 | benjamin.peterson | 2008-12-11 22:16:47 -0600 (Thu, 11 Dec 2008) | 1 line

  consolidate tests
........
  r67723 | benjamin.peterson | 2008-12-12 19:49:31 -0600 (Fri, 12 Dec 2008) | 1 line

  fix name
........
  r67765 | benjamin.peterson | 2008-12-14 14:05:05 -0600 (Sun, 14 Dec 2008) | 1 line

  run fix_isinstance after fix_long and fix_unicode
........
  r67766 | benjamin.peterson | 2008-12-14 14:13:05 -0600 (Sun, 14 Dec 2008) | 1 line

  use run_order instead of order
........
  r67767 | benjamin.peterson | 2008-12-14 14:28:12 -0600 (Sun, 14 Dec 2008) | 1 line

  don't retain parenthesis if there is only one item left
........
  r67768 | benjamin.peterson | 2008-12-14 14:32:30 -0600 (Sun, 14 Dec 2008) | 1 line

  use insert_child()
........
  r67769 | benjamin.peterson | 2008-12-14 14:59:10 -0600 (Sun, 14 Dec 2008) | 1 line

  parenthesize doesn't belong in pygram or FixerBase
........
  r67770 | alexandre.vassalotti | 2008-12-14 15:15:36 -0600 (Sun, 14 Dec 2008) | 2 lines

  Fix typo: html.paser -> html.parser.
........
  r67771 | benjamin.peterson | 2008-12-14 15:22:09 -0600 (Sun, 14 Dec 2008) | 1 line

  altering .children needs to call changed()
........
  r67774 | benjamin.peterson | 2008-12-14 15:55:38 -0600 (Sun, 14 Dec 2008) | 1 line

  employ an evil hack to fix multiple names in the same import statement
........
  r67776 | benjamin.peterson | 2008-12-14 16:22:38 -0600 (Sun, 14 Dec 2008) | 1 line

  make a common mixin class for Test_imports and friends
........
  r67778 | alexandre.vassalotti | 2008-12-14 17:48:20 -0600 (Sun, 14 Dec 2008) | 2 lines

  Make fix_imports refactor multiple imports as.
........

11 files changed, 134 insertions, 41 deletions

diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py
index faede68..5af13b7 100644
--- a/Lib/lib2to3/fixes/fix_apply.py
+++ b/Lib/lib2to3/fixes/fix_apply.py
@@ -9,7 +9,7 @@ This converts apply(func, v, k) into (func)(*v, **k)."""
 from .. import pytree
 from ..pgen2 import token
 from .. import fixer_base
-from ..fixer_util import Call, Comma
+from ..fixer_util import Call, Comma, parenthesize
 
 class FixApply(fixer_base.BaseFix):
 
@@ -39,7 +39,7 @@ class FixApply(fixer_base.BaseFix):
             (func.type != syms.power or
              func.children[-2].type == token.DOUBLESTAR)):
             # Need to parenthesize
-            func = self.parenthesize(func)
+            func = parenthesize(func)
         func.set_prefix("")
         args = args.clone()
         args.set_prefix("")
diff --git a/Lib/lib2to3/fixes/fix_has_key.py b/Lib/lib2to3/fixes/fix_has_key.py
index fb7b07b..482f27d 100644
--- a/Lib/lib2to3/fixes/fix_has_key.py
+++ b/Lib/lib2to3/fixes/fix_has_key.py
@@ -33,7 +33,7 @@ CAVEATS:
 from .. import pytree
 from ..pgen2 import token
 from .. import fixer_base
-from ..fixer_util import Name
+from ..fixer_util import Name, parenthesize
 
 
 class FixHasKey(fixer_base.BaseFix):
@@ -86,7 +86,7 @@ class FixHasKey(fixer_base.BaseFix):
             after = [n.clone() for n in after]
         if arg.type in (syms.comparison, syms.not_test, syms.and_test,
                         syms.or_test, syms.test, syms.lambdef, syms.argument):
-            arg = self.parenthesize(arg)
+            arg = parenthesize(arg)
         if len(before) == 1:
             before = before[0]
         else:
@@ -98,12 +98,12 @@ class FixHasKey(fixer_base.BaseFix):
             n_op = pytree.Node(syms.comp_op, (n_not, n_op))
         new = pytree.Node(syms.comparison, (arg, n_op, before))
         if after:
-            new = self.parenthesize(new)
+            new = parenthesize(new)
             new = pytree.Node(syms.power, (new,) + tuple(after))
         if node.parent.type in (syms.comparison, syms.expr, syms.xor_expr,
                                 syms.and_expr, syms.shift_expr,
                                 syms.arith_expr, syms.term,
                                 syms.factor, syms.power):
-            new = self.parenthesize(new)
+            new = parenthesize(new)
         new.set_prefix(prefix)
         return new
diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py
index e48c4f0..75770c9 100644
--- a/Lib/lib2to3/fixes/fix_imports.py
+++ b/Lib/lib2to3/fixes/fix_imports.py
@@ -42,6 +42,8 @@ MAPPING = {'StringIO':  'io',
            'DocXMLRPCServer': 'xmlrpc.server',
            'SimpleXMLRPCServer': 'xmlrpc.server',
            'httplib': 'http.client',
+           'htmlentitydefs' : 'html.entities',
+           'HTMLParser' : 'html.parser',
            'Cookie': 'http.cookies',
            'cookielib': 'http.cookiejar',
            'BaseHTTPServer': 'http.server',
@@ -64,16 +66,17 @@ def build_pattern(mapping=MAPPING):
     mod_list = ' | '.join(["module_name='%s'" % key for key in mapping])
     bare_names = alternates(mapping.keys())
 
-    yield """name_import=import_name< 'import' ((%s)
-                          | dotted_as_names< any* (%s) any* >) >
+    yield """name_import=import_name< 'import' ((%s) |
+               multiple_imports=dotted_as_names< any* (%s) any* >) >
           """ % (mod_list, mod_list)
     yield """import_from< 'from' (%s) 'import' ['(']
               ( any | import_as_name< any 'as' any > |
                 import_as_names< any* >)  [')'] >
           """ % mod_list
-    yield """import_name< 'import'
-                          dotted_as_name< (%s) 'as' any > >
-          """ % mod_list
+    yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > |
+               multiple_imports=dotted_as_names<
+                 any* dotted_as_name< (%s) 'as' any > any* >) >
+          """ % (mod_list, mod_list)
 
     # Find usages of module members in code e.g. thread.foo(bar)
     yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names
@@ -100,8 +103,8 @@ class FixImports(fixer_base.BaseFix):
         match = super(FixImports, self).match
         results = match(node)
         if results:
-            # Module usage could be in the trailier of an attribute lookup, so
-            # we might have nested matches when "bare_with_attr" is present.
+            # Module usage could be in the trailer of an attribute lookup, so we
+            # might have nested matches when "bare_with_attr" is present.
             if "bare_with_attr" not in results and \
                     any([match(obj) for obj in attr_chain(node, "parent")]):
                 return False
@@ -116,11 +119,21 @@ class FixImports(fixer_base.BaseFix):
         import_mod = results.get("module_name")
         if import_mod:
             new_name = self.mapping[(import_mod or mod_name).value]
+            import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
             if "name_import" in results:
                 # If it's not a "from x import x, y" or "import x as y" import,
                 # marked its usage to be replaced.
                 self.replace[import_mod.value] = new_name
-            import_mod.replace(Name(new_name, prefix=import_mod.get_prefix()))
+            if "multiple_imports" in results:
+                # This is a nasty hack to fix multiple imports on a
+                # line (e.g., "import StringIO, urlparse"). The problem is that I
+                # can't figure out an easy way to make a pattern recognize the
+                # keys of MAPPING randomly sprinkled in an import statement.
+                while True:
+                    results = self.match(node)
+                    if not results:
+                        break
+                    self.transform(node, results)
         else:
             # Replace usage of the module.
             bare_name = results["bare_with_attr"][0]
diff --git a/Lib/lib2to3/fixes/fix_imports2.py b/Lib/lib2to3/fixes/fix_imports2.py
index bcd7aa6..477bafd 100644
--- a/Lib/lib2to3/fixes/fix_imports2.py
+++ b/Lib/lib2to3/fixes/fix_imports2.py
@@ -11,6 +11,6 @@ MAPPING = {
 
 class FixImports2(fix_imports.FixImports):
 
-    order = "post"
+    run_order = 6
 
     mapping = MAPPING
diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py
index 921ba59..66c616e 100644
--- a/Lib/lib2to3/fixes/fix_intern.py
+++ b/Lib/lib2to3/fixes/fix_intern.py
@@ -8,7 +8,7 @@ intern(s) -> sys.intern(s)"""
 # Local imports
 from .. import pytree
 from .. import fixer_base
-from ..fixer_util import Name, Attr
+from ..fixer_util import Name, Attr, touch_import
 
 
 class FixIntern(fixer_base.BaseFix):
@@ -40,4 +40,5 @@ class FixIntern(fixer_base.BaseFix):
                                         newarglist,
                                         results["rpar"].clone()])] + after)
         new.set_prefix(node.get_prefix())
+        touch_import(None, 'sys', node)
         return new
diff --git a/Lib/lib2to3/fixes/fix_isinstance.py b/Lib/lib2to3/fixes/fix_isinstance.py
new file mode 100644
index 0000000..295577a
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_isinstance.py
@@ -0,0 +1,52 @@
+# Copyright 2008 Armin Ronacher.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Fixer that cleans up a tuple argument to isinstance after the tokens
+in it were fixed.  This is mainly used to remove double occurrences of
+tokens as a leftover of the long -> int / unicode -> str conversion.
+
+eg.  isinstance(x, (int, long)) -> isinstance(x, (int, int))
+       -> isinstance(x, int)
+"""
+
+from .. import fixer_base
+from ..fixer_util import token
+
+
+class FixIsinstance(fixer_base.BaseFix):
+
+    PATTERN = """
+    power<
+        'isinstance'
+        trailer< '(' arglist< any ',' atom< '('
+            args=testlist_gexp< any+ >
+        ')' > > ')' >
+    >
+    """
+
+    run_order = 6
+
+    def transform(self, node, results):
+        names_inserted = set()
+        testlist = results["args"]
+        args = testlist.children
+        new_args = []
+        iterator = enumerate(args)
+        for idx, arg in iterator:
+            if arg.type == token.NAME and arg.value in names_inserted:
+                if idx < len(args) - 1 and args[idx + 1].type == token.COMMA:
+                    iterator.next()
+                    continue
+            else:
+                new_args.append(arg)
+                if arg.type == token.NAME:
+                    names_inserted.add(arg.value)
+        if new_args and new_args[-1].type == token.COMMA:
+            del new_args[-1]
+        if len(new_args) == 1:
+            atom = testlist.parent
+            new_args[0].set_prefix(atom.get_prefix())
+            atom.replace(new_args[0])
+        else:
+            args[:] = new_args
+            node.changed()
diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py
index f67f026..5fd6af5 100644
--- a/Lib/lib2to3/fixes/fix_long.py
+++ b/Lib/lib2to3/fixes/fix_long.py
@@ -2,8 +2,6 @@
 # Licensed to PSF under a Contributor Agreement.
 
 """Fixer that turns 'long' into 'int' everywhere.
-
-This also strips the trailing 'L' or 'l' from long loterals.
 """
 
 # Local imports
@@ -14,22 +12,13 @@ from ..fixer_util import Name, Number
 
 class FixLong(fixer_base.BaseFix):
 
-    PATTERN = """
-    (long_type = 'long' | number = NUMBER)
-    """
+    PATTERN = "'long'"
 
     static_long = Name("long")
     static_int = Name("int")
 
     def transform(self, node, results):
-        long_type = results.get("long_type")
-        number = results.get("number")
-        new = None
-        if long_type:
-            assert node == self.static_long, node
-            new = self.static_int.clone()
-        if number and node.value[-1] in ("l", "L"):
-            new = Number(node.value[:-1])
-        if new is not None:
-            new.set_prefix(node.get_prefix())
-            return new
+        assert node == self.static_long, node
+        new = self.static_int.clone()
+        new.set_prefix(node.get_prefix())
+        return new
diff --git a/Lib/lib2to3/fixes/fix_reduce.py b/Lib/lib2to3/fixes/fix_reduce.py
new file mode 100644
index 0000000..89fa2b4
--- /dev/null
+++ b/Lib/lib2to3/fixes/fix_reduce.py
@@ -0,0 +1,33 @@
+# Copyright 2008 Armin Ronacher.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Fixer for reduce().
+
+Makes sure reduce() is imported from the functools module if reduce is
+used in that module.
+"""
+
+from .. import pytree
+from .. import fixer_base
+from ..fixer_util import Name, Attr, touch_import
+
+
+
+class FixReduce(fixer_base.BaseFix):
+
+    PATTERN = """
+    power< 'reduce'
+        trailer< '('
+            arglist< (
+                (not(argument<any '=' any>) any ','
+                 not(argument<any '=' any>) any) |
+                (not(argument<any '=' any>) any ','
+                 not(argument<any '=' any>) any ','
+                 not(argument<any '=' any>) any)
+            ) >
+        ')' >
+    >
+    """
+
+    def transform(self, node, results):
+        touch_import('functools', 'reduce', node)
diff --git a/Lib/lib2to3/fixes/fix_repr.py b/Lib/lib2to3/fixes/fix_repr.py
index 99e7722..0bc6ba6 100644
--- a/Lib/lib2to3/fixes/fix_repr.py
+++ b/Lib/lib2to3/fixes/fix_repr.py
@@ -5,7 +5,7 @@
 
 # Local imports
 from .. import fixer_base
-from ..fixer_util import Call, Name
+from ..fixer_util import Call, Name, parenthesize
 
 
 class FixRepr(fixer_base.BaseFix):
@@ -18,5 +18,5 @@ class FixRepr(fixer_base.BaseFix):
         expr = results["expr"].clone()
 
         if expr.type == self.syms.testlist1:
-            expr = self.parenthesize(expr)
+            expr = parenthesize(expr)
         return Call(Name("repr"), [expr], prefix=node.get_prefix())
diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py
index ea7e9ca..1e74bff 100644
--- a/Lib/lib2to3/fixes/fix_urllib.py
+++ b/Lib/lib2to3/fixes/fix_urllib.py
@@ -29,7 +29,7 @@ MAPPING = {'urllib':  [
                      'AbstractBasicAuthHandler',
                      'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler',
                      'AbstractDigestAuthHandler',
-                     'HTTPDigestAuthHander', 'ProxyDigestAuthHandler',
+                     'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler',
                      'HTTPHandler', 'HTTPSHandler', 'FileHandler',
                      'FTPHandler', 'CacheFTPHandler',
                      'UnknownHandler']),
diff --git a/Lib/lib2to3/fixes/fix_xrange.py b/Lib/lib2to3/fixes/fix_xrange.py
index 85efcd0..ca8f21a 100644
--- a/Lib/lib2to3/fixes/fix_xrange.py
+++ b/Lib/lib2to3/fixes/fix_xrange.py
@@ -12,7 +12,9 @@ from .. import patcomp
 class FixXrange(fixer_base.BaseFix):
 
     PATTERN = """
-              power< (name='range'|name='xrange') trailer< '(' [any] ')' > any* >
+              power<
+                 (name='range'|name='xrange') trailer< '(' args=any ')' >
+              rest=any* >
               """
 
     def transform(self, node, results):
@@ -30,11 +32,14 @@ class FixXrange(fixer_base.BaseFix):
 
     def transform_range(self, node, results):
         if not self.in_special_context(node):
-            arg = node.clone()
-            arg.set_prefix("")
-            call = Call(Name("list"), [arg])
-            call.set_prefix(node.get_prefix())
-            return call
+            range_call = Call(Name("range"), [results["args"].clone()])
+            # Encase the range call in list().
+            list_call = Call(Name("list"), [range_call],
+                             prefix=node.get_prefix())
+            # Put things that were after the range() call after the list call.
+            for n in results["rest"]:
+                list_call.append_child(n)
+            return list_call
         return node
 
     P1 = "power< func=NAME trailer< '(' node=any ')' > any* >"