Merged revisions 76160-76161,76250,76252,76447,76506 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r76160 | benjamin.peterson | 2009-11-08 18:53:48 -0600 (Sun, 08 Nov 2009) | 1 line

  undeprecate the -p option; it's useful for converting python3 sources
........
  r76161 | benjamin.peterson | 2009-11-08 19:05:37 -0600 (Sun, 08 Nov 2009) | 1 line

  simplify condition
........
  r76250 | benjamin.peterson | 2009-11-13 16:56:48 -0600 (Fri, 13 Nov 2009) | 1 line

  fix handling of a utf-8 bom #7313
........
  r76252 | benjamin.peterson | 2009-11-13 16:58:36 -0600 (Fri, 13 Nov 2009) | 1 line

  remove pdb turd
........
  r76447 | benjamin.peterson | 2009-11-22 18:17:40 -0600 (Sun, 22 Nov 2009) | 1 line

  #7375 fix nested transformations in fix_urllib
........
  r76506 | benjamin.peterson | 2009-11-24 18:34:31 -0600 (Tue, 24 Nov 2009) | 1 line

  use generator expressions in any()
........

10 files changed, 68 insertions, 50 deletions

diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py
index 9c815e1..e1ad667 100644
--- a/Lib/lib2to3/fixes/fix_imports.py
+++ b/Lib/lib2to3/fixes/fix_imports.py
@@ -108,7 +108,7 @@ class FixImports(fixer_base.BaseFix):
             # Module usage could be in the trailer of an attribute lookup, so we
             # might have nested matches when "bare_with_attr" is present.
             if "bare_with_attr" not in results and \
-                    any([match(obj) for obj in attr_chain(node, "parent")]):
+                    any(match(obj) for obj in attr_chain(node, "parent")):
                 return False
             return results
         return False
diff --git a/Lib/lib2to3/fixes/fix_next.py b/Lib/lib2to3/fixes/fix_next.py
index c70f72b..9514798 100644
--- a/Lib/lib2to3/fixes/fix_next.py
+++ b/Lib/lib2to3/fixes/fix_next.py
@@ -99,4 +99,4 @@ def find_assign(node):
 def is_subtree(root, node):
     if root == node:
         return True
-    return any([is_subtree(c, node) for c in root.children])
+    return any(is_subtree(c, node) for c in root.children)
diff --git a/Lib/lib2to3/fixes/fix_renames.py b/Lib/lib2to3/fixes/fix_renames.py
index ae398c7..a45345a 100644
--- a/Lib/lib2to3/fixes/fix_renames.py
+++ b/Lib/lib2to3/fixes/fix_renames.py
@@ -49,7 +49,7 @@ class FixRenames(fixer_base.BaseFix):
         match = super(FixRenames, self).match
         results = match(node)
         if results:
-            if any([match(obj) for obj in attr_chain(node, "parent")]):
+            if any(match(obj) for obj in attr_chain(node, "parent")):
                 return False
             return results
         return False
diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py
index d11220c..db18ca8 100644
--- a/Lib/lib2to3/fixes/fix_urllib.py
+++ b/Lib/lib2to3/fixes/fix_urllib.py
@@ -63,7 +63,8 @@ def build_pattern():
             yield """import_name< 'import'
                                   dotted_as_name< module_as=%r 'as' any > >
                   """ % old_module
-            yield """power< module_dot=%r trailer< '.' member=%s > any* >
+            # bare_with_attr has a special significance for FixImports.match().
+            yield """power< bare_with_attr=%r trailer< '.' member=%s > any* >
                   """ % (old_module, members)
 
 
@@ -150,12 +151,11 @@ class FixUrllib(FixImports):
 
     def transform_dot(self, node, results):
         """Transform for calls to module members in code."""
-        module_dot = results.get('module_dot')
+        module_dot = results.get('bare_with_attr')
         member = results.get('member')
-        # this may be a list of length one, or just a node
+        new_name = None
         if isinstance(member, list):
             member = member[0]
-        new_name = None
         for change in MAPPING[module_dot.value]:
             if member.value in change[1]:
                 new_name = change[0]
@@ -171,7 +171,7 @@ class FixUrllib(FixImports):
             self.transform_import(node, results)
         elif results.get('mod_member'):
             self.transform_member(node, results)
-        elif results.get('module_dot'):
+        elif results.get('bare_with_attr'):
             self.transform_dot(node, results)
         # Renaming and star imports are not supported for these modules.
         elif results.get('module_star'):
diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
index 9238807..736d5a6 100644
--- a/Lib/lib2to3/main.py
+++ b/Lib/lib2to3/main.py
@@ -91,8 +91,7 @@ def main(fixer_pkg, args=None):
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations (fixes/fix_*.py)")
     parser.add_option("-p", "--print-function", action="store_true",
-                      help="DEPRECATED Modify the grammar so that print() is "
-                          "a function")
+                      help="Modify the grammar so that print() is a function")
     parser.add_option("-v", "--verbose", action="store_true",
                       help="More verbose logging")
     parser.add_option("--no-diffs", action="store_true",
@@ -104,12 +103,10 @@ def main(fixer_pkg, args=None):
 
     # Parse command line arguments
     refactor_stdin = False
+    flags = {}
     options, args = parser.parse_args(args)
     if not options.write and options.no_diffs:
         warn("not writing files and not printing diffs; that's not very useful")
-    if options.print_function:
-        warn("-p is deprecated; "
-             "detection of from __future__ import print_function is automatic")
     if not options.write and options.nobackups:
         parser.error("Can't use -n without -w")
     if options.list_fixes:
@@ -127,6 +124,8 @@ def main(fixer_pkg, args=None):
         if options.write:
             print >> sys.stderr, "Can't write to stdin."
             return 2
+    if options.print_function:
+        flags["print_function"] = True
 
     # Set up logging handler
     level = logging.DEBUG if options.verbose else logging.INFO
@@ -147,7 +146,7 @@ def main(fixer_pkg, args=None):
     else:
         requested = avail_fixes.union(explicit)
     fixer_names = requested.difference(unwanted_fixes)
-    rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit),
+    rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
                                options.nobackups, not options.no_diffs)
 
     # Refactor all files and directories passed as arguments
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index fb41a0a..163c561 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -281,9 +281,13 @@ def detect_encoding(readline):
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
-        if bom_found and codec.name != 'utf-8':
-            # This behaviour mimics the Python interpreter
-            raise SyntaxError('encoding problem: utf-8')
+        if bom_found:
+            if codec.name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            else:
+                # Allow it to be properly encoded and decoded.
+                encoding = 'utf-8-sig'
         return encoding
 
     first = read_or_stop()
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index 97a540d..df5456e 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -18,7 +18,6 @@ import logging
 import operator
 import collections
 import StringIO
-import warnings
 from itertools import chain
 
 # Local imports
@@ -139,26 +138,23 @@ def _detect_future_print(source):
                 if have_docstring:
                     break
                 have_docstring = True
-            elif tp == token.NAME:
-                if value == u"from":
+            elif tp == token.NAME and value == u"from":
+                tp, value = advance()
+                if tp != token.NAME and value != u"__future__":
+                    break
+                tp, value = advance()
+                if tp != token.NAME and value != u"import":
+                    break
+                tp, value = advance()
+                if tp == token.OP and value == u"(":
                     tp, value = advance()
-                    if tp != token.NAME and value != u"__future__":
-                        break
+                while tp == token.NAME:
+                    if value == u"print_function":
+                        return True
                     tp, value = advance()
-                    if tp != token.NAME and value != u"import":
+                    if tp != token.OP and value != u",":
                         break
                     tp, value = advance()
-                    if tp == token.OP and value == u"(":
-                        tp, value = advance()
-                    while tp == token.NAME:
-                        if value == u"print_function":
-                            return True
-                        tp, value = advance()
-                        if tp != token.OP and value != u",":
-                            break
-                        tp, value = advance()
-                else:
-                    break
             else:
                 break
     except StopIteration:
@@ -172,7 +168,7 @@ class FixerError(Exception):
 
 class RefactoringTool(object):
 
-    _default_options = {}
+    _default_options = {"print_function" : False}
 
     CLASS_PREFIX = "Fix" # The prefix for fixer classes
     FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -189,15 +185,16 @@ class RefactoringTool(object):
         self.explicit = explicit or []
         self.options = self._default_options.copy()
         if options is not None:
-            if "print_function" in options:
-                warnings.warn("the 'print_function' option is deprecated",
-                              DeprecationWarning)
             self.options.update(options)
+        if self.options["print_function"]:
+            self.grammar = pygram.python_grammar_no_print_statement
+        else:
+            self.grammar = pygram.python_grammar
         self.errors = []
         self.logger = logging.getLogger("RefactoringTool")
         self.fixer_log = []
         self.wrote = False
-        self.driver = driver.Driver(pygram.python_grammar,
+        self.driver = driver.Driver(self.grammar,
                                     convert=pytree.convert,
                                     logger=self.logger)
         self.pre_order, self.post_order = self.get_fixers()
@@ -353,7 +350,7 @@ class RefactoringTool(object):
                            name, err.__class__.__name__, err)
             return
         finally:
-            self.driver.grammar = pygram.python_grammar
+            self.driver.grammar = self.grammar
         self.log_debug("Refactoring %s", name)
         self.refactor_tree(tree, name)
         return tree
diff --git a/Lib/lib2to3/tests/data/bom.py b/Lib/lib2to3/tests/data/bom.py
new file mode 100644
index 0000000..ecb782a
--- /dev/null
+++ b/Lib/lib2to3/tests/data/bom.py
@@ -0,0 +1,3 @@
+# coding: utf-8
+print "BOM BOOM!"
+
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index cad2a7d..eb0b4b4 100755
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -1753,6 +1753,8 @@ class Test_urllib(FixerTestCase):
         for old, changes in self.modules.items():
             for new, members in changes:
                 for member in members:
+                    new_import = ", ".join([n for (n, mems)
+                                            in self.modules[old]])
                     b = """
                         import %s
                         foo(%s.%s)
@@ -1760,9 +1762,16 @@ class Test_urllib(FixerTestCase):
                     a = """
                         import %s
                         foo(%s.%s)
-                        """ % (", ".join([n for (n, mems)
-                                           in self.modules[old]]),
-                                         new, member)
+                        """ % (new_import, new, member)
+                    self.check(b, a)
+                    b = """
+                        import %s
+                        %s.%s(%s.%s)
+                        """ % (old, old, member, old, member)
+                    a = """
+                        import %s
+                        %s.%s(%s.%s)
+                        """ % (new_import, new, member, new, member)
                     self.check(b, a)
 
 
diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py
index cc47b9d..1729027 100644
--- a/Lib/lib2to3/tests/test_refactor.py
+++ b/Lib/lib2to3/tests/test_refactor.py
@@ -4,6 +4,7 @@ Unit tests for refactor.py.
 
 import sys
 import os
+import codecs
 import operator
 import StringIO
 import tempfile
@@ -45,12 +46,10 @@ class TestRefactoringTool(unittest.TestCase):
         return refactor.RefactoringTool(fixers, options, explicit)
 
     def test_print_function_option(self):
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter("always", DeprecationWarning)
-            refactor.RefactoringTool(_DEFAULT_FIXERS, {"print_function" : True})
-        self.assertEqual(len(w), 1)
-        msg, = w
-        self.assertTrue(msg.category is DeprecationWarning)
+        rt = self.rt({"print_function" : True})
+        self.assertTrue(rt.grammar is pygram.python_grammar_no_print_statement)
+        self.assertTrue(rt.driver.grammar is
+                        pygram.python_grammar_no_print_statement)
 
     def test_fixer_loading_helpers(self):
         contents = ["explicit", "first", "last", "parrot", "preorder"]
@@ -179,10 +178,12 @@ from __future__ import print_function"""
 
         try:
             rt.refactor_file(test_file, True)
-            self.assertNotEqual(old_contents, read_file())
+            new_contents = read_file()
+            self.assertNotEqual(old_contents, new_contents)
         finally:
             with open(test_file, "wb") as fp:
                 fp.write(old_contents)
+        return new_contents
 
     def test_refactor_file(self):
         test_file = os.path.join(FIXER_DIR, "parrot_example.py")
@@ -223,6 +224,11 @@ from __future__ import print_function"""
         fn = os.path.join(TEST_DATA_DIR, "different_encoding.py")
         self.check_file_refactoring(fn)
 
+    def test_bom(self):
+        fn = os.path.join(TEST_DATA_DIR, "bom.py")
+        data = self.check_file_refactoring(fn)
+        self.assertTrue(data.startswith(codecs.BOM_UTF8))
+
     def test_crlf_newlines(self):
         old_sep = os.linesep
         os.linesep = "\r\n"