Merged revisions 76160-76161,76250,76252,76447,76506 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r76160 | benjamin.peterson | 2009-11-08 18:53:48 -0600 (Sun, 08 Nov 2009) | 1 line

  undeprecate the -p option; it's useful for converting python3 sources
........
  r76161 | benjamin.peterson | 2009-11-08 19:05:37 -0600 (Sun, 08 Nov 2009) | 1 line

  simplify condition
........
  r76250 | benjamin.peterson | 2009-11-13 16:56:48 -0600 (Fri, 13 Nov 2009) | 1 line

  fix handling of a utf-8 bom #7313
........
  r76252 | benjamin.peterson | 2009-11-13 16:58:36 -0600 (Fri, 13 Nov 2009) | 1 line

  remove pdb turd
........
  r76447 | benjamin.peterson | 2009-11-22 18:17:40 -0600 (Sun, 22 Nov 2009) | 1 line

  #7375 fix nested transformations in fix_urllib
........
  r76506 | benjamin.peterson | 2009-11-24 18:34:31 -0600 (Tue, 24 Nov 2009) | 1 line

  use generator expressions in any()
........

1 files changed, 7 insertions, 3 deletions

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index fb41a0a..163c561 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -281,9 +281,13 @@ def detect_encoding(readline):
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
-        if bom_found and codec.name != 'utf-8':
-            # This behaviour mimics the Python interpreter
-            raise SyntaxError('encoding problem: utf-8')
+        if bom_found:
+            if codec.name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            else:
+                # Allow it to be properly encoded and decoded.
+                encoding = 'utf-8-sig'
         return encoding
 
     first = read_or_stop()