Issue #10335: Add tokenize.open(), detect the file encoding using

tokenize.detect_encoding() and open it in read only mode.

1 files changed, 15 insertions, 0 deletions

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index eb58831..7745412 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -29,6 +29,7 @@ import sys
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
+from io import TextIOWrapper
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
@@ -335,6 +336,20 @@ def detect_encoding(readline):
     return default, [first, second]
 
 
+_builtin_open = open
+
+def open(filename):
+    """Open a file in read only mode using the encoding detected by
+    detect_encoding().
+    """
+    buffer = _builtin_open(filename, 'rb')
+    encoding, lines = detect_encoding(buffer.readline)
+    buffer.seek(0)
+    text = TextIOWrapper(buffer, encoding, line_buffering=True)
+    text.mode = 'r'
+    return text
+
+
 def tokenize(readline):
     """
     The tokenize() generator requires one argment, readline, which