- Issue #719888: Updated tokenize to use a bytes API. generate_tokens has been

  renamed tokenize and now works with bytes rather than strings. A new
  detect_encoding function has been added for determining source file encoding
  according to PEP-0263. Token sequences returned by tokenize always start
  with an ENCODING token which specifies the encoding used to decode the file.
  This token is used to encode the output of untokenize back to bytes.

Credit goes to Michael "I'm-going-to-name-my-first-child-unittest" Foord from Resolver Systems for this work.

3 files changed, 9 insertions, 3 deletions

diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py
index bdf52e0..69a19ef 100755
--- a/Tools/i18n/pygettext.py
+++ b/Tools/i18n/pygettext.py
@@ -631,7 +631,9 @@ def main():
         try:
             eater.set_filename(filename)
             try:
-                tokenize.tokenize(fp.readline, eater)
+                tokens = tokenize.generate_tokens(fp.readline)
+                for _token in tokens:
+                    eater(*_token)
             except tokenize.TokenError as e:
                 print('%s: %s, line %d, column %d' % (
                     e.args[0], filename, e.args[1][0], e.args[1][1]),
diff --git a/Tools/scripts/checkappend.py b/Tools/scripts/checkappend.py
index 8953285..4c74ee5 100755
--- a/Tools/scripts/checkappend.py
+++ b/Tools/scripts/checkappend.py
@@ -103,7 +103,9 @@ class AppendChecker:
 
     def run(self):
         try:
-            tokenize.tokenize(self.file.readline, self.tokeneater)
+            tokens = tokenize.generate_tokens(self.file.readline)
+            for _token in tokens:
+                self.tokeneater(*_token)
         except tokenize.TokenError as msg:
             errprint("%r: Token Error: %s" % (self.fname, msg))
             self.nerrors = self.nerrors + 1
diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py
index 981f63a..9a55198 100755
--- a/Tools/scripts/reindent.py
+++ b/Tools/scripts/reindent.py
@@ -173,7 +173,9 @@ class Reindenter:
         self.stats = []
 
     def run(self):
-        tokenize.tokenize(self.getline, self.tokeneater)
+        tokens = tokenize.generate_tokens(self.getline)
+        for _token in tokens:
+            self.tokeneater(*_token)
         # Remove trailing empty lines.
         lines = self.lines
         while lines and lines[-1] == "\n":