From 6c60d099e5ed97ee0026687c1ec3401cca49c0c2 Mon Sep 17 00:00:00 2001
From: Raymond Hettinger <python@rcn.com>
Date: Thu, 9 Sep 2010 04:32:39 +0000
Subject: Improve the repr for the TokenInfo named tuple.

---
 Doc/library/tokenize.rst |  2 +-
 Lib/tokenize.py          | 29 ++++++++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst
index 446d3bb..dbd01c4 100644
--- a/Doc/library/tokenize.rst
+++ b/Doc/library/tokenize.rst
@@ -111,7 +111,7 @@ function it uses to do this is available:
                 return fp.read()
 
 
-Example of a script re-writer that transforms float literals into Decimal
+Example of a script rewriter that transforms float literals into Decimal
 objects::
 
     from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index aa4b246..c97ed8a 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -63,7 +63,9 @@ class TokenInfo(tuple):
         return result
 
     def __repr__(self):
-        return 'TokenInfo(type=%r, string=%r, start=%r, end=%r, line=%r)' % self
+        typ = self[0]
+        return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \
+               ((('%d (%s)' % (typ, tok_name[typ])),) + self[1:])
 
     def _asdict(self):
         'Return a new dict which maps field names to their values'
@@ -550,3 +552,28 @@ def _tokenize(readline, encoding):
 # library that expect to be able to use tokenize with strings
 def generate_tokens(readline):
     return _tokenize(readline, None)
+
+if __name__ == "__main__":
+    # Quick sanity check
+    s = b'''def parseline(self, line):
+            """Parse the line into a command name and a string containing
+            the arguments.  Returns a tuple containing (command, args, line).
+            'command' and 'args' may be None if the line couldn't be parsed.
+            """
+            line = line.strip()
+            if not line:
+                return None, None, line
+            elif line[0] == '?':
+                line = 'help ' + line[1:]
+            elif line[0] == '!':
+                if hasattr(self, 'do_shell'):
+                    line = 'shell ' + line[1:]
+                else:
+                    return None, None, line
+            i, n = 0, len(line)
+            while i < n and line[i] in self.identchars: i = i+1
+            cmd, arg = line[:i], line[i:].strip()
+            return cmd, arg, line
+    '''
+    for tok in tokenize(iter(s.splitlines()).__next__):
+        print(tok)
-- 
cgit v0.12