diff options
| -rw-r--r-- | Doc/library/tokenize.rst | 2 | ||||
| -rw-r--r-- | Lib/tokenize.py | 29 |
2 files changed, 29 insertions, 2 deletions
diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 446d3bb..dbd01c4 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -111,7 +111,7 @@ function it uses to do this is available: return fp.read() -Example of a script re-writer that transforms float literals into Decimal +Example of a script rewriter that transforms float literals into Decimal objects:: from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP diff --git a/Lib/tokenize.py b/Lib/tokenize.py index aa4b246..c97ed8a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -63,7 +63,9 @@ class TokenInfo(tuple): return result def __repr__(self): - return 'TokenInfo(type=%r, string=%r, start=%r, end=%r, line=%r)' % self + typ = self[0] + return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \ + ((('%d (%s)' % (typ, tok_name[typ])),) + self[1:]) def _asdict(self): 'Return a new dict which maps field names to their values' @@ -550,3 +552,28 @@ def _tokenize(readline, encoding): # library that expect to be able to use tokenize with strings def generate_tokens(readline): return _tokenize(readline, None) + +if __name__ == "__main__": + # Quick sanity check + s = b'''def parseline(self, line): + """Parse the line into a command name and a string containing + the arguments. Returns a tuple containing (command, args, line). + 'command' and 'args' may be None if the line couldn't be parsed. + """ + line = line.strip() + if not line: + return None, None, line + elif line[0] == '?': + line = 'help ' + line[1:] + elif line[0] == '!': + if hasattr(self, 'do_shell'): + line = 'shell ' + line[1:] + else: + return None, None, line + i, n = 0, len(line) + while i < n and line[i] in self.identchars: i = i+1 + cmd, arg = line[:i], line[i:].strip() + return cmd, arg, line + ''' + for tok in tokenize(iter(s.splitlines()).__next__): + print(tok) |