diff options
Diffstat (limited to 'Lib/lib2to3/pgen2/tokenize.py')
| -rw-r--r-- | Lib/lib2to3/pgen2/tokenize.py | 16 |
1 files changed, 9 insertions, 7 deletions
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index c31d549..33cfc33 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -94,8 +94,8 @@ ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + PseudoExtras = group(r'\\\r?\n', Comment, Triple) PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) -tokenprog, pseudoprog, single3prog, double3prog = map( - re.compile, (Token, PseudoToken, Single3, Double3)) +tokenprog, pseudoprog, single3prog, double3prog = list(map( + re.compile, (Token, PseudoToken, Single3, Double3))) endprogs = {"'": re.compile(Single), '"': re.compile(Double), "'''": single3prog, '"""': double3prog, "r'''": single3prog, 'r"""': double3prog, @@ -143,9 +143,11 @@ class TokenError(Exception): pass class StopTokenizing(Exception): pass -def printtoken(type, token, (srow, scol), (erow, ecol), line): # for testing - print "%d,%d-%d,%d:\t%s\t%s" % \ - (srow, scol, erow, ecol, tok_name[type], repr(token)) +def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing + (srow, scol) = xxx_todo_changeme + (erow, ecol) = xxx_todo_changeme1 + print("%d,%d-%d,%d:\t%s\t%s" % \ + (srow, scol, erow, ecol, tok_name[type], repr(token))) def tokenize(readline, tokeneater=printtoken): """ @@ -279,7 +281,7 @@ def generate_tokens(readline): if contstr: # continued string if not line: - raise TokenError, ("EOF in multi-line string", strstart) + raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) @@ -335,7 +337,7 @@ def generate_tokens(readline): else: # continued statement if not line: - raise TokenError, ("EOF in multi-line statement", (lnum, 0)) + raise TokenError("EOF in multi-line statement", (lnum, 0)) continued = 0 while pos < max: |