diff options
author | Tim Peters <tim.peters@gmail.com> | 2002-05-23 15:15:30 (GMT) |
---|---|---|
committer | Tim Peters <tim.peters@gmail.com> | 2002-05-23 15:15:30 (GMT) |
commit | 8ac1495a6a1d18111a626cec0c7f2eb67df3edb3 (patch) | |
tree | 2d91993770d3a5b3f3668857983d9bf75276b14f /Lib/tokenize.py | |
parent | f655328483b2e237cc2f71c1c308eceb2f30f6fd (diff) | |
download | cpython-8ac1495a6a1d18111a626cec0c7f2eb67df3edb3.zip cpython-8ac1495a6a1d18111a626cec0c7f2eb67df3edb3.tar.gz cpython-8ac1495a6a1d18111a626cec0c7f2eb67df3edb3.tar.bz2 |
Whitespace normalization.
Diffstat (limited to 'Lib/tokenize.py')
-rw-r--r-- | Lib/tokenize.py | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 42aafe4..22f28c4 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -124,14 +124,14 @@ def tokenize(readline, tokeneater=printtoken): """ The tokenize() function accepts two parameters: one representing the input stream, and one providing an output mechanism for tokenize(). - + The first parameter, readline, must be a callable object which provides the same interface as the readline() method of built-in file objects. - Each call to the function should return one line of input as a string. + Each call to the function should return one line of input as a string. The second parameter, tokeneater, must also be a callable object. It is called once for each token, with five arguments, corresponding to the - tuples generated by generate_tokens(). + tuples generated by generate_tokens(). """ try: tokenize_loop(readline, tokeneater) @@ -149,13 +149,13 @@ def generate_tokens(readline): must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as a string. - + The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the - logical line; continuation lines are included. + logical line; continuation lines are included. """ lnum = parenlev = continued = 0 namechars, numchars = string.ascii_letters + '_', '0123456789' |