diff options
| -rw-r--r-- | Lib/tokenize.py | 15 |
1 files changed, 5 insertions, 10 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 9d2a6bb..b8ee2c8 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -377,17 +377,12 @@ def tokenize(readline): The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ + # This import is here to avoid problems when the itertools module is not + # built yet and tokenize is imported. + from itertools import chain encoding, consumed = detect_encoding(readline) - def readline_generator(consumed): - for line in consumed: - yield line - while True: - try: - yield readline() - except StopIteration: - return - chained = readline_generator(consumed) - return _tokenize(chained.__next__, encoding) + rl_iter = iter(readline, "") + return _tokenize(chain(consumed, rl_iter).__next__, encoding) def _tokenize(readline, encoding): |