diff options
Diffstat (limited to 'Lib/tokenize.py')
| -rw-r--r-- | Lib/tokenize.py | 12 |
1 files changed, 11 insertions, 1 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 825aa90..e131ae6 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -507,8 +507,15 @@ def _tokenize(readline, encoding): # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') - while True: # loop over lines in stream + last_line = b'' + line = b'' + while True: # loop over lines in stream try: + # We capture the value of the line variable here because + # readline uses the empty string '' to signal end of input, + # hence `line` itself will always be overwritten at the end + # of this loop. + last_line = line line = readline() except StopIteration: line = b'' @@ -719,6 +726,9 @@ def _tokenize(readline, encoding): yield stashed stashed = None + # Add an implicit NEWLINE if the input doesn't end in one + if last_line and last_line[-1] not in '\r\n': + yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '') for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') |