summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDingyuan Wang <abcdoyle888@gmail.com>2015-06-22 02:01:12 (GMT)
committerDingyuan Wang <abcdoyle888@gmail.com>2015-06-22 02:01:12 (GMT)
commite411b6629fb5f7bc01bec89df75737875ce6d8f5 (patch)
tree58e5f80f1d968d314e5c73941288f37db8b894f6
parentb6d1cdda8e2160ac647b39776198bf48dc7e656f (diff)
downloadcpython-e411b6629fb5f7bc01bec89df75737875ce6d8f5.zip
cpython-e411b6629fb5f7bc01bec89df75737875ce6d8f5.tar.gz
cpython-e411b6629fb5f7bc01bec89df75737875ce6d8f5.tar.bz2
Issue #20387: Restore retention of indentation during untokenize.
-rw-r--r--Lib/tokenize.py17
1 files changed, 17 insertions, 0 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index cf18bf9..4d93a83 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -244,6 +244,8 @@ class Untokenizer:
def untokenize(self, iterable):
it = iter(iterable)
+ indents = []
+ startline = False
for t in it:
if len(t) == 2:
self.compat(t, it)
@@ -254,6 +256,21 @@ class Untokenizer:
continue
if tok_type == ENDMARKER:
break
+ if tok_type == INDENT:
+ indents.append(token)
+ continue
+ elif tok_type == DEDENT:
+ indents.pop()
+ self.prev_row, self.prev_col = end
+ continue
+ elif tok_type in (NEWLINE, NL):
+ startline = True
+ elif startline and indents:
+ indent = indents[-1]
+ if start[1] >= len(indent):
+ self.tokens.append(indent)
+ self.prev_col = len(indent)
+ startline = False
self.add_whitespace(start)
self.tokens.append(token)
self.prev_row, self.prev_col = end