diff options
Diffstat (limited to 'Lib')
| -rw-r--r-- | Lib/test/test_tokenize.py | 20 | ||||
| -rw-r--r-- | Lib/tokenize.py | 3 |
2 files changed, 22 insertions, 1 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 2ce435f..d3c1cc4 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1,4 +1,4 @@ -from test.test_support import verbose, findfile, is_resource_enabled +from test.test_support import verbose, findfile, is_resource_enabled, TestFailed import os, glob, random from tokenize import (tokenize, generate_tokens, untokenize, NUMBER, NAME, OP, STRING) @@ -41,6 +41,24 @@ for f in testfiles: test_roundtrip(f) +###### Test detecton of IndentationError ###################### + +from cStringIO import StringIO + +sampleBadText = """ +def foo(): + bar + baz +""" + +try: + for tok in generate_tokens(StringIO(sampleBadText).readline): + pass +except IndentationError: + pass +else: + raise TestFailed("Did not detect IndentationError:") + ###### Test example in the docs ############################### diff --git a/Lib/tokenize.py b/Lib/tokenize.py index b29da6b..2b40e6f 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -271,6 +271,9 @@ def generate_tokens(readline): indents.append(column) yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level") indents = indents[:-1] yield (DEDENT, '', (lnum, pos), (lnum, pos), line) |