diff options
| author | Benjamin Peterson <benjamin@python.org> | 2010-08-30 14:44:53 (GMT) |
|---|---|---|
| committer | Benjamin Peterson <benjamin@python.org> | 2010-08-30 14:44:53 (GMT) |
| commit | 66428b2e5d456e0af609de8cdbcd9bae2a76d02d (patch) | |
| tree | 51de6d4aaae68845249503d8b4835df0976f6fcf /Lib/test | |
| parent | f8a08d9d3618a8a3a53bdf1370daf07032313795 (diff) | |
| download | cpython-66428b2e5d456e0af609de8cdbcd9bae2a76d02d.zip cpython-66428b2e5d456e0af609de8cdbcd9bae2a76d02d.tar.gz cpython-66428b2e5d456e0af609de8cdbcd9bae2a76d02d.tar.bz2 | |
Merged revisions 84364 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/branches/py3k
........
r84364 | benjamin.peterson | 2010-08-30 09:41:20 -0500 (Mon, 30 Aug 2010) | 1 line
handle names starting with non-ascii characters correctly #9712
........
Diffstat (limited to 'Lib/test')
| -rw-r--r-- | Lib/test/test_tokenize.py | 13 |
1 files changed, 13 insertions, 0 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 7b91ab2..eeefce1 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -533,6 +533,7 @@ pass the '-ucompiler' option to process the full directory. True Evil tabs + >>> dump_tokens("def f():\\n\\tif x\\n \\tpass") ENCODING 'utf-8' (0, 0) (0, 0) NAME 'def' (1, 0) (1, 3) @@ -549,6 +550,18 @@ Evil tabs NAME 'pass' (3, 9) (3, 13) DEDENT '' (4, 0) (4, 0) DEDENT '' (4, 0) (4, 0) + +Non-ascii identifiers + + >>> dump_tokens("Örter = 'places'\\ngrün = 'green'") + ENCODING 'utf-8' (0, 0) (0, 0) + NAME 'Örter' (1, 0) (1, 5) + OP '=' (1, 6) (1, 7) + STRING "'places'" (1, 8) (1, 16) + NEWLINE '\\n' (1, 16) (1, 17) + NAME 'grün' (2, 0) (2, 4) + OP '=' (2, 5) (2, 6) + STRING "'green'" (2, 7) (2, 14) """ from test import support |