summaryrefslogtreecommitdiffstats
path: root/Lib/test/test_tokenize.py
diff options
context:
space:
mode:
authorArmin Ronacher <armin.ronacher@active-4.com>2012-03-04 13:07:57 (GMT)
committerArmin Ronacher <armin.ronacher@active-4.com>2012-03-04 13:07:57 (GMT)
commitc0eaecafe9809757301551285f2a41ea89f1f228 (patch)
treee4cf1b8c0894a42bbf923b737ac714fad77235f0 /Lib/test/test_tokenize.py
parent50364b4a5c8f02ec05d33928e29a8780d9acf968 (diff)
downloadcpython-c0eaecafe9809757301551285f2a41ea89f1f228.zip
cpython-c0eaecafe9809757301551285f2a41ea89f1f228.tar.gz
cpython-c0eaecafe9809757301551285f2a41ea89f1f228.tar.bz2
Updated tokenize to support the inverse byte literals new in 3.3
Diffstat (limited to 'Lib/test/test_tokenize.py')
-rw-r--r--Lib/test/test_tokenize.py12
1 files changed, 12 insertions, 0 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index dce3c6e..db87e11 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -563,6 +563,18 @@ Non-ascii identifiers
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "'green'" (2, 7) (2, 14)
+
+Legacy unicode literals:
+
+ >>> dump_tokens("Örter = u'places'\\ngrün = UR'green'")
+ ENCODING 'utf-8' (0, 0) (0, 0)
+ NAME 'Örter' (1, 0) (1, 5)
+ OP '=' (1, 6) (1, 7)
+ STRING "u'places'" (1, 8) (1, 17)
+ NEWLINE '\\n' (1, 17) (1, 18)
+ NAME 'grün' (2, 0) (2, 4)
+ OP '=' (2, 5) (2, 6)
+ STRING "UR'green'" (2, 7) (2, 16)
"""
from test import support