diff options
| author | Eric Smith <eric@trueblade.com> | 2008-03-17 19:49:19 (GMT) |
|---|---|---|
| committer | Eric Smith <eric@trueblade.com> | 2008-03-17 19:49:19 (GMT) |
| commit | 74ca557e1e092822cfb71fe1cf7080a0b1564d80 (patch) | |
| tree | 72e04740c1371e29300bbedb66e3349f7cca9e78 /Lib/test/test_tokenize.py | |
| parent | f1bb97c95f53379acb78a2ef07b9d43221e8dcf4 (diff) | |
| download | cpython-74ca557e1e092822cfb71fe1cf7080a0b1564d80.zip cpython-74ca557e1e092822cfb71fe1cf7080a0b1564d80.tar.gz cpython-74ca557e1e092822cfb71fe1cf7080a0b1564d80.tar.bz2 | |
Added tests for tokenize, blocked corresponding checkin from trunk.
Diffstat (limited to 'Lib/test/test_tokenize.py')
| -rw-r--r-- | Lib/test/test_tokenize.py | 12 |
1 files changed, 10 insertions, 2 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 9165003..2938520 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -4,7 +4,7 @@ Tests for the tokenize module. >>> import glob, random, sys The tests can be really simple. Given a small fragment of source -code, print out a table with thokens. The ENDMARK is omitted for +code, print out a table with tokens. The ENDMARK is omitted for brevity. >>> dump_tokens("1 + 1") @@ -105,7 +105,7 @@ Some error-handling code ... "else: print 'Loaded'\\n") True -Balancing contunuation +Balancing continuation >>> roundtrip("a = (3,4, \\n" ... "5,6)\\n" @@ -125,6 +125,14 @@ Ordinary integers and binary operators NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) + >>> dump_tokens("0b10 <= 255") + NUMBER '0b10' (1, 0) (1, 4) + OP '<=' (1, 5) (1, 7) + NUMBER '255' (1, 8) (1, 11) + >>> dump_tokens("0o123 <= 0O123") + NUMBER '0o123' (1, 0) (1, 5) + OP '<=' (1, 6) (1, 8) + NUMBER '0O123' (1, 9) (1, 14) >>> dump_tokens("1234567 > ~0x15") NUMBER '1234567' (1, 0) (1, 7) OP '>' (1, 8) (1, 9) |