diff options
| author | Eric Smith <eric@trueblade.com> | 2008-03-17 19:43:40 (GMT) |
|---|---|---|
| committer | Eric Smith <eric@trueblade.com> | 2008-03-17 19:43:40 (GMT) |
| commit | 0aed07ad80795bd5856ed60e7edcadeb353cf5a0 (patch) | |
| tree | 55268a52e9e0686949dea35b218275817b2a1bbe | |
| parent | 6f778cfb5ba0291ed29c51fbdd6e40760097fd36 (diff) | |
| download | cpython-0aed07ad80795bd5856ed60e7edcadeb353cf5a0.zip cpython-0aed07ad80795bd5856ed60e7edcadeb353cf5a0.tar.gz cpython-0aed07ad80795bd5856ed60e7edcadeb353cf5a0.tar.bz2 | |
Added PEP 3127 support to tokenize (with tests); added PEP 3127 to NEWS.
| -rw-r--r-- | Lib/test/test_tokenize.py | 12 | ||||
| -rw-r--r-- | Lib/tokenize.py | 5 | ||||
| -rw-r--r-- | Misc/NEWS | 4 |
3 files changed, 17 insertions, 4 deletions
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index d6cfb65..cbfafa8 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -4,7 +4,7 @@ Tests for the tokenize module. >>> import glob, random, sys The tests can be really simple. Given a small fragment of source -code, print out a table with thokens. The ENDMARK is omitted for +code, print out a table with tokens. The ENDMARK is omitted for brevity. >>> dump_tokens("1 + 1") @@ -106,7 +106,7 @@ Some error-handling code ... "else: print 'Loaded'\\n") True -Balancing contunuation +Balancing continuation >>> roundtrip("a = (3,4, \\n" ... "5,6)\\n" @@ -126,6 +126,14 @@ Ordinary integers and binary operators NUMBER '0xff' (1, 0) (1, 4) OP '<=' (1, 5) (1, 7) NUMBER '255' (1, 8) (1, 11) + >>> dump_tokens("0b10 <= 255") + NUMBER '0b10' (1, 0) (1, 4) + OP '<=' (1, 5) (1, 7) + NUMBER '255' (1, 8) (1, 11) + >>> dump_tokens("0o123 <= 0123") + NUMBER '0o123' (1, 0) (1, 5) + OP '<=' (1, 6) (1, 8) + NUMBER '0123' (1, 9) (1, 13) >>> dump_tokens("01234567 > ~0x15") NUMBER '01234567' (1, 0) (1, 8) OP '>' (1, 9) (1, 10) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1c93944..76ab430 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -51,9 +51,10 @@ Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) Name = r'[a-zA-Z_]\w*' Hexnumber = r'0[xX][\da-fA-F]+[lL]?' -Octnumber = r'0[0-7]*[lL]?' +Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?' +Binnumber = r'0[bB][01]+[lL]?' Decnumber = r'[1-9]\d*[lL]?' -Intnumber = group(Hexnumber, Octnumber, Decnumber) +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) Exponent = r'[eE][-+]?\d+' Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) Expfloat = r'\d+' + Exponent @@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 2? Core and builtins ----------------- +- PEP 3127: octal literals now start with "0o". Old-style octal literals + are still valid. There are binary literals with a prefix of "0b". + This also affects int(x, 0). + - Issue #1779871: Gnu gcc can now build Python on OS X because the flags -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd are no longer passed. |