From 0aed07ad80795bd5856ed60e7edcadeb353cf5a0 Mon Sep 17 00:00:00 2001
From: Eric Smith <eric@trueblade.com>
Date: Mon, 17 Mar 2008 19:43:40 +0000
Subject: Added PEP 3127 support to tokenize (with tests); added PEP 3127 to
 NEWS.

---
 Lib/test/test_tokenize.py | 12 ++++++++++--
 Lib/tokenize.py           |  5 +++--
 Misc/NEWS                 |  4 ++++
 3 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index d6cfb65..cbfafa8 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -4,7 +4,7 @@ Tests for the tokenize module.
     >>> import glob, random, sys
 
 The tests can be really simple. Given a small fragment of source
-code, print out a table with thokens. The ENDMARK is omitted for
+code, print out a table with tokens. The ENDMARK is omitted for
 brevity.
 
     >>> dump_tokens("1 + 1")
@@ -106,7 +106,7 @@ Some error-handling code
     ...           "else:   print 'Loaded'\\n")
     True
 
-Balancing contunuation
+Balancing continuation
 
     >>> roundtrip("a = (3,4, \\n"
     ...           "5,6)\\n"
@@ -126,6 +126,14 @@ Ordinary integers and binary operators
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
+    >>> dump_tokens("0b10 <= 255")
+    NUMBER     '0b10'        (1, 0) (1, 4)
+    OP         '<='          (1, 5) (1, 7)
+    NUMBER     '255'         (1, 8) (1, 11)
+    >>> dump_tokens("0o123 <= 0123")
+    NUMBER     '0o123'       (1, 0) (1, 5)
+    OP         '<='          (1, 6) (1, 8)
+    NUMBER     '0123'        (1, 9) (1, 13)
     >>> dump_tokens("01234567 > ~0x15")
     NUMBER     '01234567'    (1, 0) (1, 8)
     OP         '>'           (1, 9) (1, 10)
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1c93944..76ab430 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -51,9 +51,10 @@ Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
 Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'0[0-7]*[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
 Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Octnumber, Decnumber)
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?\d+'
 Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
 Expfloat = r'\d+' + Exponent
diff --git a/Misc/NEWS b/Misc/NEWS
index 1963ad5..1fc6691 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 2?
 Core and builtins
 -----------------
 
+- PEP 3127: octal literals now start with "0o". Old-style octal literals
+  are still valid. There are binary literals with a prefix of "0b".
+  This also affects int(x, 0).
+
 - Issue #1779871: Gnu gcc can now build Python on OS X because the
   flags -Wno-long-double, -no-cpp-precomp, and -mno-fused-madd are no
   longer passed.
-- 
cgit v0.12