Merge p3yk branch with the trunk up to revision 45595. This breaks a fair

number of tests, all because of the codecs/_multibytecodecs issue described
here (it's not a Py3K issue, just something Py3K discovers):
http://mail.python.org/pipermail/python-dev/2006-April/064051.html

Hye-Shik Chang promised to look for a fix, so no need to fix it here. The
tests that are expected to break are:

test_codecencodings_cn
test_codecencodings_hk
test_codecencodings_jp
test_codecencodings_kr
test_codecencodings_tw
test_codecs
test_multibytecodec

This merge fixes an actual test failure (test_weakref) in this branch,
though, so I believe merging is the right thing to do anyway.

13 files changed, 294 insertions, 46 deletions

diff --git a/Demo/parser/unparse.py b/Demo/parser/unparse.py
index dd75c22..510cdb0 100644
--- a/Demo/parser/unparse.py
+++ b/Demo/parser/unparse.py
@@ -1,5 +1,8 @@
 "Usage: unparse.py <path to source file>"
 import sys
+import _ast
+import cStringIO
+import os
 
 class Unparser:
     """Methods in this class recursively traverse an AST and
@@ -70,6 +73,18 @@ class Unparser:
             if a.asname:
                 self.write(" as "+a.asname)
 
+    def _ImportFrom(self, t):
+        self.fill("from ")
+        self.write(t.module)
+        self.write(" import ")
+        for i, a in enumerate(t.names):
+            if i == 0:
+                self.write(", ")
+            self.write(a.name)
+            if a.asname:
+                self.write(" as "+a.asname)
+        # XXX(jpe) what is level for?
+
     def _Assign(self, t):
         self.fill()
         for target in t.targets:
@@ -88,6 +103,36 @@ class Unparser:
         if t.value:
             self.dispatch(t.value)
 
+    def _Pass(self, t):
+        self.fill("pass")
+
+    def _Break(self, t):
+        self.fill("break")
+
+    def _Continue(self, t):
+        self.fill("continue")
+
+    def _Delete(self, t):
+        self.fill("del ")
+        self.dispatch(t.targets)
+
+    def _Assert(self, t):
+        self.fill("assert ")
+        self.dispatch(t.test)
+        if t.msg:
+            self.write(", ")
+            self.dispatch(t.msg)
+
+    def _Exec(self, t):
+        self.fill("exec ")
+        self.dispatch(t.body)
+        if t.globals:
+            self.write(" in ")
+            self.dispatch(t.globals)
+        if t.locals:
+            self.write(", ")
+            self.dispatch(t.locals)
+
     def _Print(self, t):
         self.fill("print ")
         do_comma = False
@@ -102,6 +147,67 @@ class Unparser:
         if not t.nl:
             self.write(",")
 
+    def _Global(self, t):
+        self.fill("global")
+        for i, n in enumerate(t.names):
+            if i != 0:
+                self.write(",")
+            self.write(" " + n)
+
+    def _Yield(self, t):
+        self.fill("yield")
+        if t.value:
+            self.write(" (")
+            self.dispatch(t.value)
+            self.write(")")
+
+    def _Raise(self, t):
+        self.fill('raise ')
+        if t.type:
+            self.dispatch(t.type)
+        if t.inst:
+            self.write(", ")
+            self.dispatch(t.inst)
+        if t.tback:
+            self.write(", ")
+            self.dispatch(t.tback)
+
+    def _TryExcept(self, t):
+        self.fill("try")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+        for ex in t.handlers:
+            self.dispatch(ex)
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave()
+
+    def _TryFinally(self, t):
+        self.fill("try")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+        self.fill("finally")
+        self.enter()
+        self.dispatch(t.finalbody)
+        self.leave()
+
+    def _excepthandler(self, t):
+        self.fill("except ")
+        if t.type:
+            self.dispatch(t.type)
+        if t.name:
+            self.write(", ")
+            self.dispatch(t.name)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
     def _ClassDef(self, t):
         self.write("\n")
         self.fill("class "+t.name)
@@ -119,9 +225,24 @@ class Unparser:
         self.write("\n")
         self.fill("def "+t.name + "(")
         self.dispatch(t.args)
+        self.write(")")
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
+    def _For(self, t):
+        self.fill("for ")
+        self.dispatch(t.target)
+        self.write(" in ")
+        self.dispatch(t.iter)
         self.enter()
         self.dispatch(t.body)
         self.leave()
+        if t.orelse:
+            self.fill("else")
+            self.enter()
+            self.dispatch(t.orelse)
+            self.leave
 
     def _If(self, t):
         self.fill("if ")
@@ -136,11 +257,9 @@ class Unparser:
             self.dispatch(t.orelse)
             self.leave()
 
-    def _For(self, t):
-        self.fill("for ")
-        self.dispatch(t.target)
-        self.write(" in ")
-        self.dispatch(t.iter)
+    def _While(self, t):
+        self.fill("while ")
+        self.dispatch(t.test)
         self.enter()
         self.dispatch(t.body)
         self.leave()
@@ -150,6 +269,16 @@ class Unparser:
             self.dispatch(t.orelse)
             self.leave
 
+    def _With(self, t):
+        self.fill("with ")
+        self.dispatch(t.context_expr)
+        if t.optional_vars:
+            self.write(" as ")
+            self.dispatch(t.optional_vars)
+        self.enter()
+        self.dispatch(t.body)
+        self.leave()
+
     # expr
     def _Str(self, tree):
         self.write(repr(tree.s))
@@ -157,6 +286,11 @@ class Unparser:
     def _Name(self, t):
         self.write(t.id)
 
+    def _Repr(self, t):
+        self.write("`")
+        self.dispatch(t.value)
+        self.write("`")
+
     def _Num(self, t):
         self.write(repr(t.n))
 
@@ -167,6 +301,37 @@ class Unparser:
             self.write(", ")
         self.write("]")
 
+    def _ListComp(self, t):
+        self.write("[")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write("]")
+
+    def _GeneratorExp(self, t):
+        self.write("(")
+        self.dispatch(t.elt)
+        for gen in t.generators:
+            self.dispatch(gen)
+        self.write(")")
+
+    def _comprehension(self, t):
+        self.write(" for ")
+        self.dispatch(t.target)
+        self.write(" in ")
+        self.dispatch(t.iter)
+        for if_clause in t.ifs:
+            self.write(" if ")
+            self.dispatch(if_clause)
+
+    def _IfExp(self, t):
+        self.dispatch(t.body)
+        self.write(" if ")
+        self.dispatch(t.test)
+        if t.orelse:
+            self.write(" else ")
+            self.dispatch(t.orelse)
+
     def _Dict(self, t):
         self.write("{")
         for k,v in zip(t.keys, t.values):
@@ -194,8 +359,8 @@ class Unparser:
         self.write(")")
 
     binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
-                    "RShift":"<<", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
-                    "FloorDiv":"//"}
+                    "LShift":">>", "RShift":"<<", "BitOr":"|", "BitXor":"^", "BitAnd":"&",
+                    "FloorDiv":"//", "Pow": "**"}
     def _BinOp(self, t):
         self.write("(")
         self.dispatch(t.left)
@@ -213,6 +378,15 @@ class Unparser:
             self.dispatch(e)
             self.write(")")
 
+    boolops = {_ast.And: 'and', _ast.Or: 'or'}
+    def _BoolOp(self, t):
+        self.write("(")
+        self.dispatch(t.values[0])
+        for v in t.values[1:]:
+            self.write(" %s " % self.boolops[t.op.__class__])
+            self.dispatch(v)
+        self.write(")")
+
     def _Attribute(self,t):
         self.dispatch(t.value)
         self.write(".")
@@ -234,12 +408,12 @@ class Unparser:
             if comma: self.write(", ")
             else: comma = True
             self.write("*")
-            self.dispatch(t.stararg)
+            self.dispatch(t.starargs)
         if t.kwargs:
             if comma: self.write(", ")
             else: comma = True
             self.write("**")
-            self.dispatch(t.stararg)
+            self.dispatch(t.kwargs)
         self.write(")")
 
     def _Subscript(self, t):
@@ -249,6 +423,9 @@ class Unparser:
         self.write("]")
 
     # slice
+    def _Ellipsis(self, t):
+        self.write("...")
+
     def _Index(self, t):
         self.dispatch(t.value)
 
@@ -262,6 +439,12 @@ class Unparser:
             self.write(":")
             self.dispatch(t.step)
 
+    def _ExtSlice(self, t):
+        for i, d in enumerate(t.dims):
+            if i != 0:
+                self.write(': ')
+            self.dispatch(d)
+
     # others
     def _arguments(self, t):
         first = True
@@ -283,13 +466,51 @@ class Unparser:
         if t.kwarg:
             if first:first = False
             else: self.write(", ")
-            self.write("**"+self.kwarg)
-        self.write(")")
+            self.write("**"+t.kwarg)
+
+    def _keyword(self, t):
+        self.write(t.arg)
+        self.write("=")
+        self.dispatch(t.value)
+
+    def _Lambda(self, t):
+        self.write("lambda ")
+        self.dispatch(t.args)
+        self.write(": ")
+        self.dispatch(t.body)
 
-def roundtrip(filename):
+def roundtrip(filename, output=sys.stdout):
     source = open(filename).read()
     tree = compile(source, filename, "exec", 0x400)
-    Unparser(tree)
+    Unparser(tree, output)
+
+
+
+def testdir(a):
+    try:
+        names = [n for n in os.listdir(a) if n.endswith('.py')]
+    except OSError:
+        print >> sys.stderr, "Directory not readable: %s" % a
+    else:
+        for n in names:
+            fullname = os.path.join(a, n)
+            if os.path.isfile(fullname):
+                output = cStringIO.StringIO()
+                print 'Testing %s' % fullname
+                try:
+                    roundtrip(fullname, output)
+                except Exception, e:
+                    print '  Failed to compile, exception is %s' % repr(e)
+            elif os.path.isdir(fullname):
+                testdir(fullname)
+
+def main(args):
+    if args[0] == '--testdir':
+        for a in args[1:]:
+            testdir(a)
+    else:
+        for a in args:
+            roundtrip(a)
 
 if __name__=='__main__':
-    roundtrip(sys.argv[1])
+    main(sys.argv[1:])
diff --git a/Demo/pdist/makechangelog.py b/Demo/pdist/makechangelog.py
index b26f30b..1ffa588 100755
--- a/Demo/pdist/makechangelog.py
+++ b/Demo/pdist/makechangelog.py
@@ -6,7 +6,7 @@
 
 import sys
 import string
-import regex
+import re
 import getopt
 import time
 
@@ -35,9 +35,9 @@ def main():
     for rev in allrevs:
         formatrev(rev, prefix)
 
-parsedateprog = regex.compile(
-    '^date: \([0-9]+\)/\([0-9]+\)/\([0-9]+\) ' +
-    '\([0-9]+\):\([0-9]+\):\([0-9]+\);  author: \([^ ;]+\)')
+parsedateprog = re.compile(
+    '^date: ([0-9]+)/([0-9]+)/([0-9]+) ' +
+    '([0-9]+):([0-9]+):([0-9]+);  author: ([^ ;]+)')
 
 authormap = {
     'guido': 'Guido van Rossum  <guido@cnri.reston.va.us>',
@@ -70,7 +70,7 @@ def formatrev(rev, prefix):
         print
         print
 
-startprog = regex.compile("^Working file: \(.*\)$")
+startprog = re.compile("^Working file: (.*)$")
 
 def getnextfile(f):
     while 1:
diff --git a/Demo/pdist/rcsbump b/Demo/pdist/rcsbump
index e4e9ed5..4fa078e 100755
--- a/Demo/pdist/rcsbump
+++ b/Demo/pdist/rcsbump
@@ -6,12 +6,12 @@
 # Python script for bumping up an RCS major revision number.
 
 import sys
-import regex
+import re
 import rcslib
 import string
 
 WITHLOCK = 1
-majorrev_re = regex.compile('^[0-9]+')
+majorrev_re = re.compile('^[0-9]+')
 
 dir = rcslib.RCS()
 
diff --git a/Demo/pdist/rcslib.py b/Demo/pdist/rcslib.py
index d5f7b65..3e63869 100755
--- a/Demo/pdist/rcslib.py
+++ b/Demo/pdist/rcslib.py
@@ -8,7 +8,7 @@ files and (possibly) corresponding work files.
 
 import fnmatch
 import os
-import regsub
+import re
 import string
 import tempfile
 
@@ -150,7 +150,7 @@ class RCS:
             cmd = 'ci %s%s -t%s %s %s' % \
                   (lockflag, rev, f.name, otherflags, name)
         else:
-            message = regsub.gsub('\([\\"$`]\)', '\\\\\\1', message)
+            message = re.sub(r'([\"$`])', r'\\\1', message)
             cmd = 'ci %s%s -m"%s" %s %s' % \
                   (lockflag, rev, message, otherflags, name)
         return self._system(cmd)
diff --git a/Demo/scripts/eqfix.py b/Demo/scripts/eqfix.py
index 165ca49..35c43aa 100755
--- a/Demo/scripts/eqfix.py
+++ b/Demo/scripts/eqfix.py
@@ -29,7 +29,7 @@
 # into a program for a different change to Python programs...
 
 import sys
-import regex
+import re
 import os
 from stat import *
 import string
@@ -53,7 +53,7 @@ def main():
             if fix(arg): bad = 1
     sys.exit(bad)
 
-ispythonprog = regex.compile('^[a-zA-Z0-9_]+\.py$')
+ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$')
 def ispython(name):
     return ispythonprog.match(name) >= 0
 
@@ -104,7 +104,7 @@ def fix(filename):
         if lineno == 1 and g is None and line[:2] == '#!':
             # Check for non-Python scripts
             words = string.split(line[2:])
-            if words and regex.search('[pP]ython', words[0]) < 0:
+            if words and re.search('[pP]ython', words[0]) < 0:
                 msg = filename + ': ' + words[0]
                 msg = msg + ' script; not fixed\n'
                 err(msg)
diff --git a/Demo/scripts/ftpstats.py b/Demo/scripts/ftpstats.py
index b37a58d..5c1599e 100755
--- a/Demo/scripts/ftpstats.py
+++ b/Demo/scripts/ftpstats.py
@@ -13,12 +13,12 @@
 
 import os
 import sys
-import regex
+import re
 import string
 import getopt
 
-pat = '^\([a-zA-Z0-9 :]*\)!\(.*\)!\(.*\)!\([<>].*\)!\([0-9]+\)!\([0-9]+\)$'
-prog = regex.compile(pat)
+pat = '^([a-zA-Z0-9 :]*)!(.*)!(.*)!([<>].*)!([0-9]+)!([0-9]+)$'
+prog = re.compile(pat)
 
 def main():
     maxitems = 25
diff --git a/Demo/scripts/mboxconvert.py b/Demo/scripts/mboxconvert.py
index 502d774..8c462f3 100755
--- a/Demo/scripts/mboxconvert.py
+++ b/Demo/scripts/mboxconvert.py
@@ -10,7 +10,7 @@ import time
 import os
 import stat
 import getopt
-import regex
+import re
 
 def main():
     dofile = mmdf
@@ -45,7 +45,7 @@ def main():
     if sts:
         sys.exit(sts)
 
-numeric = regex.compile('[1-9][0-9]*')
+numeric = re.compile('[1-9][0-9]*')
 
 def mh(dir):
     sts = 0
diff --git a/Demo/scripts/update.py b/Demo/scripts/update.py
index 32ad6c8..c936026 100755
--- a/Demo/scripts/update.py
+++ b/Demo/scripts/update.py
@@ -8,10 +8,10 @@
 
 import os
 import sys
-import regex
+import re
 
-pat = '^\([^: \t\n]+\):\([1-9][0-9]*\):'
-prog = regex.compile(pat)
+pat = '^([^: \t\n]+):([1-9][0-9]*):'
+prog = re.compile(pat)
 
 class FileObj:
     def __init__(self, filename):
diff --git a/Demo/sockets/mcast.py b/Demo/sockets/mcast.py
index 122dad7..1abd305 100755
--- a/Demo/sockets/mcast.py
+++ b/Demo/sockets/mcast.py
@@ -13,7 +13,6 @@ MYGROUP = '225.0.0.250'
 import sys
 import time
 import struct
-import regsub
 from socket import *
 
 
diff --git a/Demo/tix/grid.py b/Demo/tix/grid.py
new file mode 100644
index 0000000..07ca87f
--- /dev/null
+++ b/Demo/tix/grid.py
@@ -0,0 +1,28 @@
+###
+import Tix as tk
+from pprint import pprint
+
+r= tk.Tk()
+r.title("test")
+
+l=tk.Label(r, name="a_label")
+l.pack()
+
+class MyGrid(tk.Grid):
+    def __init__(self, *args, **kwargs):
+        kwargs['editnotify']= self.editnotify
+        tk.Grid.__init__(self, *args, **kwargs)
+    def editnotify(self, x, y):
+        return True
+
+g = MyGrid(r, name="a_grid",
+selectunit="cell")
+g.pack(fill=tk.BOTH)
+for x in xrange(5):
+    for y in xrange(5):
+        g.set(x,y,text=str((x,y)))
+
+c = tk.Button(r, text="Close", command=r.destroy)
+c.pack()
+
+tk.mainloop()
diff --git a/Demo/tkinter/guido/ManPage.py b/Demo/tkinter/guido/ManPage.py
index 911961e..221af88 100755
--- a/Demo/tkinter/guido/ManPage.py
+++ b/Demo/tkinter/guido/ManPage.py
@@ -1,6 +1,6 @@
 # Widget to display a man page
 
-import regex
+import re
 from Tkinter import *
 from Tkinter import _tkinter
 from ScrolledText import ScrolledText
@@ -11,10 +11,10 @@ ITALICFONT = '*-Courier-Medium-O-Normal-*-120-*'
 
 # XXX Recognizing footers is system dependent
 # (This one works for IRIX 5.2 and Solaris 2.2)
-footerprog = regex.compile(
+footerprog = re.compile(
         '^     Page [1-9][0-9]*[ \t]+\|^.*Last change:.*[1-9][0-9]*\n')
-emptyprog = regex.compile('^[ \t]*\n')
-ulprog = regex.compile('^[ \t]*[Xv!_][Xv!_ \t]*\n')
+emptyprog = re.compile('^[ \t]*\n')
+ulprog = re.compile('^[ \t]*[Xv!_][Xv!_ \t]*\n')
 
 # Basic Man Page class -- does not disable editing
 class EditableManPage(ScrolledText):
diff --git a/Demo/tkinter/guido/mbox.py b/Demo/tkinter/guido/mbox.py
index 9b16f6b..3c36d88 100755
--- a/Demo/tkinter/guido/mbox.py
+++ b/Demo/tkinter/guido/mbox.py
@@ -4,7 +4,7 @@
 
 import os
 import sys
-import regex
+import re
 import getopt
 import string
 import mhlib
@@ -157,7 +157,7 @@ def scan_unpost(e):
     scanmenu.unpost()
     scanmenu.invoke('active')
 
-scanparser = regex.compile('^ *\([0-9]+\)')
+scanparser = re.compile('^ *([0-9]+)')
 
 def open_folder(e=None):
     global folder, mhf
diff --git a/Demo/tkinter/guido/tkman.py b/Demo/tkinter/guido/tkman.py
index 11d9690..6b0b641 100755
--- a/Demo/tkinter/guido/tkman.py
+++ b/Demo/tkinter/guido/tkman.py
@@ -5,7 +5,7 @@
 import sys
 import os
 import string
-import regex
+import re
 from Tkinter import *
 from ManPage import ManPage
 
@@ -208,15 +208,15 @@ class SelectionBox:
             print 'Empty search string'
             return
         if not self.casevar.get():
-            map = regex.casefold
+            map = re.IGNORECASE
         else:
             map = None
         try:
             if map:
-                prog = regex.compile(search, map)
+                prog = re.compile(search, map)
             else:
-                prog = regex.compile(search)
-        except regex.error, msg:
+                prog = re.compile(search)
+        except re.error, msg:
             self.frame.bell()
             print 'Regex error:', msg
             return