Merge ast-branch to head

This change implements a new bytecode compiler, based on a
transformation of the parse tree to an abstract syntax defined in
Parser/Python.asdl.

The compiler implementation is not complete, but it is in stable
enough shape to run the entire test suite excepting two disabled
tests.

12 files changed, 214 insertions, 70 deletions

diff --git a/Lib/test/output/test_grammar b/Lib/test/output/test_grammar
index 6174e7a..fed4197 100644
--- a/Lib/test/output/test_grammar
+++ b/Lib/test/output/test_grammar
@@ -34,6 +34,7 @@ continue + try/except ok
 continue + try/finally ok
 testing continue and break in try/except in loop
 return_stmt
+yield_stmt
 raise_stmt
 import_name
 import_from
diff --git a/Lib/test/output/test_profile b/Lib/test/output/test_profile
index b46bb6a..e745075 100644
--- a/Lib/test/output/test_profile
+++ b/Lib/test/output/test_profile
@@ -7,7 +7,7 @@ test_profile
        12    0.000    0.000    0.012    0.001 :0(hasattr)
         8    0.000    0.000    0.000    0.000 :0(range)
         1    0.000    0.000    0.000    0.000 :0(setprofile)
-        1    0.000    0.000    1.000    1.000 <string>:1(?)
+        1    0.000    0.000    1.000    1.000 <string>:1(<module>)
         0    0.000             0.000          profile:0(profiler)
         1    0.000    0.000    1.000    1.000 profile:0(testfunc())
         1    0.400    0.400    1.000    1.000 test_profile.py:23(testfunc)
diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
new file mode 100644
index 0000000..ff95c6a
--- /dev/null
+++ b/Lib/test/test_code.py
@@ -0,0 +1,85 @@
+"""This module includes tests of the code object representation.
+
+>>> def f(x):
+...     def g(y):
+...         return x + y
+...     return g
+...
+
+>>> dump(f.func_code)
+name: f
+argcount: 1
+names: ()
+varnames: ('x', 'g')
+cellvars: ('x',)
+freevars: ()
+nlocals: 2
+flags: 3
+consts: ('None', '<code object g>')
+
+>>> dump(f(4).func_code)
+name: g
+argcount: 1
+names: ()
+varnames: ('y',)
+cellvars: ()
+freevars: ('x',)
+nlocals: 1
+flags: 19
+consts: ('None',)
+
+>>> def h(x, y):
+...     a = x + y
+...     b = x - y
+...     c = a * b
+...     return c
+... 
+>>> dump(h.func_code)
+name: h
+argcount: 2
+names: ()
+varnames: ('x', 'y', 'a', 'b', 'c')
+cellvars: ()
+freevars: ()
+nlocals: 5
+flags: 67
+consts: ('None',)
+
+>>> def attrs(obj):
+...     print obj.attr1
+...     print obj.attr2
+...     print obj.attr3
+
+>>> dump(attrs.func_code)
+name: attrs
+argcount: 1
+names: ('attr1', 'attr2', 'attr3')
+varnames: ('obj',)
+cellvars: ()
+freevars: ()
+nlocals: 1
+flags: 67
+consts: ('None',)
+
+"""
+
+def consts(t):
+    """Yield a doctest-safe sequence of object reprs."""
+    for elt in t:
+        r = repr(elt)
+        if r.startswith("<code object"):
+            yield "<code object %s>" % elt.co_name
+        else:
+            yield r
+
+def dump(co):
+    """Print out a text representation of a code object."""
+    for attr in ["name", "argcount", "names", "varnames", "cellvars",
+                 "freevars", "nlocals", "flags"]:
+        print "%s: %s" % (attr, getattr(co, "co_" + attr))
+    print "consts:", tuple(consts(co.co_consts))
+
+def test_main(verbose=None):
+    from test.test_support import run_doctest
+    from test import test_code
+    run_doctest(test_code, verbose)
diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py
index fc4841a..9c39ee8 100644
--- a/Lib/test/test_doctest.py
+++ b/Lib/test/test_doctest.py
@@ -1559,11 +1559,11 @@ Run the debugger on the docstring, and then restore sys.stdin.
 
     >>> try: doctest.debug_src(s)
     ... finally: sys.stdin = real_stdin
-    > <string>(1)?()
+    > <string>(1)<module>()
     (Pdb) next
     12
     --Return--
-    > <string>(1)?()->None
+    > <string>(1)<module>()->None
     (Pdb) print x
     12
     (Pdb) continue
@@ -1601,7 +1601,7 @@ def test_pdb_set_trace():
       >>> try: runner.run(test)
       ... finally: sys.stdin = real_stdin
       --Return--
-      > <doctest foo[1]>(1)?()->None
+      > <doctest foo[1]>(1)<module>()->None
       -> import pdb; pdb.set_trace()
       (Pdb) print x
       42
@@ -1637,7 +1637,7 @@ def test_pdb_set_trace():
       (Pdb) print y
       2
       (Pdb) up
-      > <doctest foo[1]>(1)?()
+      > <doctest foo[1]>(1)<module>()
       -> calls_set_trace()
       (Pdb) print x
       1
@@ -1686,7 +1686,7 @@ def test_pdb_set_trace():
       [EOF]
       (Pdb) next
       --Return--
-      > <doctest foo[2]>(1)?()->None
+      > <doctest foo[2]>(1)<module>()->None
       -> f(3)
       (Pdb) list
         1  -> f(3)
@@ -1779,7 +1779,7 @@ def test_pdb_set_trace_nested():
     (Pdb) print y
     1
     (Pdb) up
-    > <doctest foo[1]>(1)?()
+    > <doctest foo[1]>(1)<module>()
     -> calls_set_trace()
     (Pdb) print foo
     *** NameError: name 'foo' is not defined
diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
index 683649d..aae3518 100644
--- a/Lib/test/test_eof.py
+++ b/Lib/test/test_eof.py
@@ -7,21 +7,21 @@ from test import test_support
 
 class EOFTestCase(unittest.TestCase):
     def test_EOFC(self):
+        expect = "EOL while scanning single-quoted string (<string>, line 1)"
         try:
             eval("""'this is a test\
             """)
         except SyntaxError, msg:
-            self.assertEqual(str(msg),
-                             "EOL while scanning single-quoted string (line 1)")
+            self.assertEqual(str(msg), expect)
         else:
             raise test_support.TestFailed
 
     def test_EOFS(self):
+        expect = "EOF while scanning triple-quoted string (<string>, line 1)"
         try:
             eval("""'''this is a test""")
         except SyntaxError, msg:
-            self.assertEqual(str(msg),
-                             "EOF while scanning triple-quoted string (line 1)")
+            self.assertEqual(str(msg), expect)
         else:
             raise test_support.TestFailed
 
diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index d226043..cb7e992 100644
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -774,7 +774,7 @@ These are fine:
 ...         try:
 ...             1//0
 ...         except ZeroDivisionError:
-...             yield 666  # bad because *outer* try has finally
+...             yield 666 
 ...         except:
 ...             pass
 ...     finally:
diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py
index 7c6fe4a..894ce6a 100644
--- a/Lib/test/test_genexps.py
+++ b/Lib/test/test_genexps.py
@@ -125,13 +125,12 @@ Verify that syntax error's are raised for genexps used as lvalues
     >>> (y for y in (1,2)) = 10
     Traceback (most recent call last):
        ...
-    SyntaxError: assign to generator expression not possible
+    SyntaxError: assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[38]>, line 1)
 
     >>> (y for y in (1,2)) += 10
     Traceback (most recent call last):
        ...
-    SyntaxError: augmented assign to tuple literal, yield, or generator expression not possible
-
+    SyntaxError: augmented assignment to generator expression not possible (<doctest test.test_genexps.__test__.doctests[39]>, line 1)
 
 
 ########### Tests borrowed from or inspired by test_generators.py ############
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 1b4a506..820fab5 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -8,7 +8,7 @@
 # regression test, the filterwarnings() call has been added to
 # regrtest.py.
 
-from test.test_support import TestFailed, verify, check_syntax
+from test.test_support import TestFailed, verify, vereq, check_syntax
 import sys
 
 print '1. Parser'
@@ -157,28 +157,31 @@ def f2(one_argument): pass
 def f3(two, arguments): pass
 def f4(two, (compound, (argument, list))): pass
 def f5((compound, first), two): pass
-verify(f2.func_code.co_varnames == ('one_argument',))
-verify(f3.func_code.co_varnames == ('two', 'arguments'))
+vereq(f2.func_code.co_varnames, ('one_argument',))
+vereq(f3.func_code.co_varnames, ('two', 'arguments'))
 if sys.platform.startswith('java'):
-    verify(f4.func_code.co_varnames ==
+    vereq(f4.func_code.co_varnames,
            ('two', '(compound, (argument, list))', 'compound', 'argument',
                         'list',))
-    verify(f5.func_code.co_varnames ==
+    vereq(f5.func_code.co_varnames,
            ('(compound, first)', 'two', 'compound', 'first'))
 else:
-    verify(f4.func_code.co_varnames == ('two', '.2', 'compound',
-                                        'argument',  'list'))
-    verify(f5.func_code.co_varnames == ('.0', 'two', 'compound', 'first'))
+    vereq(f4.func_code.co_varnames,
+          ('two', '.1', 'compound', 'argument',  'list'))
+    vereq(f5.func_code.co_varnames,
+          ('.0', 'two', 'compound', 'first'))
 def a1(one_arg,): pass
 def a2(two, args,): pass
 def v0(*rest): pass
 def v1(a, *rest): pass
 def v2(a, b, *rest): pass
 def v3(a, (b, c), *rest): return a, b, c, rest
+# ceval unpacks the formal arguments into the first argcount names;
+# thus, the names nested inside tuples must appear after these names.
 if sys.platform.startswith('java'):
     verify(v3.func_code.co_varnames == ('a', '(b, c)', 'rest', 'b', 'c'))
 else:
-    verify(v3.func_code.co_varnames == ('a', '.2', 'rest', 'b', 'c'))
+    vereq(v3.func_code.co_varnames, ('a', '.1', 'rest', 'b', 'c'))
 verify(v3(1, (2, 3), 4) == (1, 2, 3, (4,)))
 def d01(a=1): pass
 d01()
@@ -410,6 +413,10 @@ def g1(): return
 def g2(): return 1
 g1()
 x = g2()
+check_syntax("class foo:return 1")
+
+print 'yield_stmt'
+check_syntax("class foo:yield 1")
 
 print 'raise_stmt' # 'raise' test [',' test]
 try: raise RuntimeError, 'just testing'
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index b89f09b..72f27fa 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -192,3 +192,16 @@ def test_failing_reload():
             del sys.modules[TESTFN]
 
 test_failing_reload()
+
+def test_import_name_binding():
+    # import x.y.z binds x in the current namespace
+    import test as x
+    import test.test_support
+    assert x is test, x.__name__
+    assert hasattr(test.test_support, "__file__")
+
+    # import x.y.z as w binds z as w
+    import test.test_support as y
+    assert y is test.test_support, y.__name__
+
+test_import_name_binding()
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index d1ace67..771fe9d 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -411,10 +411,32 @@ class IllegalSyntaxTestCase(unittest.TestCase):
                 (0, ''))
         self.check_bad_tree(tree, "malformed global ast")
 
+
+class CompileTestCase(unittest.TestCase):
+
+    # These tests are very minimal. :-(
+
+    def test_compile_expr(self):
+        st = parser.expr('2 + 3')
+        code = parser.compilest(st)
+        self.assertEquals(eval(code), 5)
+
+    def test_compile_suite(self):
+        st = parser.suite('x = 2; y = x + 3')
+        code = parser.compilest(st)
+        globs = {}
+        exec code in globs
+        self.assertEquals(globs['y'], 5)
+
+    def test_compile_error(self):
+        st = parser.suite('1 = 3 + 4')
+        self.assertRaises(SyntaxError, parser.compilest, st)
+
 def test_main():
     test_support.run_unittest(
         RoundtripLegalSyntaxTestCase,
-        IllegalSyntaxTestCase
+        IllegalSyntaxTestCase,
+        CompileTestCase,
     )
 
 
diff --git a/Lib/test/test_repr.py b/Lib/test/test_repr.py
index 4ded484..a139473 100644
--- a/Lib/test/test_repr.py
+++ b/Lib/test/test_repr.py
@@ -123,7 +123,7 @@ class ReprTests(unittest.TestCase):
 
     def test_lambda(self):
         self.failUnless(repr(lambda x: x).startswith(
-            "<function <lambda"))
+            "<function lambda"))
         # XXX anonymous functions?  see func_repr
 
     def test_builtin_function(self):
diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py
index bf9a658..34801bd 100644
--- a/Lib/test/test_scope.py
+++ b/Lib/test/test_scope.py
@@ -1,4 +1,4 @@
-from test.test_support import verify, TestFailed, check_syntax
+from test.test_support import verify, TestFailed, check_syntax, vereq
 
 import warnings
 warnings.filterwarnings("ignore", r"import \*", SyntaxWarning, "<string>")
@@ -13,8 +13,8 @@ def make_adder(x):
 inc = make_adder(1)
 plus10 = make_adder(10)
 
-verify(inc(1) == 2)
-verify(plus10(-2) == 8)
+vereq(inc(1), 2)
+vereq(plus10(-2), 8)
 
 print "2. extra nesting"
 
@@ -28,8 +28,8 @@ def make_adder2(x):
 inc = make_adder2(1)
 plus10 = make_adder2(10)
 
-verify(inc(1) == 2)
-verify(plus10(-2) == 8)
+vereq(inc(1), 2)
+vereq(plus10(-2), 8)
 
 print "3. simple nesting + rebinding"
 
@@ -42,8 +42,8 @@ def make_adder3(x):
 inc = make_adder3(0)
 plus10 = make_adder3(9)
 
-verify(inc(1) == 2)
-verify(plus10(-2) == 8)
+vereq(inc(1), 2)
+vereq(plus10(-2), 8)
 
 print "4. nesting with global but no free"
 
@@ -58,10 +58,10 @@ def make_adder4(): # XXX add exta level of indirection
 
 global_x = 1
 adder = make_adder4()
-verify(adder(1) == 2)
+vereq(adder(1), 2)
 
 global_x = 10
-verify(adder(-2) == 8)
+vereq(adder(-2), 8)
 
 print "5. nesting through class"
 
@@ -74,8 +74,8 @@ def make_adder5(x):
 inc = make_adder5(1)
 plus10 = make_adder5(10)
 
-verify(inc(1) == 2)
-verify(plus10(-2) == 8)
+vereq(inc(1), 2)
+vereq(plus10(-2), 8)
 
 print "6. nesting plus free ref to global"
 
@@ -89,8 +89,8 @@ def make_adder6(x):
 inc = make_adder6(1)
 plus10 = make_adder6(10)
 
-verify(inc(1) == 11) # there's only one global
-verify(plus10(-2) == 8)
+vereq(inc(1), 11) # there's only one global
+vereq(plus10(-2), 8)
 
 print "7. nearest enclosing scope"
 
@@ -103,7 +103,7 @@ def f(x):
     return g(2)
 
 test_func = f(10)
-verify(test_func(5) == 47)
+vereq(test_func(5), 47)
 
 print "8. mixed freevars and cellvars"
 
@@ -123,7 +123,7 @@ def f(x, y, z):
 
 g = f(1, 2, 3)
 h = g(2, 4, 6)
-verify(h() == 39)
+vereq(h(), 39)
 
 print "9. free variable in method"
 
@@ -141,9 +141,9 @@ def test():
     return Test()
 
 t = test()
-verify(t.test() == "var")
-verify(t.method_and_var() == "method")
-verify(t.actual_global() == "global")
+vereq(t.test(), "var")
+vereq(t.method_and_var(), "method")
+vereq(t.actual_global(), "global")
 
 method_and_var = "var"
 class Test:
@@ -158,9 +158,9 @@ class Test:
         return str(self)
 
 t = Test()
-verify(t.test() == "var")
-verify(t.method_and_var() == "method")
-verify(t.actual_global() == "global")
+vereq(t.test(), "var")
+vereq(t.method_and_var(), "method")
+vereq(t.actual_global(), "global")
 
 print "10. recursion"
 
@@ -175,7 +175,7 @@ def f(x):
     else:
         raise ValueError, "x must be >= 0"
 
-verify(f(6) == 720)
+vereq(f(6), 720)
 
 
 print "11. unoptimized namespaces"
@@ -252,24 +252,24 @@ print "12. lambdas"
 f1 = lambda x: lambda y: x + y
 inc = f1(1)
 plus10 = f1(10)
-verify(inc(1) == 2)
-verify(plus10(5) == 15)
+vereq(inc(1), 2)
+vereq(plus10(5), 15)
 
 f2 = lambda x: (lambda : lambda y: x + y)()
 inc = f2(1)
 plus10 = f2(10)
-verify(inc(1) == 2)
-verify(plus10(5) == 15)
+vereq(inc(1), 2)
+vereq(plus10(5), 15)
 
 f3 = lambda x: lambda y: global_x + y
 global_x = 1
 inc = f3(None)
-verify(inc(2) == 3)
+vereq(inc(2), 3)
 
 f8 = lambda x, y, z: lambda a, b, c: lambda : z * (b + y)
 g = f8(1, 2, 3)
 h = g(2, 4, 6)
-verify(h() == 18)
+vereq(h(), 18)
 
 print "13. UnboundLocal"
 
@@ -306,21 +306,21 @@ def makeReturner(*lst):
         return lst
     return returner
 
-verify(makeReturner(1,2,3)() == (1,2,3))
+vereq(makeReturner(1,2,3)(), (1,2,3))
 
 def makeReturner2(**kwargs):
     def returner():
         return kwargs
     return returner
 
-verify(makeReturner2(a=11)()['a'] == 11)
+vereq(makeReturner2(a=11)()['a'], 11)
 
 def makeAddPair((a, b)):
     def addPair((c, d)):
         return (a + c, b + d)
     return addPair
 
-verify(makeAddPair((1, 2))((100, 200)) == (101,202))
+vereq(makeAddPair((1, 2))((100, 200)), (101,202))
 
 print "15. scope of global statements"
 # Examples posted by Samuele Pedroni to python-dev on 3/1/2001
@@ -337,8 +337,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 7)
-verify(x == 7)
+vereq(f(), 7)
+vereq(x, 7)
 
 # II
 x = 7
@@ -352,8 +352,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
-verify(x == 7)
+vereq(f(), 2)
+vereq(x, 7)
 
 # III
 x = 7
@@ -368,8 +368,8 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
-verify(x == 2)
+vereq(f(), 2)
+vereq(x, 2)
 
 # IV
 x = 7
@@ -384,8 +384,25 @@ def f():
             return h()
         return i()
     return g()
-verify(f() == 2)
-verify(x == 2)
+vereq(f(), 2)
+vereq(x, 2)
+
+# XXX what about global statements in class blocks?
+# do they affect methods?
+
+x = 12
+class Global:
+    global x
+    x = 13
+    def set(self, val):
+        x = val
+    def get(self):
+        return x
+
+g = Global()
+vereq(g.get(), 13)
+g.set(15)
+vereq(g.get(), 13)
 
 print "16. check leaks"
 
@@ -407,7 +424,7 @@ def f1():
 for i in range(100):
     f1()
 
-verify(Foo.count == 0)
+vereq(Foo.count, 0)
 
 print "17. class and global"
 
@@ -419,9 +436,9 @@ def test(x):
     return Foo()
 
 x = 0
-verify(test(6)(2) == 8)
+vereq(test(6)(2), 8)
 x = -1
-verify(test(3)(2) == 5)
+vereq(test(3)(2), 5)
 
 print "18. verify that locals() works"
 
@@ -437,7 +454,7 @@ def f(x):
 d = f(2)(4)
 verify(d.has_key('h'))
 del d['h']
-verify(d == {'x': 2, 'y': 7, 'w': 6})
+vereq(d, {'x': 2, 'y': 7, 'w': 6})
 
 print "19. var is bound and free in class"
 
@@ -449,7 +466,7 @@ def f(x):
     return C
 
 inst = f(3)()
-verify(inst.a == inst.m())
+vereq(inst.a, inst.m())
 
 print "20. interaction with trace function"