Merge ast-branch to head

This change implements a new bytecode compiler, based on a
transformation of the parse tree to an abstract syntax defined in
Parser/Python.asdl.

The compiler implementation is not complete, but it is in stable
enough shape to run the entire test suite excepting two disabled
tests.

9 files changed, 654 insertions, 128 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
new file mode 100644
index 0000000..0859acf
--- /dev/null
+++ b/Include/Python-ast.h
@@ -0,0 +1,418 @@
+/* File automatically generated by ../Parser/asdl_c.py */
+
+#include "asdl.h"
+
+typedef struct _mod *mod_ty;
+
+typedef struct _stmt *stmt_ty;
+
+typedef struct _expr *expr_ty;
+
+typedef enum _expr_context { Load=1, Store=2, Del=3, AugLoad=4, AugStore=5,
+                             Param=6 } expr_context_ty;
+
+typedef struct _slice *slice_ty;
+
+typedef enum _boolop { And=1, Or=2 } boolop_ty;
+
+typedef enum _operator { Add=1, Sub=2, Mult=3, Div=4, Mod=5, Pow=6, LShift=7,
+                         RShift=8, BitOr=9, BitXor=10, BitAnd=11, FloorDiv=12 }
+                         operator_ty;
+
+typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty;
+
+typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8,
+                      In=9, NotIn=10 } cmpop_ty;
+
+typedef struct _comprehension *comprehension_ty;
+
+typedef struct _excepthandler *excepthandler_ty;
+
+typedef struct _arguments *arguments_ty;
+
+typedef struct _keyword *keyword_ty;
+
+typedef struct _alias *alias_ty;
+
+struct _mod {
+        enum { Module_kind=1, Interactive_kind=2, Expression_kind=3,
+               Suite_kind=4 } kind;
+        union {
+                struct {
+                        asdl_seq *body;
+                } Module;
+                
+                struct {
+                        asdl_seq *body;
+                } Interactive;
+                
+                struct {
+                        expr_ty body;
+                } Expression;
+                
+                struct {
+                        asdl_seq *body;
+                } Suite;
+                
+        } v;
+};
+
+struct _stmt {
+        enum { FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
+               Delete_kind=4, Assign_kind=5, AugAssign_kind=6, Print_kind=7,
+               For_kind=8, While_kind=9, If_kind=10, Raise_kind=11,
+               TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
+               Import_kind=15, ImportFrom_kind=16, Exec_kind=17,
+               Global_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
+               Continue_kind=22 } kind;
+        union {
+                struct {
+                        identifier name;
+                        arguments_ty args;
+                        asdl_seq *body;
+                        asdl_seq *decorators;
+                } FunctionDef;
+                
+                struct {
+                        identifier name;
+                        asdl_seq *bases;
+                        asdl_seq *body;
+                } ClassDef;
+                
+                struct {
+                        expr_ty value;
+                } Return;
+                
+                struct {
+                        asdl_seq *targets;
+                } Delete;
+                
+                struct {
+                        asdl_seq *targets;
+                        expr_ty value;
+                } Assign;
+                
+                struct {
+                        expr_ty target;
+                        operator_ty op;
+                        expr_ty value;
+                } AugAssign;
+                
+                struct {
+                        expr_ty dest;
+                        asdl_seq *values;
+                        bool nl;
+                } Print;
+                
+                struct {
+                        expr_ty target;
+                        expr_ty iter;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } For;
+                
+                struct {
+                        expr_ty test;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } While;
+                
+                struct {
+                        expr_ty test;
+                        asdl_seq *body;
+                        asdl_seq *orelse;
+                } If;
+                
+                struct {
+                        expr_ty type;
+                        expr_ty inst;
+                        expr_ty tback;
+                } Raise;
+                
+                struct {
+                        asdl_seq *body;
+                        asdl_seq *handlers;
+                        asdl_seq *orelse;
+                } TryExcept;
+                
+                struct {
+                        asdl_seq *body;
+                        asdl_seq *finalbody;
+                } TryFinally;
+                
+                struct {
+                        expr_ty test;
+                        expr_ty msg;
+                } Assert;
+                
+                struct {
+                        asdl_seq *names;
+                } Import;
+                
+                struct {
+                        identifier module;
+                        asdl_seq *names;
+                } ImportFrom;
+                
+                struct {
+                        expr_ty body;
+                        expr_ty globals;
+                        expr_ty locals;
+                } Exec;
+                
+                struct {
+                        asdl_seq *names;
+                } Global;
+                
+                struct {
+                        expr_ty value;
+                } Expr;
+                
+        } v;
+        int lineno;
+};
+
+struct _expr {
+        enum { BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
+               Dict_kind=5, ListComp_kind=6, GeneratorExp_kind=7, Yield_kind=8,
+               Compare_kind=9, Call_kind=10, Repr_kind=11, Num_kind=12,
+               Str_kind=13, Attribute_kind=14, Subscript_kind=15, Name_kind=16,
+               List_kind=17, Tuple_kind=18 } kind;
+        union {
+                struct {
+                        boolop_ty op;
+                        asdl_seq *values;
+                } BoolOp;
+                
+                struct {
+                        expr_ty left;
+                        operator_ty op;
+                        expr_ty right;
+                } BinOp;
+                
+                struct {
+                        unaryop_ty op;
+                        expr_ty operand;
+                } UnaryOp;
+                
+                struct {
+                        arguments_ty args;
+                        expr_ty body;
+                } Lambda;
+                
+                struct {
+                        asdl_seq *keys;
+                        asdl_seq *values;
+                } Dict;
+                
+                struct {
+                        expr_ty elt;
+                        asdl_seq *generators;
+                } ListComp;
+                
+                struct {
+                        expr_ty elt;
+                        asdl_seq *generators;
+                } GeneratorExp;
+                
+                struct {
+                        expr_ty value;
+                } Yield;
+                
+                struct {
+                        expr_ty left;
+                        asdl_seq *ops;
+                        asdl_seq *comparators;
+                } Compare;
+                
+                struct {
+                        expr_ty func;
+                        asdl_seq *args;
+                        asdl_seq *keywords;
+                        expr_ty starargs;
+                        expr_ty kwargs;
+                } Call;
+                
+                struct {
+                        expr_ty value;
+                } Repr;
+                
+                struct {
+                        object n;
+                } Num;
+                
+                struct {
+                        string s;
+                } Str;
+                
+                struct {
+                        expr_ty value;
+                        identifier attr;
+                        expr_context_ty ctx;
+                } Attribute;
+                
+                struct {
+                        expr_ty value;
+                        slice_ty slice;
+                        expr_context_ty ctx;
+                } Subscript;
+                
+                struct {
+                        identifier id;
+                        expr_context_ty ctx;
+                } Name;
+                
+                struct {
+                        asdl_seq *elts;
+                        expr_context_ty ctx;
+                } List;
+                
+                struct {
+                        asdl_seq *elts;
+                        expr_context_ty ctx;
+                } Tuple;
+                
+        } v;
+        int lineno;
+};
+
+struct _slice {
+        enum { Ellipsis_kind=1, Slice_kind=2, ExtSlice_kind=3, Index_kind=4 }
+               kind;
+        union {
+                struct {
+                        expr_ty lower;
+                        expr_ty upper;
+                        expr_ty step;
+                } Slice;
+                
+                struct {
+                        asdl_seq *dims;
+                } ExtSlice;
+                
+                struct {
+                        expr_ty value;
+                } Index;
+                
+        } v;
+};
+
+struct _comprehension {
+        expr_ty target;
+        expr_ty iter;
+        asdl_seq *ifs;
+};
+
+struct _excepthandler {
+        expr_ty type;
+        expr_ty name;
+        asdl_seq *body;
+};
+
+struct _arguments {
+        asdl_seq *args;
+        identifier vararg;
+        identifier kwarg;
+        asdl_seq *defaults;
+};
+
+struct _keyword {
+        identifier arg;
+        expr_ty value;
+};
+
+struct _alias {
+        identifier name;
+        identifier asname;
+};
+
+mod_ty Module(asdl_seq * body);
+mod_ty Interactive(asdl_seq * body);
+mod_ty Expression(expr_ty body);
+mod_ty Suite(asdl_seq * body);
+stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
+                    asdl_seq * decorators, int lineno);
+stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
+                 lineno);
+stmt_ty Return(expr_ty value, int lineno);
+stmt_ty Delete(asdl_seq * targets, int lineno);
+stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno);
+stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno);
+stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno);
+stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
+            int lineno);
+stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
+stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
+stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno);
+stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
+                  lineno);
+stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno);
+stmt_ty Assert(expr_ty test, expr_ty msg, int lineno);
+stmt_ty Import(asdl_seq * names, int lineno);
+stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno);
+stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno);
+stmt_ty Global(asdl_seq * names, int lineno);
+stmt_ty Expr(expr_ty value, int lineno);
+stmt_ty Pass(int lineno);
+stmt_ty Break(int lineno);
+stmt_ty Continue(int lineno);
+expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno);
+expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno);
+expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno);
+expr_ty Lambda(arguments_ty args, expr_ty body, int lineno);
+expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno);
+expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno);
+expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno);
+expr_ty Yield(expr_ty value, int lineno);
+expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
+                lineno);
+expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
+             starargs, expr_ty kwargs, int lineno);
+expr_ty Repr(expr_ty value, int lineno);
+expr_ty Num(object n, int lineno);
+expr_ty Str(string s, int lineno);
+expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
+                  lineno);
+expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
+                  lineno);
+expr_ty Name(identifier id, expr_context_ty ctx, int lineno);
+expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno);
+expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno);
+slice_ty Ellipsis(void);
+slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step);
+slice_ty ExtSlice(asdl_seq * dims);
+slice_ty Index(expr_ty value);
+comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs);
+excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body);
+arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
+                       asdl_seq * defaults);
+keyword_ty keyword(identifier arg, expr_ty value);
+alias_ty alias(identifier name, identifier asname);
+void free_mod(mod_ty);
+void free_stmt(stmt_ty);
+void free_expr(expr_ty);
+void free_expr_context(expr_context_ty);
+void free_slice(slice_ty);
+void free_boolop(boolop_ty);
+void free_operator(operator_ty);
+void free_unaryop(unaryop_ty);
+void free_cmpop(cmpop_ty);
+void free_comprehension(comprehension_ty);
+void free_excepthandler(excepthandler_ty);
+void free_arguments(arguments_ty);
+void free_keyword(keyword_ty);
+void free_alias(alias_ty);
+int marshal_write_mod(PyObject **, int *, mod_ty);
+int marshal_write_stmt(PyObject **, int *, stmt_ty);
+int marshal_write_expr(PyObject **, int *, expr_ty);
+int marshal_write_expr_context(PyObject **, int *, expr_context_ty);
+int marshal_write_slice(PyObject **, int *, slice_ty);
+int marshal_write_boolop(PyObject **, int *, boolop_ty);
+int marshal_write_operator(PyObject **, int *, operator_ty);
+int marshal_write_unaryop(PyObject **, int *, unaryop_ty);
+int marshal_write_cmpop(PyObject **, int *, cmpop_ty);
+int marshal_write_comprehension(PyObject **, int *, comprehension_ty);
+int marshal_write_excepthandler(PyObject **, int *, excepthandler_ty);
+int marshal_write_arguments(PyObject **, int *, arguments_ty);
+int marshal_write_keyword(PyObject **, int *, keyword_ty);
+int marshal_write_alias(PyObject **, int *, alias_ty);
diff --git a/Include/Python.h b/Include/Python.h
index 2d48d2e..f941cba 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -128,8 +128,7 @@
 #include "pystrtod.h"
 
 /* _Py_Mangle is defined in compile.c */
-PyAPI_FUNC(int) _Py_Mangle(char *p, char *name, \
-				 char *buffer, size_t maxlen);
+PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
 
 /* PyArg_GetInt is deprecated and should not be used, use PyArg_Parse(). */
 #define PyArg_GetInt(v, a)	PyArg_Parse((v), "i", (a))
diff --git a/Include/asdl.h b/Include/asdl.h
new file mode 100644
index 0000000..8ad46fa
--- /dev/null
+++ b/Include/asdl.h
@@ -0,0 +1,54 @@
+#ifndef Py_ASDL_H
+#define Py_ASDL_H
+
+typedef PyObject * identifier;
+typedef PyObject * string;
+typedef PyObject * object;
+
+typedef enum {false, true} bool;
+
+/* It would be nice if the code generated by asdl_c.py was completely
+   independent of Python, but it is a goal the requires too much work
+   at this stage.  So, for example, I'll represent identifiers as
+   interned Python strings.
+*/
+
+/* XXX A sequence should be typed so that its use can be typechecked. */
+
+/* XXX We shouldn't pay for offset when we don't need APPEND. */
+
+typedef struct {
+    int size;
+    int offset;
+    void *elements[1];
+} asdl_seq;
+
+asdl_seq *asdl_seq_new(int size);
+void asdl_seq_free(asdl_seq *);
+
+#ifdef Py_DEBUG
+#define asdl_seq_GET(S, I) (S)->elements[(I)]
+#define asdl_seq_SET(S, I, V) { \
+        int _asdl_i = (I); \
+        assert((S) && _asdl_i < (S)->size); \
+        (S)->elements[_asdl_i] = (V); \
+}
+#define asdl_seq_APPEND(S, V) { \
+        assert((S) && (S)->offset < (S)->size); \
+        (S)->elements[(S)->offset++] = (V); \
+}
+#else
+#define asdl_seq_GET(S, I) (S)->elements[(I)]
+#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
+#define asdl_seq_APPEND(S, V) (S)->elements[(S)->offset++] = (V)
+#endif
+#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
+
+/* Routines to marshal the basic types. */
+int marshal_write_int(PyObject **, int *, int);
+int marshal_write_bool(PyObject **, int *, bool);
+int marshal_write_identifier(PyObject **, int *, identifier);
+int marshal_write_string(PyObject **, int *, string);
+int marshal_write_object(PyObject **, int *, object);
+
+#endif /* !Py_ASDL_H */
diff --git a/Include/ast.h b/Include/ast.h
new file mode 100644
index 0000000..8912f38
--- /dev/null
+++ b/Include/ast.h
@@ -0,0 +1,13 @@
+#ifndef Py_AST_H
+#define Py_AST_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern DL_IMPORT(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
+					const char *);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_AST_H */
diff --git a/Include/code.h b/Include/code.h
new file mode 100644
index 0000000..e6478b0
--- /dev/null
+++ b/Include/code.h
@@ -0,0 +1,73 @@
+/* Definitions for bytecode */
+
+#ifndef Py_CODE_H
+#define Py_CODE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Bytecode object */
+typedef struct {
+    PyObject_HEAD
+    int co_argcount;		/* #arguments, except *args */
+    int co_nlocals;		/* #local variables */
+    int co_stacksize;		/* #entries needed for evaluation stack */
+    int co_flags;		/* CO_..., see below */
+    PyObject *co_code;		/* instruction opcodes */
+    PyObject *co_consts;	/* list (constants used) */
+    PyObject *co_names;		/* list of strings (names used) */
+    PyObject *co_varnames;	/* tuple of strings (local variable names) */
+    PyObject *co_freevars;	/* tuple of strings (free variable names) */
+    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
+    /* The rest doesn't count for hash/cmp */
+    PyObject *co_filename;	/* string (where it was loaded from) */
+    PyObject *co_name;		/* string (name, for reference) */
+    int co_firstlineno;		/* first source line number */
+    PyObject *co_lnotab;	/* string (encoding addr<->lineno mapping) */
+} PyCodeObject;
+
+/* Masks for co_flags above */
+#define CO_OPTIMIZED	0x0001
+#define CO_NEWLOCALS	0x0002
+#define CO_VARARGS	0x0004
+#define CO_VARKEYWORDS	0x0008
+#define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
+/* The CO_NOFREE flag is set if there are no free or cell variables.
+   This information is redundant, but it allows a single flag test
+   to determine whether there is any extra work to be done when the
+   call frame it setup.
+*/
+#define CO_NOFREE       0x0040
+/* XXX Temporary hack.  Until generators are a permanent part of the
+   language, we need a way for a code object to record that generators
+   were *possible* when it was compiled.  This is so code dynamically
+   compiled *by* a code object knows whether to allow yield stmts.  In
+   effect, this passes on the "from __future__ import generators" state
+   in effect when the code block was compiled. */
+#define CO_GENERATOR_ALLOWED    0x1000 /* no longer used in an essential way */
+#define CO_FUTURE_DIVISION    	0x2000
+
+#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
+
+extern DL_IMPORT(PyTypeObject) PyCode_Type;
+
+#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
+#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
+
+/* Public interface */
+DL_IMPORT(PyCodeObject *) PyCode_New(
+	int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
+	PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *); 
+        /* same as struct above */
+DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
+
+/* for internal use only */
+#define _PyCode_GETCODEPTR(co, pp) \
+	((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \
+	 ((co)->co_code, 0, (void **)(pp)))
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CODE_H */
diff --git a/Include/compile.h b/Include/compile.h
index 82bf708..27a3f76 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -1,5 +1,6 @@
-
-/* Definitions for bytecode */
+#ifndef Py_CODE_H
+#include "code.h"
+#endif
 
 #ifndef Py_COMPILE_H
 #define Py_COMPILE_H
@@ -7,55 +8,6 @@
 extern "C" {
 #endif
 
-/* Bytecode object */
-typedef struct {
-    PyObject_HEAD
-    int co_argcount;		/* #arguments, except *args */
-    int co_nlocals;		/* #local variables */
-    int co_stacksize;		/* #entries needed for evaluation stack */
-    int co_flags;		/* CO_..., see below */
-    PyObject *co_code;		/* instruction opcodes */
-    PyObject *co_consts;	/* list (constants used) */
-    PyObject *co_names;		/* list of strings (names used) */
-    PyObject *co_varnames;	/* tuple of strings (local variable names) */
-    PyObject *co_freevars;	/* tuple of strings (free variable names) */
-    PyObject *co_cellvars;      /* tuple of strings (cell variable names) */
-    /* The rest doesn't count for hash/cmp */
-    PyObject *co_filename;	/* string (where it was loaded from) */
-    PyObject *co_name;		/* string (name, for reference) */
-    int co_firstlineno;		/* first source line number */
-    PyObject *co_lnotab;	/* string (encoding addr<->lineno mapping) */
-} PyCodeObject;
-
-/* Masks for co_flags above */
-#define CO_OPTIMIZED	0x0001
-#define CO_NEWLOCALS	0x0002
-#define CO_VARARGS	0x0004
-#define CO_VARKEYWORDS	0x0008
-#define CO_NESTED       0x0010
-#define CO_GENERATOR    0x0020
-/* The CO_NOFREE flag is set if there are no free or cell variables.
-   This information is redundant, but it allows a single flag test
-   to determine whether there is any extra work to be done when the
-   call frame it setup.
-*/
-#define CO_NOFREE       0x0040
-/* XXX Temporary hack.  Until generators are a permanent part of the
-   language, we need a way for a code object to record that generators
-   were *possible* when it was compiled.  This is so code dynamically
-   compiled *by* a code object knows whether to allow yield stmts.  In
-   effect, this passes on the "from __future__ import generators" state
-   in effect when the code block was compiled. */
-#define CO_GENERATOR_ALLOWED    0x1000 /* no longer used in an essential way */
-#define CO_FUTURE_DIVISION    	0x2000
-
-PyAPI_DATA(PyTypeObject) PyCode_Type;
-
-#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
-#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
-
-#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
-
 /* Public interface */
 struct _node; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
@@ -68,19 +20,22 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
 /* Future feature support */
 
 typedef struct {
-    int ff_found_docstring;
-    int ff_last_lineno;
-    int ff_features;
+    int ff_features;      /* flags set by future statements */
+    int ff_lineno;        /* line number of last future statement */
 } PyFutureFeatures;
 
-PyAPI_FUNC(PyFutureFeatures *) PyNode_Future(struct _node *, const char *);
-PyAPI_FUNC(PyCodeObject *) PyNode_CompileFlags(struct _node *, const char *,
-					      PyCompilerFlags *);
-
 #define FUTURE_NESTED_SCOPES "nested_scopes"
 #define FUTURE_GENERATORS "generators"
 #define FUTURE_DIVISION "division"
 
+struct _mod; /* Declare the existence of this type */
+DL_IMPORT(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
+					PyCompilerFlags *);
+DL_IMPORT(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+
+#define ERR_LATE_FUTURE \
+"from __future__ imports must occur at the beginning of the file"
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/pyport.h b/Include/pyport.h
index 2440c55..ea2091b 100644
--- a/Include/pyport.h
+++ b/Include/pyport.h
@@ -583,6 +583,7 @@ typedef	struct fd_set {
 
 #ifndef INT_MAX
 #define INT_MAX 2147483647
+#define INT_MIN (-INT_MAX - 1)
 #endif
 
 #ifndef LONG_MAX
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index f461d13..490613e 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -29,46 +29,37 @@ PyAPI_FUNC(int) Py_IsInitialized(void);
 PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
 PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 
-PyAPI_FUNC(int) PyRun_AnyFile(FILE *, const char *);
-PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *, const char *, int);
-
-PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-
-PyAPI_FUNC(int) PyRun_SimpleString(const char *);
+PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, char *, PyCompilerFlags *);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, char *, int, PyCompilerFlags *);
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFile(FILE *, const char *);
-PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *, const char *, int);
 PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *, const char *);
 PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *, const char *);
 PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
 
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseString(const char *, int);
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseFile(FILE *, const char *, int);
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int);
-PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *,
-								  const char *,
-								  int,
-								  int);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, 
+						 int, PyCompilerFlags *flags);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int, 
+					       char *, char *,
+                                               PyCompilerFlags *, int *);
+#define PyParser_SimpleParseString(S, B) \
+        PyParser_SimpleParseStringFlags(S, B, 0)
+#define PyParser_SimpleParseFile(FP, S, B) \
+        PyParser_SimpleParseFileFlags(FP, S, B, 0)
+PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, 
+							  int);
 PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
 							int, int);
 
-PyAPI_FUNC(PyObject *) PyRun_String(const char *, int, PyObject *, PyObject *);
-PyAPI_FUNC(PyObject *) PyRun_File(FILE *, const char *, int, PyObject *, PyObject *);
-PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *, const char *, int,
-				   PyObject *, PyObject *, int);
-PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *,
-					PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *, const char *, int, PyObject *, 
-				      PyObject *, PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, PyObject *, 
-					PyObject *, int, PyCompilerFlags *);
-
-PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
+PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, 
+					 PyObject *, PyCompilerFlags *);
+
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, 
+					 PyObject *, PyObject *, int, 
+					 PyCompilerFlags *);
+
+#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL)
 PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int,
-					    PyCompilerFlags *);
+					     PyCompilerFlags *);
 PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
 
 PyAPI_FUNC(void) PyErr_Print(void);
@@ -84,6 +75,25 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
 /* Bootstrap */
 PyAPI_FUNC(int) Py_Main(int argc, char **argv);
 
+/* Use macros for a bunch of old variants */
+#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
+#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL)
+#define PyRun_AnyFileEx(fp, name, closeit) \
+	PyRun_AnyFileExFlags(fp, name, closeit, NULL)
+#define PyRun_AnyFileFlags(fp, name, flags) \
+	PyRun_AnyFileExFlags(fp, name, 0, flags)
+#define PyRun_SimpleString(s, f) PyRunSimpleStringFlags(s, f, NULL)
+#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL)
+#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL)
+#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL)
+#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL)
+#define PyRun_File(fp, p, s, g, l) \
+        PyRun_FileExFlags(fp, p, s, g, l, 0, NULL)
+#define PyRun_FileEx(fp, p, s, g, l, c) \
+        PyRun_FileExFlags(fp, p, s, g, l, c, NULL)
+#define PyRun_FileFlags(fp, p, s, g, l, flags) \
+        PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
+
 /* In getpath.c */
 PyAPI_FUNC(char *) Py_GetProgramFullPath(void);
 PyAPI_FUNC(char *) Py_GetPrefix(void);
diff --git a/Include/symtable.h b/Include/symtable.h
index 628c3e6..646602c 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -4,64 +4,59 @@
 extern "C" {
 #endif
 
-/* A symbol table is constructed each time PyNode_Compile() is
-   called.  The table walks the entire parse tree and identifies each
-   use or definition of a variable. 
-
-   The symbol table contains a dictionary for each code block in a
-   module: The symbol dictionary for the block.  They keys of these
-   dictionaries are the name of all variables used or defined in the
-   block; the integer values are used to store several flags,
-   e.g. DEF_PARAM indicates that a variable is a parameter to a
-   function. 
-*/
+typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock }
+    block_ty;
 
 struct _symtable_entry;
 
 struct symtable {
-	int st_pass;             /* pass == 1 or 2 */
 	const char *st_filename; /* name of file being compiled */
 	struct _symtable_entry *st_cur; /* current symbol table entry */
+	struct _symtable_entry *st_top; /* module entry */
 	PyObject *st_symbols;    /* dictionary of symbol table entries */
         PyObject *st_stack;      /* stack of namespace info */
 	PyObject *st_global;     /* borrowed ref to MODULE in st_symbols */
-	int st_nscopes;          /* number of scopes */
-	int st_errors;           /* number of errors */
+	int st_nblocks;          /* number of blocks */
 	char *st_private;        /* name of current class or NULL */
+        int st_tmpname;          /* temporary name counter */
 	PyFutureFeatures *st_future; /* module's future features */
 };
 
 typedef struct _symtable_entry {
 	PyObject_HEAD
-	PyObject *ste_id;        /* int: key in st_symbols) */
-	PyObject *ste_symbols;   /* dict: name to flags) */
-	PyObject *ste_name;      /* string: name of scope */
+	PyObject *ste_id;        /* int: key in st_symbols */
+	PyObject *ste_symbols;   /* dict: name to flags */
+	PyObject *ste_name;      /* string: name of block */
 	PyObject *ste_varnames;  /* list of variable names */
 	PyObject *ste_children;  /* list of child ids */
-	int ste_type;            /* module, class, or function */
-	int ste_lineno;          /* first line of scope */
-	int ste_optimized;       /* true if namespace can't be optimized */
-	int ste_nested;          /* true if scope is nested */
-	int ste_child_free;      /* true if a child scope has free variables,
+	block_ty ste_type;       /* module, class, or function */
+	int ste_unoptimized;     /* false if namespace is optimized */
+	int ste_nested : 1;      /* true if block is nested */
+	int ste_free : 1;        /* true if block has free variables */
+	int ste_child_free : 1;  /* true if a child block has free variables,
 				    including free refs to globals */
-	int ste_generator;       /* true if namespace is a generator */
+	int ste_generator : 1;   /* true if namespace is a generator */
+	int ste_varargs : 1;     /* true if block has varargs */
+	int ste_varkeywords : 1; /* true if block has varkeywords */
+	int ste_lineno;          /* first line of block */
 	int ste_opt_lineno;      /* lineno of last exec or import * */
-	int ste_tmpname;         /* temporary name counter */
+	int ste_tmpname;         /* counter for listcomp temp vars */
 	struct symtable *ste_table;
-} PySymtableEntryObject;
-
-PyAPI_DATA(PyTypeObject) PySymtableEntry_Type;
+} PySTEntryObject;
 
-#define PySymtableEntry_Check(op) ((op)->ob_type == &PySymtableEntry_Type)
+PyAPI_DATA(PyTypeObject) PySTEntry_Type;
 
-PyAPI_FUNC(PyObject *) PySymtableEntry_New(struct symtable *,
-						 char *, int, int);
+#define PySTEntry_Check(op) ((op)->ob_type == &PySTEntry_Type)
 
-PyAPI_FUNC(struct symtable *) PyNode_CompileSymtable(struct _node *, const char *);
-PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
+PyAPI_FUNC(PySTEntryObject *) \
+	PySTEntry_New(struct symtable *, identifier, block_ty, void *, int);
+PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
 
+PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *, 
+					      PyFutureFeatures *);
+PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
 
-#define TOP "global"
+PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 
 /* Flags for def-use information */
 
@@ -72,16 +67,19 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_STAR 2<<3          /* parameter is star arg */
 #define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
 #define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
-#define DEF_FREE 2<<6          /* name used but not defined in nested scope */
+#define DEF_FREE 2<<6          /* name used but not defined in nested block */
 #define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
 #define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
 #define DEF_IMPORT 2<<9        /* assignment occurred via import */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 
-#define TYPE_FUNCTION 1
-#define TYPE_CLASS 2
-#define TYPE_MODULE 3
+/* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
+   table.  GLOBAL is returned from PyST_GetScope() for either of them. 
+   It is stored in ste_symbols at bits 12-14.
+*/
+#define SCOPE_OFF 11
+#define SCOPE_MASK 7
 
 #define LOCAL 1
 #define GLOBAL_EXPLICIT 2
@@ -89,9 +87,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define FREE 4
 #define CELL 5
 
+/* The following three names are used for the ste_unoptimized bit field */
 #define OPT_IMPORT_STAR 1
 #define OPT_EXEC 2
 #define OPT_BARE_EXEC 4
+#define OPT_TOPLEVEL 8  /* top-level names, including eval and exec */
+
+#define GENERATOR 1
+#define GENERATOR_EXPRESSION 2
 
 #define GENERATOR 1
 #define GENERATOR_EXPRESSION 2