Merge ast-branch to head

This change implements a new bytecode compiler, based on a
transformation of the parse tree to an abstract syntax defined in
Parser/Python.asdl.

The compiler implementation is not complete, but it is in stable
enough shape to run the entire test suite excepting two disabled
tests.

1 files changed, 124 insertions, 152 deletions

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index e007f98..ad837d2 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -3,13 +3,16 @@
 
 #include "Python.h"
 
+#include "Python-ast.h"
 #include "grammar.h"
 #include "node.h"
 #include "token.h"
 #include "parsetok.h"
 #include "errcode.h"
+#include "code.h"
 #include "compile.h"
 #include "symtable.h"
+#include "ast.h"
 #include "eval.h"
 #include "marshal.h"
 
@@ -32,9 +35,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */
 /* Forward */
 static void initmain(void);
 static void initsite(void);
-static PyObject *run_err_node(node *, const char *, PyObject *, PyObject *,
+static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *,
 			      PyCompilerFlags *);
-static PyObject *run_node(node *, const char *, PyObject *, PyObject *,
+static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
 			  PyCompilerFlags *);
 static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
 			      PyCompilerFlags *);
@@ -634,25 +637,7 @@ initsite(void)
 /* Parse input from a file and execute it */
 
 int
-PyRun_AnyFile(FILE *fp, const char *filename)
-{
-	return PyRun_AnyFileExFlags(fp, filename, 0, NULL);
-}
-
-int
-PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
-{
-	return PyRun_AnyFileExFlags(fp, filename, 0, flags);
-}
-
-int
-PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit)
-{
-	return PyRun_AnyFileExFlags(fp, filename, closeit, NULL);
-}
-
-int
-PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
+PyRun_AnyFileExFlags(FILE *fp, char *filename, int closeit, 
 		     PyCompilerFlags *flags)
 {
 	if (filename == NULL)
@@ -668,12 +653,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
 }
 
 int
-PyRun_InteractiveLoop(FILE *fp, const char *filename)
-{
-	return PyRun_InteractiveLoopFlags(fp, filename, NULL);
-}
-
-int
 PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
 	PyObject *v;
@@ -708,12 +687,6 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 	}
 }
 
-int
-PyRun_InteractiveOne(FILE *fp, const char *filename)
-{
-	return PyRun_InteractiveOneFlags(fp, filename, NULL);
-}
-
 /* compute parser flags based on compiler flags */
 #define PARSER_FLAGS(flags) \
 	(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
@@ -723,9 +696,9 @@ int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
 	PyObject *m, *d, *v, *w;
-	node *n;
-	perrdetail err;
+	mod_ty mod;
 	char *ps1 = "", *ps2 = "";
+	int errcode = 0;
 
 	v = PySys_GetObject("ps1");
 	if (v != NULL) {
@@ -743,26 +716,25 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 		else if (PyString_Check(w))
 			ps2 = PyString_AsString(w);
 	}
-	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
-			    	    Py_single_input, ps1, ps2, &err,
-			    	    PARSER_FLAGS(flags));
+	mod = PyParser_ASTFromFile(fp, filename, 
+				   Py_single_input, ps1, ps2,
+				   flags, &errcode);
 	Py_XDECREF(v);
 	Py_XDECREF(w);
-	if (n == NULL) {
-		if (err.error == E_EOF) {
-			if (err.text)
-				PyMem_DEL(err.text);
+	if (mod == NULL) {
+		if (errcode == E_EOF) {
+			PyErr_Clear();
 			return E_EOF;
 		}
-		err_input(&err);
 		PyErr_Print();
-		return err.error;
+		return -1;
 	}
 	m = PyImport_AddModule("__main__");
 	if (m == NULL)
 		return -1;
 	d = PyModule_GetDict(m);
-	v = run_node(n, filename, d, d, flags);
+	v = run_mod(mod, filename, d, d, flags);
+	free_mod(mod);
 	if (v == NULL) {
 		PyErr_Print();
 		return -1;
@@ -773,12 +745,6 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 	return 0;
 }
 
-int
-PyRun_SimpleFile(FILE *fp, const char *filename)
-{
-	return PyRun_SimpleFileEx(fp, filename, 0);
-}
-
 /* Check whether a file maybe a pyc file: Look at the extension,
    the file type, and, if we may close it, at the first few bytes. */
 
@@ -820,12 +786,6 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
 }
 
 int
-PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit)
-{
-	return PyRun_SimpleFileExFlags(fp, filename, closeit, NULL);
-}
-
-int
 PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 			PyCompilerFlags *flags)
 {
@@ -874,12 +834,6 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 }
 
 int
-PyRun_SimpleString(const char *command)
-{
-	return PyRun_SimpleStringFlags(command, NULL);
-}
-
-int
 PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
 {
 	PyObject *m, *d, *v;
@@ -1054,6 +1008,8 @@ PyErr_PrintEx(int set_sys_last_vars)
 		handle_system_exit();
 	}
 	PyErr_Fetch(&exception, &v, &tb);
+	if (exception == NULL)
+		return;
 	PyErr_NormalizeException(&exception, &v, &tb);
 	if (exception == NULL)
 		return;
@@ -1195,74 +1151,48 @@ void PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb)
 }
 
 PyObject *
-PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals)
-{
-	return run_err_node(PyParser_SimpleParseString(str, start),
-			    "<string>", globals, locals, NULL);
-}
-
-PyObject *
-PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals,
-	   PyObject *locals)
+PyRun_StringFlags(const char *str, int start, PyObject *globals, 
+		  PyObject *locals, PyCompilerFlags *flags)
 {
-	return PyRun_FileEx(fp, filename, start, globals, locals, 0);
-}
-
-PyObject *
-PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals,
-	     PyObject *locals, int closeit)
-{
-	node *n = PyParser_SimpleParseFile(fp, filename, start);
-	if (closeit)
-		fclose(fp);
-	return run_err_node(n, filename, globals, locals, NULL);
-}
-
-PyObject *
-PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals,
-		  PyCompilerFlags *flags)
-{
-	return run_err_node(PyParser_SimpleParseStringFlags(
-				    str, start, PARSER_FLAGS(flags)),
-			    "<string>", globals, locals, flags);
-}
-
-PyObject *
-PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals,
-		PyObject *locals, PyCompilerFlags *flags)
-{
-	return PyRun_FileExFlags(fp, filename, start, globals, locals, 0,
-				 flags);
+	PyObject *ret;
+	mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags);
+	ret = run_err_mod(mod, "<string>", globals, locals, flags);
+	free_mod(mod);
+	return ret;
 }
 
 PyObject *
 PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
 		  PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
-	node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
-						PARSER_FLAGS(flags));
+	PyObject *ret;
+	mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
+					  flags, NULL);
+	if (mod == NULL)
+		return NULL;
 	if (closeit)
 		fclose(fp);
-	return run_err_node(n, filename, globals, locals, flags);
+	ret = run_err_mod(mod, filename, globals, locals, flags);
+	free_mod(mod);
+	return ret;
 }
 
 static PyObject *
-run_err_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
-	     PyCompilerFlags *flags)
+run_err_mod(mod_ty mod, const char *filename, PyObject *globals, 
+	    PyObject *locals, PyCompilerFlags *flags)
 {
-	if (n == NULL)
+	if (mod == NULL)
 		return  NULL;
-	return run_node(n, filename, globals, locals, flags);
+	return run_mod(mod, filename, globals, locals, flags);
 }
 
 static PyObject *
-run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
+run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
 	 PyCompilerFlags *flags)
 {
 	PyCodeObject *co;
 	PyObject *v;
-	co = PyNode_CompileFlags(n, filename, flags);
-	PyNode_Free(n);
+	co = PyAST_Compile(mod, filename, flags);
 	if (co == NULL)
 		return NULL;
 	v = PyEval_EvalCode(co, globals, locals);
@@ -1271,8 +1201,8 @@ run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
 }
 
 static PyObject *
-run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals,
-	     PyCompilerFlags *flags)
+run_pyc_file(FILE *fp, const char *filename, PyObject *globals, 
+	     PyObject *locals, PyCompilerFlags *flags)
 {
 	PyCodeObject *co;
 	PyObject *v;
@@ -1303,41 +1233,77 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals
 }
 
 PyObject *
-Py_CompileString(const char *str, const char *filename, int start)
-{
-	return Py_CompileStringFlags(str, filename, start, NULL);
-}
-
-PyObject *
 Py_CompileStringFlags(const char *str, const char *filename, int start,
 		      PyCompilerFlags *flags)
 {
-	node *n;
+	mod_ty mod;
 	PyCodeObject *co;
-
-	n = PyParser_SimpleParseStringFlagsFilename(str, filename, start,
-						    PARSER_FLAGS(flags));
-	if (n == NULL)
+	mod = PyParser_ASTFromString(str, filename, start, flags);
+	if (mod == NULL)
 		return NULL;
-	co = PyNode_CompileFlags(n, filename, flags);
-	PyNode_Free(n);
+	co = PyAST_Compile(mod, filename, flags);
+	free_mod(mod);
 	return (PyObject *)co;
 }
 
 struct symtable *
 Py_SymtableString(const char *str, const char *filename, int start)
 {
-	node *n;
+	mod_ty mod;
 	struct symtable *st;
-	n = PyParser_SimpleParseStringFlagsFilename(str, filename,
-						    start, 0);
-	if (n == NULL)
+
+	mod = PyParser_ASTFromString(str, filename, start, NULL);
+	if (mod == NULL)
 		return NULL;
-	st = PyNode_CompileSymtable(n, filename);
-	PyNode_Free(n);
+	st = PySymtable_Build(mod, filename, 0);
+	free_mod(mod);
 	return st;
 }
 
+/* Preferred access to parser is through AST. */
+mod_ty
+PyParser_ASTFromString(const char *s, const char *filename, int start, 
+		       PyCompilerFlags *flags)
+{
+	node *n;
+	mod_ty mod;
+	perrdetail err;
+	n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar,
+					      start, &err, 
+					      PARSER_FLAGS(flags));
+	if (n) {
+		mod = PyAST_FromNode(n, flags, filename);
+		PyNode_Free(n);
+		return mod;
+	}
+	else {
+		err_input(&err);
+		return NULL;
+	}
+}
+
+mod_ty
+PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, 
+		     char *ps2, PyCompilerFlags *flags, int *errcode)
+{
+	node *n;
+	mod_ty mod;
+	perrdetail err;
+	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start, 
+				    ps1, ps2, &err, PARSER_FLAGS(flags));
+	if (n) {
+		mod = PyAST_FromNode(n, flags, filename);
+		PyNode_Free(n);
+		return mod;
+	}
+	else {
+		err_input(&err);
+		if (errcode)
+			*errcode = err.error;
+		return NULL;
+	}
+}
+
 /* Simplified interface to parsefile -- return node or set exception */
 
 node *
@@ -1349,15 +1315,10 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla
 					(char *)0, (char *)0, &err, flags);
 	if (n == NULL)
 		err_input(&err);
+		
 	return n;
 }
 
-node *
-PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
-{
-	return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
-}
-
 /* Simplified interface to parsestring -- return node or set exception */
 
 node *
@@ -1373,12 +1334,6 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags)
 }
 
 node *
-PyParser_SimpleParseString(const char *str, int start)
-{
-	return PyParser_SimpleParseStringFlags(str, start, 0);
-}
-
-node *
 PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
 					int start, int flags)
 {
@@ -1418,12 +1373,6 @@ err_input(perrdetail *err)
 	PyObject* u = NULL;
 	char *msg = NULL;
 	errtype = PyExc_SyntaxError;
-	v = Py_BuildValue("(ziiz)", err->filename,
-			    err->lineno, err->offset, err->text);
-	if (err->text != NULL) {
-		PyMem_DEL(err->text);
-		err->text = NULL;
-	}
 	switch (err->error) {
 	case E_SYNTAX:
 		errtype = PyExc_IndentationError;
@@ -1450,11 +1399,9 @@ err_input(perrdetail *err)
 	case E_INTR:
 		if (!PyErr_Occurred())
 			PyErr_SetNone(PyExc_KeyboardInterrupt);
-		Py_XDECREF(v);
 		return;
 	case E_NOMEM:
 		PyErr_NoMemory();
-		Py_XDECREF(v);
 		return;
 	case E_EOF:
 		msg = "unexpected EOF while parsing";
@@ -1498,7 +1445,15 @@ err_input(perrdetail *err)
 		msg = "unknown parsing error";
 		break;
 	}
-	w = Py_BuildValue("(sO)", msg, v);
+	v = Py_BuildValue("(ziiz)", err->filename,
+			  err->lineno, err->offset, err->text);
+ 	if (err->text != NULL) {
+		PyMem_DEL(err->text);
+		err->text = NULL;
+	}
+	w = NULL;
+	if (v != NULL)
+		w = Py_BuildValue("(sO)", msg, v);
 	Py_XDECREF(u);
 	Py_XDECREF(v);
 	PyErr_SetObject(errtype, w);
@@ -1687,3 +1642,20 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler)
 	return oldhandler;
 #endif
 }
+
+/* Deprecated C API functions still provided for binary compatiblity */
+
+#undef PyParser_SimpleParseFile
+#undef PyParser_SimpleParseString
+
+node *
+PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
+{
+	return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
+}
+
+node *
+PyParser_SimpleParseString(const char *str, int start)
+{
+	return PyParser_SimpleParseStringFlags(str, start, 0);
+}