Move a bunch of definitions that were internal to compile.c to

symtable.h, so that they can be used by external module.

Improve error handling in symtable_enter_scope(), which return an
error code that went unchecked by most callers. XXX The error handling
in symtable code is sloppy in general.

Modify symtable to record the line number that begins each scope.
This can help to identify which code block is being referred to when
multiple blocks are bound to the same name.

Add st_scopes dict that is used to preserve scope info when
PyNode_CompileSymtable() is called.  Otherwise, this information is
tossed as soon as it is no longer needed.

Add Py_SymtableString() to pythonrun; analogous to Py_CompileString().

4 files changed, 210 insertions, 116 deletions

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index b963a06..13ed471 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -37,6 +37,7 @@ DL_IMPORT(PyObject *) PyRun_FileEx(FILE *, char *, int,
 				   PyObject *, PyObject *, int);
 
 DL_IMPORT(PyObject *) Py_CompileString(char *, char *, int);
+DL_IMPORT(struct symtable *) Py_SymtableString(char *, char *, int);
 
 DL_IMPORT(void) PyErr_Print(void);
 DL_IMPORT(void) PyErr_PrintEx(int);
diff --git a/Include/symtable.h b/Include/symtable.h
new file mode 100644
index 0000000..2d427aa
--- /dev/null
+++ b/Include/symtable.h
@@ -0,0 +1,99 @@
+#ifndef Py_SYMTABLE_H
+#define Py_SYMTABLE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* A symbol table is constructed each time PyNode_Compile() is
+   called.  The table walks the entire parse tree and identifies each
+   use or definition of a variable. 
+
+   The symbol table contains a dictionary for each code block in a
+   module: The symbol dictionary for the block.  They keys of these
+   dictionaries are the name of all variables used or defined in the
+   block; the integer values are used to store several flags,
+   e.g. DEF_PARAM indicates that a variable is a parameter to a
+   function. 
+
+   The slots st_cur_XXX pointers always refer to the current code
+   block.  The st_cur slot is the symbol dictionary.  The st_cur_id
+   slot is the id is the key in st_symbols.  The st_cur_name slot is
+   the name of the current scope. The st_cur_type slot is one of
+   TYPE_FUNCTION, TYPE_CLASS, or TYPE_MODULE.  The st_cur_children is
+   a list of the ids of the current node's children.
+
+   The st_symbols slot is a dictionary that maps code block ids to
+   symbol dictionaries.  The keys are generated by a counter that is
+   incremented each time a new code block is found.  The counter is
+   identifies a specific scope, because both passes walk the parse
+   tree in the same order.
+
+   The st_varnames slot is a dictionary that maps code block ids to
+   parameter lists.  The st_global slot always refers to the symbol 
+   dictionary for the module.
+
+   The st_children slot is a dictionary that maps ids to a list
+   containing the ids of its children.
+
+   If st_keep is true then the namespace info pushed on st_stack will
+   also be stored in st_scopes.  This is useful if the symbol table is
+   being passed to something other than the compiler.
+*/
+
+struct symtable {
+	int st_pass;             /* pass == 1 or 2 */
+	int st_keep;             /* true if symtable will be returned */
+	PyObject *st_symbols;    /* dictionary of symbol tables */
+	PyObject *st_varnames;   /* dictionary of parameter lists */
+        PyObject *st_stack;      /* stack of namespace info */
+	PyObject *st_scopes;     /* dictionary of namespace info */
+	PyObject *st_children;   /* dictionary (id=[ids]) */
+	PyObject *st_cur;        /* borrowed ref to dict in st_symbols */
+	PyObject *st_cur_name;   /* string, name of current scope */
+	PyObject *st_cur_id;     /* int id of current code block */
+	PyObject *st_cur_children; /* ref to current children list */
+	int st_cur_type;         /* type of current scope */ 
+	int st_cur_lineno;       /* line number where current scope begins */
+	PyObject *st_global;     /* borrowed ref to MODULE in st_symbols */
+	int st_nscopes;          /* number of scopes */
+	int st_errors;           /* number of errors */
+	char *st_private;        /* name of current class or NULL */
+	int st_tmpname;          /* temporary name counter */
+	int st_nested;           /* bool (true if nested scope) */
+};
+
+DL_IMPORT(struct symtable *) PyNode_CompileSymtable(struct _node *, char *);
+DL_IMPORT(void) PySymtable_Free(struct symtable *);
+
+
+#define TOP "global"
+#define NOOPT ".noopt"
+
+/* Flags for def-use information */
+
+#define DEF_GLOBAL 1           /* global stmt */
+#define DEF_LOCAL 2            /* assignment in code block */
+#define DEF_PARAM 2<<1         /* formal parameter */
+#define USE 2<<2               /* name is used */
+#define DEF_STAR 2<<3          /* parameter is star arg */
+#define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
+#define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
+#define DEF_FREE 2<<6          /* name used by not defined in nested scope */
+#define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
+#define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
+#define DEF_IMPORT 2<<9        /* assignment occurred via import */
+
+#define TYPE_FUNCTION 1
+#define TYPE_CLASS 2
+#define TYPE_MODULE 3
+
+#define LOCAL 1
+#define GLOBAL_EXPLICIT 2
+#define GLOBAL_IMPLICIT 3
+#define FREE 4
+#define CELL 5
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_SYMTABLE_H */
diff --git a/Python/compile.c b/Python/compile.c
index 3f12e41..acad667 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -16,6 +16,7 @@
 #include "token.h"
 #include "graminit.h"
 #include "compile.h"
+#include "symtable.h"
 #include "opcode.h"
 #include "structmember.h"
 
@@ -45,16 +46,6 @@ int Py_OptimizeFlag = 0;
 #define VAR_STORE 1
 #define VAR_DELETE 2
 
-#define TYPE_FUNCTION 1
-#define TYPE_CLASS 2
-#define TYPE_MODULE 3
-
-#define LOCAL 1
-#define GLOBAL_EXPLICIT 2
-#define GLOBAL_IMPLICIT 3
-#define FREE 4
-#define CELL 5
-
 #define DEL_CLOSURE_ERROR \
 "can not delete variable '%.400s' referenced in nested scope"
 
@@ -367,74 +358,6 @@ struct compiling {
 	struct symtable *c_symtable;   /* pointer to module symbol table */
 };
 
-/* A symbol table is constructed each time PyNode_Compile() is
-   called.  The table walks the entire parse tree and identifies each
-   use or definition of a variable. 
-
-   The symbol table contains a dictionary for each code block in a
-   module: The symbol dictionary for the block.  They keys of these
-   dictionaries are the name of all variables used or defined in the
-   block; the integer values are used to store several flags,
-   e.g. DEF_PARAM indicates that a variable is a parameter to a
-   function. 
-
-   The slots st_cur_XXX pointers always refer to the current code
-   block.  The st_cur slot is the symbol dictionary.  The st_cur_id
-   slot is the id is the key in st_symbols.  The st_cur_name slot is
-   the name of the current scope. The st_cur_type slot is one of
-   TYPE_FUNCTION, TYPE_CLASS, or TYPE_MODULE.  The st_cur_children is
-   a list of the ids of the current node's children.
-
-   The st_symbols slot is a dictionary that maps code block ids to
-   symbol dictionaries.  The keys are generated by a counter that is
-   incremented each time a new code block is found.  The counter is
-   identifies a specific scope, because both passes walk the parse
-   tree in the same order.
-
-   The st_varnames slot is a dictionary that maps code block ids to
-   parameter lists.  The st_global slot always refers to the symbol 
-   dictionary for the module.
-
-   The st_children slot is a dictionary that maps ids to a list
-   containing the ids of its children.
-*/
-
-struct symtable {
-	int st_pass;             /* pass == 1 or 2 */
-	PyObject *st_symbols;    /* dictionary of symbol tables */
-	PyObject *st_varnames;   /* dictionary of parameter lists */
-        PyObject *st_stack;      /* stack of namespace info */
-	PyObject *st_children;   /* dictionary (id=[ids]) */
-	PyObject *st_cur;        /* borrowed ref to dict in st_symbols */
-	PyObject *st_cur_name;   /* string, name of current scope */
-	PyObject *st_cur_id;     /* int id of current code block */
-	PyObject *st_cur_children; /* ref to current children list */
-	int st_cur_type;         /* type of current scope */ 
-	PyObject *st_global;     /* borrowed ref to MODULE in st_symbols */
-	int st_scopes;           /* number of scopes */
-	int st_errors;           /* number of errors */
-	char *st_private;        /* name of current class or NULL */
-	int st_tmpname;          /* temporary name counter */
-	int st_nested;           /* bool (true if nested scope) */
-};
-
-#define TOP "global"
-#define NOOPT ".noopt"
-
-/* Flags for def-use information */
-
-#define DEF_GLOBAL 1           /* global stmt */
-#define DEF_LOCAL 2            /* assignment in code block */
-#define DEF_PARAM 2<<1         /* formal parameter */
-#define USE 2<<2               /* name is used */
-#define DEF_STAR 2<<3          /* parameter is star arg */
-#define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
-#define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
-#define DEF_FREE 2<<6          /* name used by not defined in nested scope */
-#define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
-#define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
-#define DEF_IMPORT 2<<9        /* assignment occurred via import */
-
 int is_free(int v)
 {
 	if ((v & (USE | DEF_FREE))
@@ -553,9 +476,8 @@ static int get_ref_type(struct compiling *, char *);
 /* symtable operations */
 static int symtable_build(struct compiling *, node *);
 static int symtable_load_symbols(struct compiling *);
-static struct symtable *symtable_init(void);
-static void symtable_free(struct symtable *);
-static int symtable_enter_scope(struct symtable *, char *, int);
+static struct symtable *symtable_init(int);
+static void symtable_enter_scope(struct symtable *, char *, int, int);
 static int symtable_exit_scope(struct symtable *);
 static int symtable_update_cur(struct symtable *);
 static int symtable_add_def(struct symtable *, char *, int);
@@ -2217,7 +2139,8 @@ com_test(struct compiling *c, node *n)
 		PyObject *co;
 		int i, closure;
 		int ndefs = com_argdefs(c, CHILD(n, 0));
-		symtable_enter_scope(c->c_symtable, "lambda", lambdef);
+		symtable_enter_scope(c->c_symtable, "lambda", lambdef,
+				     n->n_lineno);
 		co = (PyObject *) icompile(CHILD(n, 0), c);
 		symtable_exit_scope(c->c_symtable);
 		if (co == NULL) {
@@ -3335,7 +3258,8 @@ com_funcdef(struct compiling *c, node *n)
 	int ndefs;
 	REQ(n, funcdef); /* funcdef: 'def' NAME parameters ':' suite */
 	ndefs = com_argdefs(c, n);
-	symtable_enter_scope(c->c_symtable, STR(CHILD(n, 1)), TYPE(n));
+	symtable_enter_scope(c->c_symtable, STR(CHILD(n, 1)), TYPE(n),
+			     n->n_lineno);
 	co = (PyObject *)icompile(n, c);
 	symtable_exit_scope(c->c_symtable);
 	if (co == NULL)
@@ -3395,7 +3319,7 @@ com_classdef(struct compiling *c, node *n)
 	else
 		com_bases(c, CHILD(n, 3));
 	name = STR(CHILD(n, 1));
-	symtable_enter_scope(c->c_symtable, name, TYPE(n));
+	symtable_enter_scope(c->c_symtable, name, TYPE(n), n->n_lineno);
 	co = (PyObject *)icompile(n, c);
 	symtable_exit_scope(c->c_symtable);
 	if (co == NULL)
@@ -3872,6 +3796,27 @@ PyNode_Compile(node *n, char *filename)
 	return jcompile(n, filename, NULL);
 }
 
+struct symtable *
+PyNode_CompileSymtable(node *n, char *filename)
+{
+	struct symtable *st;
+
+	st = symtable_init(1);
+	if (st == NULL)
+		return NULL;
+	symtable_enter_scope(st, TOP, TYPE(n), n->n_lineno);
+	if (st->st_errors > 0) {
+		PySymtable_Free(st);
+		return NULL;
+	}
+	symtable_node(st, n);
+	if (st->st_errors > 0) {
+		PySymtable_Free(st);
+		return NULL;
+	}
+	return st;
+}
+
 static PyCodeObject *
 icompile(node *n, struct compiling *base)
 {
@@ -3948,7 +3893,7 @@ jcompile(node *n, char *filename, struct compiling *base)
 	}
  exit:
 	if (base == NULL)
-		symtable_free(sc.c_symtable);
+		PySymtable_Free(sc.c_symtable);
 	com_free(&sc);
 	return co;
 }
@@ -4005,15 +3950,16 @@ get_ref_type(struct compiling *c, char *name)
 static int
 symtable_build(struct compiling *c, node *n)
 {
-	if ((c->c_symtable = symtable_init()) == NULL)
+	if ((c->c_symtable = symtable_init(0)) == NULL)
 		return -1;
-	if (symtable_enter_scope(c->c_symtable, TOP, TYPE(n)) < 0)
+	symtable_enter_scope(c->c_symtable, TOP, TYPE(n), n->n_lineno);
+	if (c->c_symtable->st_errors > 0)
 		return -1;
 	symtable_node(c->c_symtable, n);
 	if (c->c_symtable->st_errors > 0)
 		return -1;
 	/* reset for second pass */
-	c->c_symtable->st_scopes = 1;
+	c->c_symtable->st_nscopes = 1;
 	c->c_symtable->st_pass = 2;
 	return 0;
 }
@@ -4192,7 +4138,7 @@ symtable_load_symbols(struct compiling *c)
 }
 
 static struct symtable *
-symtable_init()
+symtable_init(int keep)
 {
 	struct symtable *st;
 	PyObject *d;
@@ -4201,6 +4147,7 @@ symtable_init()
 	if (st == NULL)
 		return NULL;
 	st->st_pass = 1;
+	st->st_keep = keep;
 	if ((st->st_stack = PyList_New(0)) == NULL)
 		goto fail;
 	if ((st->st_symbols = PyDict_New()) == NULL)
@@ -4214,41 +4161,49 @@ symtable_init()
 	if (PyDict_SetItemString(st->st_symbols, TOP, d) < 0)
 		goto fail;
 	Py_DECREF(d);
-	st->st_global = d;
+	if (keep) {
+		if ((d = PyDict_New()) == NULL)
+			goto fail;
+		st->st_scopes = d;
+	} else 
+		st->st_scopes = NULL;
+	st->st_global = d; /* use ref borrowed from st->st_symbols */
 	st->st_cur = NULL;
 	st->st_cur_id = NULL;
 	st->st_cur_name = NULL;
 	st->st_cur_children = NULL;
 	st->st_cur_type = 0;
 	st->st_nested = 0;
-	st->st_scopes = 0;
+	st->st_nscopes = 0;
 	st->st_errors = 0;
 	st->st_tmpname = 0;
 	st->st_private = NULL;
 	return st;
  fail:
-	symtable_free(st);
+	PySymtable_Free(st);
 	return NULL;
 }
 
-static void
-symtable_free(struct symtable *st)
+void
+PySymtable_Free(struct symtable *st)
 {
 	Py_XDECREF(st->st_symbols);
 	Py_XDECREF(st->st_varnames);
 	Py_XDECREF(st->st_children);
 	Py_XDECREF(st->st_stack);
+	Py_XDECREF(st->st_scopes);
 	Py_XDECREF(st->st_cur_id);
 	Py_XDECREF(st->st_cur_name);
 	PyMem_Free((void *)st);
 }
 
 static PyObject *
-make_scope_info(PyObject *id, PyObject *name, int nested, int type)
+make_scope_info(PyObject *id, PyObject *name, int nested, int type,
+		int lineno)
 {
-	PyObject *t, *i1 = NULL, *i2 = NULL;
+	PyObject *t, *i1 = NULL, *i2 = NULL, *i3 = NULL;
 
-	t = PyTuple_New(4);
+	t = PyTuple_New(5);
 	if (t == NULL)
 		return NULL;
 	i1 = PyInt_FromLong(nested);
@@ -4257,6 +4212,9 @@ make_scope_info(PyObject *id, PyObject *name, int nested, int type)
 	i2 = PyInt_FromLong(type);
 	if (i2 == NULL)
 		goto fail;
+	i3 = PyInt_FromLong(lineno);
+	if (i3 == NULL)
+		goto fail;
 
 	Py_INCREF(name);
 	Py_INCREF(id);
@@ -4265,11 +4223,13 @@ make_scope_info(PyObject *id, PyObject *name, int nested, int type)
 	/* i1 & i2 alloced here; don't need incref */
 	PyTuple_SET_ITEM(t, 2, i1);
 	PyTuple_SET_ITEM(t, 3, i2);
+	PyTuple_SET_ITEM(t, 4, i3);
 	return t;
  fail:
 	Py_XDECREF(t);
 	Py_XDECREF(i1);
 	Py_XDECREF(i2);
+	Py_XDECREF(i3);
 	return NULL;
 }
 
@@ -4378,7 +4338,6 @@ symtable_undo_free(struct symtable *st, PyObject *id,
 	return 0;
 }
 
-
 static int
 symtable_exit_scope(struct symtable *st)
 {
@@ -4402,26 +4361,40 @@ symtable_exit_scope(struct symtable *st)
 	return symtable_update_cur(st);
 }
 
-static int
-symtable_enter_scope(struct symtable *st, char *name, int type)
+static void
+symtable_enter_scope(struct symtable *st, char *name, int type,
+		     int lineno)
 {
 	PyObject *o;
 
 	if (st->st_cur) {
 		/* push current scope info on stack */
 		o = make_scope_info(st->st_cur_id, st->st_cur_name, 
-				    st->st_nested, st->st_cur_type);
-		if (o == NULL)
-			return -1;
+				    st->st_nested, st->st_cur_type,
+				    st->st_cur_lineno);
+		if (o == NULL) {
+			st->st_errors++;
+			return;
+		}
 		if (PyList_Append(st->st_stack, o) < 0) {
 			Py_DECREF(o);
-			return -1;
+			st->st_errors++;
+			return;
+		}
+		if (st->st_keep) {
+			if (PyDict_SetItem(st->st_scopes,
+					   st->st_cur_id, o) < 0) {
+				Py_DECREF(o);
+				st->st_errors++;
+				return;
+			}
 		}
 		Py_DECREF(o);
 	}
 	st->st_cur_name = PyString_FromString(name);
 	if (st->st_nested || st->st_cur_type == TYPE_FUNCTION)
 		st->st_nested = 1;
+	st->st_cur_lineno = lineno;
 	switch (type) {
 	case funcdef:
 	case lambdef:
@@ -4437,30 +4410,37 @@ symtable_enter_scope(struct symtable *st, char *name, int type)
 		break;
 	default:
 		fprintf(stderr, "invalid symtable scope: %d\n", type);
-		return -1;
+		st->st_errors++;
+		return;
 	}
 	/* update st_cur_id and parent's st_cur_children */
-	o = PyInt_FromLong(st->st_scopes++);
-	if (o == NULL)
-		return -1;
+	o = PyInt_FromLong(st->st_nscopes++);
+	if (o == NULL) {
+		st->st_errors++;
+		return;
+	}
 	if (st->st_cur_children) {
 		if (PyList_Append(st->st_cur_children, o) < 0) {
 			Py_DECREF(o);
-			return -1;
+			st->st_errors++;
+			return;
 		}
 	}
 	st->st_cur_id = o;
 	/* create st_cur_children list */
 	o = PyList_New(0);
-	if (o == NULL)
-		return -1;
+	if (o == NULL) {
+		st->st_errors++;
+		return;
+	}
 	if (PyDict_SetItem(st->st_children, st->st_cur_id, o) < 0) {
 		Py_DECREF(o);
-		return -1;
+		st->st_errors++;
+		return;
 	}
 	Py_DECREF(o);
 
-	return symtable_update_cur(st);
+	symtable_update_cur(st);
 }
 
 static int
@@ -4575,7 +4555,7 @@ symtable_node(struct symtable *st, node *n)
 		char *func_name = STR(CHILD(n, 1));
 		symtable_add_def(st, func_name, DEF_LOCAL);
 		symtable_default_args(st, CHILD(n, 2));
-		symtable_enter_scope(st, func_name, TYPE(n));
+		symtable_enter_scope(st, func_name, TYPE(n), n->n_lineno);
 		symtable_funcdef(st, n);
 		symtable_exit_scope(st);
 		break;
@@ -4583,7 +4563,7 @@ symtable_node(struct symtable *st, node *n)
 	case lambdef:
 		if (NCH(n) == 4)
 			symtable_default_args(st, CHILD(n, 1));
-		symtable_enter_scope(st, "lambda", TYPE(n));
+		symtable_enter_scope(st, "lambda", TYPE(n), n->n_lineno);
 		symtable_funcdef(st, n);
 		symtable_exit_scope(st);
 		break;
@@ -4597,7 +4577,7 @@ symtable_node(struct symtable *st, node *n)
 				symtable_node(st, CHILD(bases, i));
 			}
 		}
-		symtable_enter_scope(st, class_name, TYPE(n));
+		symtable_enter_scope(st, class_name, TYPE(n), n->n_lineno);
 		tmp = st->st_private;
 		st->st_private = class_name;
 		symtable_node(st, CHILD(n, NCH(n) - 1));
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 5f90797..5e0377d 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -9,6 +9,7 @@
 #include "parsetok.h"
 #include "errcode.h"
 #include "compile.h"
+#include "symtable.h"
 #include "eval.h"
 #include "marshal.h"
 
@@ -963,6 +964,19 @@ Py_CompileString(char *str, char *filename, int start)
 	return (PyObject *)co;
 }
 
+struct symtable *
+Py_SymtableString(char *str, char *filename, int start)
+{
+	node *n;
+	struct symtable *st;
+	n = PyParser_SimpleParseString(str, start);
+	if (n == NULL)
+		return NULL;
+	st = PyNode_CompileSymtable(n, filename);
+	PyNode_Free(n);
+	return st;
+}
+
 /* Simplified interface to parsefile -- return node or set exception */
 
 node *