Preliminary support for "from __future__ import generators" to enable

the yield statement.  I figure we have to have this in before I can
release 2.2a1 on Wednesday.

Note: test_generators is currently broken, I'm counting on Tim to fix
this.

2 files changed, 36 insertions, 3 deletions

diff --git a/Parser/parser.c b/Parser/parser.c
index 6eaa925..753c43a 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -79,6 +79,7 @@ PyParser_New(grammar *g, int start)
 	if (ps == NULL)
 		return NULL;
 	ps->p_grammar = g;
+	ps->p_generators = 0;
 	ps->p_tree = PyNode_New(start);
 	if (ps->p_tree == NULL) {
 		PyMem_DEL(ps);
@@ -131,8 +132,9 @@ push(register stack *s, int type, dfa *d, int newstate, int lineno)
 /* PARSER PROPER */
 
 static int
-classify(grammar *g, int type, char *str)
+classify(parser_state *ps, int type, char *str)
 {
+	grammar *g = ps->p_grammar;
 	register int n = g->g_ll.ll_nlabels;
 	
 	if (type == NAME) {
@@ -143,6 +145,10 @@ classify(grammar *g, int type, char *str)
 			if (l->lb_type == NAME && l->lb_str != NULL &&
 					l->lb_str[0] == s[0] &&
 					strcmp(l->lb_str, s) == 0) {
+				if (!ps->p_generators &&
+				    s[0] == 'y' &&
+				    strcmp(s, "yield") == 0)
+					break; /* not a keyword */
 				D(printf("It's a keyword\n"));
 				return n - i;
 			}
@@ -164,6 +170,22 @@ classify(grammar *g, int type, char *str)
 	return -1;
 }
 
+static void
+future_hack(parser_state *ps)
+{
+	node *n = ps->p_stack.s_top->s_parent;
+	node *ch;
+
+	if (strcmp(STR(CHILD(n, 0)), "from") != 0)
+		return;
+	ch = CHILD(n, 1);
+	if (strcmp(STR(CHILD(ch, 0)), "__future__") != 0)
+		return;
+	ch = CHILD(n, 3);
+	if (NCH(ch) == 1 && strcmp(STR(CHILD(ch, 0)), "generators") == 0)
+		ps->p_generators = 1;
+}
+
 int
 PyParser_AddToken(register parser_state *ps, register int type, char *str,
 	          int lineno, int *expected_ret)
@@ -174,7 +196,7 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 	D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str));
 	
 	/* Find out which label this token is */
-	ilabel = classify(ps->p_grammar, type, str);
+	ilabel = classify(ps, type, str);
 	if (ilabel < 0)
 		return E_SYNTAX;
 	
@@ -217,7 +239,14 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 				while (s = &d->d_state
 						[ps->p_stack.s_top->s_state],
 					s->s_accept && s->s_narcs == 1) {
-					D(printf("  Direct pop.\n"));
+					D(printf("  DFA '%s', state %d: "
+						 "Direct pop.\n",
+						 d->d_name,
+						 ps->p_stack.s_top->s_state));
+					if (d->d_name[0] == 'i' &&
+					    strcmp(d->d_name,
+						   "import_stmt") == 0)
+						future_hack(ps);
 					s_pop(&ps->p_stack);
 					if (s_empty(&ps->p_stack)) {
 						D(printf("  ACCEPT.\n"));
@@ -230,6 +259,9 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
 		}
 		
 		if (s->s_accept) {
+			if (d->d_name[0] == 'i' &&
+			    strcmp(d->d_name, "import_stmt") == 0)
+				future_hack(ps);
 			/* Pop this dfa and try again */
 			s_pop(&ps->p_stack);
 			D(printf(" Pop ...\n"));
diff --git a/Parser/parser.h b/Parser/parser.h
index cf8d318..b0c9a1e 100644
--- a/Parser/parser.h
+++ b/Parser/parser.h
@@ -25,6 +25,7 @@ typedef struct {
 	stack	 	p_stack;	/* Stack of parser states */
 	grammar		*p_grammar;	/* Grammar to use */
 	node		*p_tree;	/* Top of parse tree */
+	int		p_generators;	/* 1 if yield is a keyword */
 } parser_state;
 
 parser_state *PyParser_New(grammar *g, int start);