Implement PEP 380 - 'yield from' (closes #11682)

6 files changed, 140 insertions, 127 deletions

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 9389049..4e21674e 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -245,6 +245,7 @@ struct _expr {
                 } GeneratorExp;
                 
                 struct {
+                        int is_from;
                         expr_ty value;
                 } Yield;
                 
@@ -487,8 +488,9 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
 #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
                          col_offset, PyArena *arena);
-#define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)
-expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, PyArena *arena);
+#define Yield(a0, a1, a2, a3, a4) _Py_Yield(a0, a1, a2, a3, a4)
+expr_ty _Py_Yield(int is_from, expr_ty value, int lineno, int col_offset,
+                  PyArena *arena);
 #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
 expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
                     int lineno, int col_offset, PyArena *arena);
diff --git a/Include/frameobject.h b/Include/frameobject.h
index 1fb64bb..55447b7 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -9,45 +9,46 @@ extern "C" {
 #endif
 
 typedef struct {
-    int b_type;			/* what kind of block this is */
-    int b_handler;		/* where to jump to find handler */
-    int b_level;		/* value stack level to pop to */
+    int b_type;                 /* what kind of block this is */
+    int b_handler;              /* where to jump to find handler */
+    int b_level;                /* value stack level to pop to */
 } PyTryBlock;
 
 typedef struct _frame {
     PyObject_VAR_HEAD
-    struct _frame *f_back;	/* previous frame, or NULL */
-    PyCodeObject *f_code;	/* code segment */
-    PyObject *f_builtins;	/* builtin symbol table (PyDictObject) */
-    PyObject *f_globals;	/* global symbol table (PyDictObject) */
-    PyObject *f_locals;		/* local symbol table (any mapping) */
-    PyObject **f_valuestack;	/* points after the last local */
+    struct _frame *f_back;      /* previous frame, or NULL */
+    PyCodeObject *f_code;       /* code segment */
+    PyObject *f_builtins;       /* builtin symbol table (PyDictObject) */
+    PyObject *f_globals;        /* global symbol table (PyDictObject) */
+    PyObject *f_locals;         /* local symbol table (any mapping) */
+    PyObject **f_valuestack;    /* points after the last local */
     /* Next free slot in f_valuestack.  Frame creation sets to f_valuestack.
        Frame evaluation usually NULLs it, but a frame that yields sets it
        to the current stack top. */
     PyObject **f_stacktop;
-    PyObject *f_trace;		/* Trace function */
-
-	/* In a generator, we need to be able to swap between the exception
-	   state inside the generator and the exception state of the calling
-	   frame (which shouldn't be impacted when the generator "yields"
-	   from an except handler).
-	   These three fields exist exactly for that, and are unused for
-	   non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE
-	   macros in ceval.c for details of their use. */
+    PyObject *f_trace;          /* Trace function */
+    PyObject *f_yieldfrom;      /* Iterator being delegated to by yield from */
+
+        /* In a generator, we need to be able to swap between the exception
+           state inside the generator and the exception state of the calling
+           frame (which shouldn't be impacted when the generator "yields"
+           from an except handler).
+           These three fields exist exactly for that, and are unused for
+           non-generator frames. See the SAVE_EXC_STATE and SWAP_EXC_STATE
+           macros in ceval.c for details of their use. */
     PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
 
     PyThreadState *f_tstate;
-    int f_lasti;		/* Last instruction if called */
+    int f_lasti;                /* Last instruction if called */
     /* Call PyFrame_GetLineNumber() instead of reading this field
        directly.  As of 2.3 f_lineno is only valid when tracing is
        active (i.e. when f_trace is set).  At other times we use
        PyCode_Addr2Line to calculate the line from the current
        bytecode index. */
-    int f_lineno;		/* Current line number */
-    int f_iblock;		/* index in f_blockstack */
+    int f_lineno;               /* Current line number */
+    int f_iblock;               /* index in f_blockstack */
     PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
-    PyObject *f_localsplus[1];	/* locals+stack, dynamically sized */
+    PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
 } PyFrameObject;
 
 
diff --git a/Include/genobject.h b/Include/genobject.h
index d29fb1e..13a3856 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -11,20 +11,20 @@ extern "C" {
 struct _frame; /* Avoid including frameobject.h */
 
 typedef struct {
-	PyObject_HEAD
-	/* The gi_ prefix is intended to remind of generator-iterator. */
+        PyObject_HEAD
+        /* The gi_ prefix is intended to remind of generator-iterator. */
 
-	/* Note: gi_frame can be NULL if the generator is "finished" */
-	struct _frame *gi_frame;
+        /* Note: gi_frame can be NULL if the generator is "finished" */
+        struct _frame *gi_frame;
 
-	/* True if generator is being executed. */
-	int gi_running;
+        /* True if generator is being executed. */
+        int gi_running;
     
-	/* The code object backing the generator */
-	PyObject *gi_code;
+        /* The code object backing the generator */
+        PyObject *gi_code;
 
-	/* List of weak reference. */
-	PyObject *gi_weakreflist;
+        /* List of weak reference. */
+        PyObject *gi_weakreflist;
 } PyGenObject;
 
 PyAPI_DATA(PyTypeObject) PyGen_Type;
@@ -34,6 +34,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
 PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
+PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **);
 
 #ifdef __cplusplus
 }
diff --git a/Include/graminit.h b/Include/graminit.h
index e0e27f9..3ec949a 100644
--- a/Include/graminit.h
+++ b/Include/graminit.h
@@ -81,3 +81,4 @@
 #define comp_if 334
 #define encoding_decl 335
 #define yield_expr 336
+#define yield_arg 337
diff --git a/Include/opcode.h b/Include/opcode.h
index ece713e..a90184d 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -7,116 +7,117 @@ extern "C" {
 
 /* Instruction opcodes for compiled code */
 
-#define POP_TOP		1
-#define ROT_TWO		2
-#define ROT_THREE	3
-#define DUP_TOP		4
+#define POP_TOP         1
+#define ROT_TWO         2
+#define ROT_THREE       3
+#define DUP_TOP         4
 #define DUP_TOP_TWO     5
-#define NOP		9
+#define NOP             9
 
-#define UNARY_POSITIVE	10
-#define UNARY_NEGATIVE	11
-#define UNARY_NOT	12
+#define UNARY_POSITIVE  10
+#define UNARY_NEGATIVE  11
+#define UNARY_NOT       12
 
-#define UNARY_INVERT	15
+#define UNARY_INVERT    15
 
-#define BINARY_POWER	19
+#define BINARY_POWER    19
 
-#define BINARY_MULTIPLY	20
+#define BINARY_MULTIPLY 20
 
-#define BINARY_MODULO	22
-#define BINARY_ADD	23
-#define BINARY_SUBTRACT	24
-#define BINARY_SUBSCR	25
+#define BINARY_MODULO   22
+#define BINARY_ADD      23
+#define BINARY_SUBTRACT 24
+#define BINARY_SUBSCR   25
 #define BINARY_FLOOR_DIVIDE 26
 #define BINARY_TRUE_DIVIDE 27
 #define INPLACE_FLOOR_DIVIDE 28
 #define INPLACE_TRUE_DIVIDE 29
 
-#define STORE_MAP	54
-#define INPLACE_ADD	55
-#define INPLACE_SUBTRACT	56
-#define INPLACE_MULTIPLY	57
-
-#define INPLACE_MODULO	59
-#define STORE_SUBSCR	60
-#define DELETE_SUBSCR	61
-
-#define BINARY_LSHIFT	62
-#define BINARY_RSHIFT	63
-#define BINARY_AND	64
-#define BINARY_XOR	65
-#define BINARY_OR	66
-#define INPLACE_POWER	67
-#define GET_ITER	68
-#define STORE_LOCALS	69
-#define PRINT_EXPR	70
+#define STORE_MAP       54
+#define INPLACE_ADD     55
+#define INPLACE_SUBTRACT        56
+#define INPLACE_MULTIPLY        57
+
+#define INPLACE_MODULO  59
+#define STORE_SUBSCR    60
+#define DELETE_SUBSCR   61
+
+#define BINARY_LSHIFT   62
+#define BINARY_RSHIFT   63
+#define BINARY_AND      64
+#define BINARY_XOR      65
+#define BINARY_OR       66
+#define INPLACE_POWER   67
+#define GET_ITER        68
+#define STORE_LOCALS    69
+#define PRINT_EXPR      70
 #define LOAD_BUILD_CLASS 71
-
-#define INPLACE_LSHIFT	75
-#define INPLACE_RSHIFT	76
-#define INPLACE_AND	77
-#define INPLACE_XOR	78
-#define INPLACE_OR	79
-#define BREAK_LOOP	80
+#define YIELD_FROM      72
+
+#define INPLACE_LSHIFT  75
+#define INPLACE_RSHIFT  76
+#define INPLACE_AND     77
+#define INPLACE_XOR     78
+#define INPLACE_OR      79
+#define BREAK_LOOP      80
 #define WITH_CLEANUP    81
 
-#define RETURN_VALUE	83
-#define IMPORT_STAR	84
+#define RETURN_VALUE    83
+#define IMPORT_STAR     84
 
-#define YIELD_VALUE	86
-#define POP_BLOCK	87
-#define END_FINALLY	88
-#define POP_EXCEPT	89
+#define YIELD_VALUE     86
+#define POP_BLOCK       87
+#define END_FINALLY     88
+#define POP_EXCEPT      89
 
-#define HAVE_ARGUMENT	90	/* Opcodes from here have an argument: */
+#define HAVE_ARGUMENT   90      /* Opcodes from here have an argument: */
 
-#define STORE_NAME	90	/* Index in name list */
-#define DELETE_NAME	91	/* "" */
-#define UNPACK_SEQUENCE	92	/* Number of sequence items */
-#define FOR_ITER	93
+#define STORE_NAME      90      /* Index in name list */
+#define DELETE_NAME     91      /* "" */
+#define UNPACK_SEQUENCE 92      /* Number of sequence items */
+#define FOR_ITER        93
 #define UNPACK_EX       94      /* Num items before variable part +
                                    (Num items after variable part << 8) */
 
-#define STORE_ATTR	95	/* Index in name list */
-#define DELETE_ATTR	96	/* "" */
-#define STORE_GLOBAL	97	/* "" */
-#define DELETE_GLOBAL	98	/* "" */
-
-#define LOAD_CONST	100	/* Index in const list */
-#define LOAD_NAME	101	/* Index in name list */
-#define BUILD_TUPLE	102	/* Number of tuple items */
-#define BUILD_LIST	103	/* Number of list items */
-#define BUILD_SET	104     /* Number of set items */
-#define BUILD_MAP	105	/* Always zero for now */
-#define LOAD_ATTR	106	/* Index in name list */
-#define COMPARE_OP	107	/* Comparison operator */
-#define IMPORT_NAME	108	/* Index in name list */
-#define IMPORT_FROM	109	/* Index in name list */
-
-#define JUMP_FORWARD	110	/* Number of bytes to skip */
-#define JUMP_IF_FALSE_OR_POP 111	/* Target byte offset from beginning of code */
-#define JUMP_IF_TRUE_OR_POP 112	/* "" */
-#define JUMP_ABSOLUTE	113	/* "" */
-#define POP_JUMP_IF_FALSE 114	/* "" */
-#define POP_JUMP_IF_TRUE 115	/* "" */
-
-#define LOAD_GLOBAL	116	/* Index in name list */
-
-#define CONTINUE_LOOP	119	/* Start of loop (absolute) */
-#define SETUP_LOOP	120	/* Target address (relative) */
-#define SETUP_EXCEPT	121	/* "" */
-#define SETUP_FINALLY	122	/* "" */
-
-#define LOAD_FAST	124	/* Local variable number */
-#define STORE_FAST	125	/* Local variable number */
-#define DELETE_FAST	126	/* Local variable number */
-
-#define RAISE_VARARGS	130	/* Number of raise arguments (1, 2 or 3) */
+#define STORE_ATTR      95      /* Index in name list */
+#define DELETE_ATTR     96      /* "" */
+#define STORE_GLOBAL    97      /* "" */
+#define DELETE_GLOBAL   98      /* "" */
+
+#define LOAD_CONST      100     /* Index in const list */
+#define LOAD_NAME       101     /* Index in name list */
+#define BUILD_TUPLE     102     /* Number of tuple items */
+#define BUILD_LIST      103     /* Number of list items */
+#define BUILD_SET       104     /* Number of set items */
+#define BUILD_MAP       105     /* Always zero for now */
+#define LOAD_ATTR       106     /* Index in name list */
+#define COMPARE_OP      107     /* Comparison operator */
+#define IMPORT_NAME     108     /* Index in name list */
+#define IMPORT_FROM     109     /* Index in name list */
+
+#define JUMP_FORWARD    110     /* Number of bytes to skip */
+#define JUMP_IF_FALSE_OR_POP 111        /* Target byte offset from beginning of code */
+#define JUMP_IF_TRUE_OR_POP 112 /* "" */
+#define JUMP_ABSOLUTE   113     /* "" */
+#define POP_JUMP_IF_FALSE 114   /* "" */
+#define POP_JUMP_IF_TRUE 115    /* "" */
+
+#define LOAD_GLOBAL     116     /* Index in name list */
+
+#define CONTINUE_LOOP   119     /* Start of loop (absolute) */
+#define SETUP_LOOP      120     /* Target address (relative) */
+#define SETUP_EXCEPT    121     /* "" */
+#define SETUP_FINALLY   122     /* "" */
+
+#define LOAD_FAST       124     /* Local variable number */
+#define STORE_FAST      125     /* Local variable number */
+#define DELETE_FAST     126     /* Local variable number */
+
+#define RAISE_VARARGS   130     /* Number of raise arguments (1, 2 or 3) */
 /* CALL_FUNCTION_XXX opcodes defined below depend on this definition */
-#define CALL_FUNCTION	131	/* #args + (#kwargs<<8) */
-#define MAKE_FUNCTION	132	/* #defaults + #kwdefaults<<8 + #annotations<<16 */
-#define BUILD_SLICE 	133	/* Number of items */
+#define CALL_FUNCTION   131     /* #args + (#kwargs<<8) */
+#define MAKE_FUNCTION   132     /* #defaults + #kwdefaults<<8 + #annotations<<16 */
+#define BUILD_SLICE     133     /* Number of items */
 
 #define MAKE_CLOSURE    134     /* same as MAKE_FUNCTION */
 #define LOAD_CLOSURE    135     /* Load free variable from closure */
@@ -126,9 +127,9 @@ extern "C" {
 
 /* The next 3 opcodes must be contiguous and satisfy
    (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1  */
-#define CALL_FUNCTION_VAR          140	/* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_KW           141	/* #args + (#kwargs<<8) */
-#define CALL_FUNCTION_VAR_KW       142	/* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_VAR          140  /* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_KW           141  /* #args + (#kwargs<<8) */
+#define CALL_FUNCTION_VAR_KW       142  /* #args + (#kwargs<<8) */
 
 #define SETUP_WITH 143
 
@@ -148,7 +149,7 @@ extern "C" {
 
 
 enum cmp_op {PyCmp_LT=Py_LT, PyCmp_LE=Py_LE, PyCmp_EQ=Py_EQ, PyCmp_NE=Py_NE, PyCmp_GT=Py_GT, PyCmp_GE=Py_GE,
-	     PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
+             PyCmp_IN, PyCmp_NOT_IN, PyCmp_IS, PyCmp_IS_NOT, PyCmp_EXC_MATCH, PyCmp_BAD};
 
 #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT)
 
diff --git a/Include/pyerrors.h b/Include/pyerrors.h
index 44eb3d9..1bd0442 100644
--- a/Include/pyerrors.h
+++ b/Include/pyerrors.h
@@ -51,6 +51,11 @@ typedef struct {
     Py_ssize_t written;   /* only for BlockingIOError, -1 otherwise */
 } PyOSErrorObject;
 
+typedef struct {
+    PyException_HEAD
+    PyObject *value;
+} PyStopIterationObject;
+
 /* Compatibility typedefs */
 typedef PyOSErrorObject PyEnvironmentErrorObject;
 #ifdef MS_WINDOWS
@@ -380,6 +385,8 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason(
     const char *reason          /* UTF-8 encoded string */
     );
 
+/* create a StopIteration exception with the given value */
+PyAPI_FUNC(PyObject *) PyStopIteration_Create(PyObject *);
 
 /* These APIs aren't really part of the error implementation, but
    often needed to format error messages; the native C lib APIs are