From 8170e8c0d12cb9414f3a3d3ca81a447b4afc5f26 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 9 May 2015 11:44:30 -0400 Subject: PEP 479: Change StopIteration handling inside generators. Closes issue #22906. --- Doc/howto/functional.rst | 8 ++++---- Doc/library/__future__.rst | 3 +++ Doc/library/exceptions.rst | 8 ++++++++ Doc/reference/expressions.rst | 12 ++++++------ Include/code.h | 1 + Include/compile.h | 1 + Include/pythonrun.h | 3 ++- Lib/__future__.py | 6 ++++++ Lib/contextlib.py | 11 +++++++++-- Lib/difflib.py | 3 +-- Lib/test/test_contextlib.py | 34 ++++++++++++++++++++++++++++++++++ Misc/NEWS | 2 ++ Objects/genobject.c | 24 ++++++++++++++++++++++++ Python/future.c | 2 ++ 14 files changed, 103 insertions(+), 15 deletions(-) diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 1969b32..945a240 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -481,10 +481,10 @@ Here's a sample usage of the ``generate_ints()`` generator: You could equally write ``for i in generate_ints(5)``, or ``a,b,c = generate_ints(3)``. -Inside a generator function, ``return value`` is semantically equivalent to -``raise StopIteration(value)``. If no value is returned or the bottom of the -function is reached, the procession of values ends and the generator cannot -return any further values. +Inside a generator function, ``return value`` causes ``StopIteration(value)`` +to be raised from the :meth:`~generator.__next__` method. Once this happens, or +the bottom of the function is reached, the procession of values ends and the +generator cannot yield any further values. You could achieve the effect of generators manually by writing your own class and storing all the local variables of the generator as instance variables. For diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 72f2963..73d8b6b 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -87,6 +87,9 @@ language using this mechanism: | unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | | | | | *Bytes literals in Python 3000* | +------------------+-------------+--------------+---------------------------------------------+ +| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | +| | | | *StopIteration handling inside generators* | ++------------------+-------------+--------------+---------------------------------------------+ .. seealso:: diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index bddd0ed..e7768be 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -310,10 +310,18 @@ The following exceptions are the exceptions that are usually raised. raised, and the value returned by the function is used as the :attr:`value` parameter to the constructor of the exception. + If a generator function defined in the presence of a ``from __future__ + import generator_stop`` directive raises :exc:`StopIteration`, it will be + converted into a :exc:`RuntimeError` (retaining the :exc:`StopIteration` + as the new exception's cause). + .. versionchanged:: 3.3 Added ``value`` attribute and the ability for generator functions to use it to return a value. + .. versionchanged:: 3.5 + Introduced the RuntimeError transformation. + .. exception:: SyntaxError Raised when the parser encounters a syntax error. This may occur in an diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index efe4d0b..b252e26 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -443,12 +443,12 @@ is already executing raises a :exc:`ValueError` exception. .. method:: generator.close() Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then raises :exc:`StopIteration` (by - exiting normally, or due to already being closed) or :exc:`GeneratorExit` (by - not catching the exception), close returns to its caller. If the generator - yields a value, a :exc:`RuntimeError` is raised. If the generator raises any - other exception, it is propagated to the caller. :meth:`close` does nothing - if the generator has already exited due to an exception or normal exit. + paused. If the generator function then exits gracefully, is already closed, + or raises :exc:`GeneratorExit` (by not catching the exception), close + returns to its caller. If the generator yields a value, a + :exc:`RuntimeError` is raised. If the generator raises any other exception, + it is propagated to the caller. :meth:`close` does nothing if the generator + has already exited due to an exception or normal exit. .. index:: single: yield; examples diff --git a/Include/code.h b/Include/code.h index ff2b97e..131de50 100644 --- a/Include/code.h +++ b/Include/code.h @@ -62,6 +62,7 @@ typedef struct { #define CO_FUTURE_UNICODE_LITERALS 0x20000 #define CO_FUTURE_BARRY_AS_BDFL 0x40000 +#define CO_FUTURE_GENERATOR_STOP 0x80000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. The maximum number of diff --git a/Include/compile.h b/Include/compile.h index c6650d7..ecd8dc1 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -27,6 +27,7 @@ typedef struct { #define FUTURE_PRINT_FUNCTION "print_function" #define FUTURE_UNICODE_LITERALS "unicode_literals" #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" +#define FUTURE_GENERATOR_STOP "generator_stop" struct _mod; /* Declare the existence of this type */ #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 1d9b71b..f92148d 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -9,7 +9,8 @@ extern "C" { #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ - CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL) + CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ + CO_FUTURE_GENERATOR_STOP) #define PyCF_MASK_OBSOLETE (CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 diff --git a/Lib/__future__.py b/Lib/__future__.py index 3b2d5ec..63b2be3 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -56,6 +56,7 @@ all_feature_names = [ "print_function", "unicode_literals", "barry_as_FLUFL", + "generator_stop", ] __all__ = ["all_feature_names"] + all_feature_names @@ -72,6 +73,7 @@ CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals CO_FUTURE_BARRY_AS_BDFL = 0x40000 +CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): @@ -132,3 +134,7 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2), barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2), (3, 9, 0, "alpha", 0), CO_FUTURE_BARRY_AS_BDFL) + +generator_stop = _Feature((3, 5, 0, "beta", 1), + (3, 7, 0, "alpha", 0), + CO_FUTURE_GENERATOR_STOP) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 2fbc90c..379c251 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -77,10 +77,17 @@ class _GeneratorContextManager(ContextDecorator): self.gen.throw(type, value, traceback) raise RuntimeError("generator didn't stop after throw()") except StopIteration as exc: - # Suppress the exception *unless* it's the same exception that + # Suppress StopIteration *unless* it's the same exception that # was passed to throw(). This prevents a StopIteration - # raised inside the "with" statement from being suppressed + # raised inside the "with" statement from being suppressed. return exc is not value + except RuntimeError as exc: + # Likewise, avoid suppressing if a StopIteration exception + # was passed to throw() and later wrapped into a RuntimeError + # (see PEP 479). + if exc.__cause__ is value: + return False + raise except: # only re-raise if it's *not* the exception that was # passed to throw(), because __exit__() must not raise diff --git a/Lib/difflib.py b/Lib/difflib.py index 96fd9ab..aa98436 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1596,8 +1596,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, # them up without doing anything else with them. line_pair_iterator = _line_pair_iterator() if context is None: - while True: - yield next(line_pair_iterator) + yield from line_pair_iterator # Handle case where user wants context differencing. We must do some # storage of lines until we know for sure that they are to be yielded. else: diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index c52066b..a5d68a9 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -83,6 +83,40 @@ class ContextManagerTestCase(unittest.TestCase): raise ZeroDivisionError(999) self.assertEqual(state, [1, 42, 999]) + def test_contextmanager_except_stopiter(self): + stop_exc = StopIteration('spam') + @contextmanager + def woohoo(): + yield + try: + with woohoo(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail('StopIteration was suppressed') + + def test_contextmanager_except_pep479(self): + code = """\ +from __future__ import generator_stop +from contextlib import contextmanager +@contextmanager +def woohoo(): + yield +""" + locals = {} + exec(code, locals, locals) + woohoo = locals['woohoo'] + + stop_exc = StopIteration('spam') + try: + with woohoo(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail('StopIteration was suppressed') + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): diff --git a/Misc/NEWS b/Misc/NEWS index ac065f7..8c40764 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -33,6 +33,8 @@ Core and Builtins - Issue #9951: Added a hex() method to bytes, bytearray, and memoryview. +- Issue #22906: PEP 479: Change StopIteration handling inside generators. + Library ------- diff --git a/Objects/genobject.c b/Objects/genobject.c index 87060f3..5ae8512 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -130,6 +130,30 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc) } Py_CLEAR(result); } + else if (!result) { + /* Check for __future__ generator_stop and conditionally turn + * a leaking StopIteration into RuntimeError (with its cause + * set appropriately). */ + if ((((PyCodeObject *)gen->gi_code)->co_flags & + CO_FUTURE_GENERATOR_STOP) + && PyErr_ExceptionMatches(PyExc_StopIteration)) + { + PyObject *exc, *val, *val2, *tb; + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) + PyException_SetTraceback(val, tb); + Py_DECREF(exc); + Py_XDECREF(tb); + PyErr_SetString(PyExc_RuntimeError, + "generator raised StopIteration"); + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); + } + } if (!result || f->f_stacktop == NULL) { /* generator can't be rerun, so release the frame */ diff --git a/Python/future.c b/Python/future.c index 81eab54..163f87f 100644 --- a/Python/future.c +++ b/Python/future.c @@ -40,6 +40,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, PyObject *filename) continue; } else if (strcmp(feature, FUTURE_BARRY_AS_BDFL) == 0) { ff->ff_features |= CO_FUTURE_BARRY_AS_BDFL; + } else if (strcmp(feature, FUTURE_GENERATOR_STOP) == 0) { + ff->ff_features |= CO_FUTURE_GENERATOR_STOP; } else if (strcmp(feature, "braces") == 0) { PyErr_SetString(PyExc_SyntaxError, "not a chance"); -- cgit v0.12