From 4296396db017d782d3aa16100b366748c9ea4a04 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 19 Nov 2021 18:51:50 +0000 Subject: [3.9] bpo-45806: Fix recovery from stack overflow for 3.9. Again. (GH-29640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Include/internal/pycore_ceval.h | 16 -------- Lib/test/test_exceptions.py | 48 +++++++++++++++++++++- Lib/test/test_sys.py | 34 +-------------- .../2021-11-19-19-21-48.bpo-45806.DflDMe.rst | 5 +++ Python/ceval.c | 20 +++++---- Python/errors.c | 3 ++ Python/sysmodule.c | 6 +-- 7 files changed, 69 insertions(+), 63 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 18c8f02..e7ace9b 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -90,24 +90,8 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) { #define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where) -/* Compute the "lower-water mark" for a recursion limit. When - * Py_LeaveRecursiveCall() is called with a recursion depth below this mark, - * the overflowed flag is reset to 0. */ -static inline int _Py_RecursionLimitLowerWaterMark(int limit) { - if (limit > 200) { - return (limit - 50); - } - else { - return (3 * (limit >> 2)); - } -} - static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { tstate->recursion_depth--; - int limit = tstate->interp->ceval.recursion_limit; - if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) { - tstate->overflowed = 0; - } } static inline void _Py_LeaveRecursiveCall_inline(void) { diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 4a7ca60..5168b0b 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1213,7 +1213,7 @@ class ExceptionTests(unittest.TestCase): # tstate->recursion_depth is equal to (recursion_limit - 1) # and is equal to recursion_limit when _gen_throw() calls # PyErr_NormalizeException(). - recurse(setrecursionlimit(depth + 2) - depth - 1) + recurse(setrecursionlimit(depth + 2) - depth) finally: sys.setrecursionlimit(recursionlimit) print('Done.') @@ -1243,6 +1243,52 @@ class ExceptionTests(unittest.TestCase): b'while normalizing an exception', err) self.assertIn(b'Done.', out) + def test_recursion_in_except_handler(self): + + def set_relative_recursion_limit(n): + depth = 1 + while True: + try: + sys.setrecursionlimit(depth) + except RecursionError: + depth += 1 + else: + break + sys.setrecursionlimit(depth+n) + + def recurse_in_except(): + try: + 1/0 + except: + recurse_in_except() + + def recurse_after_except(): + try: + 1/0 + except: + pass + recurse_after_except() + + def recurse_in_body_and_except(): + try: + recurse_in_body_and_except() + except: + recurse_in_body_and_except() + + recursionlimit = sys.getrecursionlimit() + try: + set_relative_recursion_limit(10) + for func in (recurse_in_except, recurse_after_except, recurse_in_body_and_except): + with self.subTest(func=func): + try: + func() + except RecursionError: + pass + else: + self.fail("Should have raised a RecursionError") + finally: + sys.setrecursionlimit(recursionlimit) + @cpython_only def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 450e392..2f1e5e9 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -260,42 +260,10 @@ class SysModuleTest(unittest.TestCase): sys.setrecursionlimit(1000) for limit in (10, 25, 50, 75, 100, 150, 200): - # formula extracted from _Py_RecursionLimitLowerWaterMark() - if limit > 200: - depth = limit - 50 - else: - depth = limit * 3 // 4 - set_recursion_limit_at_depth(depth, limit) + set_recursion_limit_at_depth(limit, limit) finally: sys.setrecursionlimit(oldlimit) - # The error message is specific to CPython - @test.support.cpython_only - def test_recursionlimit_fatalerror(self): - # A fatal error occurs if a second recursion limit is hit when recovering - # from a first one. - code = textwrap.dedent(""" - import sys - - def f(): - try: - f() - except RecursionError: - f() - - sys.setrecursionlimit(%d) - f()""") - with test.support.SuppressCrashReport(): - for i in (50, 1000): - sub = subprocess.Popen([sys.executable, '-c', code % i], - stderr=subprocess.PIPE) - err = sub.communicate()[1] - self.assertTrue(sub.returncode, sub.returncode) - self.assertIn( - b"Fatal Python error: _Py_CheckRecursiveCall: " - b"Cannot recover from stack overflow", - err) - def test_getwindowsversion(self): # Raise SkipTest if sys doesn't have getwindowsversion attribute test.support.get_attribute(sys, "getwindowsversion") diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst new file mode 100644 index 0000000..f8c47ca --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-19-19-21-48.bpo-45806.DflDMe.rst @@ -0,0 +1,5 @@ +Re-introduced fix that allows recovery from stack overflow without crashing +the interpreter. The original fix as part of :issue:`42500` was reverted +(see release notes for Python 3.9.4) since it introduced an ABI change in a +bugfix release which is not allowed. The new fix doesn't introduce any ABI +changes. Patch by Mark Shannon. diff --git a/Python/ceval.c b/Python/ceval.c index 86a5d81..9a61f8a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -797,19 +797,21 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) /* Somebody asked that we don't check for recursion. */ return 0; if (tstate->overflowed) { - if (tstate->recursion_depth > recursion_limit + 50) { + if (tstate->recursion_depth > recursion_limit + 50 || tstate->overflowed > 50) { /* Overflowing while handling an overflow. Give up. */ Py_FatalError("Cannot recover from stack overflow."); } - return 0; } - if (tstate->recursion_depth > recursion_limit) { - --tstate->recursion_depth; - tstate->overflowed = 1; - _PyErr_Format(tstate, PyExc_RecursionError, - "maximum recursion depth exceeded%s", - where); - return -1; + else { + if (tstate->recursion_depth > recursion_limit) { + tstate->overflowed++; + _PyErr_Format(tstate, PyExc_RecursionError, + "maximum recursion depth exceeded%s", + where); + tstate->overflowed--; + --tstate->recursion_depth; + return -1; + } } return 0; } diff --git a/Python/errors.c b/Python/errors.c index 40d8e68..7927876 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -317,12 +317,14 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, PyObject **val, PyObject **tb) { int recursion_depth = 0; + tstate->overflowed++; PyObject *type, *value, *initial_tb; restart: type = *exc; if (type == NULL) { /* There was no exception, so nothing to do. */ + tstate->overflowed--; return; } @@ -374,6 +376,7 @@ _PyErr_NormalizeException(PyThreadState *tstate, PyObject **exc, } *exc = type; *val = value; + tstate->overflowed--; return; error: diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 3e4115f..a52b299 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -17,7 +17,7 @@ Data members: #include "Python.h" #include "code.h" #include "frameobject.h" // PyFrame_GetBack() -#include "pycore_ceval.h" // _Py_RecursionLimitLowerWaterMark() +#include "pycore_ceval.h" #include "pycore_initconfig.h" #include "pycore_object.h" #include "pycore_pathconfig.h" @@ -1160,7 +1160,6 @@ static PyObject * sys_setrecursionlimit_impl(PyObject *module, int new_limit) /*[clinic end generated code: output=35e1c64754800ace input=b0f7a23393924af3]*/ { - int mark; PyThreadState *tstate = _PyThreadState_GET(); if (new_limit < 1) { @@ -1178,8 +1177,7 @@ sys_setrecursionlimit_impl(PyObject *module, int new_limit) Reject too low new limit if the current recursion depth is higher than the new low-water mark. Otherwise it may not be possible anymore to reset the overflowed flag to 0. */ - mark = _Py_RecursionLimitLowerWaterMark(new_limit); - if (tstate->recursion_depth >= mark) { + if (tstate->recursion_depth >= new_limit) { _PyErr_Format(tstate, PyExc_RecursionError, "cannot set the recursion limit to %i at " "the recursion depth %i: the limit is too low", -- cgit v0.12