gh-117180: Complete call sequence when trace stack overflow (GH-117184)

---------

Co-authored-by: Peter Lazorchak <lazorchakp@gmail.com>
Co-authored-by: Guido van Rossum <gvanrossum@users.noreply.github.com>
Co-authored-by: Guido van Rossum <gvanrossum@gmail.com>

2 files changed, 27 insertions, 0 deletions

diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py
index b0859a3..a1dc03d 100644
--- a/Lib/test/test_capi/test_opt.py
+++ b/Lib/test/test_capi/test_opt.py
@@ -955,6 +955,32 @@ class TestUopsOptimization(unittest.TestCase):
         _, ex = self._run_with_optimizer(testfunc, 16)
         self.assertIsNone(ex)
 
+    def test_many_nested(self):
+        # overflow the trace_stack
+        def dummy_a(x):
+            return x
+        def dummy_b(x):
+            return dummy_a(x)
+        def dummy_c(x):
+            return dummy_b(x)
+        def dummy_d(x):
+            return dummy_c(x)
+        def dummy_e(x):
+            return dummy_d(x)
+        def dummy_f(x):
+            return dummy_e(x)
+        def dummy_g(x):
+            return dummy_f(x)
+        def dummy_h(x):
+            return dummy_g(x)
+        def testfunc(n):
+            a = 0
+            for _ in range(n):
+                a += dummy_h(n)
+            return a
+
+        self._run_with_optimizer(testfunc, 32)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Python/optimizer.c b/Python/optimizer.c
index 177ad34..f8c1390 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -476,6 +476,7 @@ BRANCH_TO_GUARD[4][2] = {
     if (trace_stack_depth >= TRACE_STACK_SIZE) { \
         DPRINTF(2, "Trace stack overflow\n"); \
         OPT_STAT_INC(trace_stack_overflow); \
+        ADD_TO_TRACE(uop, oparg, operand, target); \
         ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); \
         goto done; \
     } \