GH-117066: Tier 2 optimizer: Don't throw away good traces if we can't optimize them perfectly. (GH-117067)

1 files changed, 10 insertions, 6 deletions

diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c
index 0c95616..603ac68 100644
--- a/Python/optimizer_analysis.c
+++ b/Python/optimizer_analysis.c
@@ -406,24 +406,28 @@ optimize_uops(
 out_of_space:
     DPRINTF(3, "\n");
     DPRINTF(1, "Out of space in abstract interpreter\n");
-    _Py_uop_abstractcontext_fini(ctx);
-    return 0;
-
+    goto done;
 error:
     DPRINTF(3, "\n");
     DPRINTF(1, "Encountered error in abstract interpreter\n");
     _Py_uop_abstractcontext_fini(ctx);
-    return 0;
+    return -1;
 
 hit_bottom:
     // Attempted to push a "bottom" (contradition) symbol onto the stack.
     // This means that the abstract interpreter has hit unreachable code.
-    // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but it's
-    // simpler to just admit failure and not create the executor.
+    // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting
+    // bottom indicates type instability, so we are probably better off
+    // retrying later.
     DPRINTF(3, "\n");
     DPRINTF(1, "Hit bottom in abstract interpreter\n");
     _Py_uop_abstractcontext_fini(ctx);
     return 0;
+done:
+    /* Cannot optimize further, but there would be no benefit
+     * in retrying later */
+    _Py_uop_abstractcontext_fini(ctx);
+    return 1;
 }