GH-111848: Tidy up tier 2 handling of FOR_ITER specialization by using DEOPT_IF instead of jumps. (GH-111849)

4 files changed, 33 insertions, 113 deletions

diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h
index c6ebea0..384a112 100644
--- a/Python/abstract_interp_cases.c.h
+++ b/Python/abstract_interp_cases.c.h
@@ -628,9 +628,7 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_LIST: {
-            STACK_GROW(1);
-            PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
+        case _GUARD_NOT_EXHAUSTED_LIST: {
             break;
         }
 
@@ -644,9 +642,7 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_TUPLE: {
-            STACK_GROW(1);
-            PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
+        case _GUARD_NOT_EXHAUSTED_TUPLE: {
             break;
         }
 
@@ -660,9 +656,7 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_RANGE: {
-            STACK_GROW(1);
-            PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true);
+        case _GUARD_NOT_EXHAUSTED_RANGE: {
             break;
         }
 
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index 8e1d318..f879ea5 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2580,7 +2580,7 @@ dummy_func(
             DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type);
         }
 
-        op(_ITER_JUMP_LIST, (iter -- iter)) {
+        replaced op(_ITER_JUMP_LIST, (iter -- iter)) {
             _PyListIterObject *it = (_PyListIterObject *)iter;
             assert(Py_TYPE(iter) == &PyListIter_Type);
             STAT_INC(FOR_ITER, hit);
@@ -2599,21 +2599,12 @@ dummy_func(
         }
 
         // Only used by Tier 2
-        op(_IS_ITER_EXHAUSTED_LIST, (iter -- iter, exhausted)) {
+        op(_GUARD_NOT_EXHAUSTED_LIST, (iter -- iter)) {
             _PyListIterObject *it = (_PyListIterObject *)iter;
             assert(Py_TYPE(iter) == &PyListIter_Type);
             PyListObject *seq = it->it_seq;
-            if (seq == NULL) {
-                exhausted = Py_True;
-            }
-            else if (it->it_index >= PyList_GET_SIZE(seq)) {
-                Py_DECREF(seq);
-                it->it_seq = NULL;
-                exhausted = Py_True;
-            }
-            else {
-                exhausted = Py_False;
-            }
+            DEOPT_IF(seq == NULL);
+            DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq));
         }
 
         op(_ITER_NEXT_LIST, (iter -- iter, next)) {
@@ -2635,7 +2626,7 @@ dummy_func(
             DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type);
         }
 
-        op(_ITER_JUMP_TUPLE, (iter -- iter)) {
+        replaced op(_ITER_JUMP_TUPLE, (iter -- iter)) {
             _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
             assert(Py_TYPE(iter) == &PyTupleIter_Type);
             STAT_INC(FOR_ITER, hit);
@@ -2654,21 +2645,12 @@ dummy_func(
         }
 
         // Only used by Tier 2
-        op(_IS_ITER_EXHAUSTED_TUPLE, (iter -- iter, exhausted)) {
+        op(_GUARD_NOT_EXHAUSTED_TUPLE, (iter -- iter)) {
             _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
             assert(Py_TYPE(iter) == &PyTupleIter_Type);
             PyTupleObject *seq = it->it_seq;
-            if (seq == NULL) {
-                exhausted = Py_True;
-            }
-            else if (it->it_index >= PyTuple_GET_SIZE(seq)) {
-                Py_DECREF(seq);
-                it->it_seq = NULL;
-                exhausted = Py_True;
-            }
-            else {
-                exhausted = Py_False;
-            }
+            DEOPT_IF(seq == NULL);
+            DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq));
         }
 
         op(_ITER_NEXT_TUPLE, (iter -- iter, next)) {
@@ -2691,7 +2673,7 @@ dummy_func(
             DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type);
         }
 
-        op(_ITER_JUMP_RANGE, (iter -- iter)) {
+        replaced op(_ITER_JUMP_RANGE, (iter -- iter)) {
             _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
             assert(Py_TYPE(r) == &PyRangeIter_Type);
             STAT_INC(FOR_ITER, hit);
@@ -2705,10 +2687,10 @@ dummy_func(
         }
 
         // Only used by Tier 2
-        op(_IS_ITER_EXHAUSTED_RANGE, (iter -- iter, exhausted)) {
+        op(_GUARD_NOT_EXHAUSTED_RANGE, (iter -- iter)) {
             _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
             assert(Py_TYPE(r) == &PyRangeIter_Type);
-            exhausted = r->len <= 0 ? Py_True : Py_False;
+            DEOPT_IF(r->len <= 0);
         }
 
         op(_ITER_NEXT_RANGE, (iter -- iter, next)) {
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index d9e9ad1..d94a7cc 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -2138,26 +2138,14 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_LIST: {
+        case _GUARD_NOT_EXHAUSTED_LIST: {
             PyObject *iter;
-            PyObject *exhausted;
             iter = stack_pointer[-1];
             _PyListIterObject *it = (_PyListIterObject *)iter;
             assert(Py_TYPE(iter) == &PyListIter_Type);
             PyListObject *seq = it->it_seq;
-            if (seq == NULL) {
-                exhausted = Py_True;
-            }
-            else if (it->it_index >= PyList_GET_SIZE(seq)) {
-                Py_DECREF(seq);
-                it->it_seq = NULL;
-                exhausted = Py_True;
-            }
-            else {
-                exhausted = Py_False;
-            }
-            STACK_GROW(1);
-            stack_pointer[-1] = exhausted;
+            DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_LIST);
+            DEOPT_IF(it->it_index >= PyList_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_LIST);
             break;
         }
 
@@ -2183,26 +2171,14 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_TUPLE: {
+        case _GUARD_NOT_EXHAUSTED_TUPLE: {
             PyObject *iter;
-            PyObject *exhausted;
             iter = stack_pointer[-1];
             _PyTupleIterObject *it = (_PyTupleIterObject *)iter;
             assert(Py_TYPE(iter) == &PyTupleIter_Type);
             PyTupleObject *seq = it->it_seq;
-            if (seq == NULL) {
-                exhausted = Py_True;
-            }
-            else if (it->it_index >= PyTuple_GET_SIZE(seq)) {
-                Py_DECREF(seq);
-                it->it_seq = NULL;
-                exhausted = Py_True;
-            }
-            else {
-                exhausted = Py_False;
-            }
-            STACK_GROW(1);
-            stack_pointer[-1] = exhausted;
+            DEOPT_IF(seq == NULL, _GUARD_NOT_EXHAUSTED_TUPLE);
+            DEOPT_IF(it->it_index >= PyTuple_GET_SIZE(seq), _GUARD_NOT_EXHAUSTED_TUPLE);
             break;
         }
 
@@ -2229,15 +2205,12 @@
             break;
         }
 
-        case _IS_ITER_EXHAUSTED_RANGE: {
+        case _GUARD_NOT_EXHAUSTED_RANGE: {
             PyObject *iter;
-            PyObject *exhausted;
             iter = stack_pointer[-1];
             _PyRangeIterObject *r = (_PyRangeIterObject *)iter;
             assert(Py_TYPE(r) == &PyRangeIter_Type);
-            exhausted = r->len <= 0 ? Py_True : Py_False;
-            STACK_GROW(1);
-            stack_pointer[-1] = exhausted;
+            DEOPT_IF(r->len <= 0, _GUARD_NOT_EXHAUSTED_RANGE);
             break;
         }
 
diff --git a/Python/optimizer.c b/Python/optimizer.c
index a332fd1..065e127 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -384,6 +384,14 @@ PyTypeObject _PyUOpExecutor_Type = {
     .tp_methods = executor_methods,
 };
 
+/* TO DO -- Generate this table */
+static const uint16_t
+_PyUop_Replacements[OPCODE_METADATA_SIZE] = {
+    [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE,
+    [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST,
+    [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,
+};
+
 #define TRACE_STACK_SIZE 5
 
 /* Returns 1 on success,
@@ -586,46 +594,6 @@ pop_jump_if_bool:
                 break;
             }
 
-            case FOR_ITER_LIST:
-            case FOR_ITER_TUPLE:
-            case FOR_ITER_RANGE:
-            {
-                RESERVE(4, 3);
-                int check_op, exhausted_op, next_op;
-                switch (opcode) {
-                    case FOR_ITER_LIST:
-                        check_op = _ITER_CHECK_LIST;
-                        exhausted_op = _IS_ITER_EXHAUSTED_LIST;
-                        next_op = _ITER_NEXT_LIST;
-                        break;
-                    case FOR_ITER_TUPLE:
-                        check_op = _ITER_CHECK_TUPLE;
-                        exhausted_op = _IS_ITER_EXHAUSTED_TUPLE;
-                        next_op = _ITER_NEXT_TUPLE;
-                        break;
-                    case FOR_ITER_RANGE:
-                        check_op = _ITER_CHECK_RANGE;
-                        exhausted_op = _IS_ITER_EXHAUSTED_RANGE;
-                        next_op = _ITER_NEXT_RANGE;
-                        break;
-                    default:
-                        Py_UNREACHABLE();
-                }
-                // Assume jump unlikely (can a for-loop exit be likely?)
-                _Py_CODEUNIT *target_instr =  // +1 at the end skips over END_FOR
-                    instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + oparg + 1;
-                max_length -= 3;  // Really the start of the stubs
-                ADD_TO_TRACE(check_op, 0, 0);
-                ADD_TO_TRACE(exhausted_op, 0, 0);
-                ADD_TO_TRACE(_POP_JUMP_IF_TRUE, max_length, 0);
-                ADD_TO_TRACE(next_op, 0, 0);
-
-                ADD_TO_STUB(max_length + 0, POP_TOP, 0, 0);
-                ADD_TO_STUB(max_length + 1, _SET_IP, INSTR_IP(target_instr, code), 0);
-                ADD_TO_STUB(max_length + 2, _EXIT_TRACE, 0, 0);
-                break;
-            }
-
             default:
             {
                 const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode];
@@ -661,6 +629,9 @@ pop_jump_if_bool:
                                         oparg += extras;
                                     }
                                 }
+                                if (_PyUop_Replacements[uop]) {
+                                    uop = _PyUop_Replacements[uop];
+                                }
                                 break;
                             case OPARG_CACHE_1:
                                 operand = read_u16(&instr[offset].cache);