GH-111848: Convert remaining jumps to deopts into tier 2 code. (GH-112045)

5 files changed, 92 insertions, 69 deletions

diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h
index 8892d26..a2f6aa8 100644
--- a/Python/abstract_interp_cases.c.h
+++ b/Python/abstract_interp_cases.c.h
@@ -914,12 +914,22 @@
             break;
         }
 
-        case _POP_JUMP_IF_FALSE: {
+        case _GUARD_IS_TRUE_POP: {
             STACK_SHRINK(1);
             break;
         }
 
-        case _POP_JUMP_IF_TRUE: {
+        case _GUARD_IS_FALSE_POP: {
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case _GUARD_IS_NONE_POP: {
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case _GUARD_IS_NOT_NONE_POP: {
             STACK_SHRINK(1);
             break;
         }
diff --git a/Python/bytecodes.c b/Python/bytecodes.c
index bb8123e..8a7dcb8 100644
--- a/Python/bytecodes.c
+++ b/Python/bytecodes.c
@@ -2368,7 +2368,7 @@ dummy_func(
             goto enter_tier_one;
         }
 
-        inst(POP_JUMP_IF_FALSE, (unused/1, cond -- )) {
+       replaced op(_POP_JUMP_IF_FALSE, (unused/1, cond -- )) {
             assert(PyBool_Check(cond));
             int flag = Py_IsFalse(cond);
             #if ENABLE_SPECIALIZATION
@@ -2377,7 +2377,7 @@ dummy_func(
             JUMPBY(oparg * flag);
         }
 
-        inst(POP_JUMP_IF_TRUE, (unused/1, cond -- )) {
+        replaced op(_POP_JUMP_IF_TRUE, (unused/1, cond -- )) {
             assert(PyBool_Check(cond));
             int flag = Py_IsTrue(cond);
             #if ENABLE_SPECIALIZATION
@@ -2396,9 +2396,13 @@ dummy_func(
             }
         }
 
-        macro(POP_JUMP_IF_NONE) = _IS_NONE + POP_JUMP_IF_TRUE;
+        macro(POP_JUMP_IF_TRUE) = _POP_JUMP_IF_TRUE;
 
-        macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + POP_JUMP_IF_FALSE;
+        macro(POP_JUMP_IF_FALSE) = _POP_JUMP_IF_FALSE;
+
+        macro(POP_JUMP_IF_NONE) = _IS_NONE + _POP_JUMP_IF_TRUE;
+
+        macro(POP_JUMP_IF_NOT_NONE) = _IS_NONE + _POP_JUMP_IF_FALSE;
 
         inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) {
             /* This bytecode is used in the `yield from` or `await` loop.
@@ -3963,16 +3967,23 @@ dummy_func(
 
         ///////// Tier-2 only opcodes /////////
 
-        op(_POP_JUMP_IF_FALSE, (flag -- )) {
-            if (Py_IsFalse(flag)) {
-                next_uop = current_executor->trace + oparg;
-            }
+        op (_GUARD_IS_TRUE_POP, (flag -- )) {
+            DEOPT_IF(Py_IsFalse(flag));
+            assert(Py_IsTrue(flag));
         }
 
-        op(_POP_JUMP_IF_TRUE, (flag -- )) {
-            if (Py_IsTrue(flag)) {
-                next_uop = current_executor->trace + oparg;
-            }
+        op (_GUARD_IS_FALSE_POP, (flag -- )) {
+            DEOPT_IF(Py_IsTrue(flag));
+            assert(Py_IsFalse(flag));
+        }
+
+        op (_GUARD_IS_NONE_POP, (val -- )) {
+            DEOPT_IF(!Py_IsNone(val));
+        }
+
+        op (_GUARD_IS_NOT_NONE_POP, (val -- )) {
+            DEOPT_IF(Py_IsNone(val));
+            Py_DECREF(val);
         }
 
         op(_JUMP_TO_TOP, (--)) {
diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h
index ac542a5..4e29fb9 100644
--- a/Python/executor_cases.c.h
+++ b/Python/executor_cases.c.h
@@ -3185,22 +3185,37 @@
             break;
         }
 
-        case _POP_JUMP_IF_FALSE: {
+        case _GUARD_IS_TRUE_POP: {
             PyObject *flag;
             flag = stack_pointer[-1];
-            if (Py_IsFalse(flag)) {
-                next_uop = current_executor->trace + oparg;
-            }
+            DEOPT_IF(Py_IsFalse(flag), _GUARD_IS_TRUE_POP);
+            assert(Py_IsTrue(flag));
             STACK_SHRINK(1);
             break;
         }
 
-        case _POP_JUMP_IF_TRUE: {
+        case _GUARD_IS_FALSE_POP: {
             PyObject *flag;
             flag = stack_pointer[-1];
-            if (Py_IsTrue(flag)) {
-                next_uop = current_executor->trace + oparg;
-            }
+            DEOPT_IF(Py_IsTrue(flag), _GUARD_IS_FALSE_POP);
+            assert(Py_IsFalse(flag));
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case _GUARD_IS_NONE_POP: {
+            PyObject *val;
+            val = stack_pointer[-1];
+            DEOPT_IF(!Py_IsNone(val), _GUARD_IS_NONE_POP);
+            STACK_SHRINK(1);
+            break;
+        }
+
+        case _GUARD_IS_NOT_NONE_POP: {
+            PyObject *val;
+            val = stack_pointer[-1];
+            DEOPT_IF(Py_IsNone(val), _GUARD_IS_NOT_NONE_POP);
+            Py_DECREF(val);
             STACK_SHRINK(1);
             break;
         }
diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h
index 6cbc54c..fe0cbfe 100644
--- a/Python/generated_cases.c.h
+++ b/Python/generated_cases.c.h
@@ -3435,14 +3435,14 @@
             goto enter_tier_one;
         }
 
-        TARGET(POP_JUMP_IF_FALSE) {
+        TARGET(POP_JUMP_IF_TRUE) {
             _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
             next_instr += 2;
-            INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
+            INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
             PyObject *cond;
             cond = stack_pointer[-1];
             assert(PyBool_Check(cond));
-            int flag = Py_IsFalse(cond);
+            int flag = Py_IsTrue(cond);
             #if ENABLE_SPECIALIZATION
             this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
@@ -3451,14 +3451,14 @@
             DISPATCH();
         }
 
-        TARGET(POP_JUMP_IF_TRUE) {
+        TARGET(POP_JUMP_IF_FALSE) {
             _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr;
             next_instr += 2;
-            INSTRUCTION_STATS(POP_JUMP_IF_TRUE);
+            INSTRUCTION_STATS(POP_JUMP_IF_FALSE);
             PyObject *cond;
             cond = stack_pointer[-1];
             assert(PyBool_Check(cond));
-            int flag = Py_IsTrue(cond);
+            int flag = Py_IsFalse(cond);
             #if ENABLE_SPECIALIZATION
             this_instr[1].cache = (this_instr[1].cache << 1) | flag;
             #endif
@@ -3485,7 +3485,7 @@
                     Py_DECREF(value);
                 }
             }
-            // POP_JUMP_IF_TRUE
+            // _POP_JUMP_IF_TRUE
             cond = b;
             {
                 assert(PyBool_Check(cond));
@@ -3517,7 +3517,7 @@
                     Py_DECREF(value);
                 }
             }
-            // POP_JUMP_IF_FALSE
+            // _POP_JUMP_IF_FALSE
             cond = b;
             {
                 assert(PyBool_Check(cond));
diff --git a/Python/optimizer.c b/Python/optimizer.c
index e142bd0..bc518d0 100644
--- a/Python/optimizer.c
+++ b/Python/optimizer.c
@@ -385,7 +385,7 @@ PyTypeObject _PyUOpExecutor_Type = {
     .tp_methods = executor_methods,
 };
 
-/* TO DO -- Generate this table */
+/* TO DO -- Generate these tables */
 static const uint16_t
 _PyUop_Replacements[OPCODE_METADATA_SIZE] = {
     [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE,
@@ -393,6 +393,18 @@ _PyUop_Replacements[OPCODE_METADATA_SIZE] = {
     [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE,
 };
 
+static const uint16_t
+BRANCH_TO_GUARD[4][2] = {
+    [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP,
+    [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP,
+    [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP,
+    [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP,
+    [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP,
+    [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP,
+    [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP,
+    [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP,
+};
+
 #define TRACE_STACK_SIZE 5
 
 /* Returns 1 on success,
@@ -528,45 +540,23 @@ top:  // Jump here after _PUSH_FRAME or likely branches
         }
 
         switch (opcode) {
-
             case POP_JUMP_IF_NONE:
-            {
-                RESERVE(2, 2);
-                ADD_TO_TRACE(_IS_NONE, 0, 0);
-                opcode = POP_JUMP_IF_TRUE;
-                goto pop_jump_if_bool;
-            }
-
             case POP_JUMP_IF_NOT_NONE:
-            {
-                RESERVE(2, 2);
-                ADD_TO_TRACE(_IS_NONE, 0, 0);
-                opcode = POP_JUMP_IF_FALSE;
-                goto pop_jump_if_bool;
-            }
-
             case POP_JUMP_IF_FALSE:
             case POP_JUMP_IF_TRUE:
             {
-pop_jump_if_bool:
-                RESERVE(1, 2);
-                max_length -= 2;  // Really the start of the stubs
+                RESERVE(1, 0);
                 int counter = instr[1].cache;
                 int bitcount = _Py_popcount32(counter);
-                bool jump_likely = bitcount > 8;
-                bool jump_sense = opcode == POP_JUMP_IF_TRUE;
-                uint32_t uopcode = jump_sense ^ jump_likely ?
-                    _POP_JUMP_IF_TRUE : _POP_JUMP_IF_FALSE;
+                int jump_likely = bitcount > 8;
+                uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely];
                 _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]];
-                _Py_CODEUNIT *target_instr = next_instr + oparg;
-                _Py_CODEUNIT *stub_target = jump_likely ? next_instr : target_instr;
-                DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, sense=%d, uopcode=%s\n",
+                DPRINTF(4, "%s(%d): counter=%x, bitcount=%d, likely=%d, uopcode=%s\n",
                         uop_name(opcode), oparg,
-                        counter, bitcount, jump_likely, jump_sense, uop_name(uopcode));
+                        counter, bitcount, jump_likely, uop_name(uopcode));
                 ADD_TO_TRACE(uopcode, max_length, 0);
-                ADD_TO_STUB(max_length, _SET_IP, INSTR_IP(stub_target, code), 0);
-                ADD_TO_STUB(max_length + 1, _EXIT_TRACE, 0, 0);
                 if (jump_likely) {
+                    _Py_CODEUNIT *target_instr = next_instr + oparg;
                     DPRINTF(2, "Jump likely (%x = %d bits), continue at byte offset %d\n",
                             instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code));
                     instr = target_instr;
@@ -797,16 +787,13 @@ compute_used(_PyUOpInstruction *buffer, uint32_t *used)
         }
         /* All other micro-ops fall through, so i+1 is reachable */
         SET_BIT(used, i+1);
-        switch(opcode) {
-            case NOP:
-                /* Don't count NOPs as used */
-                count--;
-                UNSET_BIT(used, i);
-                break;
-            case _POP_JUMP_IF_FALSE:
-            case _POP_JUMP_IF_TRUE:
-                /* Mark target as reachable */
-                SET_BIT(used, buffer[i].oparg);
+        if (OPCODE_HAS_JUMP(opcode)) {
+            /* Mark target as reachable */
+            SET_BIT(used, buffer[i].oparg);
+        }
+        if (opcode == NOP) {
+            count--;
+            UNSET_BIT(used, i);
         }
     }
     return count;