GH-116017: Get rid of _COLD_EXITs (GH-120960)

5 files changed, 122 insertions, 141 deletions

diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h
index 0bcca1e..3db3aa3 100644
--- a/Include/internal/pycore_backoff.h
+++ b/Include/internal/pycore_backoff.h
@@ -122,14 +122,14 @@ initial_jump_backoff_counter(void)
  * otherwise when a side exit warms up we may construct
  * a new trace before the Tier 1 code has properly re-specialized.
  * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */
-#define COLD_EXIT_INITIAL_VALUE 64
-#define COLD_EXIT_INITIAL_BACKOFF 6
+#define SIDE_EXIT_INITIAL_VALUE 64
+#define SIDE_EXIT_INITIAL_BACKOFF 6
 
 static inline _Py_BackoffCounter
 initial_temperature_backoff_counter(void)
 {
-    return make_backoff_counter(COLD_EXIT_INITIAL_VALUE,
-                                COLD_EXIT_INITIAL_BACKOFF);
+    return make_backoff_counter(SIDE_EXIT_INITIAL_VALUE,
+                                SIDE_EXIT_INITIAL_BACKOFF);
 }
 
 /* Unreachable backoff counter. */
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index 81caaa5..cf2cc7f 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -535,7 +535,7 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 #define ADAPTIVE_COOLDOWN_BACKOFF 0
 
 // Can't assert this in pycore_backoff.h because of header order dependencies
-static_assert(COLD_EXIT_INITIAL_VALUE > ADAPTIVE_COOLDOWN_VALUE,
+static_assert(SIDE_EXIT_INITIAL_VALUE > ADAPTIVE_COOLDOWN_VALUE,
     "Cold exit value should be larger than adaptive cooldown value");
 
 static inline _Py_BackoffCounter
diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h
index 1007f83..bcbb8b7 100644
--- a/Include/internal/pycore_optimizer.h
+++ b/Include/internal/pycore_optimizer.h
@@ -41,24 +41,18 @@ typedef struct {
  * the 32 bits between the oparg and operand are:
  * UOP_FORMAT_TARGET:
  *    uint32_t target;
- * UOP_FORMAT_EXIT
- *    uint16_t exit_index;
- *    uint16_t error_target;
  * UOP_FORMAT_JUMP
  *    uint16_t jump_target;
  *    uint16_t error_target;
  */
 typedef struct {
-    uint16_t opcode:14;
-    uint16_t format:2;
+    uint16_t opcode:15;
+    uint16_t format:1;
     uint16_t oparg;
     union {
         uint32_t target;
         struct {
-            union {
-                uint16_t exit_index;
-                uint16_t jump_target;
-            };
+            uint16_t jump_target;
             uint16_t error_target;
         };
     };
@@ -160,9 +154,7 @@ struct _Py_UopsSymbol {
 };
 
 #define UOP_FORMAT_TARGET 0
-#define UOP_FORMAT_EXIT 1
-#define UOP_FORMAT_JUMP 2
-#define UOP_FORMAT_UNUSED 3
+#define UOP_FORMAT_JUMP 1
 
 static inline uint32_t uop_get_target(const _PyUOpInstruction *inst)
 {
@@ -170,12 +162,6 @@ static inline uint32_t uop_get_target(const _PyUOpInstruction *inst)
     return inst->target;
 }
 
-static inline uint16_t uop_get_exit_index(const _PyUOpInstruction *inst)
-{
-    assert(inst->format == UOP_FORMAT_EXIT);
-    return inst->exit_index;
-}
-
 static inline uint16_t uop_get_jump_target(const _PyUOpInstruction *inst)
 {
     assert(inst->format == UOP_FORMAT_JUMP);
diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h
index 85e689c..bd1d27b 100644
--- a/Include/internal/pycore_uop_ids.h
+++ b/Include/internal/pycore_uop_ids.h
@@ -72,12 +72,11 @@ extern "C" {
 #define _CHECK_STACK_SPACE_OPERAND 337
 #define _CHECK_VALIDITY 338
 #define _CHECK_VALIDITY_AND_SET_IP 339
-#define _COLD_EXIT 340
-#define _COMPARE_OP 341
-#define _COMPARE_OP_FLOAT 342
-#define _COMPARE_OP_INT 343
-#define _COMPARE_OP_STR 344
-#define _CONTAINS_OP 345
+#define _COMPARE_OP 340
+#define _COMPARE_OP_FLOAT 341
+#define _COMPARE_OP_INT 342
+#define _COMPARE_OP_STR 343
+#define _CONTAINS_OP 344
 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT
 #define _CONTAINS_OP_SET CONTAINS_OP_SET
 #define _CONVERT_VALUE CONVERT_VALUE
@@ -89,53 +88,53 @@ extern "C" {
 #define _DELETE_GLOBAL DELETE_GLOBAL
 #define _DELETE_NAME DELETE_NAME
 #define _DELETE_SUBSCR DELETE_SUBSCR
-#define _DEOPT 346
+#define _DEOPT 345
 #define _DICT_MERGE DICT_MERGE
 #define _DICT_UPDATE DICT_UPDATE
-#define _DYNAMIC_EXIT 347
+#define _DYNAMIC_EXIT 346
 #define _END_SEND END_SEND
-#define _ERROR_POP_N 348
+#define _ERROR_POP_N 347
 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK
-#define _EXPAND_METHOD 349
-#define _FATAL_ERROR 350
+#define _EXPAND_METHOD 348
+#define _FATAL_ERROR 349
 #define _FORMAT_SIMPLE FORMAT_SIMPLE
 #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC
-#define _FOR_ITER 351
-#define _FOR_ITER_GEN_FRAME 352
-#define _FOR_ITER_TIER_TWO 353
+#define _FOR_ITER 350
+#define _FOR_ITER_GEN_FRAME 351
+#define _FOR_ITER_TIER_TWO 352
 #define _GET_AITER GET_AITER
 #define _GET_ANEXT GET_ANEXT
 #define _GET_AWAITABLE GET_AWAITABLE
 #define _GET_ITER GET_ITER
 #define _GET_LEN GET_LEN
 #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER
-#define _GUARD_BOTH_FLOAT 354
-#define _GUARD_BOTH_INT 355
-#define _GUARD_BOTH_UNICODE 356
-#define _GUARD_BUILTINS_VERSION 357
-#define _GUARD_DORV_NO_DICT 358
-#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 359
-#define _GUARD_GLOBALS_VERSION 360
-#define _GUARD_IS_FALSE_POP 361
-#define _GUARD_IS_NONE_POP 362
-#define _GUARD_IS_NOT_NONE_POP 363
-#define _GUARD_IS_TRUE_POP 364
-#define _GUARD_KEYS_VERSION 365
-#define _GUARD_NOS_FLOAT 366
-#define _GUARD_NOS_INT 367
-#define _GUARD_NOT_EXHAUSTED_LIST 368
-#define _GUARD_NOT_EXHAUSTED_RANGE 369
-#define _GUARD_NOT_EXHAUSTED_TUPLE 370
-#define _GUARD_TOS_FLOAT 371
-#define _GUARD_TOS_INT 372
-#define _GUARD_TYPE_VERSION 373
-#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 374
-#define _INIT_CALL_PY_EXACT_ARGS 375
-#define _INIT_CALL_PY_EXACT_ARGS_0 376
-#define _INIT_CALL_PY_EXACT_ARGS_1 377
-#define _INIT_CALL_PY_EXACT_ARGS_2 378
-#define _INIT_CALL_PY_EXACT_ARGS_3 379
-#define _INIT_CALL_PY_EXACT_ARGS_4 380
+#define _GUARD_BOTH_FLOAT 353
+#define _GUARD_BOTH_INT 354
+#define _GUARD_BOTH_UNICODE 355
+#define _GUARD_BUILTINS_VERSION 356
+#define _GUARD_DORV_NO_DICT 357
+#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 358
+#define _GUARD_GLOBALS_VERSION 359
+#define _GUARD_IS_FALSE_POP 360
+#define _GUARD_IS_NONE_POP 361
+#define _GUARD_IS_NOT_NONE_POP 362
+#define _GUARD_IS_TRUE_POP 363
+#define _GUARD_KEYS_VERSION 364
+#define _GUARD_NOS_FLOAT 365
+#define _GUARD_NOS_INT 366
+#define _GUARD_NOT_EXHAUSTED_LIST 367
+#define _GUARD_NOT_EXHAUSTED_RANGE 368
+#define _GUARD_NOT_EXHAUSTED_TUPLE 369
+#define _GUARD_TOS_FLOAT 370
+#define _GUARD_TOS_INT 371
+#define _GUARD_TYPE_VERSION 372
+#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 373
+#define _INIT_CALL_PY_EXACT_ARGS 374
+#define _INIT_CALL_PY_EXACT_ARGS_0 375
+#define _INIT_CALL_PY_EXACT_ARGS_1 376
+#define _INIT_CALL_PY_EXACT_ARGS_2 377
+#define _INIT_CALL_PY_EXACT_ARGS_3 378
+#define _INIT_CALL_PY_EXACT_ARGS_4 379
 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL
 #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX
 #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW
@@ -152,65 +151,65 @@ extern "C" {
 #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST
 #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE
 #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE
-#define _INTERNAL_INCREMENT_OPT_COUNTER 381
-#define _IS_NONE 382
+#define _INTERNAL_INCREMENT_OPT_COUNTER 380
+#define _IS_NONE 381
 #define _IS_OP IS_OP
-#define _ITER_CHECK_LIST 383
-#define _ITER_CHECK_RANGE 384
-#define _ITER_CHECK_TUPLE 385
-#define _ITER_JUMP_LIST 386
-#define _ITER_JUMP_RANGE 387
-#define _ITER_JUMP_TUPLE 388
-#define _ITER_NEXT_LIST 389
-#define _ITER_NEXT_RANGE 390
-#define _ITER_NEXT_TUPLE 391
-#define _JUMP_TO_TOP 392
+#define _ITER_CHECK_LIST 382
+#define _ITER_CHECK_RANGE 383
+#define _ITER_CHECK_TUPLE 384
+#define _ITER_JUMP_LIST 385
+#define _ITER_JUMP_RANGE 386
+#define _ITER_JUMP_TUPLE 387
+#define _ITER_NEXT_LIST 388
+#define _ITER_NEXT_RANGE 389
+#define _ITER_NEXT_TUPLE 390
+#define _JUMP_TO_TOP 391
 #define _LIST_APPEND LIST_APPEND
 #define _LIST_EXTEND LIST_EXTEND
-#define _LOAD_ATTR 393
-#define _LOAD_ATTR_CLASS 394
-#define _LOAD_ATTR_CLASS_0 395
-#define _LOAD_ATTR_CLASS_1 396
+#define _LOAD_ATTR 392
+#define _LOAD_ATTR_CLASS 393
+#define _LOAD_ATTR_CLASS_0 394
+#define _LOAD_ATTR_CLASS_1 395
 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN
-#define _LOAD_ATTR_INSTANCE_VALUE 397
-#define _LOAD_ATTR_INSTANCE_VALUE_0 398
-#define _LOAD_ATTR_INSTANCE_VALUE_1 399
-#define _LOAD_ATTR_METHOD_LAZY_DICT 400
-#define _LOAD_ATTR_METHOD_NO_DICT 401
-#define _LOAD_ATTR_METHOD_WITH_VALUES 402
-#define _LOAD_ATTR_MODULE 403
-#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 404
-#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 405
+#define _LOAD_ATTR_INSTANCE_VALUE 396
+#define _LOAD_ATTR_INSTANCE_VALUE_0 397
+#define _LOAD_ATTR_INSTANCE_VALUE_1 398
+#define _LOAD_ATTR_METHOD_LAZY_DICT 399
+#define _LOAD_ATTR_METHOD_NO_DICT 400
+#define _LOAD_ATTR_METHOD_WITH_VALUES 401
+#define _LOAD_ATTR_MODULE 402
+#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 403
+#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 404
 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY
-#define _LOAD_ATTR_SLOT 406
-#define _LOAD_ATTR_SLOT_0 407
-#define _LOAD_ATTR_SLOT_1 408
-#define _LOAD_ATTR_WITH_HINT 409
+#define _LOAD_ATTR_SLOT 405
+#define _LOAD_ATTR_SLOT_0 406
+#define _LOAD_ATTR_SLOT_1 407
+#define _LOAD_ATTR_WITH_HINT 408
 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS
 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT
 #define _LOAD_CONST LOAD_CONST
-#define _LOAD_CONST_INLINE 410
-#define _LOAD_CONST_INLINE_BORROW 411
-#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 412
-#define _LOAD_CONST_INLINE_WITH_NULL 413
+#define _LOAD_CONST_INLINE 409
+#define _LOAD_CONST_INLINE_BORROW 410
+#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 411
+#define _LOAD_CONST_INLINE_WITH_NULL 412
 #define _LOAD_DEREF LOAD_DEREF
-#define _LOAD_FAST 414
-#define _LOAD_FAST_0 415
-#define _LOAD_FAST_1 416
-#define _LOAD_FAST_2 417
-#define _LOAD_FAST_3 418
-#define _LOAD_FAST_4 419
-#define _LOAD_FAST_5 420
-#define _LOAD_FAST_6 421
-#define _LOAD_FAST_7 422
+#define _LOAD_FAST 413
+#define _LOAD_FAST_0 414
+#define _LOAD_FAST_1 415
+#define _LOAD_FAST_2 416
+#define _LOAD_FAST_3 417
+#define _LOAD_FAST_4 418
+#define _LOAD_FAST_5 419
+#define _LOAD_FAST_6 420
+#define _LOAD_FAST_7 421
 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR
 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK
 #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST
 #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF
 #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS
-#define _LOAD_GLOBAL 423
-#define _LOAD_GLOBAL_BUILTINS 424
-#define _LOAD_GLOBAL_MODULE 425
+#define _LOAD_GLOBAL 422
+#define _LOAD_GLOBAL_BUILTINS 423
+#define _LOAD_GLOBAL_MODULE 424
 #define _LOAD_LOCALS LOAD_LOCALS
 #define _LOAD_NAME LOAD_NAME
 #define _LOAD_SPECIAL LOAD_SPECIAL
@@ -225,51 +224,51 @@ extern "C" {
 #define _MATCH_SEQUENCE MATCH_SEQUENCE
 #define _NOP NOP
 #define _POP_EXCEPT POP_EXCEPT
-#define _POP_JUMP_IF_FALSE 426
-#define _POP_JUMP_IF_TRUE 427
+#define _POP_JUMP_IF_FALSE 425
+#define _POP_JUMP_IF_TRUE 426
 #define _POP_TOP POP_TOP
-#define _POP_TOP_LOAD_CONST_INLINE_BORROW 428
+#define _POP_TOP_LOAD_CONST_INLINE_BORROW 427
 #define _PUSH_EXC_INFO PUSH_EXC_INFO
-#define _PUSH_FRAME 429
+#define _PUSH_FRAME 428
 #define _PUSH_NULL PUSH_NULL
-#define _PY_FRAME_GENERAL 430
-#define _REPLACE_WITH_TRUE 431
+#define _PY_FRAME_GENERAL 429
+#define _REPLACE_WITH_TRUE 430
 #define _RESUME_CHECK RESUME_CHECK
 #define _RETURN_GENERATOR RETURN_GENERATOR
 #define _RETURN_VALUE RETURN_VALUE
-#define _SAVE_RETURN_OFFSET 432
-#define _SEND 433
+#define _SAVE_RETURN_OFFSET 431
+#define _SEND 432
 #define _SEND_GEN SEND_GEN
 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS
 #define _SET_ADD SET_ADD
 #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE
 #define _SET_UPDATE SET_UPDATE
-#define _START_EXECUTOR 434
-#define _STORE_ATTR 435
-#define _STORE_ATTR_INSTANCE_VALUE 436
-#define _STORE_ATTR_SLOT 437
-#define _STORE_ATTR_WITH_HINT 438
+#define _START_EXECUTOR 433
+#define _STORE_ATTR 434
+#define _STORE_ATTR_INSTANCE_VALUE 435
+#define _STORE_ATTR_SLOT 436
+#define _STORE_ATTR_WITH_HINT 437
 #define _STORE_DEREF STORE_DEREF
-#define _STORE_FAST 439
-#define _STORE_FAST_0 440
-#define _STORE_FAST_1 441
-#define _STORE_FAST_2 442
-#define _STORE_FAST_3 443
-#define _STORE_FAST_4 444
-#define _STORE_FAST_5 445
-#define _STORE_FAST_6 446
-#define _STORE_FAST_7 447
+#define _STORE_FAST 438
+#define _STORE_FAST_0 439
+#define _STORE_FAST_1 440
+#define _STORE_FAST_2 441
+#define _STORE_FAST_3 442
+#define _STORE_FAST_4 443
+#define _STORE_FAST_5 444
+#define _STORE_FAST_6 445
+#define _STORE_FAST_7 446
 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST
 #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST
 #define _STORE_GLOBAL STORE_GLOBAL
 #define _STORE_NAME STORE_NAME
 #define _STORE_SLICE STORE_SLICE
-#define _STORE_SUBSCR 448
+#define _STORE_SUBSCR 447
 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT
 #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT
 #define _SWAP SWAP
-#define _TIER2_RESUME_CHECK 449
-#define _TO_BOOL 450
+#define _TIER2_RESUME_CHECK 448
+#define _TO_BOOL 449
 #define _TO_BOOL_BOOL TO_BOOL_BOOL
 #define _TO_BOOL_INT TO_BOOL_INT
 #define _TO_BOOL_LIST TO_BOOL_LIST
@@ -279,13 +278,13 @@ extern "C" {
 #define _UNARY_NEGATIVE UNARY_NEGATIVE
 #define _UNARY_NOT UNARY_NOT
 #define _UNPACK_EX UNPACK_EX
-#define _UNPACK_SEQUENCE 451
+#define _UNPACK_SEQUENCE 450
 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST
 #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE
 #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE
 #define _WITH_EXCEPT_START WITH_EXCEPT_START
 #define _YIELD_VALUE YIELD_VALUE
-#define MAX_UOP_ID 451
+#define MAX_UOP_ID 450
 
 #ifdef __cplusplus
 }
diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h
index 5aef6ba..2a2d6e9 100644
--- a/Include/internal/pycore_uop_metadata.h
+++ b/Include/internal/pycore_uop_metadata.h
@@ -244,7 +244,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_SET_IP] = 0,
     [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG,
     [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG,
-    [_EXIT_TRACE] = 0,
+    [_EXIT_TRACE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_CHECK_VALIDITY] = HAS_DEOPT_FLAG,
     [_LOAD_CONST_INLINE] = HAS_PURE_FLAG,
     [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG,
@@ -253,7 +253,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {
     [_LOAD_CONST_INLINE_BORROW_WITH_NULL] = HAS_PURE_FLAG,
     [_CHECK_FUNCTION] = HAS_DEOPT_FLAG,
     [_INTERNAL_INCREMENT_OPT_COUNTER] = 0,
-    [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG,
     [_START_EXECUTOR] = HAS_DEOPT_FLAG,
     [_FATAL_ERROR] = 0,
@@ -325,7 +324,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = {
     [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND",
     [_CHECK_VALIDITY] = "_CHECK_VALIDITY",
     [_CHECK_VALIDITY_AND_SET_IP] = "_CHECK_VALIDITY_AND_SET_IP",
-    [_COLD_EXIT] = "_COLD_EXIT",
     [_COMPARE_OP] = "_COMPARE_OP",
     [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT",
     [_COMPARE_OP_INT] = "_COMPARE_OP_INT",
@@ -984,8 +982,6 @@ int _PyUop_num_popped(int opcode, int oparg)
             return 0;
         case _INTERNAL_INCREMENT_OPT_COUNTER:
             return 1;
-        case _COLD_EXIT:
-            return 0;
         case _DYNAMIC_EXIT:
             return 0;
         case _START_EXECUTOR: