GH-98686: Quicken everything (GH-98687)

4 files changed, 52 insertions, 75 deletions

diff --git a/Include/cpython/code.h b/Include/cpython/code.h
index 893ff93..ebac0b1 100644
--- a/Include/cpython/code.h
+++ b/Include/cpython/code.h
@@ -70,7 +70,6 @@ typedef struct {
     PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
                                       table */                                 \
     int co_flags;                  /* CO_..., see below */                     \
-    short co_warmup;                 /* Warmup counter for quickening */       \
     short _co_linearray_entry_size;  /* Size of each entry in _co_linearray */ \
                                                                                \
     /* The rest are not so impactful on performance. */                        \
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index bf59454..cf8573a 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -91,28 +91,8 @@ typedef struct {
 
 #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache)
 
-#define QUICKENING_WARMUP_DELAY 8
-
-/* We want to compare to zero for efficiency, so we offset values accordingly */
-#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
-
-void _PyCode_Quicken(PyCodeObject *code);
-
-static inline void
-_PyCode_Warmup(PyCodeObject *code)
-{
-    if (code->co_warmup != 0) {
-        code->co_warmup++;
-        if (code->co_warmup == 0) {
-            _PyCode_Quicken(code);
-        }
-    }
-}
-
 extern uint8_t _PyOpcode_Adaptive[256];
 
-extern Py_ssize_t _Py_QuickenedCount;
-
 // Borrowed references to common callables:
 struct callable_cache {
     PyObject *isinstance;
@@ -252,10 +232,10 @@ extern void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr,
                                           int oparg);
 extern void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr);
 
-/* Deallocator function for static codeobjects used in deepfreeze.py */
-extern void _PyStaticCode_Dealloc(PyCodeObject *co);
-/* Function to intern strings of codeobjects */
-extern int _PyStaticCode_InternStrings(PyCodeObject *co);
+/* Finalizer function for static codeobjects used in deepfreeze.py */
+extern void _PyStaticCode_Fini(PyCodeObject *co);
+/* Function to intern strings of codeobjects and quicken the bytecode */
+extern int _PyStaticCode_Init(PyCodeObject *co);
 
 #ifdef Py_STATS
 
@@ -397,8 +377,8 @@ write_location_entry_start(uint8_t *ptr, int code, int length)
 
 /* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
 #define ADAPTIVE_BACKOFF_BITS 4
-/* The initial counter value is 31 == 2**ADAPTIVE_BACKOFF_START - 1 */
-#define ADAPTIVE_BACKOFF_START 5
+/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */
+#define ADAPTIVE_BACKOFF_START 1
 
 #define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)
 
diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h
index 33617b4..0ef677f 100644
--- a/Include/internal/pycore_opcode.h
+++ b/Include/internal/pycore_opcode.h
@@ -142,7 +142,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [IS_OP] = IS_OP,
     [JUMP_BACKWARD] = JUMP_BACKWARD,
     [JUMP_BACKWARD_NO_INTERRUPT] = JUMP_BACKWARD_NO_INTERRUPT,
-    [JUMP_BACKWARD_QUICK] = JUMP_BACKWARD,
     [JUMP_FORWARD] = JUMP_FORWARD,
     [JUMP_IF_FALSE_OR_POP] = JUMP_IF_FALSE_OR_POP,
     [JUMP_IF_TRUE_OR_POP] = JUMP_IF_TRUE_OR_POP,
@@ -200,7 +199,6 @@ const uint8_t _PyOpcode_Deopt[256] = {
     [RAISE_VARARGS] = RAISE_VARARGS,
     [RERAISE] = RERAISE,
     [RESUME] = RESUME,
-    [RESUME_QUICK] = RESUME,
     [RETURN_GENERATOR] = RETURN_GENERATOR,
     [RETURN_VALUE] = RETURN_VALUE,
     [SEND] = SEND,
@@ -313,21 +311,21 @@ static const char *const _PyOpcode_OpName[263] = {
     [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER",
     [PRINT_EXPR] = "PRINT_EXPR",
     [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS",
-    [JUMP_BACKWARD_QUICK] = "JUMP_BACKWARD_QUICK",
     [LOAD_ATTR_ADAPTIVE] = "LOAD_ATTR_ADAPTIVE",
+    [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
     [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR",
     [RETURN_GENERATOR] = "RETURN_GENERATOR",
-    [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS",
     [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
     [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE",
     [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE",
     [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY",
     [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT",
+    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
     [LIST_TO_TUPLE] = "LIST_TO_TUPLE",
     [RETURN_VALUE] = "RETURN_VALUE",
     [IMPORT_STAR] = "IMPORT_STAR",
     [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS",
-    [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT",
+    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
     [ASYNC_GEN_WRAP] = "ASYNC_GEN_WRAP",
     [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR",
     [POP_EXCEPT] = "POP_EXCEPT",
@@ -354,7 +352,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [JUMP_FORWARD] = "JUMP_FORWARD",
     [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP",
     [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP",
-    [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT",
+    [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
     [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE",
     [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE",
     [LOAD_GLOBAL] = "LOAD_GLOBAL",
@@ -362,7 +360,7 @@ static const char *const _PyOpcode_OpName[263] = {
     [CONTAINS_OP] = "CONTAINS_OP",
     [RERAISE] = "RERAISE",
     [COPY] = "COPY",
-    [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT",
+    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
     [BINARY_OP] = "BINARY_OP",
     [SEND] = "SEND",
     [LOAD_FAST] = "LOAD_FAST",
@@ -382,9 +380,9 @@ static const char *const _PyOpcode_OpName[263] = {
     [STORE_DEREF] = "STORE_DEREF",
     [DELETE_DEREF] = "DELETE_DEREF",
     [JUMP_BACKWARD] = "JUMP_BACKWARD",
-    [LOAD_ATTR_METHOD_WITH_DICT] = "LOAD_ATTR_METHOD_WITH_DICT",
-    [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
     [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES",
+    [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX",
+    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [EXTENDED_ARG] = "EXTENDED_ARG",
     [LIST_APPEND] = "LIST_APPEND",
     [SET_ADD] = "SET_ADD",
@@ -394,28 +392,26 @@ static const char *const _PyOpcode_OpName[263] = {
     [YIELD_VALUE] = "YIELD_VALUE",
     [RESUME] = "RESUME",
     [MATCH_CLASS] = "MATCH_CLASS",
-    [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST",
     [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST",
+    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [FORMAT_VALUE] = "FORMAT_VALUE",
     [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP",
     [BUILD_STRING] = "BUILD_STRING",
-    [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST",
     [LOAD_GLOBAL_ADAPTIVE] = "LOAD_GLOBAL_ADAPTIVE",
     [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN",
     [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE",
+    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
     [LIST_EXTEND] = "LIST_EXTEND",
     [SET_UPDATE] = "SET_UPDATE",
     [DICT_MERGE] = "DICT_MERGE",
     [DICT_UPDATE] = "DICT_UPDATE",
-    [RESUME_QUICK] = "RESUME_QUICK",
-    [STORE_ATTR_ADAPTIVE] = "STORE_ATTR_ADAPTIVE",
     [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE",
     [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT",
     [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT",
-    [CALL] = "CALL",
-    [KW_NAMES] = "KW_NAMES",
     [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST",
     [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST",
+    [CALL] = "CALL",
+    [KW_NAMES] = "KW_NAMES",
     [STORE_SUBSCR_ADAPTIVE] = "STORE_SUBSCR_ADAPTIVE",
     [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT",
     [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT",
@@ -423,6 +419,8 @@ static const char *const _PyOpcode_OpName[263] = {
     [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST",
     [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE",
     [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE",
+    [180] = "<180>",
+    [181] = "<181>",
     [182] = "<182>",
     [183] = "<183>",
     [184] = "<184>",
@@ -508,6 +506,8 @@ static const char *const _PyOpcode_OpName[263] = {
 #endif
 
 #define EXTRA_CASES \
+    case 180: \
+    case 181: \
     case 182: \
     case 183: \
     case 184: \
diff --git a/Include/opcode.h b/Include/opcode.h
index 9b9414c..661b587 100644
--- a/Include/opcode.h
+++ b/Include/opcode.h
@@ -167,39 +167,37 @@ extern "C" {
 #define FOR_ITER_ADAPTIVE                       65
 #define FOR_ITER_LIST                           66
 #define FOR_ITER_RANGE                          67
-#define JUMP_BACKWARD_QUICK                     72
-#define LOAD_ATTR_ADAPTIVE                      73
-#define LOAD_ATTR_CLASS                         76
-#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       77
-#define LOAD_ATTR_INSTANCE_VALUE                78
-#define LOAD_ATTR_MODULE                        79
-#define LOAD_ATTR_PROPERTY                      80
-#define LOAD_ATTR_SLOT                          81
-#define LOAD_ATTR_WITH_HINT                     86
-#define LOAD_ATTR_METHOD_LAZY_DICT             113
-#define LOAD_ATTR_METHOD_NO_DICT               121
-#define LOAD_ATTR_METHOD_WITH_DICT             141
-#define LOAD_ATTR_METHOD_WITH_VALUES           143
-#define LOAD_CONST__LOAD_FAST                  153
-#define LOAD_FAST__LOAD_CONST                  154
-#define LOAD_FAST__LOAD_FAST                   158
-#define LOAD_GLOBAL_ADAPTIVE                   159
-#define LOAD_GLOBAL_BUILTIN                    160
-#define LOAD_GLOBAL_MODULE                     161
-#define RESUME_QUICK                           166
-#define STORE_ATTR_ADAPTIVE                    167
-#define STORE_ATTR_INSTANCE_VALUE              168
-#define STORE_ATTR_SLOT                        169
-#define STORE_ATTR_WITH_HINT                   170
-#define STORE_FAST__LOAD_FAST                  173
-#define STORE_FAST__STORE_FAST                 174
-#define STORE_SUBSCR_ADAPTIVE                  175
-#define STORE_SUBSCR_DICT                      176
-#define STORE_SUBSCR_LIST_INT                  177
-#define UNPACK_SEQUENCE_ADAPTIVE               178
-#define UNPACK_SEQUENCE_LIST                   179
-#define UNPACK_SEQUENCE_TUPLE                  180
-#define UNPACK_SEQUENCE_TWO_TUPLE              181
+#define LOAD_ATTR_ADAPTIVE                      72
+#define LOAD_ATTR_CLASS                         73
+#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN       76
+#define LOAD_ATTR_INSTANCE_VALUE                77
+#define LOAD_ATTR_MODULE                        78
+#define LOAD_ATTR_PROPERTY                      79
+#define LOAD_ATTR_SLOT                          80
+#define LOAD_ATTR_WITH_HINT                     81
+#define LOAD_ATTR_METHOD_LAZY_DICT              86
+#define LOAD_ATTR_METHOD_NO_DICT               113
+#define LOAD_ATTR_METHOD_WITH_DICT             121
+#define LOAD_ATTR_METHOD_WITH_VALUES           141
+#define LOAD_CONST__LOAD_FAST                  143
+#define LOAD_FAST__LOAD_CONST                  153
+#define LOAD_FAST__LOAD_FAST                   154
+#define LOAD_GLOBAL_ADAPTIVE                   158
+#define LOAD_GLOBAL_BUILTIN                    159
+#define LOAD_GLOBAL_MODULE                     160
+#define STORE_ATTR_ADAPTIVE                    161
+#define STORE_ATTR_INSTANCE_VALUE              166
+#define STORE_ATTR_SLOT                        167
+#define STORE_ATTR_WITH_HINT                   168
+#define STORE_FAST__LOAD_FAST                  169
+#define STORE_FAST__STORE_FAST                 170
+#define STORE_SUBSCR_ADAPTIVE                  173
+#define STORE_SUBSCR_DICT                      174
+#define STORE_SUBSCR_LIST_INT                  175
+#define UNPACK_SEQUENCE_ADAPTIVE               176
+#define UNPACK_SEQUENCE_LIST                   177
+#define UNPACK_SEQUENCE_TUPLE                  178
+#define UNPACK_SEQUENCE_TWO_TUPLE              179
 #define DO_TRACING                             255
 
 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\