GH-98686: Quicken everything (GH-98687)

5 files changed, 25 insertions, 86 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 100aa3d..2b42c68 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -143,7 +143,7 @@ lltrace_instruction(_PyInterpreterFrame *frame,
     const char *opname = _PyOpcode_OpName[opcode];
     assert(opname != NULL);
     int offset = (int)(next_instr - _PyCode_CODE(frame->f_code));
-    if (HAS_ARG(opcode)) {
+    if (HAS_ARG(_PyOpcode_Deopt[opcode])) {
         printf("%d: %s %d\n", offset * 2, opname, oparg);
     }
     else {
@@ -1165,14 +1165,7 @@ handle_eval_breaker:
         TARGET(NOP) {
             DISPATCH();
         }
-
         TARGET(RESUME) {
-            _PyCode_Warmup(frame->f_code);
-            GO_TO_INSTRUCTION(RESUME_QUICK);
-        }
-
-        TARGET(RESUME_QUICK) {
-            PREDICTED(RESUME_QUICK);
             assert(tstate->cframe == &cframe);
             assert(frame == cframe.current_frame);
             if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
@@ -1710,7 +1703,7 @@ handle_eval_breaker:
             PyObject *list = PEEK(oparg);
             if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
                 goto error;
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -1722,7 +1715,7 @@ handle_eval_breaker:
             Py_DECREF(v);
             if (err != 0)
                 goto error;
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -2908,7 +2901,7 @@ handle_eval_breaker:
             if (_PyDict_SetItem_Take2((PyDictObject *)map, key, value) != 0) {
                 goto error;
             }
-            PREDICT(JUMP_BACKWARD_QUICK);
+            PREDICT(JUMP_BACKWARD);
             DISPATCH();
         }
 
@@ -3576,8 +3569,11 @@ handle_eval_breaker:
         }
 
         TARGET(JUMP_BACKWARD) {
-            _PyCode_Warmup(frame->f_code);
-            GO_TO_INSTRUCTION(JUMP_BACKWARD_QUICK);
+            PREDICTED(JUMP_BACKWARD);
+            assert(oparg < INSTR_OFFSET());
+            JUMPBY(-oparg);
+            CHECK_EVAL_BREAKER();
+            DISPATCH();
         }
 
         TARGET(POP_JUMP_IF_FALSE) {
@@ -3707,14 +3703,6 @@ handle_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(JUMP_BACKWARD_QUICK) {
-            PREDICTED(JUMP_BACKWARD_QUICK);
-            assert(oparg < INSTR_OFFSET());
-            JUMPBY(-oparg);
-            CHECK_EVAL_BREAKER();
-            DISPATCH();
-        }
-
         TARGET(GET_LEN) {
             // PUSH(len(TOS))
             Py_ssize_t len_i = PyObject_Length(TOP());
diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h
index 3dc7aa8..5678d0a 100644
--- a/Python/clinic/sysmodule.c.h
+++ b/Python/clinic/sysmodule.c.h
@@ -884,33 +884,6 @@ exit:
 
 #endif /* defined(Py_REF_DEBUG) */
 
-PyDoc_STRVAR(sys__getquickenedcount__doc__,
-"_getquickenedcount($module, /)\n"
-"--\n"
-"\n");
-
-#define SYS__GETQUICKENEDCOUNT_METHODDEF    \
-    {"_getquickenedcount", (PyCFunction)sys__getquickenedcount, METH_NOARGS, sys__getquickenedcount__doc__},
-
-static Py_ssize_t
-sys__getquickenedcount_impl(PyObject *module);
-
-static PyObject *
-sys__getquickenedcount(PyObject *module, PyObject *Py_UNUSED(ignored))
-{
-    PyObject *return_value = NULL;
-    Py_ssize_t _return_value;
-
-    _return_value = sys__getquickenedcount_impl(module);
-    if ((_return_value == -1) && PyErr_Occurred()) {
-        goto exit;
-    }
-    return_value = PyLong_FromSsize_t(_return_value);
-
-exit:
-    return return_value;
-}
-
 PyDoc_STRVAR(sys_getallocatedblocks__doc__,
 "getallocatedblocks($module, /)\n"
 "--\n"
@@ -1345,4 +1318,4 @@ sys_is_stack_trampoline_active(PyObject *module, PyObject *Py_UNUSED(ignored))
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=2b5e1bc24a3348bd input=a9049054013a1b77]*/
+/*[clinic end generated code: output=79228e569529129c input=a9049054013a1b77]*/
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 53b2770..9004bba 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -71,21 +71,21 @@ static void *opcode_targets[256] = {
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
-    &&TARGET_JUMP_BACKWARD_QUICK,
     &&TARGET_LOAD_ATTR_ADAPTIVE,
+    &&TARGET_LOAD_ATTR_CLASS,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
-    &&TARGET_LOAD_ATTR_CLASS,
     &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN,
     &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
     &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_LOAD_ATTR_PROPERTY,
     &&TARGET_LOAD_ATTR_SLOT,
+    &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
-    &&TARGET_LOAD_ATTR_WITH_HINT,
+    &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
     &&TARGET_ASYNC_GEN_WRAP,
     &&TARGET_PREP_RERAISE_STAR,
     &&TARGET_POP_EXCEPT,
@@ -112,7 +112,7 @@ static void *opcode_targets[256] = {
     &&TARGET_JUMP_FORWARD,
     &&TARGET_JUMP_IF_FALSE_OR_POP,
     &&TARGET_JUMP_IF_TRUE_OR_POP,
-    &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT,
+    &&TARGET_LOAD_ATTR_METHOD_NO_DICT,
     &&TARGET_POP_JUMP_IF_FALSE,
     &&TARGET_POP_JUMP_IF_TRUE,
     &&TARGET_LOAD_GLOBAL,
@@ -120,7 +120,7 @@ static void *opcode_targets[256] = {
     &&TARGET_CONTAINS_OP,
     &&TARGET_RERAISE,
     &&TARGET_COPY,
-    &&TARGET_LOAD_ATTR_METHOD_NO_DICT,
+    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
     &&TARGET_BINARY_OP,
     &&TARGET_SEND,
     &&TARGET_LOAD_FAST,
@@ -140,9 +140,9 @@ static void *opcode_targets[256] = {
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
     &&TARGET_JUMP_BACKWARD,
-    &&TARGET_LOAD_ATTR_METHOD_WITH_DICT,
-    &&TARGET_CALL_FUNCTION_EX,
     &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES,
+    &&TARGET_CALL_FUNCTION_EX,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
@@ -152,28 +152,26 @@ static void *opcode_targets[256] = {
     &&TARGET_YIELD_VALUE,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
     &&TARGET_BUILD_STRING,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
     &&TARGET_LOAD_GLOBAL_MODULE,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_LIST_EXTEND,
     &&TARGET_SET_UPDATE,
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
-    &&TARGET_RESUME_QUICK,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
     &&TARGET_STORE_ATTR_WITH_HINT,
-    &&TARGET_CALL,
-    &&TARGET_KW_NAMES,
     &&TARGET_STORE_FAST__LOAD_FAST,
     &&TARGET_STORE_FAST__STORE_FAST,
+    &&TARGET_CALL,
+    &&TARGET_KW_NAMES,
     &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_STORE_SUBSCR_LIST_INT,
@@ -254,5 +252,7 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
+    &&_unknown_opcode,
+    &&_unknown_opcode,
     &&TARGET_DO_TRACING
 };
diff --git a/Python/specialize.c b/Python/specialize.c
index 6e2fa4e..70a456c 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -31,7 +31,6 @@ uint8_t _PyOpcode_Adaptive[256] = {
     [FOR_ITER] = FOR_ITER_ADAPTIVE,
 };
 
-Py_ssize_t _Py_QuickenedCount = 0;
 #ifdef Py_STATS
 PyStats _py_stats_struct = { 0 };
 PyStats *_py_stats = &_py_stats_struct;
@@ -280,16 +279,14 @@ do { \
 void
 _PyCode_Quicken(PyCodeObject *code)
 {
-    _Py_QuickenedCount++;
     int previous_opcode = -1;
     _Py_CODEUNIT *instructions = _PyCode_CODE(code);
     for (int i = 0; i < Py_SIZE(code); i++) {
-        int opcode = _Py_OPCODE(instructions[i]);
+        int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])];
         uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
         if (adaptive_opcode) {
             _Py_SET_OPCODE(instructions[i], adaptive_opcode);
-            // Make sure the adaptive counter is zero:
-            assert(instructions[i + 1] == 0);
+            instructions[i + 1] = adaptive_counter_start();
             previous_opcode = -1;
             i += _PyOpcode_Caches[opcode];
         }
@@ -299,12 +296,6 @@ _PyCode_Quicken(PyCodeObject *code)
                 case EXTENDED_ARG:
                     _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
                     break;
-                case JUMP_BACKWARD:
-                    _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
-                    break;
-                case RESUME:
-                    _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
-                    break;
                 case LOAD_FAST:
                     switch(previous_opcode) {
                         case LOAD_FAST:
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index f73d332..55cd059 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -17,7 +17,6 @@ Data members:
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_SetAsyncGenFinalizer()
-#include "pycore_code.h"          // _Py_QuickenedCount
 #include "pycore_frame.h"         // _PyInterpreterFrame
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
 #include "pycore_long.h"          // _PY_LONG_MAX_STR_DIGITS_THRESHOLD
@@ -1856,17 +1855,6 @@ sys_gettotalrefcount_impl(PyObject *module)
 #endif /* Py_REF_DEBUG */
 
 /*[clinic input]
-sys._getquickenedcount -> Py_ssize_t
-[clinic start generated code]*/
-
-static Py_ssize_t
-sys__getquickenedcount_impl(PyObject *module)
-/*[clinic end generated code: output=1ab259e7f91248a2 input=249d448159eca912]*/
-{
-    return _Py_QuickenedCount;
-}
-
-/*[clinic input]
 sys.getallocatedblocks -> Py_ssize_t
 
 Return the number of memory blocks currently allocated.
@@ -2216,7 +2204,6 @@ static PyMethodDef sys_methods[] = {
     SYS_GETALLOCATEDBLOCKS_METHODDEF
     SYS_GETFILESYSTEMENCODING_METHODDEF
     SYS_GETFILESYSTEMENCODEERRORS_METHODDEF
-    SYS__GETQUICKENEDCOUNT_METHODDEF
 #ifdef Py_TRACE_REFS
     {"getobjects", _Py_GetObjects, METH_VARARGS},
 #endif