bpo-46329: Split calls into precall and call instructions. (GH-30855)

* Add PRECALL_FUNCTION opcode.

* Move 'call shape' varaibles into struct.

* Replace CALL_NO_KW and CALL_KW with KW_NAMES and CALL instructions.

* Specialize for builtin methods taking using the METH_FASTCALL | METH_KEYWORDS protocol.

* Allow kwnames for specialized calls to builtin types.

* Specialize calls to tuple(arg) and str(arg).

5 files changed, 540 insertions, 331 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 106e408..cd05780 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1636,6 +1636,17 @@ pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
     return prev_frame;
 }
 
+/* It is only between a PRECALL_METHOD/FUNCTION instruction and the following CALL,
+ * that these values have any meaning.
+ */
+typedef struct {
+    PyObject *callable;
+    PyObject *kwnames;
+    int total_args;
+    int postcall_shrink;
+} CallShape;
+
+
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
 {
@@ -1654,22 +1665,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
     _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;
 
     CFrame cframe;
-
-    /* Variables used for making calls */
-    PyObject *kwnames;
-    int nargs;
-    /*
-     * It is only between a PRECALL_METHOD instruction and the following instruction,
-     * that these two values can be anything other than their defaults. */
-    int postcall_shrink = 1;
-    int extra_args = 0;
-#define RESET_STACK_ADJUST_FOR_CALLS \
-    do { \
-        postcall_shrink = 1; \
-        extra_args = 0; \
-    } while (0)
-#define STACK_ADJUST_IS_RESET \
-    (postcall_shrink == 1 && extra_args == 0)
+    CallShape call_shape;
+    call_shape.kwnames = NULL; // Borrowed reference
+    call_shape.postcall_shrink = 0;
+    call_shape.total_args = 0;
+    call_shape.callable = NULL; // Strong reference
 
     /* WARNING: Because the CFrame lives on the C stack,
      * but can be accessed from a heap allocated object (tstate)
@@ -2532,12 +2532,12 @@ handle_eval_breaker:
             PyObject *iter = _PyCoro_GetAwaitableIter(iterable);
 
             if (iter == NULL) {
-                int opcode_at_minus_3 = 0;
-                if ((next_instr - first_instr) > 2) {
-                    opcode_at_minus_3 = _Py_OPCODE(next_instr[-3]);
+                int opcode_at_minus_4 = 0;
+                if ((next_instr - first_instr) > 4) {
+                    opcode_at_minus_4 = _Py_OPCODE(next_instr[-4]);
                 }
                 format_awaitable_error(tstate, Py_TYPE(iterable),
-                                       opcode_at_minus_3,
+                                       opcode_at_minus_4,
                                        _Py_OPCODE(next_instr[-2]));
             }
 
@@ -4173,7 +4173,6 @@ handle_eval_breaker:
             if (iter == NULL)
                 goto error;
             PREDICT(FOR_ITER);
-            PREDICT(CALL_NO_KW);
             DISPATCH();
         }
 
@@ -4494,6 +4493,16 @@ handle_eval_breaker:
             NOTRACE_DISPATCH();
         }
 
+        TARGET(PRECALL_FUNCTION) {
+            /* Move ownership of reference from stack to call_shape */
+            call_shape.callable = PEEK(oparg + 1);
+            call_shape.postcall_shrink = 1;
+
+            call_shape.total_args = oparg;
+            call_shape.kwnames = NULL;
+            DISPATCH();
+        }
+
         TARGET(PRECALL_METHOD) {
             /* Designed to work in tamdem with LOAD_METHOD. */
             /* `meth` is NULL when LOAD_METHOD thinks that it's not
@@ -4522,54 +4531,57 @@ handle_eval_breaker:
                make it accept the `self` as a first argument.
             */
             int is_method = (PEEK(oparg + 2) != NULL);
-            extra_args = is_method;
-            postcall_shrink = 2-is_method;
+            int nargs = oparg + is_method;
+            /* Move ownership of reference from stack to call_shape
+             * and make sure that NULL is cleared from stack */
+            call_shape.callable = PEEK(nargs + 1);
+            call_shape.postcall_shrink = 2-is_method;
+
+            call_shape.total_args = nargs;
+            call_shape.kwnames = NULL;
             DISPATCH();
         }
 
-        TARGET(CALL_KW) {
-            kwnames = POP();
-            oparg += extra_args;
-            extra_args = 0;
-            nargs = oparg - (int)PyTuple_GET_SIZE(kwnames);
-            goto call_function;
+        TARGET(KW_NAMES) {
+            assert(call_shape.kwnames == NULL);
+            assert(oparg < PyTuple_GET_SIZE(consts));
+            call_shape.kwnames = GETITEM(consts, oparg);
+            DISPATCH();
         }
 
-        TARGET(CALL_NO_KW) {
+        TARGET(CALL) {
+            PREDICTED(CALL);
             PyObject *function;
-            PREDICTED(CALL_NO_KW);
-            kwnames = NULL;
-            oparg += extra_args;
-            nargs = oparg;
+            assert((oparg == 0 && call_shape.kwnames == NULL)
+                || (oparg != 0 && oparg == PyTuple_GET_SIZE(call_shape.kwnames)));
         call_function:
-            function = PEEK(oparg + 1);
+            function = call_shape.callable;
             if (Py_TYPE(function) == &PyMethod_Type) {
                 PyObject *meth = ((PyMethodObject *)function)->im_func;
                 PyObject *self = ((PyMethodObject *)function)->im_self;
                 Py_INCREF(meth);
                 Py_INCREF(self);
-                PEEK(oparg + 1) = self;
+                PEEK(call_shape.total_args + 1) = self;
                 Py_DECREF(function);
                 function = meth;
-                oparg++;
-                nargs++;
-                assert(postcall_shrink >= 1);
-                postcall_shrink--;
+                call_shape.total_args++;
+                assert(call_shape.postcall_shrink >= 1);
+                call_shape.postcall_shrink--;
             }
+            int total_args = call_shape.total_args;
+            int positional_args = total_args - oparg;
             // Check if the call can be inlined or not
             if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
                 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
                 PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
-                STACK_SHRINK(oparg);
+                STACK_SHRINK(total_args);
                 InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
                     tstate, (PyFunctionObject *)function, locals,
-                    stack_pointer, nargs, kwnames
+                    stack_pointer, positional_args, call_shape.kwnames
                 );
-                STACK_SHRINK(postcall_shrink);
-                RESET_STACK_ADJUST_FOR_CALLS;
+                STACK_SHRINK(call_shape.postcall_shrink);
                 // The frame has stolen all the arguments from the stack,
                 // so there is no need to clean them up.
-                Py_XDECREF(kwnames);
                 Py_DECREF(function);
                 if (new_frame == NULL) {
                     goto error;
@@ -4582,22 +4594,24 @@ handle_eval_breaker:
             /* Callable is not a normal Python function */
             PyObject *res;
             if (cframe.use_tracing) {
-                res = trace_call_function(tstate, function, stack_pointer-oparg, nargs, kwnames);
+                res = trace_call_function(
+                    tstate, function, stack_pointer-total_args,
+                    positional_args, call_shape.kwnames);
             }
             else {
-                res = PyObject_Vectorcall(function, stack_pointer-oparg,
-                                          nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
+                res = PyObject_Vectorcall(
+                    function, stack_pointer-total_args,
+                    positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET,
+                    call_shape.kwnames);
             }
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
             Py_DECREF(function);
-            Py_XDECREF(kwnames);
             /* Clear the stack */
-            STACK_SHRINK(oparg);
-            for (int i = 0; i < oparg; i++) {
+            STACK_SHRINK(total_args);
+            for (int i = 0; i < total_args; i++) {
                 Py_DECREF(stack_pointer[i]);
             }
-            STACK_SHRINK(postcall_shrink);
-            RESET_STACK_ADJUST_FOR_CALLS;
+            STACK_SHRINK(call_shape.postcall_shrink);
             PUSH(res);
             if (res == NULL) {
                 goto error;
@@ -4606,65 +4620,87 @@ handle_eval_breaker:
             DISPATCH();
         }
 
-        TARGET(CALL_NO_KW_ADAPTIVE) {
+        TARGET(CALL_ADAPTIVE) {
             SpecializedCacheEntry *cache = GET_CACHE();
-            oparg = cache->adaptive.original_oparg;
+            int named_args = cache->adaptive.original_oparg;
+            assert((named_args == 0 && call_shape.kwnames == NULL)
+                || (named_args != 0 && named_args == PyTuple_GET_SIZE(call_shape.kwnames)));
             if (cache->adaptive.counter == 0) {
                 next_instr--;
-                int nargs = oparg+extra_args;
-                if (_Py_Specialize_CallNoKw(
-                    PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) {
+                int nargs = call_shape.total_args;
+                int err = _Py_Specialize_CallNoKw(
+                    call_shape.callable, next_instr, nargs,
+                    call_shape.kwnames, cache, BUILTINS());
+                if (err < 0) {
                     goto error;
                 }
                 DISPATCH();
             }
             else {
-                STAT_INC(CALL_NO_KW, deferred);
+                STAT_INC(CALL, deferred);
                 cache->adaptive.counter--;
-                kwnames = NULL;
-                oparg += extra_args;
-                nargs = oparg;
+                oparg = named_args;
                 goto call_function;
             }
         }
 
-        TARGET(CALL_NO_KW_PY_SIMPLE) {
+        TARGET(CALL_PY_EXACT_ARGS) {
             SpecializedCacheEntry *caches = GET_CACHE();
-            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
-            int argcount = cache0->original_oparg + extra_args;
-            DEOPT_IF(argcount != cache0->index, CALL_NO_KW);
+            int argcount = call_shape.total_args;
+            DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL);
             _PyCallCache *cache1 = &caches[-1].call;
-            PyObject *callable = PEEK(argcount+1);
-            DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW);
-            PyFunctionObject *func = (PyFunctionObject *)callable;
-            DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW);
-            /* PEP 523 */
-            DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
+            PyFunctionObject *func = (PyFunctionObject *)call_shape.callable;
+            DEOPT_IF(func->func_version != cache1->func_version, CALL);
             PyCodeObject *code = (PyCodeObject *)func->func_code;
-            size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
-            InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
+            DEOPT_IF(code->co_argcount != argcount, CALL);
+            InterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
             if (new_frame == NULL) {
-                RESET_STACK_ADJUST_FOR_CALLS;
                 goto error;
             }
-            _PyFrame_InitializeSpecials(new_frame, func,
-                                        NULL, code->co_nlocalsplus);
             STACK_SHRINK(argcount);
             for (int i = 0; i < argcount; i++) {
                 new_frame->localsplus[i] = stack_pointer[i];
             }
-            int deflen = cache1->defaults_len;
-            for (int i = 0; i < deflen; i++) {
-                PyObject *def = PyTuple_GET_ITEM(func->func_defaults, cache1->defaults_start+i);
+            for (int i = argcount; i < code->co_nlocalsplus; i++) {
+                new_frame->localsplus[i] = NULL;
+            }
+            STACK_SHRINK(call_shape.postcall_shrink);
+            Py_DECREF(func);
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            new_frame->previous = frame;
+            frame = cframe.current_frame = new_frame;
+            goto start_frame;
+        }
+
+        TARGET(CALL_PY_WITH_DEFAULTS) {
+            SpecializedCacheEntry *caches = GET_CACHE();
+            int argcount = call_shape.total_args;
+            DEOPT_IF(!PyFunction_Check(call_shape.callable), CALL);
+            _PyCallCache *cache1 = &caches[-1].call;
+            PyFunctionObject *func = (PyFunctionObject *)call_shape.callable;
+            DEOPT_IF(func->func_version != cache1->func_version, CALL);
+            PyCodeObject *code = (PyCodeObject *)func->func_code;
+            DEOPT_IF(argcount > code->co_argcount, CALL);
+            int minargs = cache1->min_args;
+            DEOPT_IF(argcount < minargs, CALL);
+            InterpreterFrame *new_frame = _PyFrame_Push(tstate, func);
+            if (new_frame == NULL) {
+                goto error;
+            }
+            STACK_SHRINK(argcount);
+            for (int i = 0; i < argcount; i++) {
+                new_frame->localsplus[i] = stack_pointer[i];
+            }
+            int def_offset = cache1->defaults_len - code->co_argcount;
+            for (int i = argcount; i < code->co_argcount; i++) {
+                PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i + def_offset);
                 Py_INCREF(def);
-                new_frame->localsplus[argcount+i] = def;
+                new_frame->localsplus[i] = def;
             }
-            for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) {
+            for (int i = code->co_argcount; i < code->co_nlocalsplus; i++) {
                 new_frame->localsplus[i] = NULL;
             }
-            STACK_SHRINK(postcall_shrink);
-            RESET_STACK_ADJUST_FOR_CALLS;
+            STACK_SHRINK(call_shape.postcall_shrink);
             Py_DECREF(func);
             _PyFrame_SetStackPointer(frame, stack_pointer);
             new_frame->previous = frame;
@@ -4674,35 +4710,75 @@ handle_eval_breaker:
 
         TARGET(CALL_NO_KW_TYPE_1) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
-            assert(GET_CACHE()->adaptive.original_oparg == 1);
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            assert(call_shape.kwnames == NULL);
             PyObject *obj = TOP();
             PyObject *callable = SECOND();
-            DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL_NO_KW);
+            DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL);
             PyObject *res = Py_NewRef(Py_TYPE(obj));
-            STACK_SHRINK(1);
             Py_DECREF(callable);
             Py_DECREF(obj);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
             NOTRACE_DISPATCH();
         }
 
-        TARGET(CALL_NO_KW_BUILTIN_CLASS_1) {
+        TARGET(CALL_NO_KW_STR_1) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
-            SpecializedCacheEntry *caches = GET_CACHE();
-            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
-            assert(cache0->original_oparg == 1);
-            PyObject *callable = SECOND();
+            DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
+            PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            DEOPT_IF(tp != &PyUnicode_Type, CALL);
+            STAT_INC(CALL, hit);
+            assert(call_shape.kwnames == NULL);
             PyObject *arg = TOP();
-            DEOPT_IF(!PyType_Check(callable), CALL_NO_KW);
-            PyTypeObject *tp = (PyTypeObject *)callable;
-            DEOPT_IF(tp->tp_version_tag != cache0->version, CALL_NO_KW);
-            STACK_SHRINK(1);
-            PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, 1, NULL);
+            PyObject *res = PyObject_Str(arg);
+            Py_DECREF(arg);
+            Py_DECREF(&PyUnicode_Type);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
-            Py_DECREF(tp);
+            if (res == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(CALL_NO_KW_TUPLE_1) {
+            DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
+            PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            DEOPT_IF(tp != &PyTuple_Type, CALL);
+            STAT_INC(CALL, hit);
+            assert(call_shape.kwnames == NULL);
+            PyObject *arg = TOP();
+            PyObject *res = PySequence_Tuple(arg);
             Py_DECREF(arg);
+            Py_DECREF(&PyTuple_Type);
+            STACK_SHRINK(call_shape.postcall_shrink);
+            SET_TOP(res);
+            if (res == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(CALL_BUILTIN_CLASS) {
+            DEOPT_IF(!PyType_Check(call_shape.callable), CALL);
+            PyTypeObject *tp = (PyTypeObject *)call_shape.callable;
+            DEOPT_IF(tp->tp_vectorcall == NULL, CALL);
+            STAT_INC(CALL, hit);
+            int kwnames_len = GET_CACHE()->adaptive.original_oparg;
+
+            int nargs = call_shape.total_args - kwnames_len;
+            STACK_SHRINK(call_shape.total_args);
+            PyObject *res = tp->tp_vectorcall((PyObject *)tp, stack_pointer, nargs, call_shape.kwnames);
+            /* Free the arguments. */
+            for (int i = 0; i < call_shape.total_args; i++) {
+                Py_DECREF(stack_pointer[i]);
+            }
+            Py_DECREF(tp);
+            STACK_SHRINK(call_shape.postcall_shrink-1);
+            SET_TOP(res);
             if (res == NULL) {
                 goto error;
             }
@@ -4711,13 +4787,13 @@ handle_eval_breaker:
 
         TARGET(CALL_NO_KW_BUILTIN_O) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
             /* Builtin METH_O functions */
-
-            PyObject *callable = SECOND();
-            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
-            DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
+            assert(call_shape.kwnames == NULL);
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            PyObject *callable = call_shape.callable;
+            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
+            DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL);
+            STAT_INC(CALL, hit);
 
             PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
             // This is slower but CPython promises to check all non-vectorcall
@@ -4725,14 +4801,14 @@ handle_eval_breaker:
             if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
                 goto error;
             }
-            PyObject *arg = POP();
+            PyObject *arg = TOP();
             PyObject *res = cfunc(PyCFunction_GET_SELF(callable), arg);
             _Py_LeaveRecursiveCall(tstate);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
 
-            /* Clear the stack of the function object. */
             Py_DECREF(arg);
             Py_DECREF(callable);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
             if (res == NULL) {
                 goto error;
@@ -4742,32 +4818,31 @@ handle_eval_breaker:
 
         TARGET(CALL_NO_KW_BUILTIN_FAST) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
             /* Builtin METH_FASTCALL functions, without keywords */
-            SpecializedCacheEntry *caches = GET_CACHE();
-            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
-            int nargs = cache0->original_oparg;
-            PyObject **pfunc = &PEEK(nargs + 1);
-            PyObject *callable = *pfunc;
-            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW);
+            assert(call_shape.kwnames == NULL);
+            PyObject *callable = call_shape.callable;
+            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
             DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL,
-                CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
+                CALL);
+            STAT_INC(CALL, hit);
 
+            int nargs = call_shape.total_args;
             PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable);
+            STACK_SHRINK(nargs);
             /* res = func(self, args, nargs) */
             PyObject *res = ((_PyCFunctionFast)(void(*)(void))cfunc)(
                 PyCFunction_GET_SELF(callable),
-                &PEEK(nargs),
+                stack_pointer,
                 nargs);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
 
-            /* Clear the stack of the function object. */
-            while (stack_pointer > pfunc) {
-                PyObject *x = POP();
-                Py_DECREF(x);
+            /* Free the arguments. */
+            for (int i = 0; i < nargs; i++) {
+                Py_DECREF(stack_pointer[i]);
             }
+            STACK_SHRINK(call_shape.postcall_shrink);
             PUSH(res);
+            Py_DECREF(callable);
             if (res == NULL) {
                 /* Not deopting because this doesn't mean our optimization was
                    wrong. `res` can be NULL for valid reasons. Eg. getattr(x,
@@ -4779,29 +4854,72 @@ handle_eval_breaker:
             DISPATCH();
         }
 
+        TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) {
+            assert(cframe.use_tracing == 0);
+            /* Builtin METH_FASTCALL | METH_KEYWORDS functions */
+            PyObject *callable = call_shape.callable;
+            DEOPT_IF(!PyCFunction_CheckExact(callable), CALL);
+            DEOPT_IF(PyCFunction_GET_FLAGS(callable) !=
+                (METH_FASTCALL | METH_KEYWORDS), CALL);
+            STAT_INC(CALL, hit);
+            int kwnames_len = GET_CACHE()->adaptive.original_oparg;
+            assert(
+                (call_shape.kwnames == NULL && kwnames_len == 0) ||
+                (call_shape.kwnames != NULL &&
+                 PyTuple_GET_SIZE(call_shape.kwnames) == kwnames_len)
+            );
+            int nargs = call_shape.total_args - kwnames_len;
+            STACK_SHRINK(call_shape.total_args);
+            /* res = func(self, args, nargs, kwnames) */
+            _PyCFunctionFastWithKeywords cfunc =
+                (_PyCFunctionFastWithKeywords)(void(*)(void))
+                PyCFunction_GET_FUNCTION(callable);
+            PyObject *res = cfunc(
+                PyCFunction_GET_SELF(callable),
+                stack_pointer,
+                nargs,
+                call_shape.kwnames
+            );
+            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+
+            /* Free the arguments. */
+            for (int i = 0; i < call_shape.total_args; i++) {
+                Py_DECREF(stack_pointer[i]);
+            }
+            STACK_SHRINK(call_shape.postcall_shrink);
+            PUSH(res);
+            Py_DECREF(callable);
+            if (res == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
         TARGET(CALL_NO_KW_LEN) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
+            assert(call_shape.kwnames == NULL);
             /* len(o) */
             SpecializedCacheEntry *caches = GET_CACHE();
-            assert(caches[0].adaptive.original_oparg == 1);
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            assert(caches[0].adaptive.original_oparg == 0);
             _PyObjectCache *cache1 = &caches[-1].obj;
 
-            PyObject *callable = SECOND();
-            DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
+            PyObject *callable = call_shape.callable;
+            DEOPT_IF(callable != cache1->obj, CALL);
+            STAT_INC(CALL, hit);
 
-            Py_ssize_t len_i = PyObject_Length(TOP());
+            PyObject *arg = TOP();
+            Py_ssize_t len_i = PyObject_Length(arg);
             if (len_i < 0) {
                 goto error;
             }
             PyObject *res = PyLong_FromSsize_t(len_i);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
 
-            /* Clear the stack of the function object. */
-            Py_DECREF(POP());
-            Py_DECREF(callable);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
+            Py_DECREF(callable);
+            Py_DECREF(arg);
             if (res == NULL) {
                 goto error;
             }
@@ -4810,28 +4928,30 @@ handle_eval_breaker:
 
         TARGET(CALL_NO_KW_ISINSTANCE) {
             assert(cframe.use_tracing == 0);
-            assert(STACK_ADJUST_IS_RESET);
+            assert(call_shape.kwnames == NULL);
             /* isinstance(o, o2) */
             SpecializedCacheEntry *caches = GET_CACHE();
-            assert(caches[0].adaptive.original_oparg == 2);
+            assert(caches[0].adaptive.original_oparg == 0);
+            DEOPT_IF(call_shape.total_args != 2, CALL);
             _PyObjectCache *cache1 = &caches[-1].obj;
 
-            PyObject *callable = THIRD();
-            DEOPT_IF(callable != cache1->obj, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
+            DEOPT_IF(call_shape.callable != cache1->obj, CALL);
+            STAT_INC(CALL, hit);
 
-            int retval = PyObject_IsInstance(SECOND(), TOP());
+            PyObject *cls = POP();
+            PyObject *inst = TOP();
+            int retval = PyObject_IsInstance(inst, cls);
             if (retval < 0) {
                 goto error;
             }
             PyObject *res = PyBool_FromLong(retval);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
 
-            /* Clear the stack of the function object. */
-            Py_DECREF(POP());
-            Py_DECREF(POP());
-            Py_DECREF(callable);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
+            Py_DECREF(inst);
+            Py_DECREF(cls);
+            Py_DECREF(call_shape.callable);
             if (res == NULL) {
                 goto error;
             }
@@ -4840,57 +4960,78 @@ handle_eval_breaker:
 
         TARGET(CALL_NO_KW_LIST_APPEND) {
             assert(cframe.use_tracing == 0);
-            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
-            assert(GET_CACHE()->adaptive.original_oparg == 1);
-            DEOPT_IF(extra_args == 0, CALL_NO_KW);
+            assert(call_shape.kwnames == NULL);
+            SpecializedCacheEntry *caches = GET_CACHE();
+            _PyObjectCache *cache1 = &caches[-1].obj;
+            DEOPT_IF(call_shape.total_args != 2, CALL);
+            DEOPT_IF(call_shape.callable != cache1->obj, CALL);
             PyObject *list = SECOND();
-            DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
-            assert(extra_args == 1);
-            extra_args = 0;
-            assert(STACK_ADJUST_IS_RESET);
+            DEOPT_IF(!PyList_Check(list), CALL);
+            STAT_INC(CALL, hit);
             PyObject *arg = TOP();
             int err = PyList_Append(list, arg);
             if (err) {
                 goto error;
             }
-            PyObject *callable = THIRD();
             Py_DECREF(arg);
             Py_DECREF(list);
+            STACK_SHRINK(call_shape.postcall_shrink+1);
             Py_INCREF(Py_None);
-            STACK_SHRINK(2);
             SET_TOP(Py_None);
-            Py_DECREF(callable);
+            Py_DECREF(call_shape.callable);
             NOTRACE_DISPATCH();
         }
 
         TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) {
-            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
-            assert(GET_CACHE()->adaptive.original_oparg == 1);
-            DEOPT_IF(extra_args == 0, CALL_NO_KW);
-            assert(extra_args == 1);
-            PyObject *callable = THIRD();
-            DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW);
-            DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
-            assert(extra_args == 1);
-            extra_args = 0;
-            assert(STACK_ADJUST_IS_RESET);
-            PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth;
+            assert(call_shape.kwnames == NULL);
+            DEOPT_IF(call_shape.total_args != 2, CALL);
+            DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
+            PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
+            DEOPT_IF(meth->ml_flags != METH_O, CALL);
+            STAT_INC(CALL, hit);
+            PyCFunction cfunc = meth->ml_meth;
             // This is slower but CPython promises to check all non-vectorcall
             // function calls.
             if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
                 goto error;
             }
-            PyObject *arg = POP();
-            PyObject *self = POP();
+            PyObject *arg = TOP();
+            PyObject *self = SECOND();
             PyObject *res = cfunc(self, arg);
             _Py_LeaveRecursiveCall(tstate);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
             Py_DECREF(self);
             Py_DECREF(arg);
+            STACK_SHRINK(call_shape.postcall_shrink+1);
             SET_TOP(res);
-            Py_DECREF(callable);
+            Py_DECREF(call_shape.callable);
+            if (res == NULL) {
+                goto error;
+            }
+            DISPATCH();
+        }
+
+        TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
+            assert(call_shape.kwnames == NULL);
+            DEOPT_IF(call_shape.total_args != 1, CALL);
+            DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
+            PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
+            DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL);
+            STAT_INC(CALL, hit);
+            PyCFunction cfunc = meth->ml_meth;
+            // This is slower but CPython promises to check all non-vectorcall
+            // function calls.
+            if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) {
+                goto error;
+            }
+            PyObject *self = TOP();
+            PyObject *res = cfunc(self, NULL);
+            _Py_LeaveRecursiveCall(tstate);
+            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+            Py_DECREF(self);
+            STACK_SHRINK(call_shape.postcall_shrink);
+            SET_TOP(res);
+            Py_DECREF(call_shape.callable);
             if (res == NULL) {
                 goto error;
             }
@@ -4898,32 +5039,26 @@ handle_eval_breaker:
         }
 
         TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) {
-            assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD);
+            assert(call_shape.kwnames == NULL);
             /* Builtin METH_FASTCALL methods, without keywords */
-            SpecializedCacheEntry *caches = GET_CACHE();
-            _PyAdaptiveEntry *cache0 = &caches[0].adaptive;
-            DEOPT_IF(extra_args == 0, CALL_NO_KW);
-            assert(extra_args == 1);
-            int nargs = cache0->original_oparg;
-            PyObject *callable = PEEK(nargs + 2);
-            DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type),  CALL_NO_KW);
-            PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
-            DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW);
-            STAT_INC(CALL_NO_KW, hit);
-            assert(extra_args == 1);
-            extra_args = 0;
-            assert(STACK_ADJUST_IS_RESET);
+            DEOPT_IF(!Py_IS_TYPE(call_shape.callable, &PyMethodDescr_Type), CALL);
+            PyMethodDef *meth = ((PyMethodDescrObject *)call_shape.callable)->d_method;
+            DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL);
+            STAT_INC(CALL, hit);
             _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth;
-            PyObject *self = PEEK(nargs+1);
-            PyObject *res = cfunc(self, &PEEK(nargs), nargs);
+            int nargs = call_shape.total_args-1;
+            STACK_SHRINK(nargs);
+            PyObject *self = TOP();
+            PyObject *res = cfunc(self, stack_pointer, nargs);
             assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
             /* Clear the stack of the arguments. */
-            STACK_SHRINK(nargs+1);
-            for (int i = 0; i <= nargs; i++) {
+            for (int i = 0; i < nargs; i++) {
                 Py_DECREF(stack_pointer[i]);
             }
+            Py_DECREF(self);
+            STACK_SHRINK(call_shape.postcall_shrink);
             SET_TOP(res);
-            Py_DECREF(callable);
+            Py_DECREF(call_shape.callable);
             if (res == NULL) {
                 goto error;
             }
@@ -5283,7 +5418,7 @@ MISS_WITH_CACHE(LOAD_ATTR)
 MISS_WITH_CACHE(STORE_ATTR)
 MISS_WITH_CACHE(LOAD_GLOBAL)
 MISS_WITH_CACHE(LOAD_METHOD)
-MISS_WITH_CACHE(CALL_NO_KW)
+MISS_WITH_CACHE(CALL)
 MISS_WITH_CACHE(BINARY_OP)
 MISS_WITH_CACHE(COMPARE_OP)
 MISS_WITH_CACHE(BINARY_SUBSCR)
@@ -7321,7 +7456,7 @@ format_exc_unbound(PyThreadState *tstate, PyCodeObject *co, int oparg)
 }
 
 static void
-format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevopcode, int prevopcode)
+format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevprevopcode, int prevopcode)
 {
     if (type->tp_as_async == NULL || type->tp_as_async->am_await == NULL) {
         if (prevopcode == BEFORE_ASYNC_WITH) {
@@ -7330,7 +7465,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop
                           "that does not implement __await__: %.100s",
                           type->tp_name);
         }
-        else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == LOAD_CONST)) {
+        else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL && prevprevprevopcode == LOAD_CONST)) {
             _PyErr_Format(tstate, PyExc_TypeError,
                           "'async with' received an object from __aexit__ "
                           "that does not implement __await__: %.100s",
diff --git a/Python/compile.c b/Python/compile.c
index 6883a4b..eda708e 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1021,11 +1021,14 @@ stack_effect(int opcode, int oparg, int jump)
 
         /* Functions and calls */
         case PRECALL_METHOD:
-            return -1;
-        case CALL_NO_KW:
-            return -oparg;
-        case CALL_KW:
             return -oparg-1;
+        case PRECALL_FUNCTION:
+            return -oparg;
+        case KW_NAMES:
+            return 0;
+        case CALL:
+            return 0;
+
         case CALL_FUNCTION_EX:
             return -1 - ((oparg & 0x01) != 0);
         case MAKE_FUNCTION:
@@ -1823,7 +1826,8 @@ compiler_call_exit_with_nones(struct compiler *c) {
     ADDOP_LOAD_CONST(c, Py_None);
     ADDOP_LOAD_CONST(c, Py_None);
     ADDOP_LOAD_CONST(c, Py_None);
-    ADDOP_I(c, CALL_NO_KW, 3);
+    ADDOP_I(c, PRECALL_FUNCTION, 3);
+    ADDOP_I(c, CALL, 0);
     return 1;
 }
 
@@ -2208,7 +2212,8 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos)
     int old_end_col_offset = c->u->u_end_col_offset;
     for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) {
         SET_LOC(c, (expr_ty)asdl_seq_GET(decos, i));
-        ADDOP_I(c, CALL_NO_KW, 1);
+        ADDOP_I(c, PRECALL_FUNCTION, 1);
+        ADDOP_I(c, CALL, 0);
     }
     c->u->u_lineno = old_lineno;
     c->u->u_end_lineno = old_end_lineno;
@@ -3903,7 +3908,8 @@ compiler_assert(struct compiler *c, stmt_ty s)
     ADDOP(c, LOAD_ASSERTION_ERROR);
     if (s->v.Assert.msg) {
         VISIT(c, expr, s->v.Assert.msg);
-        ADDOP_I(c, CALL_NO_KW, 1);
+        ADDOP_I(c, PRECALL_FUNCTION, 1);
+        ADDOP_I(c, CALL, 0);
     }
     ADDOP_I(c, RAISE_VARARGS, 1);
     compiler_use_next_block(c, end);
@@ -4723,15 +4729,16 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e)
     VISIT_SEQ(c, expr, e->v.Call.args);
 
     if (kwdsl) {
+        VISIT_SEQ(c, keyword, kwds);
+        ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl);
         if (!compiler_call_simple_kw_helper(c, kwds, kwdsl)) {
             return 0;
         };
-        ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl+1);
-        ADDOP_I(c, CALL_KW, argsl + kwdsl);
+        ADDOP_I(c, CALL, kwdsl);
     }
     else {
         ADDOP_I(c, PRECALL_METHOD, argsl);
-        ADDOP_I(c, CALL_NO_KW, argsl);
+        ADDOP_I(c, CALL, 0);
     }
     c->u->u_lineno = old_lineno;
     return 1;
@@ -4799,7 +4806,7 @@ compiler_joined_str(struct compiler *c, expr_ty e)
             ADDOP_I(c, LIST_APPEND, 1);
         }
         ADDOP_I(c, PRECALL_METHOD, 1);
-        ADDOP_I(c, CALL_NO_KW, 1);
+        ADDOP_I(c, CALL, 0);
     }
     else {
         VISIT_SEQ(c, expr, e->v.JoinedStr.values);
@@ -4900,21 +4907,15 @@ compiler_subkwargs(struct compiler *c, asdl_keyword_seq *keywords, Py_ssize_t be
 }
 
 /* Used by compiler_call_helper and maybe_optimize_method_call to emit
-LOAD_CONST kw1
-LOAD_CONST kw2
-...
-LOAD_CONST <tuple of kwnames>
-before a CALL_(FUNCTION|METHOD)_KW.
-
-Returns 1 on success, 0 on error.
-*/
+ * KW_NAMES before CALL.
+ * Returns 1 on success, 0 on error.
+ */
 static int
 compiler_call_simple_kw_helper(struct compiler *c,
                                asdl_keyword_seq *keywords,
                                Py_ssize_t nkwelts)
 {
     PyObject *names;
-    VISIT_SEQ(c, keyword, keywords);
     names = PyTuple_New(nkwelts);
     if (names == NULL) {
         return 0;
@@ -4924,7 +4925,12 @@ compiler_call_simple_kw_helper(struct compiler *c,
         Py_INCREF(kw->arg);
         PyTuple_SET_ITEM(names, i, kw->arg);
     }
-    ADDOP_LOAD_CONST_NEW(c, names);
+    Py_ssize_t arg = compiler_add_const(c, names);
+    if (arg < 0) {
+        return 0;
+    }
+    Py_DECREF(names);
+    ADDOP_I(c, KW_NAMES, arg);
     return 1;
 }
 
@@ -4968,14 +4974,17 @@ compiler_call_helper(struct compiler *c,
         VISIT(c, expr, elt);
     }
     if (nkwelts) {
+        VISIT_SEQ(c, keyword, keywords);
+        ADDOP_I(c, PRECALL_FUNCTION, n + nelts + nkwelts);
         if (!compiler_call_simple_kw_helper(c, keywords, nkwelts)) {
             return 0;
         };
-        ADDOP_I(c, CALL_KW, n + nelts + nkwelts);
+        ADDOP_I(c, CALL, nkwelts);
         return 1;
     }
     else {
-        ADDOP_I(c, CALL_NO_KW, n + nelts);
+        ADDOP_I(c, PRECALL_FUNCTION, n + nelts);
+        ADDOP_I(c, CALL, 0);
         return 1;
     }
 
@@ -5372,7 +5381,8 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
         ADDOP(c, GET_ITER);
     }
 
-    ADDOP_I(c, CALL_NO_KW, 1);
+    ADDOP_I(c, PRECALL_FUNCTION, 1);
+    ADDOP_I(c, CALL, 0);
 
     if (is_async_generator && type != COMP_GENEXP) {
         ADDOP(c, GET_AWAITABLE);
@@ -6709,7 +6719,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc)
                     // rotated = pc_stores[:rotations]
                     // del pc_stores[:rotations]
                     // pc_stores[icontrol-istores:icontrol-istores] = rotated
-                    // Do the same thing to the stack, using several 
+                    // Do the same thing to the stack, using several
                     // rotations:
                     while (rotations--) {
                         if (!pattern_helper_rotate(c, icontrol + 1)){
@@ -8786,6 +8796,8 @@ optimize_basic_block(struct compiler *c, basicblock *bb, PyObject *consts)
                 }
                 i += swaptimize(bb, i);
                 break;
+            case KW_NAMES:
+                break;
             default:
                 /* All HAS_CONST opcodes should be handled with LOAD_CONST */
                 assert (!HAS_CONST(inst->i_opcode));
@@ -9097,7 +9109,8 @@ trim_unused_consts(struct compiler *c, struct assembler *a, PyObject *consts)
     int max_const_index = 0;
     for (basicblock *b = a->a_entry; b != NULL; b = b->b_next) {
         for (int i = 0; i < b->b_iused; i++) {
-            if (b->b_instr[i].i_opcode == LOAD_CONST &&
+            if ((b->b_instr[i].i_opcode == LOAD_CONST ||
+                b->b_instr[i].i_opcode == KW_NAMES) &&
                     b->b_instr[i].i_oparg > max_const_index) {
                 max_const_index = b->b_instr[i].i_oparg;
             }
diff --git a/Python/frame.c b/Python/frame.c
index 9578747..771de75 100644
--- a/Python/frame.c
+++ b/Python/frame.c
@@ -83,7 +83,7 @@ take_ownership(PyFrameObject *f, InterpreterFrame *frame)
 }
 
 void
-_PyFrame_Clear(InterpreterFrame * frame)
+_PyFrame_Clear(InterpreterFrame *frame)
 {
     /* It is the responsibility of the owning generator/coroutine
      * to have cleared the enclosing generator, if any. */
@@ -107,3 +107,16 @@ _PyFrame_Clear(InterpreterFrame * frame)
     Py_DECREF(frame->f_func);
     Py_DECREF(frame->f_code);
 }
+
+InterpreterFrame *
+_PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func)
+{
+    PyCodeObject *code = (PyCodeObject *)func->func_code;
+    size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE;
+    InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size);
+    if (new_frame == NULL) {
+        return NULL;
+    }
+    _PyFrame_InitializeSpecials(new_frame, func, NULL, code->co_nlocalsplus);
+    return new_frame;
+}
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index c6e6d82..1a809ed 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -28,59 +28,59 @@ static void *opcode_targets[256] = {
     &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_LIST_INT,
     &&TARGET_STORE_SUBSCR_DICT,
-    &&TARGET_CALL_NO_KW_ADAPTIVE,
+    &&TARGET_CALL_ADAPTIVE,
     &&TARGET_GET_LEN,
     &&TARGET_MATCH_MAPPING,
     &&TARGET_MATCH_SEQUENCE,
     &&TARGET_MATCH_KEYS,
-    &&TARGET_CALL_NO_KW_BUILTIN_O,
+    &&TARGET_CALL_BUILTIN_CLASS,
     &&TARGET_PUSH_EXC_INFO,
+    &&TARGET_CALL_NO_KW_BUILTIN_O,
     &&TARGET_CALL_NO_KW_BUILTIN_FAST,
+    &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS,
     &&TARGET_CALL_NO_KW_LEN,
     &&TARGET_CALL_NO_KW_ISINSTANCE,
-    &&TARGET_CALL_NO_KW_PY_SIMPLE,
+    &&TARGET_CALL_PY_EXACT_ARGS,
+    &&TARGET_CALL_PY_WITH_DEFAULTS,
     &&TARGET_CALL_NO_KW_LIST_APPEND,
     &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O,
+    &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+    &&TARGET_CALL_NO_KW_STR_1,
+    &&TARGET_CALL_NO_KW_TUPLE_1,
     &&TARGET_CALL_NO_KW_TYPE_1,
-    &&TARGET_CALL_NO_KW_BUILTIN_CLASS_1,
-    &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
-    &&TARGET_JUMP_ABSOLUTE_QUICK,
-    &&TARGET_LOAD_ATTR_ADAPTIVE,
-    &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
-    &&TARGET_LOAD_ATTR_WITH_HINT,
     &&TARGET_WITH_EXCEPT_START,
     &&TARGET_GET_AITER,
     &&TARGET_GET_ANEXT,
     &&TARGET_BEFORE_ASYNC_WITH,
     &&TARGET_BEFORE_WITH,
     &&TARGET_END_ASYNC_FOR,
+    &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST,
+    &&TARGET_JUMP_ABSOLUTE_QUICK,
+    &&TARGET_LOAD_ATTR_ADAPTIVE,
+    &&TARGET_LOAD_ATTR_INSTANCE_VALUE,
+    &&TARGET_LOAD_ATTR_WITH_HINT,
+    &&TARGET_STORE_SUBSCR,
+    &&TARGET_DELETE_SUBSCR,
     &&TARGET_LOAD_ATTR_SLOT,
     &&TARGET_LOAD_ATTR_MODULE,
     &&TARGET_LOAD_GLOBAL_ADAPTIVE,
     &&TARGET_LOAD_GLOBAL_MODULE,
     &&TARGET_LOAD_GLOBAL_BUILTIN,
-    &&TARGET_STORE_SUBSCR,
-    &&TARGET_DELETE_SUBSCR,
     &&TARGET_LOAD_METHOD_ADAPTIVE,
-    &&TARGET_LOAD_METHOD_CACHED,
-    &&TARGET_LOAD_METHOD_CLASS,
-    &&TARGET_LOAD_METHOD_MODULE,
-    &&TARGET_LOAD_METHOD_NO_DICT,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
+    &&TARGET_LOAD_METHOD_CACHED,
     &&TARGET_GET_AWAITABLE,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
+    &&TARGET_LOAD_METHOD_CLASS,
+    &&TARGET_LOAD_METHOD_MODULE,
+    &&TARGET_LOAD_METHOD_NO_DICT,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
-    &&TARGET_STORE_ATTR_WITH_HINT,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
-    &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
@@ -130,7 +130,7 @@ static void *opcode_targets[256] = {
     &&TARGET_POP_JUMP_IF_NOT_NONE,
     &&TARGET_POP_JUMP_IF_NONE,
     &&TARGET_RAISE_VARARGS,
-    &&TARGET_STORE_FAST__STORE_FAST,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_MAKE_FUNCTION,
     &&TARGET_BUILD_SLICE,
     &&TARGET_JUMP_NO_INTERRUPT,
@@ -139,20 +139,20 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
-    &&_unknown_opcode,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
+    &&TARGET_STORE_FAST__LOAD_FAST,
     &&TARGET_CALL_FUNCTION_EX,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
     &&TARGET_COPY_FREE_VARS,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
-    &&_unknown_opcode,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&_unknown_opcode,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
@@ -166,12 +166,12 @@ static void *opcode_targets[256] = {
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
     &&_unknown_opcode,
-    &&_unknown_opcode,
+    &&TARGET_PRECALL_FUNCTION,
     &&TARGET_PRECALL_METHOD,
-    &&TARGET_CALL_NO_KW,
-    &&TARGET_CALL_KW,
     &&_unknown_opcode,
     &&_unknown_opcode,
+    &&TARGET_CALL,
+    &&TARGET_KW_NAMES,
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
diff --git a/Python/specialize.c b/Python/specialize.c
index 44c0062..a69b73c 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -124,7 +124,7 @@ _Py_GetSpecializationStats(void) {
     err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
     err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
     err += add_stat_dict(stats, STORE_ATTR, "store_attr");
-    err += add_stat_dict(stats, CALL_NO_KW, "call_no_kw");
+    err += add_stat_dict(stats, CALL, "call");
     err += add_stat_dict(stats, BINARY_OP, "binary_op");
     err += add_stat_dict(stats, COMPARE_OP, "compare_op");
     if (err < 0) {
@@ -251,7 +251,7 @@ static uint8_t adaptive_opcodes[256] = {
     [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
     [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
     [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
-    [CALL_NO_KW] = CALL_NO_KW_ADAPTIVE,
+    [CALL] = CALL_ADAPTIVE,
     [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
     [BINARY_OP] = BINARY_OP_ADAPTIVE,
     [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
@@ -264,7 +264,7 @@ static uint8_t cache_requirements[256] = {
     [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */
     [BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */
     [STORE_SUBSCR] = 0,
-    [CALL_NO_KW] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
+    [CALL] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */
     [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */
     [BINARY_OP] = 1,  // _PyAdaptiveEntry
     [COMPARE_OP] = 1, /* _PyAdaptiveEntry */
@@ -512,8 +512,13 @@ initial_counter_value(void) {
 #define SPEC_FAIL_CLASS 18
 #define SPEC_FAIL_PYTHON_CLASS 19
 #define SPEC_FAIL_C_METHOD_CALL 20
-#define SPEC_FAIL_METHDESCR_NON_METHOD 21
-#define SPEC_FAIL_METHOD_CALL_CLASS 22
+#define SPEC_FAIL_BOUND_METHOD 21
+#define SPEC_FAIL_CALL_STR 22
+#define SPEC_FAIL_CLASS_NO_VECTORCALL 23
+#define SPEC_FAIL_CLASS_MUTABLE 24
+#define SPEC_FAIL_KWNAMES 25
+#define SPEC_FAIL_METHOD_WRAPPER 26
+#define SPEC_FAIL_OPERATOR_WRAPPER 27
 
 /* COMPARE_OP */
 #define SPEC_FAIL_STRING_COMPARE 13
@@ -1337,50 +1342,83 @@ success:
 static int
 specialize_class_call(
     PyObject *callable, _Py_CODEUNIT *instr,
-    int nargs, SpecializedCacheEntry *cache)
+    int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
 {
     PyTypeObject *tp = _PyType_CAST(callable);
-    if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHOD_CALL_CLASS);
-        return -1;
-    }
     if (tp->tp_new == PyBaseObject_Type.tp_new) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_PYTHON_CLASS);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_PYTHON_CLASS);
         return -1;
     }
-    if (nargs == 1) {
-        if (tp == &PyType_Type) {
-            *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
-            return 0;
+    if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
+        if (nargs == 1 && kwnames == NULL) {
+            if (tp == &PyUnicode_Type) {
+                *instr = _Py_MAKECODEUNIT(CALL_NO_KW_STR_1, _Py_OPARG(*instr));
+                return 0;
+            }
+            else if (tp == &PyType_Type) {
+                *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TYPE_1, _Py_OPARG(*instr));
+                return 0;
+            }
+            else if (tp == &PyTuple_Type) {
+                *instr = _Py_MAKECODEUNIT(CALL_NO_KW_TUPLE_1, _Py_OPARG(*instr));
+                return 0;
+            }
         }
-        if ((tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) && tp->tp_vectorcall != NULL) {
-            cache->adaptive.version = tp->tp_version_tag;
-            *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_CLASS_1, _Py_OPARG(*instr));
+        if (tp->tp_vectorcall != NULL) {
+            *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_CLASS, _Py_OPARG(*instr));
             return 0;
         }
+        SPECIALIZATION_FAIL(CALL, tp == &PyUnicode_Type ?
+            SPEC_FAIL_CALL_STR : SPEC_FAIL_CLASS_NO_VECTORCALL);
+        return -1;
     }
-    SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_CLASS);
+    SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CLASS_MUTABLE);
     return -1;
 }
 
+#ifdef Py_STATS
+static int
+builtin_call_fail_kind(int ml_flags)
+{
+    switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
+        METH_KEYWORDS | METH_METHOD)) {
+        case METH_VARARGS:
+            return SPEC_FAIL_PYCFUNCTION;
+        case METH_VARARGS | METH_KEYWORDS:
+            return SPEC_FAIL_PYCFUNCTION_WITH_KEYWORDS;
+        case METH_FASTCALL | METH_KEYWORDS:
+            return SPEC_FAIL_PYCFUNCTION_FAST_WITH_KEYWORDS;
+        case METH_NOARGS:
+            return SPEC_FAIL_PYCFUNCTION_NOARGS;
+        /* This case should never happen with PyCFunctionObject -- only
+            PyMethodObject. See zlib.compressobj()'s methods for an example.
+        */
+        case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
+        default:
+            return SPEC_FAIL_BAD_CALL_FLAGS;
+    }
+}
+#endif
+
 static PyMethodDescrObject *_list_append = NULL;
 _Py_IDENTIFIER(append);
 
 static int
 specialize_method_descriptor(
     PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
-    int nargs, SpecializedCacheEntry *cache)
+    int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
 {
-    int oparg = cache->adaptive.original_oparg;
-    if (nargs - oparg != 1) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHDESCR_NON_METHOD);
+    if (kwnames) {
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
         return -1;
     }
     if (_list_append == NULL) {
         _list_append = (PyMethodDescrObject *)_PyType_LookupId(&PyList_Type, &PyId_append);
     }
-    if (oparg == 1 && descr == _list_append) {
+    assert(_list_append != NULL);
+    if (nargs == 2 && descr == _list_append) {
         assert(_Py_OPCODE(instr[-1]) == PRECALL_METHOD);
+        cache[-1].obj.obj = (PyObject *)_list_append;
         *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr));
         return 0;
     }
@@ -1388,10 +1426,19 @@ specialize_method_descriptor(
     switch (descr->d_method->ml_flags &
         (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
         METH_KEYWORDS | METH_METHOD)) {
+        case METH_NOARGS: {
+            if (nargs != 1) {
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+                return -1;
+            }
+            *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
+                _Py_OPARG(*instr));
+            return 0;
+        }
         case METH_O: {
-            if (oparg != 1) {
-                SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
-                return 1;
+            if (nargs != 2) {
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
+                return -1;
             }
             *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O,
                 _Py_OPARG(*instr));
@@ -1403,89 +1450,68 @@ specialize_method_descriptor(
             return 0;
         }
     }
-    SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OTHER);
+    SPECIALIZATION_FAIL(CALL, builtin_call_fail_kind(descr->d_method->ml_flags));
     return -1;
 }
 
 static int
 specialize_py_call(
     PyFunctionObject *func, _Py_CODEUNIT *instr,
-    int nargs, SpecializedCacheEntry *cache)
+    int nargs, PyObject *kwnames, SpecializedCacheEntry *cache)
 {
     _PyCallCache *cache1 = &cache[-1].call;
     PyCodeObject *code = (PyCodeObject *)func->func_code;
     int kind = function_kind(code);
+    if (kwnames) {
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
+        return -1;
+    }
     if (kind != SIMPLE_FUNCTION) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, kind);
+        SPECIALIZATION_FAIL(CALL, kind);
         return -1;
     }
     int argcount = code->co_argcount;
     if (argcount > 0xffff) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
         return -1;
     }
     int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
     assert(defcount <= argcount);
     int min_args = argcount-defcount;
     if (nargs > argcount || nargs < min_args) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
         return -1;
     }
     assert(nargs <= argcount && nargs >= min_args);
-    int defstart = nargs - min_args;
-    int deflen = argcount - nargs;
-    assert(defstart >= 0 && deflen >= 0);
-    assert(deflen == 0 || func->func_defaults != NULL);
-    if (defstart > 0xffff || deflen > 0xffff) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
+    assert(min_args >= 0 && defcount >= 0);
+    assert(defcount == 0 || func->func_defaults != NULL);
+    if (min_args > 0xffff || defcount > 0xffff) {
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
         return -1;
     }
     int version = _PyFunction_GetVersionForCurrentState(func);
     if (version == 0) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_VERSIONS);
+        SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
         return -1;
     }
     cache[0].adaptive.index = nargs;
     cache1->func_version = version;
-    cache1->defaults_start = defstart;
-    cache1->defaults_len = deflen;
-    *instr = _Py_MAKECODEUNIT(CALL_NO_KW_PY_SIMPLE, _Py_OPARG(*instr));
-    return 0;
-}
-
-#ifdef Py_STATS
-static int
-builtin_call_fail_kind(int ml_flags)
-{
-    switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
-        METH_KEYWORDS | METH_METHOD)) {
-        case METH_VARARGS:
-            return SPEC_FAIL_PYCFUNCTION;
-        case METH_VARARGS | METH_KEYWORDS:
-            return SPEC_FAIL_PYCFUNCTION_WITH_KEYWORDS;
-        case METH_FASTCALL | METH_KEYWORDS:
-            return SPEC_FAIL_PYCFUNCTION_FAST_WITH_KEYWORDS;
-        case METH_NOARGS:
-            return SPEC_FAIL_PYCFUNCTION_NOARGS;
-        /* This case should never happen with PyCFunctionObject -- only
-            PyMethodObject. See zlib.compressobj()'s methods for an example.
-        */
-        case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
-        default:
-            return SPEC_FAIL_BAD_CALL_FLAGS;
+    cache1->min_args = min_args;
+    cache1->defaults_len = defcount;
+    if (argcount == nargs) {
+        *instr = _Py_MAKECODEUNIT(CALL_PY_EXACT_ARGS, _Py_OPARG(*instr));
     }
+    else {
+        *instr = _Py_MAKECODEUNIT(CALL_PY_WITH_DEFAULTS, _Py_OPARG(*instr));
+    }
+    return 0;
 }
-#endif
 
 static int
 specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
-    SpecializedCacheEntry *cache, PyObject *builtins)
+    PyObject *kwnames, SpecializedCacheEntry *cache, PyObject *builtins)
 {
     _PyObjectCache *cache1 = &cache[-1].obj;
-    if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) {
-        SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_C_METHOD_CALL);
-        return -1;
-    }
     if (PyCFunction_GET_FUNCTION(callable) == NULL) {
         return 1;
     }
@@ -1493,8 +1519,12 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
         (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
         METH_KEYWORDS | METH_METHOD)) {
         case METH_O: {
+            if (kwnames) {
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
+                return -1;
+            }
             if (nargs != 1) {
-                SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE);
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
                 return 1;
             }
             /* len(o) */
@@ -1510,6 +1540,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
             return 0;
         }
         case METH_FASTCALL: {
+            if (kwnames) {
+                SPECIALIZATION_FAIL(CALL, SPEC_FAIL_KWNAMES);
+                return -1;
+            }
             if (nargs == 2) {
                 /* isinstance(o1, o2) */
                 PyObject *builtin_isinstance = PyDict_GetItemString(
@@ -1525,8 +1559,13 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
                 _Py_OPARG(*instr));
             return 0;
         }
+        case METH_FASTCALL | METH_KEYWORDS: {
+            *instr = _Py_MAKECODEUNIT(CALL_BUILTIN_FAST_WITH_KEYWORDS,
+                _Py_OPARG(*instr));
+            return 0;
+        }
         default:
-            SPECIALIZATION_FAIL(CALL_NO_KW,
+            SPECIALIZATION_FAIL(CALL,
                 builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
             return 1;
     }
@@ -1549,6 +1588,15 @@ call_fail_kind(PyObject *callable)
     else if (PyType_Check(callable)) {
         return  SPEC_FAIL_CLASS;
     }
+    else if (Py_TYPE(callable) == &PyWrapperDescr_Type) {
+        return SPEC_FAIL_OPERATOR_WRAPPER;
+    }
+    else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) {
+        return SPEC_FAIL_METHOD_WRAPPER;
+    }
+    else if (Py_TYPE(callable) == &PyMethod_Type) {
+        return SPEC_FAIL_BOUND_METHOD;
+    }
     return SPEC_FAIL_OTHER;
 }
 #endif
@@ -1559,35 +1607,35 @@ call_fail_kind(PyObject *callable)
 int
 _Py_Specialize_CallNoKw(
     PyObject *callable, _Py_CODEUNIT *instr,
-    int nargs, SpecializedCacheEntry *cache,
-    PyObject *builtins)
+    int nargs, PyObject *kwnames,
+    SpecializedCacheEntry *cache, PyObject *builtins)
 {
+    _PyAdaptiveEntry *cache0 = &cache->adaptive;
     int fail;
     if (PyCFunction_CheckExact(callable)) {
-        fail = specialize_c_call(callable, instr, nargs, cache, builtins);
+        fail = specialize_c_call(callable, instr, nargs, kwnames, cache, builtins);
     }
     else if (PyFunction_Check(callable)) {
-        fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, cache);
+        fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs, kwnames, cache);
     }
     else if (PyType_Check(callable)) {
-        fail = specialize_class_call(callable, instr, nargs, cache);
+        fail = specialize_class_call(callable, instr, nargs, kwnames, cache);
     }
     else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
         fail = specialize_method_descriptor(
-            (PyMethodDescrObject *)callable, instr, nargs, cache);
+            (PyMethodDescrObject *)callable, instr, nargs, kwnames, cache);
     }
     else {
-        SPECIALIZATION_FAIL(CALL_NO_KW, call_fail_kind(callable));
+        SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
         fail = -1;
     }
-    _PyAdaptiveEntry *cache0 = &cache->adaptive;
     if (fail) {
-        STAT_INC(CALL_NO_KW, failure);
+        STAT_INC(CALL, failure);
         assert(!PyErr_Occurred());
         cache_backoff(cache0);
     }
     else {
-        STAT_INC(CALL_NO_KW, success);
+        STAT_INC(CALL, success);
         assert(!PyErr_Occurred());
         cache0->counter = initial_counter_value();
     }