bpo-47127: Specialize calls for fastcall c methods with keywords (GH-32125)

* add PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS

3 files changed, 47 insertions, 11 deletions

diff --git a/Python/ceval.c b/Python/ceval.c
index 4824b19..8b59f42 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5090,6 +5090,38 @@ handle_eval_breaker:
             DISPATCH();
         }
 
+        TARGET(PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) {
+            int is_meth = is_method(stack_pointer, oparg);
+            int total_args = oparg + is_meth;
+            PyObject *callable = PEEK(total_args + 1);
+            DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), PRECALL);
+            PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method;
+            DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), PRECALL);
+            STAT_INC(PRECALL, hit);
+            SKIP_CALL();
+            int nargs = total_args-1;
+            STACK_SHRINK(nargs);
+            _PyCFunctionFastWithKeywords cfunc = (_PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth;
+            PyObject *self = TOP();
+            PyObject *res = cfunc(self, stack_pointer, nargs - KWNAMES_LEN(), call_shape.kwnames);
+            assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL));
+            call_shape.kwnames = NULL;
+
+            /* Free the arguments. */
+            for (int i = 0; i < nargs; i++) {
+                Py_DECREF(stack_pointer[i]);
+            }
+            Py_DECREF(self);
+            STACK_SHRINK(2-is_meth);
+            SET_TOP(res);
+            Py_DECREF(callable);
+            if (res == NULL) {
+                goto error;
+            }
+            CHECK_EVAL_BREAKER();
+            DISPATCH();
+        }
+
         TARGET(PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS) {
             assert(call_shape.kwnames == NULL);
             assert(oparg == 0 || oparg == 1);
diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h
index 2aa6471..dbcacee 100644
--- a/Python/opcode_targets.h
+++ b/Python/opcode_targets.h
@@ -64,23 +64,23 @@ static void *opcode_targets[256] = {
     &&TARGET_PRECALL_BOUND_METHOD,
     &&TARGET_PRECALL_BUILTIN_CLASS,
     &&TARGET_PRECALL_BUILTIN_FAST_WITH_KEYWORDS,
+    &&TARGET_PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS,
     &&TARGET_PRECALL_NO_KW_BUILTIN_FAST,
     &&TARGET_PRECALL_NO_KW_BUILTIN_O,
-    &&TARGET_PRECALL_NO_KW_ISINSTANCE,
     &&TARGET_GET_ITER,
     &&TARGET_GET_YIELD_FROM_ITER,
     &&TARGET_PRINT_EXPR,
     &&TARGET_LOAD_BUILD_CLASS,
+    &&TARGET_PRECALL_NO_KW_ISINSTANCE,
     &&TARGET_PRECALL_NO_KW_LEN,
-    &&TARGET_PRECALL_NO_KW_LIST_APPEND,
     &&TARGET_LOAD_ASSERTION_ERROR,
     &&TARGET_RETURN_GENERATOR,
+    &&TARGET_PRECALL_NO_KW_LIST_APPEND,
     &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST,
     &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS,
     &&TARGET_PRECALL_NO_KW_METHOD_DESCRIPTOR_O,
     &&TARGET_PRECALL_NO_KW_STR_1,
     &&TARGET_PRECALL_NO_KW_TUPLE_1,
-    &&TARGET_PRECALL_NO_KW_TYPE_1,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
@@ -139,39 +139,40 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
+    &&TARGET_PRECALL_NO_KW_TYPE_1,
     &&TARGET_PRECALL_PYFUNC,
-    &&TARGET_RESUME_QUICK,
     &&TARGET_CALL_FUNCTION_EX,
-    &&TARGET_STORE_ATTR_ADAPTIVE,
+    &&TARGET_RESUME_QUICK,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
     &&TARGET_COPY_FREE_VARS,
-    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
+    &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
+    &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
-    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_FORMAT_VALUE,
     &&TARGET_BUILD_CONST_KEY_MAP,
     &&TARGET_BUILD_STRING,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_LOAD_METHOD,
-    &&TARGET_STORE_SUBSCR_ADAPTIVE,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_LIST_EXTEND,
     &&TARGET_SET_UPDATE,
     &&TARGET_DICT_MERGE,
     &&TARGET_DICT_UPDATE,
     &&TARGET_PRECALL,
+    &&TARGET_STORE_SUBSCR_ADAPTIVE,
     &&TARGET_STORE_SUBSCR_DICT,
     &&TARGET_STORE_SUBSCR_LIST_INT,
     &&TARGET_UNPACK_SEQUENCE_ADAPTIVE,
-    &&TARGET_UNPACK_SEQUENCE_LIST,
     &&TARGET_CALL,
     &&TARGET_KW_NAMES,
+    &&TARGET_UNPACK_SEQUENCE_LIST,
     &&TARGET_UNPACK_SEQUENCE_TUPLE,
     &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE,
     &&_unknown_opcode,
@@ -253,6 +254,5 @@ static void *opcode_targets[256] = {
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,
-    &&_unknown_opcode,
     &&TARGET_DO_TRACING
 };
diff --git a/Python/specialize.c b/Python/specialize.c
index 720bc7f..5839d76 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -1446,6 +1446,10 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
             _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST);
             return 0;
         }
+        case METH_FASTCALL|METH_KEYWORDS: {
+            _Py_SET_OPCODE(*instr, PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
+            return 0;
+        }
     }
     SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(descr->d_method->ml_flags));
     return -1;