Optimize slots: avoid temporary PyMethodObject

Issue #29507: Optimize slots calling Python methods. For Python methods, get
the unbound Python function and prepend arguments with self, rather than
calling the descriptor which creates a temporary PyMethodObject.

Add a new _PyObject_FastCall_Prepend() function used to call the unbound Python
method with self. It avoids the creation of a temporary tuple to pass
positional arguments.

Avoiding temporary PyMethodObject and avoiding temporary tuple makes Python
slots up to 1.46x faster. Microbenchmark on a __getitem__() method implemented
in Python:

Median +- std dev: 121 ns +- 5 ns -> 82.8 ns +- 1.0 ns: 1.46x faster (-31%)

Co-Authored-by: INADA Naoki <songofacandy@gmail.com>

2 files changed, 160 insertions, 58 deletions

diff --git a/Objects/abstract.c b/Objects/abstract.c
index 4d7f94a..3d33048 100644
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -2367,6 +2367,41 @@ _PyObject_FastCallDict(PyObject *callable, PyObject **args, Py_ssize_t nargs,
 /* Positional arguments are obj followed by args:
    call callable(obj, *args, **kwargs) */
 PyObject *
+_PyObject_FastCall_Prepend(PyObject *callable,
+                           PyObject *obj, PyObject **args, Py_ssize_t nargs)
+{
+    PyObject *small_stack[_PY_FASTCALL_SMALL_STACK];
+    PyObject **args2;
+    PyObject *result;
+
+    nargs++;
+    if (nargs <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) {
+        args2 = small_stack;
+    }
+    else {
+        args2 = PyMem_Malloc(nargs * sizeof(PyObject *));
+        if (args2 == NULL) {
+            PyErr_NoMemory();
+            return NULL;
+        }
+    }
+
+    /* use borrowed references */
+    args2[0] = obj;
+    memcpy(&args2[1],
+           args,
+           (nargs - 1)* sizeof(PyObject *));
+
+    result = _PyObject_FastCall(callable, args2, nargs);
+    if (args2 != small_stack) {
+        PyMem_Free(args2);
+    }
+    return result;
+}
+
+
+/* Call callable(obj, *args, **kwargs). */
+PyObject *
 _PyObject_Call_Prepend(PyObject *callable,
                        PyObject *obj, PyObject *args, PyObject *kwargs)
 {
diff --git a/Objects/typeobject.c b/Objects/typeobject.c
index 0193192..f18a795 100644
--- a/Objects/typeobject.c
+++ b/Objects/typeobject.c
@@ -1394,14 +1394,18 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b)
    the method name as a C string, and the address of a
    static variable used to cache the interned Python string.
 
-   Two variants:
+   Variants:
 
    - lookup_maybe() returns NULL without raising an exception
      when the _PyType_Lookup() call fails;
 
-   - lookup_method() always raises an exception upon errors.
+   - lookup_maybe_method() and lookup_method() are similar to
+     lookup_maybe(), but can return unbound PyFunction
+     to avoid temporary method object. Pass self as first argument when
+     unbound == 1.
 
-   - _PyObject_LookupSpecial() exported for the benefit of other places.
+   - _PyObject_LookupSpecial() expose lookup_maybe for the benefit of
+     other places.
 */
 
 static PyObject *
@@ -1421,11 +1425,38 @@ lookup_maybe(PyObject *self, _Py_Identifier *attrid)
 }
 
 static PyObject *
-lookup_method(PyObject *self, _Py_Identifier *attrid)
+lookup_maybe_method(PyObject *self, _Py_Identifier *attrid, int *unbound)
 {
-    PyObject *res = lookup_maybe(self, attrid);
-    if (res == NULL && !PyErr_Occurred())
+    PyObject *res = _PyType_LookupId(Py_TYPE(self), attrid);
+    if (res == NULL) {
+        return NULL;
+    }
+
+    if (PyFunction_Check(res)) {
+        /* Avoid temporary PyMethodObject */
+        *unbound = 1;
+        Py_INCREF(res);
+    }
+    else {
+        *unbound = 0;
+        descrgetfunc f = Py_TYPE(res)->tp_descr_get;
+        if (f == NULL) {
+            Py_INCREF(res);
+        }
+        else {
+            res = f(res, self, (PyObject *)(Py_TYPE(self)));
+        }
+    }
+    return res;
+}
+
+static PyObject *
+lookup_method(PyObject *self, _Py_Identifier *attrid, int *unbound)
+{
+    PyObject *res = lookup_maybe_method(self, attrid, unbound);
+    if (res == NULL && !PyErr_Occurred()) {
         PyErr_SetObject(PyExc_AttributeError, attrid->object);
+    }
     return res;
 }
 
@@ -1435,26 +1466,49 @@ _PyObject_LookupSpecial(PyObject *self, _Py_Identifier *attrid)
     return lookup_maybe(self, attrid);
 }
 
-/* A variation of PyObject_CallMethodObjArgs that uses lookup_method()
-   instead of PyObject_GetAttrString().  This uses the same convention
-   as lookup_method to cache the interned name string object. */
+static PyObject*
+call_unbound(int unbound, PyObject *func, PyObject *self,
+             PyObject **args, Py_ssize_t nargs)
+{
+    if (unbound) {
+        return _PyObject_FastCall_Prepend(func, self, args, nargs);
+    }
+    else {
+        return _PyObject_FastCall(func, args, nargs);
+    }
+}
 
+static PyObject*
+call_unbound_noarg(int unbound, PyObject *func, PyObject *self)
+{
+    if (unbound) {
+        PyObject *args[1] = {self};
+        return _PyObject_FastCall(func, args, 1);
+    }
+    else {
+        return _PyObject_CallNoArg(func);
+    }
+}
+
+/* A variation of PyObject_CallMethodObjArgs that uses lookup_maybe_method()
+   instead of PyObject_GetAttrString().  This uses the same convention
+   as lookup_maybe_method to cache the interned name string object. */
 static PyObject *
 call_method(PyObject *obj, _Py_Identifier *name,
             PyObject **args, Py_ssize_t nargs)
 {
+    int unbound;
     PyObject *func, *retval;
 
-    func = lookup_maybe(obj, name);
+    func = lookup_maybe_method(obj, name, &unbound);
     if (func == NULL) {
         if (!PyErr_Occurred())
             PyErr_SetObject(PyExc_AttributeError, name->object);
         return NULL;
     }
 
-    retval = _PyObject_FastCall(func, args, nargs);
+    retval = call_unbound(unbound, func, obj, args, nargs);
     Py_DECREF(func);
-
     return retval;
 }
 
@@ -1464,18 +1518,18 @@ static PyObject *
 call_maybe(PyObject *obj, _Py_Identifier *name,
            PyObject **args, Py_ssize_t nargs)
 {
+    int unbound;
     PyObject *func, *retval;
 
-    func = lookup_maybe(obj, name);
+    func = lookup_maybe_method(obj, name, &unbound);
     if (func == NULL) {
         if (!PyErr_Occurred())
             Py_RETURN_NOTIMPLEMENTED;
         return NULL;
     }
 
-    retval = _PyObject_FastCall(func, args, nargs);
+    retval = call_unbound(unbound, func, obj, args, nargs);
     Py_DECREF(func);
-
     return retval;
 }
 
@@ -1830,10 +1884,12 @@ mro_invoke(PyTypeObject *type)
 
     if (custom) {
         _Py_IDENTIFIER(mro);
-        PyObject *mro_meth = lookup_method((PyObject *)type, &PyId_mro);
+        int unbound;
+        PyObject *mro_meth = lookup_method((PyObject *)type, &PyId_mro,
+                                           &unbound);
         if (mro_meth == NULL)
             return NULL;
-        mro_result = _PyObject_CallNoArg(mro_meth);
+        mro_result = call_unbound_noarg(unbound, mro_meth, (PyObject *)type);
         Py_DECREF(mro_meth);
     }
     else {
@@ -5892,10 +5948,10 @@ static int
 slot_sq_contains(PyObject *self, PyObject *value)
 {
     PyObject *func, *res;
-    int result = -1;
+    int result = -1, unbound;
     _Py_IDENTIFIER(__contains__);
 
-    func = lookup_maybe(self, &PyId___contains__);
+    func = lookup_maybe_method(self, &PyId___contains__, &unbound);
     if (func == Py_None) {
         Py_DECREF(func);
         PyErr_Format(PyExc_TypeError,
@@ -5904,7 +5960,8 @@ slot_sq_contains(PyObject *self, PyObject *value)
         return -1;
     }
     if (func != NULL) {
-        res = PyObject_CallFunctionObjArgs(func, value, NULL);
+        PyObject *args[1] = {value};
+        res = call_unbound(unbound, func, self, args, 1);
         Py_DECREF(func);
         if (res != NULL) {
             result = PyObject_IsTrue(res);
@@ -5982,17 +6039,17 @@ static int
 slot_nb_bool(PyObject *self)
 {
     PyObject *func, *value;
-    int result;
+    int result, unbound;
     int using_len = 0;
     _Py_IDENTIFIER(__bool__);
 
-    func = lookup_maybe(self, &PyId___bool__);
+    func = lookup_maybe_method(self, &PyId___bool__, &unbound);
     if (func == NULL) {
         if (PyErr_Occurred()) {
             return -1;
         }
 
-        func = lookup_maybe(self, &PyId___len__);
+        func = lookup_maybe_method(self, &PyId___len__, &unbound);
         if (func == NULL) {
             if (PyErr_Occurred()) {
                 return -1;
@@ -6002,7 +6059,7 @@ slot_nb_bool(PyObject *self)
         using_len = 1;
     }
 
-    value = _PyObject_CallNoArg(func);
+    value = call_unbound_noarg(unbound, func, self);
     if (value == NULL) {
         goto error;
     }
@@ -6078,10 +6135,11 @@ slot_tp_repr(PyObject *self)
 {
     PyObject *func, *res;
     _Py_IDENTIFIER(__repr__);
+    int unbound;
 
-    func = lookup_method(self, &PyId___repr__);
+    func = lookup_method(self, &PyId___repr__, &unbound);
     if (func != NULL) {
-        res = PyEval_CallObject(func, NULL);
+        res = call_unbound_noarg(unbound, func, self);
         Py_DECREF(func);
         return res;
     }
@@ -6090,27 +6148,16 @@ slot_tp_repr(PyObject *self)
                                Py_TYPE(self)->tp_name, self);
 }
 
-static PyObject *
-slot_tp_str(PyObject *self)
-{
-    PyObject *func, *res;
-    _Py_IDENTIFIER(__str__);
-
-    func = lookup_method(self, &PyId___str__);
-    if (func == NULL)
-        return NULL;
-    res = PyEval_CallObject(func, NULL);
-    Py_DECREF(func);
-    return res;
-}
+SLOT0(slot_tp_str, "__str__")
 
 static Py_hash_t
 slot_tp_hash(PyObject *self)
 {
     PyObject *func, *res;
     Py_ssize_t h;
+    int unbound;
 
-    func = lookup_method(self, &PyId___hash__);
+    func = lookup_method(self, &PyId___hash__, &unbound);
 
     if (func == Py_None) {
         Py_DECREF(func);
@@ -6121,7 +6168,7 @@ slot_tp_hash(PyObject *self)
         return PyObject_HashNotImplemented(self);
     }
 
-    res = PyEval_CallObject(func, NULL);
+    res = call_unbound_noarg(unbound, func, self);
     Py_DECREF(func);
     if (res == NULL)
         return -1;
@@ -6155,13 +6202,19 @@ static PyObject *
 slot_tp_call(PyObject *self, PyObject *args, PyObject *kwds)
 {
     _Py_IDENTIFIER(__call__);
-    PyObject *meth = lookup_method(self, &PyId___call__);
+    int unbound;
+    PyObject *meth = lookup_method(self, &PyId___call__, &unbound);
     PyObject *res;
 
     if (meth == NULL)
         return NULL;
 
-    res = PyObject_Call(meth, args, kwds);
+    if (unbound) {
+        res = _PyObject_Call_Prepend(meth, self, args, kwds);
+    }
+    else {
+        res = PyObject_Call(meth, args, kwds);
+    }
 
     Py_DECREF(meth);
     return res;
@@ -6280,14 +6333,17 @@ static _Py_Identifier name_op[] = {
 static PyObject *
 slot_tp_richcompare(PyObject *self, PyObject *other, int op)
 {
+    int unbound;
     PyObject *func, *res;
 
-    func = lookup_method(self, &name_op[op]);
+    func = lookup_method(self, &name_op[op], &unbound);
     if (func == NULL) {
         PyErr_Clear();
         Py_RETURN_NOTIMPLEMENTED;
     }
-    res = PyObject_CallFunctionObjArgs(func, other, NULL);
+
+    PyObject *args[1] = {other};
+    res = call_unbound(unbound, func, self, args, 1);
     Py_DECREF(func);
     return res;
 }
@@ -6295,10 +6351,11 @@ slot_tp_richcompare(PyObject *self, PyObject *other, int op)
 static PyObject *
 slot_tp_iter(PyObject *self)
 {
+    int unbound;
     PyObject *func, *res;
     _Py_IDENTIFIER(__iter__);
 
-    func = lookup_method(self, &PyId___iter__);
+    func = lookup_method(self, &PyId___iter__, &unbound);
     if (func == Py_None) {
         Py_DECREF(func);
         PyErr_Format(PyExc_TypeError,
@@ -6308,13 +6365,13 @@ slot_tp_iter(PyObject *self)
     }
 
     if (func != NULL) {
-        res = _PyObject_CallNoArg(func);
+        res = call_unbound_noarg(unbound, func, self);
         Py_DECREF(func);
         return res;
     }
 
     PyErr_Clear();
-    func = lookup_method(self, &PyId___getitem__);
+    func = lookup_method(self, &PyId___getitem__, &unbound);
     if (func == NULL) {
         PyErr_Format(PyExc_TypeError,
                      "'%.200s' object is not iterable",
@@ -6380,12 +6437,18 @@ static int
 slot_tp_init(PyObject *self, PyObject *args, PyObject *kwds)
 {
     _Py_IDENTIFIER(__init__);
-    PyObject *meth = lookup_method(self, &PyId___init__);
+    int unbound;
+    PyObject *meth = lookup_method(self, &PyId___init__, &unbound);
     PyObject *res;
 
     if (meth == NULL)
         return -1;
-    res = PyObject_Call(meth, args, kwds);
+    if (unbound) {
+        res = _PyObject_Call_Prepend(meth, self, args, kwds);
+    }
+    else {
+        res = PyObject_Call(meth, args, kwds);
+    }
     Py_DECREF(meth);
     if (res == NULL)
         return -1;
@@ -6419,6 +6482,7 @@ static void
 slot_tp_finalize(PyObject *self)
 {
     _Py_IDENTIFIER(__del__);
+    int unbound;
     PyObject *del, *res;
     PyObject *error_type, *error_value, *error_traceback;
 
@@ -6426,9 +6490,9 @@ slot_tp_finalize(PyObject *self)
     PyErr_Fetch(&error_type, &error_value, &error_traceback);
 
     /* Execute __del__ method, if any. */
-    del = lookup_maybe(self, &PyId___del__);
+    del = lookup_maybe_method(self, &PyId___del__, &unbound);
     if (del != NULL) {
-        res = PyEval_CallObject(del, NULL);
+        res = call_unbound_noarg(unbound, del, self);
         if (res == NULL)
             PyErr_WriteUnraisable(del);
         else
@@ -6443,12 +6507,13 @@ slot_tp_finalize(PyObject *self)
 static PyObject *
 slot_am_await(PyObject *self)
 {
+    int unbound;
     PyObject *func, *res;
     _Py_IDENTIFIER(__await__);
 
-    func = lookup_method(self, &PyId___await__);
+    func = lookup_method(self, &PyId___await__, &unbound);
     if (func != NULL) {
-        res = PyEval_CallObject(func, NULL);
+        res = call_unbound_noarg(unbound, func, self);
         Py_DECREF(func);
         return res;
     }
@@ -6461,12 +6526,13 @@ slot_am_await(PyObject *self)
 static PyObject *
 slot_am_aiter(PyObject *self)
 {
+    int unbound;
     PyObject *func, *res;
     _Py_IDENTIFIER(__aiter__);
 
-    func = lookup_method(self, &PyId___aiter__);
+    func = lookup_method(self, &PyId___aiter__, &unbound);
     if (func != NULL) {
-        res = PyEval_CallObject(func, NULL);
+        res = call_unbound_noarg(unbound, func, self);
         Py_DECREF(func);
         return res;
     }
@@ -6479,12 +6545,13 @@ slot_am_aiter(PyObject *self)
 static PyObject *
 slot_am_anext(PyObject *self)
 {
+    int unbound;
     PyObject *func, *res;
     _Py_IDENTIFIER(__anext__);
 
-    func = lookup_method(self, &PyId___anext__);
+    func = lookup_method(self, &PyId___anext__, &unbound);
     if (func != NULL) {
-        res = PyEval_CallObject(func, NULL);
+        res = call_unbound_noarg(unbound, func, self);
         Py_DECREF(func);
         return res;
     }