Optimize _PyCFunction_FastCallKeywords()

Issue #29259: Write fast path in _PyCFunction_FastCallKeywords() for
METH_FASTCALL, avoid the creation of a temporary dictionary for keyword
arguments.

Cleanup also _PyCFunction_FastCallDict():

* Don't dereference func before checking that it's not NULL
* Move code to raise the "no keyword argument" exception into a new
  no_keyword_error label.

Update python-gdb.py for the change.

2 files changed, 133 insertions, 37 deletions

diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 14750b6..1e9ad0d 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -89,7 +89,7 @@ PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwds)
 
     assert(kwds == NULL || PyDict_Check(kwds));
     /* PyCFunction_Call() must not be called with an exception set,
-       because it may clear it (directly or indirectly) and so the
+       because it can clear it (directly or indirectly) and so the
        caller loses its exception */
     assert(!PyErr_Occurred());
 
@@ -155,14 +155,14 @@ PyObject *
 _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
                           PyObject *kwargs)
 {
-    PyCFunctionObject *func = (PyCFunctionObject*)func_obj;
-    PyCFunction meth = PyCFunction_GET_FUNCTION(func);
-    PyObject *self = PyCFunction_GET_SELF(func);
+    PyCFunctionObject *func;
+    PyCFunction meth;
+    PyObject *self;
     PyObject *result;
     int flags;
 
-    assert(PyCFunction_Check(func));
-    assert(func != NULL);
+    assert(func_obj != NULL);
+    assert(PyCFunction_Check(func_obj));
     assert(nargs >= 0);
     assert(nargs == 0 || args != NULL);
     assert(kwargs == NULL || PyDict_Check(kwargs));
@@ -172,34 +172,28 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
        caller loses its exception */
     assert(!PyErr_Occurred());
 
+    func = (PyCFunctionObject*)func_obj;
+    meth = PyCFunction_GET_FUNCTION(func);
+    self = PyCFunction_GET_SELF(func);
     flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
 
     switch (flags)
     {
     case METH_NOARGS:
+        if (nargs != 0) {
+            goto no_keyword_error;
+        }
+
         if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
             PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
                          func->m_ml->ml_name);
             return NULL;
         }
 
-        if (nargs != 0) {
-            PyErr_Format(PyExc_TypeError,
-                "%.200s() takes no arguments (%zd given)",
-                func->m_ml->ml_name, nargs);
-            return NULL;
-        }
-
         result = (*meth) (self, NULL);
         break;
 
     case METH_O:
-        if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
-            PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments",
-                         func->m_ml->ml_name);
-            return NULL;
-        }
-
         if (nargs != 1) {
             PyErr_Format(PyExc_TypeError,
                 "%.200s() takes exactly one argument (%zd given)",
@@ -207,20 +201,22 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
             return NULL;
         }
 
+        if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
+            goto no_keyword_error;
+        }
+
         result = (*meth) (self, args[0]);
         break;
 
     case METH_VARARGS:
     case METH_VARARGS | METH_KEYWORDS:
     {
-        /* Slow-path: create a temporary tuple */
+        /* Slow-path: create a temporary tuple for positional arguments */
         PyObject *tuple;
 
-        if (!(flags & METH_KEYWORDS) && kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
-            PyErr_Format(PyExc_TypeError,
-                         "%.200s() takes no keyword arguments",
-                         func->m_ml->ml_name);
-            return NULL;
+        if (!(flags & METH_KEYWORDS)
+                && kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) {
+            goto no_keyword_error;
         }
 
         tuple = _PyStack_AsTuple(args, nargs);
@@ -267,35 +263,134 @@ _PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs,
     result = _Py_CheckFunctionResult(func_obj, result, NULL);
 
     return result;
+
+no_keyword_error:
+    PyErr_Format(PyExc_TypeError,
+            "%.200s() takes no arguments (%zd given)",
+            func->m_ml->ml_name, nargs);
+    return NULL;
 }
 
 PyObject *
-_PyCFunction_FastCallKeywords(PyObject *func, PyObject **stack,
+_PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args,
                               Py_ssize_t nargs, PyObject *kwnames)
 {
-    PyObject *kwdict, *result;
+    PyCFunctionObject *func;
+    PyCFunction meth;
+    PyObject *self, *result;
     Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames);
+    int flags;
 
-    assert(PyCFunction_Check(func));
+    assert(func_obj != NULL);
+    assert(PyCFunction_Check(func_obj));
     assert(nargs >= 0);
     assert(kwnames == NULL || PyTuple_CheckExact(kwnames));
-    assert((nargs == 0 && nkwargs == 0) || stack != NULL);
+    assert((nargs == 0 && nkwargs == 0) || args != NULL);
     /* kwnames must only contains str strings, no subclass, and all keys must
        be unique */
 
-    if (nkwargs > 0) {
-        kwdict = _PyStack_AsDict(stack + nargs, kwnames);
-        if (kwdict == NULL) {
+    /* _PyCFunction_FastCallKeywords() must not be called with an exception
+       set, because it can clear it (directly or indirectly) and so the caller
+       loses its exception */
+    assert(!PyErr_Occurred());
+
+    func = (PyCFunctionObject*)func_obj;
+    meth = PyCFunction_GET_FUNCTION(func);
+    self = PyCFunction_GET_SELF(func);
+    flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST);
+
+    switch (flags)
+    {
+    case METH_NOARGS:
+        if (nargs != 0) {
+            PyErr_Format(PyExc_TypeError,
+                "%.200s() takes no arguments (%zd given)",
+                func->m_ml->ml_name, nargs);
+            return NULL;
+        }
+
+        if (nkwargs) {
+            goto no_keyword_error;
+        }
+
+        result = (*meth) (self, NULL);
+        break;
+
+    case METH_O:
+        if (nargs != 1) {
+            PyErr_Format(PyExc_TypeError,
+                "%.200s() takes exactly one argument (%zd given)",
+                func->m_ml->ml_name, nargs);
             return NULL;
         }
+
+        if (nkwargs) {
+            goto no_keyword_error;
+        }
+
+        result = (*meth) (self, args[0]);
+        break;
+
+    case METH_FASTCALL:
+        /* Fast-path: avoid temporary dict to pass keyword arguments */
+        result = ((_PyCFunctionFast)meth) (self, args, nargs, kwnames);
+        break;
+
+    case METH_VARARGS:
+    case METH_VARARGS | METH_KEYWORDS:
+    {
+        /* Slow-path: create a temporary tuple for positional arguments
+           and a temporary dict for keyword arguments */
+        PyObject *argtuple;
+
+        if (!(flags & METH_KEYWORDS) && nkwargs) {
+            goto no_keyword_error;
+        }
+
+        argtuple = _PyStack_AsTuple(args, nargs);
+        if (argtuple == NULL) {
+            return NULL;
+        }
+
+        if (flags & METH_KEYWORDS) {
+            PyObject *kwdict;
+
+            if (nkwargs > 0) {
+                kwdict = _PyStack_AsDict(args + nargs, kwnames);
+                if (kwdict == NULL) {
+                    Py_DECREF(argtuple);
+                    return NULL;
+                }
+            }
+            else {
+                kwdict = NULL;
+            }
+
+            result = (*(PyCFunctionWithKeywords)meth) (self, argtuple, kwdict);
+            Py_XDECREF(kwdict);
+        }
+        else {
+            result = (*meth) (self, argtuple);
+        }
+        Py_DECREF(argtuple);
+        break;
     }
-    else {
-        kwdict = NULL;
+
+    default:
+        PyErr_SetString(PyExc_SystemError,
+                        "Bad call flags in _PyCFunction_FastCallKeywords. "
+                        "METH_OLDARGS is no longer supported!");
+        return NULL;
     }
 
-    result = _PyCFunction_FastCallDict(func, stack, nargs, kwdict);
-    Py_XDECREF(kwdict);
+    result = _Py_CheckFunctionResult(func_obj, result, NULL);
     return result;
+
+no_keyword_error:
+    PyErr_Format(PyExc_TypeError,
+                 "%.200s() takes no keyword arguments",
+                 func->m_ml->ml_name);
+    return NULL;
 }
 
 /* Methods (the standard built-in methods, that is) */
diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py
index cc1afbe..88fb0aa 100755
--- a/Tools/gdb/libpython.py
+++ b/Tools/gdb/libpython.py
@@ -1518,7 +1518,8 @@ class Frame(object):
             except RuntimeError:
                 return 'PyCFunction invocation (unable to read "func")'
 
-        elif caller == '_PyCFunction_FastCallDict':
+        elif caller in {'_PyCFunction_FastCallDict',
+                        '_PyCFunction_FastCallKeywords'}:
             try:
                 func = older._gdbframe.read_var('func_obj')
                 return str(func)