PEP 489: Multi-phase extension module initialization

Known limitations of the current implementation:

- documentation changes are incomplete
- there's a reference leak I haven't tracked down yet

The leak is most visible by running:

  ./python -m test -R3:3 test_importlib

However, you can also see it by running:

  ./python -X showrefcount

Importing the array or _testmultiphase modules, and
then deleting them from both sys.modules and the local
namespace shows significant increases in the total
number of active references each cycle. By contrast,
with _testcapi (which continues to use single-phase
initialisation) the global refcounts stabilise after
a couple of cycles.

1 files changed, 283 insertions, 52 deletions

diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c
index dca5e58..7a86a5b 100644
--- a/Objects/moduleobject.c
+++ b/Objects/moduleobject.c
@@ -20,7 +20,7 @@ static PyMemberDef module_members[] = {
     {0}
 };
 
-static PyTypeObject moduledef_type = {
+PyTypeObject PyModuleDef_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "moduledef",                                /* tp_name */
     sizeof(struct PyModuleDef),                 /* tp_size */
@@ -28,6 +28,20 @@ static PyTypeObject moduledef_type = {
 };
 
 
+PyObject*
+PyModuleDef_Init(struct PyModuleDef* def)
+{
+    if (PyType_Ready(&PyModuleDef_Type) < 0)
+         return NULL;
+    if (def->m_base.m_index == 0) {
+        max_module_number++;
+        Py_REFCNT(def) = 1;
+        Py_TYPE(def) = &PyModuleDef_Type;
+        def->m_base.m_index = max_module_number;
+    }
+    return (PyObject*)def;
+}
+
 static int
 module_init_dict(PyModuleObject *mod, PyObject *md_dict,
                  PyObject *name, PyObject *doc)
@@ -97,35 +111,45 @@ PyModule_New(const char *name)
     return module;
 }
 
+/* Check API/ABI version
+ * Issues a warning on mismatch, which is usually not fatal.
+ * Returns 0 if an exception is raised.
+ */
+static int
+check_api_version(const char *name, int module_api_version)
+{
+    if (module_api_version != PYTHON_API_VERSION && module_api_version != PYTHON_ABI_VERSION) {
+        int err;
+        err = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
+            "Python C API version mismatch for module %.100s: "
+            "This Python has API version %d, module %.100s has version %d.",
+             name,
+             PYTHON_API_VERSION, name, module_api_version);
+        if (err)
+            return 0;
+    }
+    return 1;
+}
 
 PyObject *
 PyModule_Create2(struct PyModuleDef* module, int module_api_version)
 {
-    PyObject *d, *v, *n;
-    PyMethodDef *ml;
     const char* name;
     PyModuleObject *m;
     PyInterpreterState *interp = PyThreadState_Get()->interp;
     if (interp->modules == NULL)
         Py_FatalError("Python import machinery not initialized");
-    if (PyType_Ready(&moduledef_type) < 0)
+    if (!PyModuleDef_Init(module))
         return NULL;
-    if (module->m_base.m_index == 0) {
-        max_module_number++;
-        Py_REFCNT(module) = 1;
-        Py_TYPE(module) = &moduledef_type;
-        module->m_base.m_index = max_module_number;
-    }
     name = module->m_name;
-    if (module_api_version != PYTHON_API_VERSION && module_api_version != PYTHON_ABI_VERSION) {
-        int err;
-        err = PyErr_WarnFormat(PyExc_RuntimeWarning, 1,
-            "Python C API version mismatch for module %.100s: "
-            "This Python has API version %d, module %.100s has version %d.",
-             name,
-             PYTHON_API_VERSION, name, module_api_version);
-        if (err)
-            return NULL;
+    if (!check_api_version(name, module_api_version)) {
+        return NULL;
+    }
+    if (module->m_slots) {
+        PyErr_Format(
+            PyExc_SystemError,
+            "module %s: PyModule_Create is incompatible with m_slots", name);
+        return NULL;
     }
     /* Make sure name is fully qualified.
 
@@ -156,53 +180,260 @@ PyModule_Create2(struct PyModuleDef* module, int module_api_version)
         memset(m->md_state, 0, module->m_size);
     }
 
-    d = PyModule_GetDict((PyObject*)m);
     if (module->m_methods != NULL) {
-        n = PyUnicode_FromString(name);
-        if (n == NULL) {
+        if (PyModule_AddFunctions((PyObject *) m, module->m_methods) != 0) {
             Py_DECREF(m);
             return NULL;
         }
-        for (ml = module->m_methods; ml->ml_name != NULL; ml++) {
-            if ((ml->ml_flags & METH_CLASS) ||
-                (ml->ml_flags & METH_STATIC)) {
-                PyErr_SetString(PyExc_ValueError,
-                                "module functions cannot set"
-                                " METH_CLASS or METH_STATIC");
-                Py_DECREF(n);
-                Py_DECREF(m);
-                return NULL;
-            }
-            v = PyCFunction_NewEx(ml, (PyObject*)m, n);
-            if (v == NULL) {
-                Py_DECREF(n);
-                Py_DECREF(m);
-                return NULL;
-            }
-            if (PyDict_SetItemString(d, ml->ml_name, v) != 0) {
-                Py_DECREF(v);
-                Py_DECREF(n);
-                Py_DECREF(m);
-                return NULL;
-            }
-            Py_DECREF(v);
-        }
-        Py_DECREF(n);
     }
     if (module->m_doc != NULL) {
-        _Py_IDENTIFIER(__doc__);
-        v = PyUnicode_FromString(module->m_doc);
-        if (v == NULL || _PyDict_SetItemId(d, &PyId___doc__, v) != 0) {
-            Py_XDECREF(v);
+        if (PyModule_SetDocString((PyObject *) m, module->m_doc) != 0) {
             Py_DECREF(m);
             return NULL;
         }
-        Py_DECREF(v);
     }
     m->md_def = module;
     return (PyObject*)m;
 }
 
+PyObject *
+PyModule_FromDefAndSpec2(struct PyModuleDef* def, PyObject *spec, int module_api_version)
+{
+    PyModuleDef_Slot* cur_slot;
+    PyObject *(*create)(PyObject *, PyModuleDef*) = NULL;
+    PyObject *nameobj;
+    PyObject *m = NULL;
+    int has_execution_slots = 0;
+    char *name;
+    int ret;
+
+    PyModuleDef_Init(def);
+
+    nameobj = PyObject_GetAttrString(spec, "name");
+    if (nameobj == NULL) {
+        return NULL;
+    }
+    name = PyUnicode_AsUTF8(nameobj);
+    if (name == NULL) {
+        goto error;
+    }
+
+    if (!check_api_version(name, module_api_version)) {
+        goto error;
+    }
+
+    if (def->m_size < 0) {
+        PyErr_Format(
+            PyExc_SystemError,
+            "module %s: m_size may not be negative for multi-phase initialization",
+            name);
+        goto error;
+    }
+
+    for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) {
+        if (cur_slot->slot == Py_mod_create) {
+            if (create) {
+                PyErr_Format(
+                    PyExc_SystemError,
+                    "module %s has multiple create slots",
+                    name);
+                goto error;
+            }
+            create = cur_slot->value;
+        } else if (cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT) {
+            PyErr_Format(
+                PyExc_SystemError,
+                "module %s uses unknown slot ID %i",
+                name, cur_slot->slot);
+            goto error;
+        } else {
+            has_execution_slots = 1;
+        }
+    }
+
+    if (create) {
+        m = create(spec, def);
+        if (m == NULL) {
+            if (!PyErr_Occurred()) {
+                PyErr_Format(
+                    PyExc_SystemError,
+                    "creation of module %s failed without setting an exception",
+                    name);
+            }
+            goto error;
+        } else {
+            if (PyErr_Occurred()) {
+                PyErr_Format(PyExc_SystemError,
+                            "creation of module %s raised unreported exception",
+                            name);
+                goto error;
+            }
+        }
+    } else {
+        m = PyModule_New(name);
+        if (m == NULL) {
+            goto error;
+        }
+    }
+
+    if (PyModule_Check(m)) {
+        ((PyModuleObject*)m)->md_state = NULL;
+        ((PyModuleObject*)m)->md_def = def;
+    } else {
+        if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) {
+            PyErr_Format(
+                PyExc_SystemError,
+                "module %s is not a module object, but requests module state",
+                name);
+            goto error;
+        }
+        if (has_execution_slots) {
+            PyErr_Format(
+                PyExc_SystemError,
+                "module %s specifies execution slots, but did not create "
+                    "a ModuleType instance",
+                name);
+            goto error;
+        }
+    }
+
+    if (def->m_methods != NULL) {
+        ret = PyModule_AddFunctions(m, def->m_methods);
+        if (ret != 0) {
+            goto error;
+        }
+    }
+
+    if (def->m_doc != NULL) {
+        ret = PyModule_SetDocString(m, def->m_doc);
+        if (ret != 0) {
+            goto error;
+        }
+    }
+
+    return m;
+
+error:
+    Py_DECREF(nameobj);
+    Py_XDECREF(m);
+    return NULL;
+}
+
+int
+PyModule_ExecDef(PyObject *module, PyModuleDef *def)
+{
+    PyModuleDef_Slot *cur_slot;
+    const char *name;
+    int ret;
+
+    name = PyModule_GetName(module);
+    if (name == NULL) {
+        return -1;
+    }
+
+    if (PyModule_Check(module) && def->m_size >= 0) {
+        PyModuleObject *md = (PyModuleObject*)module;
+        if (md->md_state == NULL) {
+            /* Always set a state pointer; this serves as a marker to skip
+             * multiple initialization (importlib.reload() is no-op) */
+            md->md_state = PyMem_MALLOC(def->m_size);
+            if (!md->md_state) {
+                PyErr_NoMemory();
+                return -1;
+            }
+            memset(md->md_state, 0, def->m_size);
+        }
+    }
+
+    if (def->m_slots == NULL) {
+        return 0;
+    }
+
+    for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) {
+        switch (cur_slot->slot) {
+            case Py_mod_create:
+                /* handled in PyModule_CreateFromSlots */
+                break;
+            case Py_mod_exec:
+                ret = ((int (*)(PyObject *))cur_slot->value)(module);
+                if (ret != 0) {
+                    if (!PyErr_Occurred()) {
+                        PyErr_Format(
+                            PyExc_SystemError,
+                            "execution of module %s failed without setting an exception",
+                            name);
+                    }
+                    return -1;
+                }
+                if (PyErr_Occurred()) {
+                    PyErr_Format(
+                        PyExc_SystemError,
+                        "execution of module %s raised unreported exception",
+                        name);
+                    return -1;
+                }
+                break;
+            default:
+                PyErr_Format(
+                    PyExc_SystemError,
+                    "module %s initialized with unknown slot %i",
+                    name, cur_slot->slot);
+                return -1;
+        }
+    }
+    return 0;
+}
+
+int
+PyModule_AddFunctions(PyObject *m, PyMethodDef *functions)
+{
+    PyObject *name, *func;
+    PyMethodDef *fdef;
+
+    name = PyModule_GetNameObject(m);
+    if (name == NULL) {
+        return -1;
+    }
+
+    for (fdef = functions; fdef->ml_name != NULL; fdef++) {
+        if ((fdef->ml_flags & METH_CLASS) ||
+            (fdef->ml_flags & METH_STATIC)) {
+            PyErr_SetString(PyExc_ValueError,
+                            "module functions cannot set"
+                            " METH_CLASS or METH_STATIC");
+            Py_DECREF(name);
+            return -1;
+        }
+        func = PyCFunction_NewEx(fdef, (PyObject*)m, name);
+        if (func == NULL) {
+            Py_DECREF(name);
+            return -1;
+        }
+        if (PyObject_SetAttrString(m, fdef->ml_name, func) != 0) {
+            Py_DECREF(func);
+            Py_DECREF(name);
+            return -1;
+        }
+        Py_DECREF(func);
+    }
+    Py_DECREF(name);
+    return 0;
+}
+
+int
+PyModule_SetDocString(PyObject *m, const char *doc)
+{
+    PyObject *v;
+    _Py_IDENTIFIER(__doc__);
+
+    v = PyUnicode_FromString(doc);
+    if (v == NULL || _PyObject_SetAttrId(m, &PyId___doc__, v) != 0) {
+        Py_XDECREF(v);
+        return -1;
+    }
+    Py_DECREF(v);
+    return 0;
+}
 
 PyObject *
 PyModule_GetDict(PyObject *m)