bpo-35713: Split _Py_InitializeCore into subfunctions (GH-11650)

* Split _Py_InitializeCore_impl() into subfunctions: add multiple pycore_init_xxx() functions
* Preliminary sys.stderr is now set earlier to get an usable
  sys.stderr ealier.
* Move code into _Py_Initialize_ReconfigureCore() to be able to call
  it from _Py_InitializeCore().
* Split _PyExc_Init(): create a new _PyBuiltins_AddExceptions()
  function.
* Call _PyExc_Init() earlier in _Py_InitializeCore_impl()
  and new_interpreter() to get working exceptions earlier.
* _Py_ReadyTypes() now returns _PyInitError rather than calling
  Py_FatalError().
* Misc code cleanup

1 files changed, 172 insertions, 83 deletions

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index f0e00ea..86d87fb 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -464,9 +464,22 @@ _Py_SetLocaleFromEnv(int category)
 */
 
 static _PyInitError
-_Py_Initialize_ReconfigureCore(PyInterpreterState *interp,
+_Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p,
                                const _PyCoreConfig *core_config)
 {
+    PyThreadState *tstate = _PyThreadState_GET();
+    if (!tstate) {
+        return _Py_INIT_ERR("failed to read thread state");
+    }
+
+    PyInterpreterState *interp = tstate->interp;
+    if (interp == NULL) {
+        return _Py_INIT_ERR("can't make main interpreter");
+    }
+    *interp_p = interp;
+
+    /* bpo-34008: For backward compatibility reasons, calling Py_Main() after
+       Py_Initialize() ignores the new configuration. */
     if (core_config->allocator != NULL) {
         const char *allocator = _PyMem_GetAllocatorsName();
         if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
@@ -492,57 +505,16 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState *interp,
 }
 
 
-/* Begin interpreter initialization
- *
- * On return, the first thread and interpreter state have been created,
- * but the compiler, signal handling, multithreading and
- * multiple interpreter support, and codec infrastructure are not yet
- * available.
- *
- * The import system will support builtin and frozen modules only.
- * The only supported io is writing to sys.stderr
- *
- * If any operation invoked by this function fails, a fatal error is
- * issued and the function does not return.
- *
- * Any code invoked from this function should *not* assume it has access
- * to the Python C API (unless the API is explicitly listed as being
- * safe to call without calling Py_Initialize first)
- *
- * The caller is responsible to call _PyCoreConfig_Read().
- */
-
 static _PyInitError
-_Py_InitializeCore_impl(PyInterpreterState **interp_p,
-                        const _PyCoreConfig *core_config)
+pycore_init_runtime(const _PyCoreConfig *core_config)
 {
-    PyInterpreterState *interp;
-    _PyInitError err;
-
-    /* bpo-34008: For backward compatibility reasons, calling Py_Main() after
-       Py_Initialize() ignores the new configuration. */
-    if (_PyRuntime.core_initialized) {
-        PyThreadState *tstate = _PyThreadState_GET();
-        if (!tstate) {
-            return _Py_INIT_ERR("failed to read thread state");
-        }
-
-        interp = tstate->interp;
-        if (interp == NULL) {
-            return _Py_INIT_ERR("can't make main interpreter");
-        }
-        *interp_p = interp;
-
-        return _Py_Initialize_ReconfigureCore(interp, core_config);
-    }
-
     if (_PyRuntime.initialized) {
         return _Py_INIT_ERR("main interpreter already initialized");
     }
 
     _PyCoreConfig_SetGlobalConfig(core_config);
 
-    err = _PyRuntime_Initialize();
+    _PyInitError err = _PyRuntime_Initialize();
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -573,8 +545,15 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
+    return _Py_INIT_OK();
+}
 
-    interp = PyInterpreterState_New();
+
+static _PyInitError
+pycore_create_interpreter(const _PyCoreConfig *core_config,
+                          PyInterpreterState **interp_p)
+{
+    PyInterpreterState *interp = PyInterpreterState_New();
     if (interp == NULL) {
         return _Py_INIT_ERR("can't make main interpreter");
     }
@@ -603,24 +582,61 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
     /* Create the GIL */
     PyEval_InitThreads();
 
-    _Py_ReadyTypes();
+    return _Py_INIT_OK();
+}
+
+
+static _PyInitError
+pycore_init_types(void)
+{
+    _PyInitError err = _Py_ReadyTypes();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
 
-    if (!_PyLong_Init())
+    err = _PyUnicode_Init();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    if (_PyStructSequence_Init() < 0) {
+        return _Py_INIT_ERR("can't initialize structseq");
+    }
+
+    if (!_PyLong_Init()) {
         return _Py_INIT_ERR("can't init longs");
+    }
 
-    if (!_PyFloat_Init())
+    err = _PyExc_Init();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    if (!_PyFloat_Init()) {
         return _Py_INIT_ERR("can't init float");
+    }
 
+    if (!_PyContext_Init()) {
+        return _Py_INIT_ERR("can't init context");
+    }
+    return _Py_INIT_OK();
+}
+
+
+static _PyInitError
+pycore_init_sys(PyInterpreterState *interp, PyObject **sysmod_p)
+{
     PyObject *modules = PyDict_New();
     if (modules == NULL)
         return _Py_INIT_ERR("can't make modules dictionary");
     interp->modules = modules;
 
     PyObject *sysmod;
-    err = _PySys_BeginInit(&sysmod);
+    _PyInitError err = _PySys_BeginInit(&sysmod);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
+    *sysmod_p = sysmod;
 
     interp->sysdict = PyModule_GetDict(sysmod);
     if (interp->sysdict == NULL) {
@@ -631,39 +647,49 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
     PyDict_SetItemString(interp->sysdict, "modules", modules);
     _PyImport_FixupBuiltin(sysmod, "sys", modules);
 
-    err = _PyUnicode_Init();
-    if (_Py_INIT_FAILED(err)) {
-        return err;
+    /* Set up a preliminary stderr printer until we have enough
+       infrastructure for the io module in place.
+
+       Use UTF-8/surrogateescape and ignore EAGAIN errors. */
+    PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
+    if (pstderr == NULL) {
+        return _Py_INIT_ERR("can't set preliminary stderr");
     }
+    _PySys_SetObjectId(&PyId_stderr, pstderr);
+    PySys_SetObject("__stderr__", pstderr);
+    Py_DECREF(pstderr);
+
+    return _Py_INIT_OK();
+}
 
-    if (_PyStructSequence_Init() < 0)
-        return _Py_INIT_ERR("can't initialize structseq");
 
+static _PyInitError
+pycore_init_builtins(PyInterpreterState *interp)
+{
     PyObject *bimod = _PyBuiltin_Init();
-    if (bimod == NULL)
+    if (bimod == NULL) {
         return _Py_INIT_ERR("can't initialize builtins modules");
-    _PyImport_FixupBuiltin(bimod, "builtins", modules);
+    }
+    _PyImport_FixupBuiltin(bimod, "builtins", interp->modules);
+
     interp->builtins = PyModule_GetDict(bimod);
-    if (interp->builtins == NULL)
+    if (interp->builtins == NULL) {
         return _Py_INIT_ERR("can't initialize builtins dict");
+    }
     Py_INCREF(interp->builtins);
 
-    /* initialize builtin exceptions */
-    err = _PyExc_Init(bimod);
+    _PyInitError err = _PyBuiltins_AddExceptions(bimod);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
+    return _Py_INIT_OK();
+}
 
-    /* Set up a preliminary stderr printer until we have enough
-       infrastructure for the io module in place. */
-    PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
-    if (pstderr == NULL)
-        return _Py_INIT_ERR("can't set preliminary stderr");
-    _PySys_SetObjectId(&PyId_stderr, pstderr);
-    PySys_SetObject("__stderr__", pstderr);
-    Py_DECREF(pstderr);
 
-    err = _PyImport_Init(interp);
+static _PyInitError
+pycore_init_import_warnings(PyInterpreterState *interp, PyObject *sysmod)
+{
+    _PyInitError err = _PyImport_Init(interp);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -678,29 +704,85 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
         return _Py_INIT_ERR("can't initialize warnings");
     }
 
-    if (!_PyContext_Init())
-        return _Py_INIT_ERR("can't init context");
-
-    if (core_config->_install_importlib) {
-        err = _PyCoreConfig_SetPathConfig(core_config);
+    if (interp->core_config._install_importlib) {
+        err = _PyCoreConfig_SetPathConfig(&interp->core_config);
         if (_Py_INIT_FAILED(err)) {
             return err;
         }
     }
 
     /* This call sets up builtin and frozen import support */
-    if (core_config->_install_importlib) {
+    if (interp->core_config._install_importlib) {
         err = initimport(interp, sysmod);
         if (_Py_INIT_FAILED(err)) {
             return err;
         }
     }
+    return _Py_INIT_OK();
+}
+
+
+static _PyInitError
+_Py_InitializeCore_impl(PyInterpreterState **interp_p,
+                        const _PyCoreConfig *core_config)
+{
+    PyInterpreterState *interp;
+
+    _PyInitError err = pycore_init_runtime(core_config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    err = pycore_create_interpreter(core_config, &interp);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+    core_config = &interp->core_config;
+    *interp_p = interp;
+
+    err = pycore_init_types();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    PyObject *sysmod;
+    err = pycore_init_sys(interp, &sysmod);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    err = pycore_init_builtins(interp);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    err = pycore_init_import_warnings(interp, sysmod);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
 
     /* Only when we get here is the runtime core fully initialized */
     _PyRuntime.core_initialized = 1;
     return _Py_INIT_OK();
 }
 
+/* Begin interpreter initialization
+ *
+ * On return, the first thread and interpreter state have been created,
+ * but the compiler, signal handling, multithreading and
+ * multiple interpreter support, and codec infrastructure are not yet
+ * available.
+ *
+ * The import system will support builtin and frozen modules only.
+ * The only supported io is writing to sys.stderr
+ *
+ * If any operation invoked by this function fails, a fatal error is
+ * issued and the function does not return.
+ *
+ * Any code invoked from this function should *not* assume it has access
+ * to the Python C API (unless the API is explicitly listed as being
+ * safe to call without calling Py_Initialize first)
+ */
 _PyInitError
 _Py_InitializeCore(PyInterpreterState **interp_p,
                    const _PyCoreConfig *src_config)
@@ -730,7 +812,12 @@ _Py_InitializeCore(PyInterpreterState **interp_p,
         goto done;
     }
 
-    err = _Py_InitializeCore_impl(interp_p, &config);
+    if (!_PyRuntime.core_initialized) {
+        err = _Py_InitializeCore_impl(interp_p, &config);
+    }
+    else {
+        err = _Py_Initialize_ReconfigureCore(interp_p, &config);
+    }
 
 done:
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -1270,6 +1357,11 @@ new_interpreter(PyThreadState **tstate_p)
         return _Py_INIT_ERR("failed to copy main interpreter config");
     }
 
+    err = _PyExc_Init();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
     /* XXX The following is lax in error checking */
     PyObject *modules = PyDict_New();
     if (modules == NULL) {
@@ -1301,18 +1393,15 @@ new_interpreter(PyThreadState **tstate_p)
         goto handle_error;
     }
 
-    /* initialize builtin exceptions */
-    err = _PyExc_Init(bimod);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
     if (bimod != NULL && sysmod != NULL) {
-        PyObject *pstderr;
+        err = _PyBuiltins_AddExceptions(bimod);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
 
         /* Set up a preliminary stderr printer until we have enough
            infrastructure for the io module in place. */
-        pstderr = PyFile_NewStdPrinter(fileno(stderr));
+        PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
         if (pstderr == NULL) {
             return _Py_INIT_ERR("can't set preliminary stderr");
         }