bpo-36443: Disable C locale coercion and UTF-8 Mode by default (GH-12589)

bpo-36443, bpo-36202: Since Python 3.7.0, calling Py_DecodeLocale()
before Py_Initialize() produces mojibake if the LC_CTYPE locale is
coerced and/or if the UTF-8 Mode is enabled by the user
configuration. This change fix the issue by disabling LC_CTYPE
coercion and UTF-8 Mode by default. They must now be enabled
explicitly (opt-in) using the new _Py_PreInitialize() API with
_PyPreConfig.

When embedding Python, set coerce_c_locale and utf8_mode attributes
of _PyPreConfig to -1 to enable automatically these parameters
depending on the LC_CTYPE locale, environment variables and command
line arguments

Alternative: Setting Py_UTF8Mode to 1 always explicitly enables the
UTF-8 Mode.

Changes:

* _PyPreConfig_INIT now sets coerce_c_locale and utf8_mode to 0 by
  default.
* _Py_InitializeFromArgs() and _Py_InitializeFromWideArgs() can now
  be called with config=NULL.

2 files changed, 18 insertions, 8 deletions

diff --git a/Python/preconfig.c b/Python/preconfig.c
index 011ed53..7ac645d 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -386,7 +386,9 @@ _PyPreConfig_GetGlobalConfig(_PyPreConfig *config)
 #ifdef MS_WINDOWS
     COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag);
 #endif
-    COPY_FLAG(utf8_mode, Py_UTF8Mode);
+    if (Py_UTF8Mode > 0) {
+        config->utf8_mode = 1;
+    }
 
 #undef COPY_FLAG
 #undef COPY_NOT_FLAG
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 7c6948e..ad14472 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -485,7 +485,7 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p,
     _PyCoreConfig_Write(core_config);
 
     if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
-        return _Py_INIT_ERR("failed to copy core config");
+        return _Py_INIT_NO_MEMORY();
     }
     core_config = &interp->core_config;
 
@@ -548,7 +548,7 @@ pycore_create_interpreter(const _PyCoreConfig *core_config,
     *interp_p = interp;
 
     if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
-        return _Py_INIT_ERR("failed to copy core config");
+        return _Py_INIT_NO_MEMORY();
     }
     core_config = &interp->core_config;
 
@@ -785,6 +785,7 @@ _Py_PreInitialize(const _PyPreConfig *src_config)
 _PyInitError
 _Py_PreInitializeFromCoreConfig(const _PyCoreConfig *coreconfig)
 {
+    assert(coreconfig != NULL);
     _PyPreConfig config = _PyPreConfig_INIT;
     _PyCoreConfig_GetCoreConfig(&config, coreconfig);
     return _Py_PreInitialize(&config);
@@ -799,8 +800,10 @@ pyinit_coreconfig(_PyCoreConfig *config,
                   const _PyArgv *args,
                   PyInterpreterState **interp_p)
 {
-    if (_PyCoreConfig_Copy(config, src_config) < 0) {
-        return _Py_INIT_ERR("failed to copy core config");
+    if (src_config) {
+        if (_PyCoreConfig_Copy(config, src_config) < 0) {
+            return _Py_INIT_NO_MEMORY();
+        }
     }
 
     _PyInitError err = _PyCoreConfig_Read(config, args);
@@ -839,9 +842,14 @@ _Py_InitializeCore(const _PyCoreConfig *src_config,
                    const _PyArgv *args,
                    PyInterpreterState **interp_p)
 {
-    assert(src_config != NULL);
+    _PyInitError err;
 
-    _PyInitError err = _Py_PreInitializeFromCoreConfig(src_config);
+    if (src_config) {
+        err = _Py_PreInitializeFromCoreConfig(src_config);
+    }
+    else {
+        err = _Py_PreInitialize(NULL);
+    }
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -1395,7 +1403,7 @@ new_interpreter(PyThreadState **tstate_p)
     }
 
     if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
-        return _Py_INIT_ERR("failed to copy core config");
+        return _Py_INIT_NO_MEMORY();
     }
     core_config = &interp->core_config;