bpo-36142: Add _PyPreConfig.utf8_mode (GH-12174)

* Move following fields from _PyCoreConfig to _PyPreConfig:

  * coerce_c_locale
  * coerce_c_locale_warn
  * legacy_windows_stdio
  * utf8_mode

* _PyPreConfig_ReadFromArgv() is now responsible to choose the
  filesystem encoding
* _PyPreConfig_Write() now sets the LC_CTYPE locale

2 files changed, 45 insertions, 22 deletions

diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h
index 7997d59..306577c 100644
--- a/Include/cpython/coreconfig.h
+++ b/Include/cpython/coreconfig.h
@@ -60,12 +60,42 @@ typedef struct {
        Set to 0 by -E command line option. If set to -1 (default), it is
        set to !Py_IgnoreEnvironmentFlag. */
     int use_environment;
+
+    int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
+    int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
+
+#ifdef MS_WINDOWS
+    /* If greater than 1, use the "mbcs" encoding instead of the UTF-8
+       encoding for the filesystem encoding.
+
+       Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is
+       set to a non-empty string. If set to -1 (default), inherit
+       Py_LegacyWindowsFSEncodingFlag value.
+
+       See PEP 529 for more details. */
+    int legacy_windows_fs_encoding;
+#endif
+
+    /* Enable UTF-8 mode?
+       Set by -X utf8 command line option and PYTHONUTF8 environment variable.
+       If set to -1 (default), inherit Py_UTF8Mode value. */
+    int utf8_mode;
 } _PyPreConfig;
 
+#ifdef MS_WINDOWS
+#  define _PyPreConfig_WINDOWS_INIT \
+        .legacy_windows_fs_encoding = -1,
+#else
+#  define _PyPreConfig_WINDOWS_INIT
+#endif
+
 #define _PyPreConfig_INIT \
     (_PyPreConfig){ \
+        _PyPreConfig_WINDOWS_INIT \
         .isolated = -1, \
-        .use_environment = -1}
+        .use_environment = -1, \
+        .coerce_c_locale = -1, \
+        .utf8_mode = -1}
 
 
 /* --- _PyCoreConfig ---------------------------------------------- */
@@ -95,8 +125,6 @@ typedef struct {
     int show_alloc_count;   /* -X showalloccount */
     int dump_refs;          /* PYTHONDUMPREFS */
     int malloc_stats;       /* PYTHONMALLOCSTATS */
-    int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
-    int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
 
     /* Python filesystem encoding and error handler:
        sys.getfilesystemencoding() and sys.getfilesystemencodeerrors().
@@ -134,11 +162,6 @@ typedef struct {
     char *filesystem_encoding;
     char *filesystem_errors;
 
-    /* Enable UTF-8 mode?
-       Set by -X utf8 command line option and PYTHONUTF8 environment variable.
-       If set to -1 (default), inherit Py_UTF8Mode value. */
-    int utf8_mode;
-
     wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
 
     wchar_t *program_name;  /* Program name, see also Py_GetProgramName() */
@@ -277,16 +300,6 @@ typedef struct {
     char *stdio_errors;
 
 #ifdef MS_WINDOWS
-    /* If greater than 1, use the "mbcs" encoding instead of the UTF-8
-       encoding for the filesystem encoding.
-
-       Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is
-       set to a non-empty string. If set to -1 (default), inherit
-       Py_LegacyWindowsFSEncodingFlag value.
-
-       See PEP 529 for more details. */
-    int legacy_windows_fs_encoding;
-
     /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys
        standard streams.
 
@@ -340,7 +353,6 @@ typedef struct {
 
 #ifdef MS_WINDOWS
 #  define _PyCoreConfig_WINDOWS_INIT \
-        .legacy_windows_fs_encoding = -1, \
         .legacy_windows_stdio = -1,
 #else
 #  define _PyCoreConfig_WINDOWS_INIT
@@ -348,13 +360,12 @@ typedef struct {
 
 #define _PyCoreConfig_INIT \
     (_PyCoreConfig){ \
+        _PyCoreConfig_WINDOWS_INIT \
         .preconfig = _PyPreConfig_INIT, \
         .install_signal_handlers = 1, \
         .use_hash_seed = -1, \
         .faulthandler = -1, \
         .tracemalloc = -1, \
-        .coerce_c_locale = -1, \
-        .utf8_mode = -1, \
         .argc = -1, \
         .nmodule_search_path = -1, \
         .site_import = -1, \
@@ -368,7 +379,6 @@ typedef struct {
         .quiet = -1, \
         .user_site_directory = -1, \
         .buffered_stdio = -1, \
-        _PyCoreConfig_WINDOWS_INIT \
         ._install_importlib = 1, \
         ._check_hash_pycs_mode = "default", \
         ._frozen = -1}
diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h
index 5135969..8df182c 100644
--- a/Include/internal/pycore_coreconfig.h
+++ b/Include/internal/pycore_coreconfig.h
@@ -36,11 +36,24 @@ PyAPI_FUNC(int) _Py_SetArgcArgv(int argc, wchar_t * const *argv);
 
 /* --- _PyPreConfig ----------------------------------------------- */
 
+PyAPI_FUNC(int) _Py_str_to_int(
+    const char *str,
+    int *result);
+PyAPI_FUNC(const wchar_t*) _Py_get_xoption(
+    int nxoption,
+    wchar_t * const *xoptions,
+    const wchar_t *name);
+
 PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
 PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
     const _PyPreConfig *config2);
 PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config);
 PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config);
+PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config,
+    const char *name);
+PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config,
+    int *flag,
+    const char *name);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config);
 PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config,
     PyObject *dict);