bpo-34523: Add _PyCoreConfig.filesystem_encoding (GH-8963)

_PyCoreConfig_Read() is now responsible to choose the filesystem
encoding and error handler. Using Py_Main(), the encoding is now
chosen even before calling Py_Initialize().

_PyCoreConfig.filesystem_encoding is now the reference, instead of
Py_FileSystemDefaultEncoding, for the Python filesystem encoding.

Changes:

* Add filesystem_encoding and filesystem_errors to _PyCoreConfig
* _PyCoreConfig_Read() now reads the locale encoding for the file
  system encoding.
* PyUnicode_EncodeFSDefault() and PyUnicode_DecodeFSDefaultAndSize()
  now use the interpreter configuration rather than
  Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors
  global configuration variables.
* Add _Py_SetFileSystemEncoding() and _Py_ClearFileSystemEncoding()
  private functions to only modify Py_FileSystemDefaultEncoding and
  Py_FileSystemDefaultEncodeErrors in coreconfig.c.
* _Py_CoerceLegacyLocale() now takes an int rather than
  _PyCoreConfig for the warning.

2 files changed, 10 insertions, 1 deletions

diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c
index fdf5013..2621a76 100644
--- a/Programs/_freeze_importlib.c
+++ b/Programs/_freeze_importlib.c
@@ -81,8 +81,15 @@ main(int argc, char *argv[])
     config.program_name = L"./_freeze_importlib";
     /* Don't install importlib, since it could execute outdated bytecode. */
     config._install_importlib = 0;
-    config.install_signal_handlers = 1;
     config._frozen = 1;
+#ifdef MS_WINDOWS
+    /* bpo-34523: initfsencoding() is not called if _install_importlib=0,
+       so interp->fscodec_initialized value remains 0.
+       PyUnicode_EncodeFSDefault() doesn't support the "surrogatepass" error
+       handler in such case, whereas it's the default error handler on Windows.
+       Force the "strict" error handler to work around this bootstrap issue. */
+    config.filesystem_errors = "strict";
+#endif
 
     _PyInitError err = _Py_InitializeFromConfig(&config);
     /* No need to call _PyCoreConfig_Clear() since we didn't allocate any
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index d569417..99772ea 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -328,6 +328,8 @@ dump_config(void)
     printf("dump_refs = %i\n", config->dump_refs);
     printf("malloc_stats = %i\n", config->malloc_stats);
 
+    printf("filesystem_encoding = %s\n", config->filesystem_encoding);
+    printf("filesystem_errors = %s\n", config->filesystem_errors);
     printf("coerce_c_locale = %i\n", config->coerce_c_locale);
     printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn);
     printf("utf8_mode = %i\n", config->utf8_mode);