bpo-28180: Implementation for PEP 538 (#659)

- new PYTHONCOERCECLOCALE config setting
- coerces legacy C locale to C.UTF-8, C.utf8 or UTF-8 by default
- always uses C.UTF-8 on Android
- uses `surrogateescape` on stdin and stdout in the coercion
  target locales
- configure option to disable locale coercion at build time
- configure option to disable C locale warning at build time

6 files changed, 315 insertions, 48 deletions

diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
index 1e74647..b3ac848 100644
--- a/Lib/test/support/script_helper.py
+++ b/Lib/test/support/script_helper.py
@@ -48,8 +48,35 @@ def interpreter_requires_environment():
     return __cached_interp_requires_environment
 
 
-_PythonRunResult = collections.namedtuple("_PythonRunResult",
-                                          ("rc", "out", "err"))
+class _PythonRunResult(collections.namedtuple("_PythonRunResult",
+                                          ("rc", "out", "err"))):
+    """Helper for reporting Python subprocess run results"""
+    def fail(self, cmd_line):
+        """Provide helpful details about failed subcommand runs"""
+        # Limit to 80 lines to ASCII characters
+        maxlen = 80 * 100
+        out, err = self.out, self.err
+        if len(out) > maxlen:
+            out = b'(... truncated stdout ...)' + out[-maxlen:]
+        if len(err) > maxlen:
+            err = b'(... truncated stderr ...)' + err[-maxlen:]
+        out = out.decode('ascii', 'replace').rstrip()
+        err = err.decode('ascii', 'replace').rstrip()
+        raise AssertionError("Process return code is %d\n"
+                             "command line: %r\n"
+                             "\n"
+                             "stdout:\n"
+                             "---\n"
+                             "%s\n"
+                             "---\n"
+                             "\n"
+                             "stderr:\n"
+                             "---\n"
+                             "%s\n"
+                             "---"
+                             % (self.rc, cmd_line,
+                                out,
+                                err))
 
 
 # Executing the interpreter in a subprocess
@@ -107,30 +134,7 @@ def run_python_until_end(*args, **env_vars):
 def _assert_python(expected_success, *args, **env_vars):
     res, cmd_line = run_python_until_end(*args, **env_vars)
     if (res.rc and expected_success) or (not res.rc and not expected_success):
-        # Limit to 80 lines to ASCII characters
-        maxlen = 80 * 100
-        out, err = res.out, res.err
-        if len(out) > maxlen:
-            out = b'(... truncated stdout ...)' + out[-maxlen:]
-        if len(err) > maxlen:
-            err = b'(... truncated stderr ...)' + err[-maxlen:]
-        out = out.decode('ascii', 'replace').rstrip()
-        err = err.decode('ascii', 'replace').rstrip()
-        raise AssertionError("Process return code is %d\n"
-                             "command line: %r\n"
-                             "\n"
-                             "stdout:\n"
-                             "---\n"
-                             "%s\n"
-                             "---\n"
-                             "\n"
-                             "stderr:\n"
-                             "---\n"
-                             "%s\n"
-                             "---"
-                             % (res.rc, cmd_line,
-                                out,
-                                err))
+        res.fail(cmd_line)
     return res
 
 def assert_python_ok(*args, **env_vars):
diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py
new file mode 100644
index 0000000..c14d820
--- /dev/null
+++ b/Lib/test/test_c_locale_coercion.py
@@ -0,0 +1,262 @@
+# Tests the attempted automatic coercion of the C locale to a UTF-8 locale
+
+import unittest
+import os
+import sys
+import sysconfig
+import shutil
+import subprocess
+from collections import namedtuple
+
+import test.support
+from test.support.script_helper import (
+    run_python_until_end,
+    interpreter_requires_environment,
+)
+
+# In order to get the warning messages to match up as expected, the candidate
+# order here must much the target locale order in Python/pylifecycle.c
+_C_UTF8_LOCALES = ("C.UTF-8", "C.utf8", "UTF-8")
+
+# There's no reliable cross-platform way of checking locale alias
+# lists, so the only way of knowing which of these locales will work
+# is to try them with locale.setlocale(). We do that in a subprocess
+# to avoid altering the locale of the test runner.
+def _set_locale_in_subprocess(locale_name):
+    cmd_fmt = "import locale; print(locale.setlocale(locale.LC_CTYPE, '{}'))"
+    cmd = cmd_fmt.format(locale_name)
+    result, py_cmd = run_python_until_end("-c", cmd, __isolated=True)
+    return result.rc == 0
+
+_EncodingDetails = namedtuple("EncodingDetails",
+                              "fsencoding stdin_info stdout_info stderr_info")
+
+class EncodingDetails(_EncodingDetails):
+    CHILD_PROCESS_SCRIPT = ";".join([
+        "import sys",
+        "print(sys.getfilesystemencoding())",
+        "print(sys.stdin.encoding + ':' + sys.stdin.errors)",
+        "print(sys.stdout.encoding + ':' + sys.stdout.errors)",
+        "print(sys.stderr.encoding + ':' + sys.stderr.errors)",
+    ])
+
+    @classmethod
+    def get_expected_details(cls, expected_fsencoding):
+        """Returns expected child process details for a given encoding"""
+        _stream = expected_fsencoding + ":{}"
+        # stdin and stdout should use surrogateescape either because the
+        # coercion triggered, or because the C locale was detected
+        stream_info = 2*[_stream.format("surrogateescape")]
+        # stderr should always use backslashreplace
+        stream_info.append(_stream.format("backslashreplace"))
+        return dict(cls(expected_fsencoding, *stream_info)._asdict())
+
+    @staticmethod
+    def _handle_output_variations(data):
+        """Adjust the output to handle platform specific idiosyncrasies
+
+        * Some platforms report ASCII as ANSI_X3.4-1968
+        * Some platforms report ASCII as US-ASCII
+        * Some platforms report UTF-8 instead of utf-8
+        """
+        data = data.replace(b"ANSI_X3.4-1968", b"ascii")
+        data = data.replace(b"US-ASCII", b"ascii")
+        data = data.lower()
+        return data
+
+    @classmethod
+    def get_child_details(cls, env_vars):
+        """Retrieves fsencoding and standard stream details from a child process
+
+        Returns (encoding_details, stderr_lines):
+
+        - encoding_details: EncodingDetails for eager decoding
+        - stderr_lines: result of calling splitlines() on the stderr output
+
+        The child is run in isolated mode if the current interpreter supports
+        that.
+        """
+        result, py_cmd = run_python_until_end(
+            "-c", cls.CHILD_PROCESS_SCRIPT,
+            __isolated=True,
+            **env_vars
+        )
+        if not result.rc == 0:
+            result.fail(py_cmd)
+        # All subprocess outputs in this test case should be pure ASCII
+        adjusted_output = cls._handle_output_variations(result.out)
+        stdout_lines = adjusted_output.decode("ascii").rstrip().splitlines()
+        child_encoding_details = dict(cls(*stdout_lines)._asdict())
+        stderr_lines = result.err.decode("ascii").rstrip().splitlines()
+        return child_encoding_details, stderr_lines
+
+
+class _ChildProcessEncodingTestCase(unittest.TestCase):
+    # Base class to check for expected encoding details in a child process
+
+    def _check_child_encoding_details(self,
+                                      env_vars,
+                                      expected_fsencoding,
+                                      expected_warning):
+        """Check the C locale handling for the given process environment
+
+        Parameters:
+            expected_fsencoding: the encoding the child is expected to report
+            allow_c_locale: setting to use for PYTHONALLOWCLOCALE
+              None: don't set the variable at all
+              str: the value set in the child's environment
+        """
+        result = EncodingDetails.get_child_details(env_vars)
+        encoding_details, stderr_lines = result
+        self.assertEqual(encoding_details,
+                         EncodingDetails.get_expected_details(
+                             expected_fsencoding))
+        self.assertEqual(stderr_lines, expected_warning)
+
+# Details of the shared library warning emitted at runtime
+LIBRARY_C_LOCALE_WARNING = (
+    "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
+    "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
+    "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
+    "locales is recommended."
+)
+
+@unittest.skipUnless(sysconfig.get_config_var("PY_WARN_ON_C_LOCALE"),
+                     "C locale runtime warning disabled at build time")
+class LocaleWarningTests(_ChildProcessEncodingTestCase):
+    # Test warning emitted when running in the C locale
+
+    def test_library_c_locale_warning(self):
+        self.maxDiff = None
+        for locale_to_set in ("C", "POSIX", "invalid.ascii"):
+            var_dict = {
+                "LC_ALL": locale_to_set
+            }
+            with self.subTest(forced_locale=locale_to_set):
+                self._check_child_encoding_details(var_dict,
+                                                   "ascii",
+                                                   [LIBRARY_C_LOCALE_WARNING])
+
+# Details of the CLI locale coercion warning emitted at runtime
+CLI_COERCION_WARNING_FMT = (
+    "Python detected LC_CTYPE=C: LC_CTYPE coerced to {} (set another locale "
+    "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
+)
+
+class _LocaleCoercionTargetsTestCase(_ChildProcessEncodingTestCase):
+    # Base class for test cases that rely on coercion targets being defined
+
+    available_targets = []
+    targets_required = True
+
+    @classmethod
+    def setUpClass(cls):
+        first_target_locale = None
+        available_targets = cls.available_targets
+        # Find the target locales available in the current system
+        for target_locale in _C_UTF8_LOCALES:
+            if _set_locale_in_subprocess(target_locale):
+                available_targets.append(target_locale)
+                if first_target_locale is None:
+                    first_target_locale = target_locale
+        if cls.targets_required and not available_targets:
+            raise unittest.SkipTest("No C-with-UTF-8 locale available")
+        # Expect coercion to use the first available locale
+        warning_msg = CLI_COERCION_WARNING_FMT.format(first_target_locale)
+        cls.EXPECTED_COERCION_WARNING = warning_msg
+
+
+class LocaleConfigurationTests(_LocaleCoercionTargetsTestCase):
+    # Test explicit external configuration via the process environment
+
+    def test_external_target_locale_configuration(self):
+        # Explicitly setting a target locale should give the same behaviour as
+        # is seen when implicitly coercing to that target locale
+        self.maxDiff = None
+
+        expected_warning = []
+        expected_fsencoding = "utf-8"
+
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+        }
+        for env_var in ("LANG", "LC_CTYPE"):
+            for locale_to_set in self.available_targets:
+                with self.subTest(env_var=env_var,
+                                  configured_locale=locale_to_set):
+                    var_dict = base_var_dict.copy()
+                    var_dict[env_var] = locale_to_set
+                    self._check_child_encoding_details(var_dict,
+                                                       expected_fsencoding,
+                                                       expected_warning)
+
+
+
+@test.support.cpython_only
+@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
+                     "C locale coercion disabled at build time")
+class LocaleCoercionTests(_LocaleCoercionTargetsTestCase):
+    # Test implicit reconfiguration of the environment during CLI startup
+
+    def _check_c_locale_coercion(self, expected_fsencoding, coerce_c_locale):
+        """Check the C locale handling for various configurations
+
+        Parameters:
+            expected_fsencoding: the encoding the child is expected to report
+            allow_c_locale: setting to use for PYTHONALLOWCLOCALE
+              None: don't set the variable at all
+              str: the value set in the child's environment
+        """
+
+        # Check for expected warning on stderr if C locale is coerced
+        self.maxDiff = None
+
+        expected_warning = []
+        if coerce_c_locale != "0":
+            expected_warning.append(self.EXPECTED_COERCION_WARNING)
+
+        base_var_dict = {
+            "LANG": "",
+            "LC_CTYPE": "",
+            "LC_ALL": "",
+        }
+        for env_var in ("LANG", "LC_CTYPE"):
+            for locale_to_set in ("", "C", "POSIX", "invalid.ascii"):
+                with self.subTest(env_var=env_var,
+                                  nominal_locale=locale_to_set,
+                                  PYTHONCOERCECLOCALE=coerce_c_locale):
+                    var_dict = base_var_dict.copy()
+                    var_dict[env_var] = locale_to_set
+                    if coerce_c_locale is not None:
+                        var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
+                    self._check_child_encoding_details(var_dict,
+                                                       expected_fsencoding,
+                                                       expected_warning)
+
+    def test_test_PYTHONCOERCECLOCALE_not_set(self):
+        # This should coerce to the first available target locale by default
+        self._check_c_locale_coercion("utf-8", coerce_c_locale=None)
+
+    def test_PYTHONCOERCECLOCALE_not_zero(self):
+        # *Any* string other that "0" is considered "set" for our purposes
+        # and hence should result in the locale coercion being enabled
+        for setting in ("", "1", "true", "false"):
+            self._check_c_locale_coercion("utf-8", coerce_c_locale=setting)
+
+    def test_PYTHONCOERCECLOCALE_set_to_zero(self):
+        # The setting "0" should result in the locale coercion being disabled
+        self._check_c_locale_coercion("ascii", coerce_c_locale="0")
+
+
+def test_main():
+    test.support.run_unittest(
+        LocaleConfigurationTests,
+        LocaleCoercionTests,
+        LocaleWarningTests
+    )
+    test.support.reap_children()
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 8ac8af9..c4a9766 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -371,14 +371,21 @@ class EmbeddingTests(unittest.TestCase):
     def tearDown(self):
         os.chdir(self.oldcwd)
 
-    def run_embedded_interpreter(self, *args):
+    def run_embedded_interpreter(self, *args, env=None):
         """Runs a test in the embedded interpreter"""
         cmd = [self.test_exe]
         cmd.extend(args)
+        if env is not None and sys.platform == 'win32':
+            # Windows requires at least the SYSTEMROOT environment variable to
+            # start Python.
+            env = env.copy()
+            env['SYSTEMROOT'] = os.environ['SYSTEMROOT']
+
         p = subprocess.Popen(cmd,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE,
-                             universal_newlines=True)
+                             universal_newlines=True,
+                             env=env)
         (out, err) = p.communicate()
         self.assertEqual(p.returncode, 0,
                          "bad returncode %d, stderr is %r" %
@@ -471,26 +478,16 @@ class EmbeddingTests(unittest.TestCase):
                 self.assertNotEqual(sub.tstate, main.tstate)
                 self.assertNotEqual(sub.modules, main.modules)
 
-    @staticmethod
-    def _get_default_pipe_encoding():
-        rp, wp = os.pipe()
-        try:
-            with os.fdopen(wp, 'w') as w:
-                default_pipe_encoding = w.encoding
-        finally:
-            os.close(rp)
-        return default_pipe_encoding
-
     def test_forced_io_encoding(self):
         # Checks forced configuration of embedded interpreter IO streams
-        out, err = self.run_embedded_interpreter("forced_io_encoding")
-        if support.verbose:
+        env = {"PYTHONIOENCODING": "utf-8:surrogateescape"}
+        out, err = self.run_embedded_interpreter("forced_io_encoding", env=env)
+        if support.verbose > 1:
             print()
             print(out)
             print(err)
-        expected_errors = sys.__stdout__.errors
-        expected_stdin_encoding = sys.__stdin__.encoding
-        expected_pipe_encoding = self._get_default_pipe_encoding()
+        expected_stream_encoding = "utf-8"
+        expected_errors = "surrogateescape"
         expected_output = '\n'.join([
         "--- Use defaults ---",
         "Expected encoding: default",
@@ -517,8 +514,8 @@ class EmbeddingTests(unittest.TestCase):
         "stdout: latin-1:replace",
         "stderr: latin-1:backslashreplace"])
         expected_output = expected_output.format(
-                                in_encoding=expected_stdin_encoding,
-                                out_encoding=expected_pipe_encoding,
+                                in_encoding=expected_stream_encoding,
+                                out_encoding=expected_stream_encoding,
                                 errors=expected_errors)
         # This is useful if we ever trip over odd platform behaviour
         self.maxDiff = None
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 5f96d61..3e20427 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -8,8 +8,9 @@ import sys
 import subprocess
 import tempfile
 from test.support import script_helper, is_android
-from test.support.script_helper import (spawn_python, kill_python, assert_python_ok,
-    assert_python_failure)
+from test.support.script_helper import (
+    spawn_python, kill_python, assert_python_ok, assert_python_failure
+)
 
 
 # XXX (ncoghlan): Move to script_helper and make consistent with run_python
@@ -150,6 +151,7 @@ class CmdLineTest(unittest.TestCase):
         env = os.environ.copy()
         # Use C locale to get ascii for the locale encoding
         env['LC_ALL'] = 'C'
+        env['PYTHONCOERCECLOCALE'] = '0'
         code = (
             b'import locale; '
             b'print(ascii("' + undecodable + b'"), '
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 7fabe6a..52b05c1 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -642,7 +642,8 @@ class ProcessTestCase(BaseTestCase):
             # on adding even when the environment in exec is empty.
             # Gentoo sandboxes also force LD_PRELOAD and SANDBOX_* to exist.
             return ('VERSIONER' in n or '__CF' in n or  # MacOS
-                    n == 'LD_PRELOAD' or n.startswith('SANDBOX'))  # Gentoo
+                    n == 'LD_PRELOAD' or n.startswith('SANDBOX') or # Gentoo
+                    n == 'LC_CTYPE') # Locale coercion triggered
 
         with subprocess.Popen([sys.executable, "-c",
                                'import os; print(list(os.environ.keys()))'],
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
index ed78e2a..3844812 100644
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -682,6 +682,7 @@ class SysModuleTest(unittest.TestCase):
         # Force the POSIX locale
         env = os.environ.copy()
         env["LC_ALL"] = "C"
+        env["PYTHONCOERCECLOCALE"] = "0"
         code = '\n'.join((
             'import sys',
             'def dump(name):',