bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972)

* The UTF-8 Mode is now also enabled by the "POSIX" locale, not only
  by the "C" locale.
* On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces
  the ASCII encoding if the LC_CTYPE locale is "POSIX", not only if
  the LC_CTYPE locale is "C".
* test_utf8_mode.test_cmd_line() checks also that the command line
  arguments are decoded from UTF-8 when the the UTF-8 Mode is enabled
  with POSIX locale or C locale.

1 files changed, 17 insertions, 6 deletions

diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py
index 3e918fd..df988c1 100644
--- a/Lib/test/test_utf8_mode.py
+++ b/Lib/test/test_utf8_mode.py
@@ -12,6 +12,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure
 
 
 MS_WINDOWS = (sys.platform == 'win32')
+POSIX_LOCALES = ('C', 'POSIX')
 
 
 class UTF8ModeTests(unittest.TestCase):
@@ -23,7 +24,7 @@ class UTF8ModeTests(unittest.TestCase):
 
     def posix_locale(self):
         loc = locale.setlocale(locale.LC_CTYPE, None)
-        return (loc == 'C')
+        return (loc in POSIX_LOCALES)
 
     def get_output(self, *args, failure=False, **kw):
         kw = dict(self.DEFAULT_ENV, **kw)
@@ -39,8 +40,10 @@ class UTF8ModeTests(unittest.TestCase):
     def test_posix_locale(self):
         code = 'import sys; print(sys.flags.utf8_mode)'
 
-        out = self.get_output('-c', code, LC_ALL='C')
-        self.assertEqual(out, '1')
+        for loc in POSIX_LOCALES:
+            with self.subTest(LC_ALL=loc):
+                out = self.get_output('-c', code, LC_ALL=loc)
+                self.assertEqual(out, '1')
 
     def test_xoption(self):
         code = 'import sys; print(sys.flags.utf8_mode)'
@@ -201,8 +204,10 @@ class UTF8ModeTests(unittest.TestCase):
         out = self.get_output('-X', 'utf8', '-c', code)
         self.assertEqual(out, 'UTF-8 UTF-8')
 
-        out = self.get_output('-X', 'utf8', '-c', code, LC_ALL='C')
-        self.assertEqual(out, 'UTF-8 UTF-8')
+        for loc in POSIX_LOCALES:
+            with self.subTest(LC_ALL=loc):
+                out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc)
+                self.assertEqual(out, 'UTF-8 UTF-8')
 
     @unittest.skipIf(MS_WINDOWS, 'test specific to Unix')
     def test_cmd_line(self):
@@ -217,13 +222,19 @@ class UTF8ModeTests(unittest.TestCase):
             self.assertEqual(args, ascii(expected), out)
 
         check('utf8', [arg_utf8])
+        for loc in POSIX_LOCALES:
+            with self.subTest(LC_ALL=loc):
+                check('utf8', [arg_utf8], LC_ALL=loc)
+
         if sys.platform == 'darwin' or support.is_android:
             c_arg = arg_utf8
         elif sys.platform.startswith("aix"):
             c_arg = arg.decode('iso-8859-1')
         else:
             c_arg = arg_ascii
-        check('utf8=0', [c_arg], LC_ALL='C')
+        for loc in POSIX_LOCALES:
+            with self.subTest(LC_ALL=loc):
+                check('utf8=0', [c_arg], LC_ALL=loc)
 
     def test_optim_level(self):
         # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag