bpo-30647: Check nl_langinfo(CODESET) in locale coercion (GH-2374)

- On some versions of FreeBSD, setting the "UTF-8" locale
  succeeds, but a subsequent "nl_langinfo(CODESET)" fails
- adding a check for this in the coercion logic means that
  coercion will happen on systems where this check succeeds,
  and will be skipped otherwise
- that way CPython should automatically adapt to changes in
  platform behaviour, rather than needing a new release to
  enable coercion at build time
- this also allows UTF-8 to be re-enabled as a coercion
  target, restoring the locale coercion behaviour on Mac OS X

1 files changed, 11 insertions, 8 deletions

diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 953bc90..3953fec 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -399,17 +399,10 @@ typedef struct _CandidateLocale {
 static _LocaleCoercionTarget _TARGET_LOCALES[] = {
     {"C.UTF-8"},
     {"C.utf8"},
-    /* {"UTF-8"}, */
+    {"UTF-8"},
     {NULL}
 };
 
-/* XXX (ncoghlan): Using UTF-8 as a target locale is currently disabled due to
- *                 problems encountered on *BSD systems with those test cases
- * For additional details see:
- *     nl_langinfo CODESET error: https://bugs.python.org/issue30647
- *     locale handling differences: https://bugs.python.org/issue30672
- */
-
 static char *
 get_default_standard_stream_error_handler(void)
 {
@@ -490,6 +483,16 @@ _Py_CoerceLegacyLocale(void)
                 const char *new_locale = setlocale(LC_CTYPE,
                                                    target->locale_name);
                 if (new_locale != NULL) {
+#if !defined(__APPLE__) && defined(HAVE_LANGINFO_H) && defined(CODESET)
+                    /* Also ensure that nl_langinfo works in this locale */
+                    char *codeset = nl_langinfo(CODESET);
+                    if (!codeset || *codeset == '\0') {
+                        /* CODESET is not set or empty, so skip coercion */
+                        new_locale = NULL;
+                        setlocale(LC_CTYPE, "");
+                        continue;
+                    }
+#endif
                     /* Successfully configured locale, so make it the default */
                     _coerce_default_locale_settings(target);
                     return;