Issue #8589: Decode PYTHONWARNINGS environment variable with the file system

encoding and surrogateespace error handler instead of the locale encoding to be
consistent with os.environ. Add PySys_AddWarnOptionUnicode() function.

5 files changed, 27 insertions, 12 deletions

diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index adadfe5..bc4cbae 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -81,6 +81,10 @@ accessible to C code.  They all work with the current interpreter thread's
 
    Append *s* to :data:`sys.warnoptions`.
 
+.. cfunction:: void PySys_AddWarnOptionUnicode(PyObject *unicode)
+
+   Append *unicode* to :data:`sys.warnoptions`.
+
 .. cfunction:: void PySys_SetPath(wchar_t *path)
 
    Set :data:`sys.path` to a list object of paths found in *path* which should
diff --git a/Include/sysmodule.h b/Include/sysmodule.h
index 5078fe0..3a3bf3e 100644
--- a/Include/sysmodule.h
+++ b/Include/sysmodule.h
@@ -21,6 +21,7 @@ PyAPI_DATA(PyObject *) _PySys_TraceFunc, *_PySys_ProfileFunc;
 
 PyAPI_FUNC(void) PySys_ResetWarnOptions(void);
 PyAPI_FUNC(void) PySys_AddWarnOption(const wchar_t *);
+PyAPI_FUNC(void) PySys_AddWarnOptionUnicode(PyObject *);
 PyAPI_FUNC(int) PySys_HasWarnOptions(void);
 
 #ifdef __cplusplus
diff --git a/Misc/NEWS b/Misc/NEWS
index 8a19de3..ceda9b7 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 3.2 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #8589: Decode PYTHONWARNINGS environment variable with the file system
+  encoding and surrogateespace error handler instead of the locale encoding to
+  be consistent with os.environ. Add PySys_AddWarnOptionUnicode() function.
+
 - PyObject_Dump() encodes unicode objects to utf8 with backslashreplace
   (instead of strict) error handler to escape surrogates
 
diff --git a/Modules/main.c b/Modules/main.c
index 92b971f..29f5fc8 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -425,7 +425,7 @@ Py_Main(int argc, wchar_t **argv)
 #else
     if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
         char *buf, *oldloc;
-        wchar_t *warning;
+        PyObject *warning;
 
         /* settle for strtok here as there's no one standard
            C89 wcstok */
@@ -437,9 +437,10 @@ Py_Main(int argc, wchar_t **argv)
         oldloc = strdup(setlocale(LC_ALL, NULL));
         setlocale(LC_ALL, "");
         for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) {
-            if ((warning = _Py_char2wchar(p)) != NULL) {
-                PySys_AddWarnOption(warning);
-                PyMem_Free(warning);
+            warning = PyUnicode_DecodeFSDefault(p);
+            if (warning != NULL) {
+                PySys_AddWarnOptionUnicode(warning);
+                Py_DECREF(warning);
             }
         }
         setlocale(LC_ALL, oldloc);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index ac14751..ce06d7d 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1048,21 +1048,26 @@ PySys_ResetWarnOptions(void)
 }
 
 void
-PySys_AddWarnOption(const wchar_t *s)
+PySys_AddWarnOptionUnicode(PyObject *unicode)
 {
-    PyObject *str;
-
     if (warnoptions == NULL || !PyList_Check(warnoptions)) {
         Py_XDECREF(warnoptions);
         warnoptions = PyList_New(0);
         if (warnoptions == NULL)
             return;
     }
-    str = PyUnicode_FromWideChar(s, -1);
-    if (str != NULL) {
-        PyList_Append(warnoptions, str);
-        Py_DECREF(str);
-    }
+    PyList_Append(warnoptions, unicode);
+}
+
+void
+PySys_AddWarnOption(const wchar_t *s)
+{
+    PyObject *unicode;
+    unicode = PyUnicode_FromWideChar(s, -1);
+    if (unicode == NULL)
+        return;
+    PySys_AddWarnOptionUnicode(unicode);
+    Py_DECREF(unicode);
 }
 
 int