bpo-38317: Fix PyConfig.warnoptions priority (GH-16478)

Fix warnings options priority: PyConfig.warnoptions has the highest
priority, as stated in the PEP 587.

* Document options order in PyConfig.warnoptions documentation.
* Make PyWideStringList_INIT macro private: replace "Py" prefix
  with "_Py".
* test_embed: add test_init_warnoptions().
(cherry picked from commit fb4ae152a9930f0e00cae8b2807f534058cf341a)

Co-authored-by: Victor Stinner <vstinner@redhat.com>

3 files changed, 85 insertions, 45 deletions

diff --git a/Python/initconfig.c b/Python/initconfig.c
index dbc9de5..fb04b4a 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -273,7 +273,7 @@ _PyWideStringList_Copy(PyWideStringList *list, const PyWideStringList *list2)
         return 0;
     }
 
-    PyWideStringList copy = PyWideStringList_INIT;
+    PyWideStringList copy = _PyWideStringList_INIT;
 
     size_t size = list2->length * sizeof(list2->items[0]);
     copy.items = PyMem_RawMalloc(size);
@@ -2095,63 +2095,83 @@ config_init_env_warnoptions(PyConfig *config, PyWideStringList *warnoptions)
 
 
 static PyStatus
-config_add_warnoption(PyConfig *config, const wchar_t *option)
+warnoptions_append(PyConfig *config, PyWideStringList *options,
+                   const wchar_t *option)
 {
+    /* config_init_warnoptions() add existing config warnoptions at the end:
+       ensure that the new option is not already present in this list to
+       prevent change the options order whne config_init_warnoptions() is
+       called twice. */
     if (_PyWideStringList_Find(&config->warnoptions, option)) {
         /* Already present: do nothing */
         return _PyStatus_OK();
     }
-    return PyWideStringList_Append(&config->warnoptions, option);
+    if (_PyWideStringList_Find(options, option)) {
+        /* Already present: do nothing */
+        return _PyStatus_OK();
+    }
+    return PyWideStringList_Append(options, option);
+}
+
+
+static PyStatus
+warnoptions_extend(PyConfig *config, PyWideStringList *options,
+                   const PyWideStringList *options2)
+{
+    const Py_ssize_t len = options2->length;
+    wchar_t *const *items = options2->items;
+
+    for (Py_ssize_t i = 0; i < len; i++) {
+        PyStatus status = warnoptions_append(config, options, items[i]);
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+    return _PyStatus_OK();
 }
 
 
 static PyStatus
 config_init_warnoptions(PyConfig *config,
                         const PyWideStringList *cmdline_warnoptions,
-                        const PyWideStringList *env_warnoptions)
+                        const PyWideStringList *env_warnoptions,
+                        const PyWideStringList *sys_warnoptions)
 {
     PyStatus status;
+    PyWideStringList options = _PyWideStringList_INIT;
 
-    /* The priority order for warnings configuration is (highest precedence
-     * first):
+    /* Priority of warnings options, lowest to highest:
      *
-     * - early PySys_AddWarnOption() calls
-     * - the BytesWarning filter, if needed ('-b', '-bb')
-     * - any '-W' command line options; then
-     * - the 'PYTHONWARNINGS' environment variable; then
-     * - the dev mode filter ('-X dev', 'PYTHONDEVMODE'); then
      * - any implicit filters added by _warnings.c/warnings.py
+     * - PyConfig.dev_mode: "default" filter
+     * - PYTHONWARNINGS environment variable
+     * - '-W' command line options
+     * - PyConfig.bytes_warning ('-b' and '-bb' command line options):
+     *   "default::BytesWarning" or "error::BytesWarning" filter
+     * - early PySys_AddWarnOption() calls
+     * - PyConfig.warnoptions
      *
-     * All settings except the last are passed to the warnings module via
-     * the `sys.warnoptions` list. Since the warnings module works on the basis
-     * of "the most recently added filter will be checked first", we add
-     * the lowest precedence entries first so that later entries override them.
+     * PyConfig.warnoptions is copied to sys.warnoptions. Since the warnings
+     * module works on the basis of "the most recently added filter will be
+     * checked first", we add the lowest precedence entries first so that later
+     * entries override them.
      */
 
     if (config->dev_mode) {
-        status = config_add_warnoption(config, L"default");
+        status = warnoptions_append(config, &options, L"default");
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    Py_ssize_t i;
-    const PyWideStringList *options;
-
-    options = env_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, env_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
-    options = cmdline_warnoptions;
-    for (i = 0; i < options->length; i++) {
-        status = config_add_warnoption(config, options->items[i]);
-        if (_PyStatus_EXCEPTION(status)) {
-            return status;
-        }
+    status = warnoptions_extend(config, &options, cmdline_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
     /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
@@ -2166,19 +2186,30 @@ config_init_warnoptions(PyConfig *config,
         else {
             filter = L"default::BytesWarning";
         }
-        status = config_add_warnoption(config, filter);
+        status = warnoptions_append(config, &options, filter);
         if (_PyStatus_EXCEPTION(status)) {
-            return status;
+            goto error;
         }
     }
 
-    /* Handle early PySys_AddWarnOption() calls */
-    status = _PySys_ReadPreinitWarnOptions(config);
+    status = warnoptions_extend(config, &options, sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
-        return status;
+        goto error;
+    }
+
+    /* Always add all PyConfig.warnoptions options */
+    status = _PyWideStringList_Extend(&options, &config->warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto error;
     }
 
+    _PyWideStringList_Clear(&config->warnoptions);
+    config->warnoptions = options;
     return _PyStatus_OK();
+
+error:
+    _PyWideStringList_Clear(&options);
+    return status;
 }
 
 
@@ -2186,7 +2217,7 @@ static PyStatus
 config_update_argv(PyConfig *config, Py_ssize_t opt_index)
 {
     const PyWideStringList *cmdline_argv = &config->argv;
-    PyWideStringList config_argv = PyWideStringList_INIT;
+    PyWideStringList config_argv = _PyWideStringList_INIT;
 
     /* Copy argv to be able to modify it (to force -c/-m) */
     if (cmdline_argv->length <= opt_index) {
@@ -2270,8 +2301,9 @@ static PyStatus
 config_read_cmdline(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList cmdline_warnoptions = PyWideStringList_INIT;
-    PyWideStringList env_warnoptions = PyWideStringList_INIT;
+    PyWideStringList cmdline_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList env_warnoptions = _PyWideStringList_INIT;
+    PyWideStringList sys_warnoptions = _PyWideStringList_INIT;
 
     if (config->parse_argv < 0) {
         config->parse_argv = 1;
@@ -2304,9 +2336,16 @@ config_read_cmdline(PyConfig *config)
         }
     }
 
+    /* Handle early PySys_AddWarnOption() calls */
+    status = _PySys_ReadPreinitWarnOptions(&sys_warnoptions);
+    if (_PyStatus_EXCEPTION(status)) {
+        goto done;
+    }
+
     status = config_init_warnoptions(config,
                                      &cmdline_warnoptions,
-                                     &env_warnoptions);
+                                     &env_warnoptions,
+                                     &sys_warnoptions);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -2316,6 +2355,7 @@ config_read_cmdline(PyConfig *config)
 done:
     _PyWideStringList_Clear(&cmdline_warnoptions);
     _PyWideStringList_Clear(&env_warnoptions);
+    _PyWideStringList_Clear(&sys_warnoptions);
     return status;
 }
 
@@ -2386,7 +2426,7 @@ PyStatus
 PyConfig_Read(PyConfig *config)
 {
     PyStatus status;
-    PyWideStringList orig_argv = PyWideStringList_INIT;
+    PyWideStringList orig_argv = _PyWideStringList_INIT;
 
     status = config_check_struct_size(config);
     if (_PyStatus_EXCEPTION(status)) {
diff --git a/Python/preconfig.c b/Python/preconfig.c
index d18b01d..01c72f5 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -75,7 +75,7 @@ _Py_SetFileSystemEncoding(const char *encoding, const char *errors)
 PyStatus
 _PyArgv_AsWstrList(const _PyArgv *args, PyWideStringList *list)
 {
-    PyWideStringList wargv = PyWideStringList_INIT;
+    PyWideStringList wargv = _PyWideStringList_INIT;
     if (args->use_bytes_argv) {
         size_t size = sizeof(wchar_t*) * args->argc;
         wargv.items = (wchar_t **)PyMem_RawMalloc(size);
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 5175158..5b0fb81 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2071,13 +2071,13 @@ _clear_preinit_entries(_Py_PreInitEntry *optionlist)
 
 
 PyStatus
-_PySys_ReadPreinitWarnOptions(PyConfig *config)
+_PySys_ReadPreinitWarnOptions(PyWideStringList *options)
 {
     PyStatus status;
     _Py_PreInitEntry entry;
 
     for (entry = _preinit_warnoptions; entry != NULL; entry = entry->next) {
-        status = PyWideStringList_Append(&config->warnoptions, entry->value);
+        status = PyWideStringList_Append(options, entry->value);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
         }