bpo-36471: Add _Py_RunMain() (GH-12618)

* Add config_read_cmdline() subfunction. Remove _PyCmdline structure.
* _PyCoreConfig_Read() now also parses config->argv command line
  arguments

5 files changed, 152 insertions, 114 deletions

diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h
index e293b04..2366c77 100644
--- a/Include/cpython/pylifecycle.h
+++ b/Include/cpython/pylifecycle.h
@@ -41,6 +41,9 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs(
     int argc,
     wchar_t **argv);
 
+PyAPI_FUNC(int) _Py_RunMain(void);
+
+
 PyAPI_FUNC(void) _Py_NO_RETURN _Py_ExitInitError(_PyInitError err);
 
 /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 164527a..cd60272 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -53,7 +53,12 @@ class EmbeddingTestsMixin:
                              stderr=subprocess.PIPE,
                              universal_newlines=True,
                              env=env)
-        (out, err) = p.communicate()
+        try:
+            (out, err) = p.communicate()
+        except:
+            p.terminate()
+            p.wait()
+            raise
         if p.returncode != 0 and support.verbose:
             print(f"--- {cmd} failed ---")
             print(f"stdout:\n{out}")
@@ -254,6 +259,11 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
         self.assertEqual(out.rstrip(), "Py_Main() after Py_Initialize: sys.argv=['-c', 'arg2']")
         self.assertEqual(err, '')
 
+    def test_run_main(self):
+        out, err = self.run_embedded_interpreter("run_main")
+        self.assertEqual(out.rstrip(), "_Py_RunMain(): sys.argv=['-c', 'arg2']")
+        self.assertEqual(err, '')
+
 
 class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     maxDiff = 4096
@@ -549,10 +559,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
             'pycache_prefix': 'conf_pycache_prefix',
             'program_name': './conf_program_name',
-            'argv': ['-c', 'pass'],
+            'argv': ['-c', 'arg2'],
             'program': 'conf_program',
             'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
             'warnoptions': ['error::ResourceWarning', 'default::BytesWarning'],
+            'run_command': 'pass\n',
 
             'site_import': 0,
             'bytes_warning': 1,
diff --git a/Modules/main.c b/Modules/main.c
index 7665769..42d2c3c 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -567,20 +567,22 @@ exit_sigint(void)
 }
 
 
-static int
-pymain_main(_PyArgv *args)
+static void _Py_NO_RETURN
+pymain_exit_error(_PyInitError err)
 {
-    _PyInitError err;
+    pymain_free();
+    _Py_ExitInitError(err);
+}
 
-    err = pymain_init(args);
-    if (_Py_INIT_FAILED(err)) {
-        goto exit_init_error;
-    }
 
+int
+_Py_RunMain(void)
+{
     int exitcode = 0;
-    err = pymain_run_python(&exitcode);
+
+    _PyInitError err = pymain_run_python(&exitcode);
     if (_Py_INIT_FAILED(err)) {
-        goto exit_init_error;
+        pymain_exit_error(err);
     }
 
     if (Py_FinalizeEx() < 0) {
@@ -596,10 +598,18 @@ pymain_main(_PyArgv *args)
     }
 
     return exitcode;
+}
 
-exit_init_error:
-    pymain_free();
-    _Py_ExitInitError(err);
+
+static int
+pymain_main(_PyArgv *args)
+{
+    _PyInitError err = pymain_init(args);
+    if (_Py_INIT_FAILED(err)) {
+        pymain_exit_error(err);
+    }
+
+    return _Py_RunMain();
 }
 
 
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index d8e12cf..7d71a96 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -447,9 +447,11 @@ static int test_init_from_config(void)
     Py_SetProgramName(L"./globalvar");
     config.program_name = L"./conf_program_name";
 
-    static wchar_t* argv[2] = {
+    static wchar_t* argv[] = {
+        L"python3",
         L"-c",
         L"pass",
+        L"arg2",
     };
     config.argv.length = Py_ARRAY_LENGTH(argv);
     config.argv.items = argv;
@@ -528,7 +530,6 @@ static int test_init_from_config(void)
     config._frozen = 1;
 
     err = _Py_InitializeFromConfig(&config);
-    /* Don't call _PyCoreConfig_Clear() since all strings are static */
     if (_Py_INIT_FAILED(err)) {
         _Py_ExitInitError(err);
     }
@@ -712,6 +713,27 @@ static int test_init_dev_mode(void)
 }
 
 
+static int test_run_main(void)
+{
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+
+    wchar_t *argv[] = {L"python3", L"-c",
+                       (L"import sys; "
+                        L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"),
+                       L"arg2"};
+    config.argv.length = Py_ARRAY_LENGTH(argv);
+    config.argv.items = argv;
+    config.program_name = L"./python3";
+
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+
+    return _Py_RunMain();
+}
+
+
 /* *********************************************************
  * List of test cases and the function that implements it.
  *
@@ -748,6 +770,7 @@ static struct TestCase TestCases[] = {
     { "init_isolated", test_init_isolated },
     { "preinit_isolated1", test_preinit_isolated1 },
     { "preinit_isolated2", test_preinit_isolated2 },
+    { "run_main", test_run_main },
     { NULL, NULL }
 };
 
diff --git a/Python/coreconfig.c b/Python/coreconfig.c
index 13aa227..7b55b06 100644
--- a/Python/coreconfig.c
+++ b/Python/coreconfig.c
@@ -1618,33 +1618,34 @@ _PyCoreConfig_Write(const _PyCoreConfig *config)
 }
 
 
-/* --- _PyCmdline ------------------------------------------------- */
-
-typedef struct {
-    _PyWstrList cmdline_warnoptions;     /* Command line -W options */
-    _PyWstrList env_warnoptions; /* PYTHONWARNINGS environment variables */
-    int print_help;              /* -h, -? options */
-    int print_version;           /* -V option */
-    int need_usage;
-} _PyCmdline;
-
+/* --- _PyCoreConfig command line parser -------------------------- */
 
 static void
-cmdline_clear(_PyCmdline *cmdline)
+config_usage(int error, const wchar_t* program)
 {
-    _PyWstrList_Clear(&cmdline->cmdline_warnoptions);
-    _PyWstrList_Clear(&cmdline->env_warnoptions);
-}
+    FILE *f = error ? stderr : stdout;
 
+    fprintf(f, usage_line, program);
+    if (error)
+        fprintf(f, "Try `python -h' for more information.\n");
+    else {
+        fputs(usage_1, f);
+        fputs(usage_2, f);
+        fputs(usage_3, f);
+        fprintf(f, usage_4, (wint_t)DELIM);
+        fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP);
+        fputs(usage_6, f);
+    }
+}
 
-/* --- _PyCoreConfig command line parser -------------------------- */
 
 /* Parse the command line arguments */
 static _PyInitError
-config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
-                     _PyPreCmdline *precmdline)
+config_parse_cmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline,
+                     _PyWstrList *warnoptions)
 {
     const _PyWstrList *argv = &precmdline->argv;
+    int print_version = 0;
 
     _PyOS_ResetGetOpt();
     do {
@@ -1698,8 +1699,8 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
             } else {
                 fprintf(stderr, "--check-hash-based-pycs must be one of "
                         "'default', 'always', or 'never'\n");
-                cmdline->need_usage = 1;
-                return _Py_INIT_OK();
+                config_usage(1, config->program);
+                return _Py_INIT_EXIT(2);
             }
             break;
 
@@ -1758,15 +1759,15 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
 
         case 'h':
         case '?':
-            cmdline->print_help++;
-            break;
+            config_usage(0, config->program);
+            return _Py_INIT_EXIT(0);
 
         case 'V':
-            cmdline->print_version++;
+            print_version++;
             break;
 
         case 'W':
-            if (_PyWstrList_Append(&cmdline->cmdline_warnoptions, _PyOS_optarg) < 0) {
+            if (_PyWstrList_Append(warnoptions, _PyOS_optarg) < 0) {
                 return _Py_INIT_NO_MEMORY();
             }
             break;
@@ -1783,11 +1784,17 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
 
         default:
             /* unknown argument: parsing failed */
-            cmdline->need_usage = 1;
-            return _Py_INIT_OK();
+            config_usage(1, config->program);
+            return _Py_INIT_EXIT(2);
         }
     } while (1);
 
+    if (print_version) {
+        printf("Python %s\n",
+                (print_version >= 2) ? Py_GetVersion() : PY_VERSION);
+        return _Py_INIT_EXIT(0);
+    }
+
     if (config->run_command == NULL && config->run_module == NULL
         && _PyOS_optind < argv->length
         && wcscmp(argv->items[_PyOS_optind], L"-") != 0
@@ -1819,7 +1826,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
 
 /* Get warning options from PYTHONWARNINGS environment variable. */
 static _PyInitError
-cmdline_init_env_warnoptions(_PyCmdline *cmdline, const _PyCoreConfig *config)
+config_init_env_warnoptions(const _PyCoreConfig *config, _PyWstrList *warnoptions)
 {
     wchar_t *env;
     int res = _PyCoreConfig_GetEnvDup(config, &env,
@@ -1838,7 +1845,7 @@ cmdline_init_env_warnoptions(_PyCmdline *cmdline, const _PyCoreConfig *config)
          warning != NULL;
          warning = WCSTOK(NULL, L",", &context))
     {
-        if (_PyWstrList_Append(&cmdline->env_warnoptions, warning) < 0) {
+        if (_PyWstrList_Append(warnoptions, warning) < 0) {
             PyMem_RawFree(env);
             return _Py_INIT_NO_MEMORY();
         }
@@ -1883,7 +1890,9 @@ config_add_warnoption(_PyCoreConfig *config, const wchar_t *option)
 
 
 static _PyInitError
-config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
+config_init_warnoptions(_PyCoreConfig *config,
+                        const _PyWstrList *cmdline_warnoptions,
+                        const _PyWstrList *env_warnoptions)
 {
     /* The priority order for warnings configuration is (highest precedence
      * first):
@@ -1909,14 +1918,14 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
     Py_ssize_t i;
     const _PyWstrList *options;
 
-    options = &cmdline->env_warnoptions;
+    options = env_warnoptions;
     for (i = 0; i < options->length; i++) {
         if (config_add_warnoption(config, options->items[i]) < 0) {
             return _Py_INIT_NO_MEMORY();
         }
     }
 
-    options = &cmdline->cmdline_warnoptions;
+    options = cmdline_warnoptions;
     for (i = 0; i < options->length; i++) {
         if (config_add_warnoption(config, options->items[i]) < 0) {
             return _Py_INIT_NO_MEMORY();
@@ -1992,36 +2001,14 @@ config_init_argv(_PyCoreConfig *config, const _PyPreCmdline *cmdline)
 }
 
 
-static void
-config_usage(int error, const wchar_t* program)
-{
-    FILE *f = error ? stderr : stdout;
-
-    fprintf(f, usage_line, program);
-    if (error)
-        fprintf(f, "Try `python -h' for more information.\n");
-    else {
-        fputs(usage_1, f);
-        fputs(usage_2, f);
-        fputs(usage_3, f);
-        fprintf(f, usage_4, (wint_t)DELIM);
-        fprintf(f, usage_5, (wint_t)DELIM, PYTHONHOMEHELP);
-        fputs(usage_6, f);
-    }
-}
-
-
 static _PyInitError
 core_read_precmdline(_PyCoreConfig *config, const _PyArgv *args,
                      _PyPreCmdline *precmdline)
 {
     _PyInitError err;
 
-    if (args) {
-        err = _PyPreCmdline_SetArgv(precmdline, args);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
+    if (_PyWstrList_Copy(&precmdline->argv, &config->argv) < 0) {
+        return _Py_INIT_NO_MEMORY();
     }
 
     _PyPreConfig preconfig = _PyPreConfig_INIT;
@@ -2040,83 +2027,88 @@ done:
 }
 
 
-/* Read the configuration into _PyCoreConfig from:
-
-   * Command line arguments
-   * Environment variables
-   * Py_xxx global configuration variables */
-_PyInitError
-_PyCoreConfig_Read(_PyCoreConfig *config, const _PyArgv *args)
+static _PyInitError
+config_read_cmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline)
 {
     _PyInitError err;
+    _PyWstrList cmdline_warnoptions = _PyWstrList_INIT;
+    _PyWstrList env_warnoptions = _PyWstrList_INIT;
 
-    err = _Py_PreInitializeFromCoreConfig(config);
+    err = config_parse_cmdline(config, precmdline, &cmdline_warnoptions);
     if (_Py_INIT_FAILED(err)) {
-        return err;
+        goto done;
     }
 
-    _PyCoreConfig_GetGlobalConfig(config);
+    err = config_init_argv(config, precmdline);
+    if (_Py_INIT_FAILED(err)) {
+        goto done;
+    }
 
-    _PyPreCmdline precmdline = _PyPreCmdline_INIT;
-    err = core_read_precmdline(config, args, &precmdline);
+    err = config_read(config, precmdline);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    if (config->program == NULL) {
-        err = config_init_program(config, &precmdline);
+    if (config->use_environment) {
+        err = config_init_env_warnoptions(config, &env_warnoptions);
         if (_Py_INIT_FAILED(err)) {
             goto done;
         }
     }
 
-    _PyCmdline cmdline;
-    memset(&cmdline, 0, sizeof(cmdline));
+    err = config_init_warnoptions(config,
+                                  &cmdline_warnoptions, &env_warnoptions);
+    if (_Py_INIT_FAILED(err)) {
+        goto done;
+    }
 
-    if (args) {
-        err = config_parse_cmdline(config, &cmdline, &precmdline);
-        if (_Py_INIT_FAILED(err)) {
-            goto done;
-        }
+    err = _Py_INIT_OK();
 
-        if (cmdline.need_usage) {
-            config_usage(1, config->program);
-            err = _Py_INIT_EXIT(2);
-            goto done;
-        }
+done:
+    _PyWstrList_Clear(&cmdline_warnoptions);
+    _PyWstrList_Clear(&env_warnoptions);
+    return err;
+}
 
-        if (cmdline.print_help) {
-            config_usage(0, config->program);
-            err = _Py_INIT_EXIT(0);
-            goto done;
-        }
 
-        if (cmdline.print_version) {
-            printf("Python %s\n",
-                   (cmdline.print_version >= 2) ? Py_GetVersion() : PY_VERSION);
-            err = _Py_INIT_EXIT(0);
-            goto done;
-        }
+/* Read the configuration into _PyCoreConfig from:
+
+   * Command line arguments
+   * Environment variables
+   * Py_xxx global configuration variables */
+_PyInitError
+_PyCoreConfig_Read(_PyCoreConfig *config, const _PyArgv *args)
+{
+    _PyInitError err;
 
-        err = config_init_argv(config, &precmdline);
+    if (args) {
+        err = _PyArgv_AsWstrList(args, &config->argv);
         if (_Py_INIT_FAILED(err)) {
-            goto done;
+            return err;
         }
     }
 
-    err = config_read(config, &precmdline);
+    err = _Py_PreInitializeFromCoreConfig(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    _PyCoreConfig_GetGlobalConfig(config);
+
+    _PyPreCmdline precmdline = _PyPreCmdline_INIT;
+    err = core_read_precmdline(config, args, &precmdline);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    if (config->use_environment) {
-        err = cmdline_init_env_warnoptions(&cmdline, config);
+    if (config->program == NULL) {
+        err = config_init_program(config, &precmdline);
         if (_Py_INIT_FAILED(err)) {
             goto done;
         }
     }
 
-    err = config_init_warnoptions(config, &cmdline);
+    err = config_read_cmdline(config, &precmdline);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
@@ -2176,7 +2168,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyArgv *args)
     err = _Py_INIT_OK();
 
 done:
-    cmdline_clear(&cmdline);
     _PyPreCmdline_Clear(&precmdline);
     return err;
 }