bpo-32030: Add pymain_get_global_config() (#4735)

* Py_Main() now starts by reading Py_xxx configuration variables to
  only work on its own private structure, and then later writes back
  the configuration into these variables.
* Replace Py_GETENV() with pymain_get_env_var() which ignores empty
  variables.
* Add _PyCoreConfig.dump_refs
* Add _PyCoreConfig.malloc_stats
* _PyObject_DebugMallocStats() is now responsible to check if debug
  hooks are installed. The function returns 1 if stats were written,
  or 0 if the hooks are disabled. Mark _PyMem_PymallocEnabled() as
  static.

7 files changed, 142 insertions, 77 deletions

diff --git a/Include/objimpl.h b/Include/objimpl.h
index 746f9c9..ed9e7a9 100644
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -109,7 +109,7 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void);
 /* Macros */
 #ifdef WITH_PYMALLOC
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
+PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out);
 #endif /* #ifndef Py_LIMITED_API */
 #endif
 
diff --git a/Include/pymem.h b/Include/pymem.h
index 09d1502..8ee0efd 100644
--- a/Include/pymem.h
+++ b/Include/pymem.h
@@ -24,10 +24,6 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);
 /* Try to get the allocators name set by _PyMem_SetupAllocators(). */
 PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);
 
-#ifdef WITH_PYMALLOC
-PyAPI_FUNC(int) _PyMem_PymallocEnabled(void);
-#endif
-
 /* Track an allocated memory block in the tracemalloc module.
    Return 0 on success, return -1 on error (failed to allocate memory to store
    the trace).
diff --git a/Include/pystate.h b/Include/pystate.h
index cf45b05..d149aeb 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -36,6 +36,8 @@ typedef struct {
     int import_time;        /* -X importtime */
     int show_ref_count;     /* -X showrefcount */
     int show_alloc_count;   /* -X showalloccount */
+    int dump_refs;          /* PYTHONDUMPREFS */
+    int malloc_stats;       /* PYTHONMALLOCSTATS */
 } _PyCoreConfig;
 
 #define _PyCoreConfig_INIT (_PyCoreConfig){.use_hash_seed = -1}
@@ -111,7 +113,7 @@ typedef struct _is {
     PyObject *after_forkers_child;
 #endif
 } PyInterpreterState;
-#endif
+#endif   /* !Py_LIMITED_API */
 
 
 /* State unique per thread */
@@ -133,7 +135,7 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
 #define PyTrace_C_EXCEPTION 5
 #define PyTrace_C_RETURN 6
 #define PyTrace_OPCODE 7
-#endif
+#endif   /* Py_LIMITED_API */
 
 #ifdef Py_LIMITED_API
 typedef struct _ts PyThreadState;
@@ -238,7 +240,7 @@ typedef struct _ts {
     /* XXX signal handlers should also be here */
 
 } PyThreadState;
-#endif
+#endif   /* !Py_LIMITED_API */
 
 
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
@@ -363,7 +365,7 @@ PyAPI_FUNC(int) PyGILState_Check(void);
    Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini()
    is called. */
 PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void);
-#endif
+#endif   /* !Py_LIMITED_API */
 
 
 /* The implementation of sys._current_frames()  Returns a dict mapping
diff --git a/Modules/main.c b/Modules/main.c
index e2ca72c..6d53c56 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -162,8 +162,8 @@ pymain_get_env_var(const char *name)
 static void
 pymain_run_startup(PyCompilerFlags *cf)
 {
-    char *startup = Py_GETENV("PYTHONSTARTUP");
-    if (startup == NULL || startup[0] == '\0') {
+    char *startup = pymain_get_env_var("PYTHONSTARTUP");
+    if (startup == NULL) {
         return;
     }
 
@@ -377,23 +377,28 @@ typedef struct {
     wchar_t *command;            /* -c argument */
     wchar_t *module;             /* -m argument */
     _Py_OptList warning_options; /* -W options */
-    PyObject *extra_options;     /* -X options */
     int print_help;              /* -h, -? options */
     int print_version;           /* -V option */
-    int bytes_warning;           /* Py_BytesWarningFlag */
-    int debug;                   /* Py_DebugFlag */
-    int inspect;                 /* Py_InspectFlag */
-    int interactive;             /* Py_InteractiveFlag */
-    int isolated;                /* Py_IsolatedFlag */
-    int optimization_level;      /* Py_OptimizeFlag */
-    int dont_write_bytecode;     /* Py_DontWriteBytecodeFlag */
-    int no_user_site_directory;  /* Py_NoUserSiteDirectory */
-    int no_site_import;          /* Py_NoSiteFlag */
-    int use_unbuffered_io;       /* Py_UnbufferedStdioFlag */
-    int verbosity;               /* Py_VerboseFlag */
-    int quiet_flag;              /* Py_QuietFlag */
+    int bytes_warning;           /* Py_BytesWarningFlag, -b */
+    int debug;                   /* Py_DebugFlag, -b, PYTHONDEBUG */
+    int inspect;                 /* Py_InspectFlag, -i, PYTHONINSPECT */
+    int interactive;             /* Py_InteractiveFlag, -i */
+    int isolated;                /* Py_IsolatedFlag, -I */
+    int optimization_level;      /* Py_OptimizeFlag, -O, PYTHONOPTIMIZE */
+    int dont_write_bytecode;     /* Py_DontWriteBytecodeFlag, -B, PYTHONDONTWRITEBYTECODE */
+    int no_user_site_directory;  /* Py_NoUserSiteDirectory, -I, -s, PYTHONNOUSERSITE */
+    int no_site_import;          /* Py_NoSiteFlag, -S */
+    int use_unbuffered_io;       /* Py_UnbufferedStdioFlag, -u, PYTHONUNBUFFERED */
+    int verbosity;               /* Py_VerboseFlag, -v, PYTHONVERBOSE */
+    int quiet_flag;              /* Py_QuietFlag, -q */
     int skip_first_line;         /* -x option */
     _Py_OptList xoptions;        /* -X options */
+#ifdef MS_WINDOWS
+    int legacy_windows_fs_encoding;  /* Py_LegacyWindowsFSEncodingFlag,
+                                        PYTHONLEGACYWINDOWSFSENCODING */
+    int legacy_windows_stdio;        /* Py_LegacyWindowsStdioFlag,
+                                        PYTHONLEGACYWINDOWSSTDIO */
+#endif
 } _Py_CommandLineDetails;
 
 /* Structure used by Py_Main() to pass data to subfunctions */
@@ -695,19 +700,6 @@ pymain_parse_cmdline_impl(_PyMain *pymain)
 }
 
 
-static void
-maybe_set_flag(int *flag, int value)
-{
-    /* Helper to set flag variables from command line options
-    *   - uses the higher of the two values if they're both set
-    *   - otherwise leaves the flag unset
-    */
-    if (*flag < value) {
-        *flag = value;
-    }
-}
-
-
 static int
 pymain_add_xoptions(_PyMain *pymain)
 {
@@ -790,9 +782,8 @@ pymain_warnings_envvar(_PyMain *pymain)
         PyMem_RawFree(buf);
     }
 #else
-    char *p;
-
-    if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
+    char *p = pymain_get_env_var("PYTHONWARNINGS");
+    if (p != NULL) {
         char *buf, *oldloc;
 
         /* settle for strtok here as there's no one standard
@@ -885,7 +876,6 @@ config_get_program_name(_PyMainInterpreterConfig *config)
     }
 
 #ifdef __APPLE__
-    char *p;
     /* On MacOS X, when the Python interpreter is embedded in an
        application bundle, it gets executed by a bootstrapping script
        that does os.execve() with an argv[0] that's different from the
@@ -895,7 +885,8 @@ config_get_program_name(_PyMainInterpreterConfig *config)
        so the actual executable path is passed in an environment variable.
        See Lib/plat-mac/bundlebuiler.py for details about the bootstrap
        script. */
-    if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') {
+    char *p = pymain_get_env_var("PYTHONEXECUTABLE");
+    if (p != NULL) {
         size_t len;
         wchar_t* program_name = Py_DecodeLocale(p, &len);
         if (program_name == NULL) {
@@ -1015,25 +1006,76 @@ pymain_set_argv(_PyMain *pymain)
 }
 
 
+static void
+pymain_get_flag(int flag, int *value)
+{
+    if (flag) {
+        *value = flag;
+    }
+}
+
+static void
+pymain_set_flag(int *flag, int value)
+{
+    /* Helper to set flag variables from command line options
+    *   - uses the higher of the two values if they're both set
+    *   - otherwise leaves the flag unset
+    */
+    if (*flag < value) {
+        *flag = value;
+    }
+}
+
+
+/* Get Py_xxx global configuration variables */
+static void
+pymain_get_global_config(_PyMain *pymain)
+{
+    _Py_CommandLineDetails *cmdline = &pymain->cmdline;
+    pymain_get_flag(Py_BytesWarningFlag, &cmdline->bytes_warning);
+    pymain_get_flag(Py_DebugFlag, &cmdline->debug);
+    pymain_get_flag(Py_InspectFlag, &cmdline->inspect);
+    pymain_get_flag(Py_InteractiveFlag, &cmdline->interactive);
+    pymain_get_flag(Py_IsolatedFlag, &cmdline->isolated);
+    pymain_get_flag(Py_OptimizeFlag, &cmdline->optimization_level);
+    pymain_get_flag(Py_DontWriteBytecodeFlag, &cmdline->dont_write_bytecode);
+    pymain_get_flag(Py_NoUserSiteDirectory, &cmdline->no_user_site_directory);
+    pymain_get_flag(Py_NoSiteFlag, &cmdline->no_site_import);
+    pymain_get_flag(Py_UnbufferedStdioFlag, &cmdline->use_unbuffered_io);
+    pymain_get_flag(Py_VerboseFlag, &cmdline->verbosity);
+    pymain_get_flag(Py_QuietFlag, &cmdline->quiet_flag);
+#ifdef MS_WINDOWS
+    pymain_get_flag(Py_LegacyWindowsFSEncodingFlag, &cmdline->legacy_windows_fs_encoding);
+    pymain_get_flag(Py_LegacyWindowsStdioFlag, &cmdline->legacy_windows_stdio);
+#endif
+
+    pymain_get_flag(Py_IgnoreEnvironmentFlag, &pymain->core_config.ignore_environment);
+}
+
+
 /* Set Py_XXX global configuration variables */
 static void
 pymain_set_global_config(_PyMain *pymain)
 {
     _Py_CommandLineDetails *cmdline = &pymain->cmdline;
-    maybe_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning);
-    maybe_set_flag(&Py_DebugFlag, cmdline->debug);
-    maybe_set_flag(&Py_InspectFlag, cmdline->inspect);
-    maybe_set_flag(&Py_InteractiveFlag, cmdline->interactive);
-    maybe_set_flag(&Py_IsolatedFlag, cmdline->isolated);
-    maybe_set_flag(&Py_OptimizeFlag, cmdline->optimization_level);
-    maybe_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode);
-    maybe_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory);
-    maybe_set_flag(&Py_NoSiteFlag, cmdline->no_site_import);
-    maybe_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io);
-    maybe_set_flag(&Py_VerboseFlag, cmdline->verbosity);
-    maybe_set_flag(&Py_QuietFlag, cmdline->quiet_flag);
-
-    maybe_set_flag(&Py_IgnoreEnvironmentFlag, pymain->core_config.ignore_environment);
+    pymain_set_flag(&Py_BytesWarningFlag, cmdline->bytes_warning);
+    pymain_set_flag(&Py_DebugFlag, cmdline->debug);
+    pymain_set_flag(&Py_InspectFlag, cmdline->inspect);
+    pymain_set_flag(&Py_InteractiveFlag, cmdline->interactive);
+    pymain_set_flag(&Py_IsolatedFlag, cmdline->isolated);
+    pymain_set_flag(&Py_OptimizeFlag, cmdline->optimization_level);
+    pymain_set_flag(&Py_DontWriteBytecodeFlag, cmdline->dont_write_bytecode);
+    pymain_set_flag(&Py_NoUserSiteDirectory, cmdline->no_user_site_directory);
+    pymain_set_flag(&Py_NoSiteFlag, cmdline->no_site_import);
+    pymain_set_flag(&Py_UnbufferedStdioFlag, cmdline->use_unbuffered_io);
+    pymain_set_flag(&Py_VerboseFlag, cmdline->verbosity);
+    pymain_set_flag(&Py_QuietFlag, cmdline->quiet_flag);
+#ifdef MS_WINDOWS
+    pymain_set_flag(&Py_LegacyWindowsFSEncodingFlag, cmdline->legacy_windows_fs_encoding);
+    pymain_set_flag(&Py_LegacyWindowsStdioFlag, cmdline->legacy_windows_stdio);
+#endif
+
+    pymain_set_flag(&Py_IgnoreEnvironmentFlag, pymain->core_config.ignore_environment);
 }
 
 
@@ -1330,24 +1372,25 @@ pymain_set_flag_from_env(int *flag, const char *name)
 static void
 pymain_set_flags_from_env(_PyMain *pymain)
 {
-    pymain_set_flag_from_env(&Py_DebugFlag,
+    _Py_CommandLineDetails *cmdline = &pymain->cmdline;
+    pymain_set_flag_from_env(&cmdline->debug,
                              "PYTHONDEBUG");
-    pymain_set_flag_from_env(&Py_VerboseFlag,
+    pymain_set_flag_from_env(&cmdline->verbosity,
                              "PYTHONVERBOSE");
-    pymain_set_flag_from_env(&Py_OptimizeFlag,
+    pymain_set_flag_from_env(&cmdline->optimization_level,
                              "PYTHONOPTIMIZE");
-    pymain_set_flag_from_env(&Py_InspectFlag,
+    pymain_set_flag_from_env(&cmdline->inspect,
                              "PYTHONINSPECT");
-    pymain_set_flag_from_env(&Py_DontWriteBytecodeFlag,
+    pymain_set_flag_from_env(&cmdline->dont_write_bytecode,
                              "PYTHONDONTWRITEBYTECODE");
-    pymain_set_flag_from_env(&Py_NoUserSiteDirectory,
+    pymain_set_flag_from_env(&cmdline->no_user_site_directory,
                              "PYTHONNOUSERSITE");
-    pymain_set_flag_from_env(&Py_UnbufferedStdioFlag,
+    pymain_set_flag_from_env(&cmdline->use_unbuffered_io,
                              "PYTHONUNBUFFERED");
 #ifdef MS_WINDOWS
-    pymain_set_flag_from_env(&Py_LegacyWindowsFSEncodingFlag,
+    pymain_set_flag_from_env(&cmdline->legacy_windows_fs_encoding,
                              "PYTHONLEGACYWINDOWSFSENCODING");
-    pymain_set_flag_from_env(&Py_LegacyWindowsStdioFlag,
+    pymain_set_flag_from_env(&cmdline->legacy_windows_stdio,
                              "PYTHONLEGACYWINDOWSSTDIO");
 #endif
 }
@@ -1485,7 +1528,7 @@ pymain_parse_envvars(_PyMain *pymain)
         return -1;
     }
 
-    core_config->allocator = Py_GETENV("PYTHONMALLOC");
+    core_config->allocator = pymain_get_env_var("PYTHONMALLOC");
 
     /* -X options */
     if (pymain_get_xoption(pymain, L"showrefcount")) {
@@ -1514,6 +1557,14 @@ pymain_parse_envvars(_PyMain *pymain)
         core_config->faulthandler = 1;
         core_config->allocator = "debug";
     }
+    if (pymain_get_env_var("PYTHONDUMPREFS")) {
+        pymain->core_config.dump_refs = 1;
+    }
+    if (pymain_get_env_var("PYTHONMALLOCSTATS")) {
+        pymain->core_config.malloc_stats = 1;
+    }
+
+
     return 0;
 }
 
@@ -1535,6 +1586,7 @@ pymain_parse_cmdline_envvars_impl(_PyMain *pymain)
         return 1;
     }
 
+    /* Set Py_IgnoreEnvironmentFlag needed by Py_GETENV() */
     pymain_set_global_config(pymain);
 
     if (pymain_parse_envvars(pymain) < 0) {
@@ -1568,6 +1620,8 @@ pymain_parse_cmdline_envvars(_PyMain *pymain)
 static int
 pymain_init_python(_PyMain *pymain)
 {
+    pymain_set_global_config(pymain);
+
     pymain_init_stdio(pymain);
 
     pymain->err = _Py_InitializeCore(&pymain->core_config);
@@ -1641,6 +1695,8 @@ pymain_impl(_PyMain *pymain)
         return -1;
     }
 
+    pymain_get_global_config(pymain);
+
     res = pymain_parse_cmdline_envvars(pymain);
     if (res < 0) {
         return -1;
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 7911028..c065541 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -379,7 +379,7 @@ _PyMem_DebugEnabled(void)
     return (_PyObject.malloc == _PyMem_DebugMalloc);
 }
 
-int
+static int
 _PyMem_PymallocEnabled(void)
 {
     if (_PyMem_DebugEnabled()) {
@@ -2467,10 +2467,17 @@ pool_is_in_list(const poolp target, poolp list)
 /* Print summary info to "out" about the state of pymalloc's structures.
  * In Py_DEBUG mode, also perform some expensive internal consistency
  * checks.
+ *
+ * Return 0 if the memory debug hooks are not installed or no statistics was
+ * writen into out, return 1 otherwise.
  */
-void
+int
 _PyObject_DebugMallocStats(FILE *out)
 {
+    if (!_PyMem_PymallocEnabled()) {
+        return 0;
+    }
+
     uint i;
     const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT;
     /* # of pools, allocated blocks, and free blocks per class index */
@@ -2603,6 +2610,7 @@ _PyObject_DebugMallocStats(FILE *out)
     total += printone(out, "# bytes lost to quantization", quantization);
     total += printone(out, "# bytes lost to arena alignment", arena_alignment);
     (void)printone(out, "Total", total);
+    return 1;
 }
 
 #endif /* #ifdef WITH_PYMALLOC */
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 504036c..0b3aa98 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1103,6 +1103,10 @@ Py_FinalizeEx(void)
     tstate = PyThreadState_GET();
     interp = tstate->interp;
 
+    /* Copy the core config to be able to use it even
+       after PyInterpreterState_Delete() */
+    _PyCoreConfig core_config = interp->core_config;
+
     /* Remaining threads (e.g. daemon threads) will automatically exit
        after taking the GIL (in PyEval_RestoreThread()). */
     _PyRuntime.finalizing = tstate;
@@ -1186,7 +1190,7 @@ Py_FinalizeEx(void)
     _PyHash_Fini();
 
 #ifdef Py_REF_DEBUG
-    if (interp->core_config.show_ref_count) {
+    if (core_config.show_ref_count) {
         _PyDebug_PrintTotalRefs();
     }
 #endif
@@ -1197,8 +1201,9 @@ Py_FinalizeEx(void)
      * Alas, a lot of stuff may still be alive now that will be cleaned
      * up later.
      */
-    if (Py_GETENV("PYTHONDUMPREFS"))
+    if (core_config.dump_refs) {
         _Py_PrintReferences(stderr);
+    }
 #endif /* Py_TRACE_REFS */
 
     /* Clear interpreter state and all thread states. */
@@ -1260,14 +1265,13 @@ Py_FinalizeEx(void)
      * An address can be used to find the repr of the object, printed
      * above by _Py_PrintReferences.
      */
-    if (Py_GETENV("PYTHONDUMPREFS"))
+    if (core_config.dump_refs) {
         _Py_PrintReferenceAddresses(stderr);
+    }
 #endif /* Py_TRACE_REFS */
 #ifdef WITH_PYMALLOC
-    if (_PyMem_PymallocEnabled()) {
-        char *opt = Py_GETENV("PYTHONMALLOCSTATS");
-        if (opt != NULL && *opt != '\0')
-            _PyObject_DebugMallocStats(stderr);
+    if (core_config.malloc_stats) {
+        _PyObject_DebugMallocStats(stderr);
     }
 #endif
 
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 64bc14e..eeeaa72 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -1368,8 +1368,7 @@ static PyObject *
 sys_debugmallocstats(PyObject *self, PyObject *args)
 {
 #ifdef WITH_PYMALLOC
-    if (_PyMem_PymallocEnabled()) {
-        _PyObject_DebugMallocStats(stderr);
+    if (_PyObject_DebugMallocStats(stderr)) {
         fputc('\n', stderr);
     }
 #endif