bpo-34170: Enhance _PyCoreConfig_Read() (GH-8468)

* Inline cmdline_get_env_flags() into config_read_env_vars():
  _PyCoreConfig_Read() now reads much more environment variables like
  PYTHONVERBOSE.
* Allow to override faulthandler and allocator even if dev_mode=1.
  PYTHONMALLOC is now the priority over PYTHONDEVMODE.
* Fix _PyCoreConfig_Copy(): copy also install_signal_handlers,
  coerce_c_locale and coerce_c_locale_warn
* _PyCoreConfig.install_signal_handlers default is now 1: install
  signals by default
* Fix also a compiler warning: don't define _PyPathConfig type twice.

5 files changed, 96 insertions, 76 deletions

diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
index 6e5d796..78f01ee 100644
--- a/Include/pylifecycle.h
+++ b/Include/pylifecycle.h
@@ -127,10 +127,9 @@ PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetPath(void);
 #ifdef Py_BUILD_CORE
 struct _PyPathConfig;
-typedef struct _PyPathConfig _PyPathConfig;
 
 PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal(
-    const _PyPathConfig *config);
+    const struct _PyPathConfig *config);
 PyAPI_FUNC(PyObject*) _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv);
 PyAPI_FUNC(int) _Py_FindEnvConfigValue(
     FILE *env_file,
diff --git a/Include/pystate.h b/Include/pystate.h
index 2c70505..f59d26b 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -26,15 +26,23 @@ typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
 
 
 typedef struct {
-    int install_signal_handlers;  /* Install signal handlers? -1 means unset */
+    /* Install signal handlers? Yes by default. */
+    int install_signal_handlers;
 
     int ignore_environment; /* -E, Py_IgnoreEnvironmentFlag, -1 means unset */
     int use_hash_seed;      /* PYTHONHASHSEED=x */
     unsigned long hash_seed;
-    const char *allocator;  /* Memory allocator: _PyMem_SetupAllocators() */
+    const char *allocator;  /* Memory allocator: PYTHONMALLOC */
     int dev_mode;           /* PYTHONDEVMODE, -X dev */
-    int faulthandler;       /* PYTHONFAULTHANDLER, -X faulthandler */
-    int tracemalloc;        /* PYTHONTRACEMALLOC, -X tracemalloc=N */
+
+    /* Enable faulthandler?
+       Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */
+    int faulthandler;
+
+    /* Enable tracemalloc?
+       Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */
+    int tracemalloc;
+
     int import_time;        /* PYTHONPROFILEIMPORTTIME, -X importtime */
     int show_ref_count;     /* -X showrefcount */
     int show_alloc_count;   /* -X showalloccount */
@@ -229,9 +237,10 @@ typedef struct {
 
 #define _PyCoreConfig_INIT \
     (_PyCoreConfig){ \
-        .install_signal_handlers = -1, \
+        .install_signal_handlers = 1, \
         .ignore_environment = -1, \
         .use_hash_seed = -1, \
+        .faulthandler = -1, \
         .tracemalloc = -1, \
         .coerce_c_locale = -1, \
         .utf8_mode = -1, \
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 0346074..21511b8 100644
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -551,9 +551,7 @@ class CmdLineTest(unittest.TestCase):
         env = dict(os.environ)
         env.pop('PYTHONWARNINGS', None)
         env.pop('PYTHONDEVMODE', None)
-        # Force malloc() to disable the debug hooks which are enabled
-        # by default for Python compiled in debug mode
-        env['PYTHONMALLOC'] = 'malloc'
+        env.pop('PYTHONMALLOC', None)
 
         if xdev:
             args = (sys.executable, '-X', 'dev', *args)
diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-07-25-19-23-33.bpo-34170.v1h_H2.rst b/Misc/NEWS.d/next/Core and Builtins/2018-07-25-19-23-33.bpo-34170.v1h_H2.rst
new file mode 100644
index 0000000..3eae903
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2018-07-25-19-23-33.bpo-34170.v1h_H2.rst
@@ -0,0 +1,2 @@
+-X dev: it is now possible to override the memory allocator using
+PYTHONMALLOC even if the developer mode is enabled.
diff --git a/Modules/main.c b/Modules/main.c
index e116dd0..e22a85f 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -701,6 +701,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
         config->LEN = config2->LEN; \
     } while (0)
 
+    COPY_ATTR(install_signal_handlers);
     COPY_ATTR(ignore_environment);
     COPY_ATTR(use_hash_seed);
     COPY_ATTR(hash_seed);
@@ -714,6 +715,9 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
     COPY_ATTR(show_alloc_count);
     COPY_ATTR(dump_refs);
     COPY_ATTR(malloc_stats);
+
+    COPY_ATTR(coerce_c_locale);
+    COPY_ATTR(coerce_c_locale_warn);
     COPY_ATTR(utf8_mode);
 
     COPY_STR_ATTR(pycache_prefix);
@@ -1244,9 +1248,7 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline)
 static _PyInitError
 cmdline_init_env_warnoptions(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline)
 {
-    if (config->ignore_environment) {
-        return _Py_INIT_OK();
-    }
+    assert(!config->ignore_environment);
 
     wchar_t *env;
     int res = config_get_env_var_dup(config, &env,
@@ -1819,37 +1821,6 @@ get_env_flag(_PyCoreConfig *config, int *flag, const char *name)
 }
 
 
-static void
-cmdline_get_env_flags(_PyMain *pymain, _PyCoreConfig *config,
-                      _PyCmdline *cmdline)
-{
-    get_env_flag(config, &config->debug, "PYTHONDEBUG");
-    get_env_flag(config, &config->verbose, "PYTHONVERBOSE");
-    get_env_flag(config, &config->optimization_level, "PYTHONOPTIMIZE");
-    get_env_flag(config, &config->inspect, "PYTHONINSPECT");
-
-    int dont_write_bytecode = 0;
-    get_env_flag(config, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE");
-    if (dont_write_bytecode) {
-        config->write_bytecode = 0;
-    }
-
-    int no_user_site_directory = 0;
-    get_env_flag(config, &no_user_site_directory, "PYTHONNOUSERSITE");
-    if (no_user_site_directory) {
-        config->user_site_directory = 0;
-    }
-
-    get_env_flag(config, &config->unbuffered_stdio, "PYTHONUNBUFFERED");
-#ifdef MS_WINDOWS
-    get_env_flag(config, &config->legacy_windows_fs_encoding,
-                 "PYTHONLEGACYWINDOWSFSENCODING");
-    get_env_flag(config, &config->legacy_windows_stdio,
-                 "PYTHONLEGACYWINDOWSSTDIO");
-#endif
-}
-
-
 static _PyInitError
 config_init_home(_PyCoreConfig *config)
 {
@@ -1944,7 +1915,37 @@ config_init_utf8_mode(_PyCoreConfig *config)
 static _PyInitError
 config_read_env_vars(_PyCoreConfig *config)
 {
-    config->allocator = config_get_env_var(config, "PYTHONMALLOC");
+    assert(!config->ignore_environment);
+
+    /* Get environment variables */
+    get_env_flag(config, &config->debug, "PYTHONDEBUG");
+    get_env_flag(config, &config->verbose, "PYTHONVERBOSE");
+    get_env_flag(config, &config->optimization_level, "PYTHONOPTIMIZE");
+    get_env_flag(config, &config->inspect, "PYTHONINSPECT");
+
+    int dont_write_bytecode = 0;
+    get_env_flag(config, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE");
+    if (dont_write_bytecode) {
+        config->write_bytecode = 0;
+    }
+
+    int no_user_site_directory = 0;
+    get_env_flag(config, &no_user_site_directory, "PYTHONNOUSERSITE");
+    if (no_user_site_directory) {
+        config->user_site_directory = 0;
+    }
+
+    get_env_flag(config, &config->unbuffered_stdio, "PYTHONUNBUFFERED");
+#ifdef MS_WINDOWS
+    get_env_flag(config, &config->legacy_windows_fs_encoding,
+                 "PYTHONLEGACYWINDOWSFSENCODING");
+    get_env_flag(config, &config->legacy_windows_stdio,
+                 "PYTHONLEGACYWINDOWSSTDIO");
+#endif
+
+    if (config->allocator == NULL) {
+        config->allocator = config_get_env_var(config, "PYTHONMALLOC");
+    }
 
     if (config_get_env_var(config, "PYTHONDUMPREFS")) {
         config->dump_refs = 1;
@@ -1998,8 +1999,6 @@ config_read_complex_options(_PyCoreConfig *config)
         config_get_env_var(config, "PYTHONDEVMODE"))
     {
         config->dev_mode = 1;
-        config->faulthandler = 1;
-        config->allocator = "debug";
     }
 
     _PyInitError err = pymain_init_tracemalloc(config);
@@ -2033,23 +2032,17 @@ pymain_read_conf_impl(_PyMain *pymain, _PyCoreConfig *config,
         return res;
     }
 
-    /* Get environment variables */
-    cmdline_get_env_flags(pymain, config, cmdline);
-
-    err = cmdline_init_env_warnoptions(pymain, config, cmdline);
-    if (_Py_INIT_FAILED(err)) {
-        pymain->err = err;
+    if (pymain_init_core_argv(pymain, config, cmdline) < 0) {
         return -1;
     }
 
-#ifdef MS_WINDOWS
-    if (config->legacy_windows_fs_encoding) {
-        config->utf8_mode = 0;
-    }
-#endif
-
-    if (pymain_init_core_argv(pymain, config, cmdline) < 0) {
-        return -1;
+    assert(config->ignore_environment >= 0);
+    if (!config->ignore_environment) {
+        err = cmdline_init_env_warnoptions(pymain, config, cmdline);
+        if (_Py_INIT_FAILED(err)) {
+            pymain->err = err;
+            return -1;
+        }
     }
 
     err = _PyCoreConfig_Read(config);
@@ -2186,14 +2179,6 @@ config_init_locale(_PyCoreConfig *config)
         }
         return;
     }
-
-    /* By default, C locale coercion and UTF-8 Mode are disabled */
-    if (config->coerce_c_locale < 0) {
-        config->coerce_c_locale = 0;
-    }
-    if (config->utf8_mode < 0) {
-        config->utf8_mode = 0;
-    }
 }
 
 
@@ -2216,9 +2201,12 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
 
     _PyCoreConfig_GetGlobalConfig(config);
 
-    err = config_read_env_vars(config);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
+    assert(config->ignore_environment >= 0);
+    if (!config->ignore_environment) {
+        err = config_read_env_vars(config);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
     }
 
     /* -X options */
@@ -2260,13 +2248,38 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
         }
     }
 
+    /* options side effects */
+    if (config->dev_mode) {
+        if (config->faulthandler < 0) {
+            config->faulthandler = 1;
+        }
+        if (config->allocator == NULL) {
+            config->allocator = "debug";
+        }
+    }
+
+#ifdef MS_WINDOWS
+    if (config->legacy_windows_fs_encoding) {
+        config->utf8_mode = 0;
+    }
+#endif
+
     /* default values */
+    if (config->use_hash_seed < 0) {
+        config->use_hash_seed = 0;
+        config->hash_seed = 0;
+    }
+    if (config->faulthandler < 0) {
+        config->faulthandler = 0;
+    }
     if (config->tracemalloc < 0) {
         config->tracemalloc = 0;
     }
-    if (config->install_signal_handlers < 0) {
-        /* Signal handlers are installed by default */
-        config->install_signal_handlers = 1;
+    if (config->coerce_c_locale < 0) {
+        config->coerce_c_locale = 0;
+    }
+    if (config->utf8_mode < 0) {
+        config->utf8_mode = 0;
     }
 
     return _Py_INIT_OK();
@@ -2599,7 +2612,6 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p)
 
     _PyCoreConfig local_config = _PyCoreConfig_INIT;
     _PyCoreConfig *config = &local_config;
-    config->install_signal_handlers = 1;
 
     _PyCoreConfig_GetGlobalConfig(config);