From b35be4b3334fbc471a39abbeb68110867b72e3e5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Mar 2019 17:37:44 +0100 Subject: bpo-36142: Add _PyPreConfig.allocator (GH-12181) * Move 'allocator' and 'dev_mode' fields from _PyCoreConfig to _PyPreConfig. * Fix InitConfigTests of test_embed: dev_mode sets allocator to "debug", add a new tests for env vars with dev mode enabled. --- Include/cpython/coreconfig.h | 6 ++-- Lib/test/test_embed.py | 52 +++++++++++++++++++---------------- Programs/_testembed.c | 29 +++++++++++++++++--- Python/coreconfig.c | 20 ++------------ Python/preconfig.c | 65 ++++++++++++++++++++++++++++++++++++++++++++ Python/pylifecycle.c | 8 +++--- Python/sysmodule.c | 2 +- 7 files changed, 129 insertions(+), 53 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index 306577c..2673576 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -80,6 +80,9 @@ typedef struct { Set by -X utf8 command line option and PYTHONUTF8 environment variable. If set to -1 (default), inherit Py_UTF8Mode value. */ int utf8_mode; + + int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */ + char *allocator; /* Memory allocator: PYTHONMALLOC */ } _PyPreConfig; #ifdef MS_WINDOWS @@ -109,9 +112,6 @@ typedef struct { int use_hash_seed; /* PYTHONHASHSEED=x */ unsigned long hash_seed; - const char *allocator; /* Memory allocator: PYTHONMALLOC */ - int dev_mode; /* PYTHONDEVMODE, -X dev */ - /* Enable faulthandler? Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ int faulthandler; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 6c245eb..1f236a9 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -561,30 +561,36 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } self.check_config("init_from_config", config) + INIT_ENV_CONFIG = { + 'use_hash_seed': 1, + 'hash_seed': 42, + 'allocator': 'malloc_debug', + 'tracemalloc': 2, + 'import_time': 1, + 'malloc_stats': 1, + 'utf8_mode': 1, + 'filesystem_encoding': 'utf-8', + 'filesystem_errors': UTF8_MODE_ERRORS, + 'inspect': 1, + 'optimization_level': 2, + 'pycache_prefix': 'env_pycache_prefix', + 'write_bytecode': 0, + 'verbose': 1, + 'buffered_stdio': 0, + 'stdio_encoding': 'iso8859-1', + 'stdio_errors': 'replace', + 'user_site_directory': 0, + 'faulthandler': 1, + } + def test_init_env(self): - config = { - 'use_hash_seed': 1, - 'hash_seed': 42, - 'allocator': 'malloc_debug', - 'tracemalloc': 2, - 'import_time': 1, - 'malloc_stats': 1, - 'utf8_mode': 1, - 'filesystem_encoding': 'utf-8', - 'filesystem_errors': self.UTF8_MODE_ERRORS, - 'inspect': 1, - 'optimization_level': 2, - 'pycache_prefix': 'env_pycache_prefix', - 'write_bytecode': 0, - 'verbose': 1, - 'buffered_stdio': 0, - 'stdio_encoding': 'iso8859-1', - 'stdio_errors': 'replace', - 'user_site_directory': 0, - 'faulthandler': 1, - 'dev_mode': 1, - } - self.check_config("init_env", config) + self.check_config("init_env", self.INIT_ENV_CONFIG) + + def test_init_env_dev_mode(self): + config = dict(self.INIT_ENV_CONFIG, + allocator='debug', + dev_mode=1) + self.check_config("init_env_dev_mode", config) def test_init_dev_mode(self): config = { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 9923f8d..70bf960 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -437,7 +437,7 @@ static int test_init_from_config(void) config.hash_seed = 123; putenv("PYTHONMALLOC=malloc"); - config.allocator = "malloc_debug"; + config.preconfig.allocator = "malloc_debug"; /* dev_mode=1 is tested in test_init_dev_mode() */ @@ -577,7 +577,6 @@ static void test_init_env_putenvs(void) putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); putenv("PYTHONNOUSERSITE=1"); putenv("PYTHONFAULTHANDLER=1"); - putenv("PYTHONDEVMODE=1"); putenv("PYTHONIOENCODING=iso8859-1:replace"); /* FIXME: test PYTHONWARNINGS */ /* FIXME: test PYTHONEXECUTABLE */ @@ -589,6 +588,15 @@ static void test_init_env_putenvs(void) } +static void test_init_env_dev_mode_putenvs(void) +{ + test_init_env_putenvs(); + putenv("PYTHONMALLOC=malloc"); + putenv("PYTHONFAULTHANDLER="); + putenv("PYTHONDEVMODE=1"); +} + + static int test_init_env(void) { /* Test initialization from environment variables */ @@ -601,6 +609,18 @@ static int test_init_env(void) } +static int test_init_env_dev_mode(void) +{ + /* Test initialization from environment variables */ + Py_IgnoreEnvironmentFlag = 0; + test_init_env_dev_mode_putenvs(); + _testembed_Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + static int test_init_isolated(void) { /* Test _PyCoreConfig.isolated=1 */ @@ -615,7 +635,7 @@ static int test_init_isolated(void) /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; - test_init_env_putenvs(); + test_init_env_dev_mode_putenvs(); _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { _Py_ExitInitError(err); @@ -631,7 +651,7 @@ static int test_init_dev_mode(void) _PyCoreConfig config = _PyCoreConfig_INIT; putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); - config.dev_mode = 1; + config.preconfig.dev_mode = 1; config.program_name = L"./_testembed"; _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { @@ -673,6 +693,7 @@ static struct TestCase TestCases[] = { { "init_global_config", test_init_global_config }, { "init_from_config", test_init_from_config }, { "init_env", test_init_env }, + { "init_env_dev_mode", test_init_env_dev_mode }, { "init_dev_mode", test_init_dev_mode }, { "init_isolated", test_init_isolated }, { NULL, NULL } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index e372de4..42441e2 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -521,8 +521,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(use_hash_seed); COPY_ATTR(hash_seed); COPY_ATTR(_install_importlib); - COPY_ATTR(allocator); - COPY_ATTR(dev_mode); COPY_ATTR(faulthandler); COPY_ATTR(tracemalloc); COPY_ATTR(import_time); @@ -931,10 +929,6 @@ config_read_env_vars(_PyCoreConfig *config) "PYTHONLEGACYWINDOWSSTDIO"); #endif - if (config->allocator == NULL) { - config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC"); - } - if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) { config->dump_refs = 1; } @@ -1059,11 +1053,6 @@ config_read_complex_options(_PyCoreConfig *config) || config_get_xoption(config, L"importtime")) { config->import_time = 1; } - if (config_get_xoption(config, L"dev" ) || - _PyCoreConfig_GetEnv(config, "PYTHONDEVMODE")) - { - config->dev_mode = 1; - } _PyInitError err; if (config->tracemalloc < 0) { @@ -1427,13 +1416,10 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig) } /* default values */ - if (config->dev_mode) { + if (config->preconfig.dev_mode) { if (config->faulthandler < 0) { config->faulthandler = 1; } - if (config->allocator == NULL) { - config->allocator = "debug"; - } } if (config->use_hash_seed < 0) { config->use_hash_seed = 0; @@ -1572,8 +1558,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(install_signal_handlers); SET_ITEM_INT(use_hash_seed); SET_ITEM_UINT(hash_seed); - SET_ITEM_STR(allocator); - SET_ITEM_INT(dev_mode); SET_ITEM_INT(faulthandler); SET_ITEM_INT(tracemalloc); SET_ITEM_INT(import_time); @@ -1950,7 +1934,7 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline) * the lowest precedence entries first so that later entries override them. */ - if (config->dev_mode) { + if (config->preconfig.dev_mode) { err = _Py_wstrlist_append(&config->nwarnoption, &config->warnoptions, L"default"); diff --git a/Python/preconfig.c b/Python/preconfig.c index 3befecf..98e0ede 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -125,6 +125,15 @@ precmdline_clear(_PyPreCmdline *cmdline) void _PyPreConfig_Clear(_PyPreConfig *config) { +#define CLEAR(ATTR) \ + do { \ + PyMem_RawFree(ATTR); \ + ATTR = NULL; \ + } while (0) + + CLEAR(config->allocator); + +#undef CLEAR } @@ -134,6 +143,15 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) _PyPreConfig_Clear(config); #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR +#define COPY_STR_ATTR(ATTR) \ + do { \ + if (config2->ATTR != NULL) { \ + config->ATTR = _PyMem_RawStrdup(config2->ATTR); \ + if (config->ATTR == NULL) { \ + return -1; \ + } \ + } \ + } while (0) COPY_ATTR(isolated); COPY_ATTR(use_environment); @@ -143,8 +161,11 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) COPY_ATTR(legacy_windows_fs_encoding); #endif COPY_ATTR(utf8_mode); + COPY_ATTR(dev_mode); + COPY_STR_ATTR(allocator); #undef COPY_ATTR +#undef COPY_STR_ATTR return 0; } @@ -345,6 +366,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline) { _PyPreConfig_GetGlobalConfig(config); + /* isolated and use_environment */ if (config->isolated > 0) { config->use_environment = 0; } @@ -354,6 +376,7 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline) config->use_environment = 0; } + /* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */ if (config->use_environment) { #ifdef MS_WINDOWS _Py_get_env_flag(config, &config->legacy_windows_fs_encoding, @@ -414,11 +437,43 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline) if (config->utf8_mode < 0) { config->utf8_mode = 0; } + if (config->coerce_c_locale < 0) { + config->coerce_c_locale = 0; + } + + /* dev_mode */ + if ((cmdline && _Py_get_xoption(cmdline->nxoption, cmdline->xoptions, L"dev")) + || _PyPreConfig_GetEnv(config, "PYTHONDEVMODE")) + { + config->dev_mode = 1; + } + if (config->dev_mode < 0) { + config->dev_mode = 0; + } + + /* allocator */ + if (config->dev_mode && config->allocator == NULL) { + config->allocator = _PyMem_RawStrdup("debug"); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + + if (config->allocator == NULL) { + const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC"); + if (allocator) { + config->allocator = _PyMem_RawStrdup(allocator); + if (config->allocator == NULL) { + return _Py_INIT_NO_MEMORY(); + } + } + } assert(config->coerce_c_locale >= 0); assert(config->utf8_mode >= 0); assert(config->isolated >= 0); assert(config->use_environment >= 0); + assert(config->dev_mode >= 0); return _Py_INIT_OK(); } @@ -448,6 +503,12 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) } while (0) #define SET_ITEM_INT(ATTR) \ SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) +#define FROM_STRING(STR) \ + ((STR != NULL) ? \ + PyUnicode_FromString(STR) \ + : (Py_INCREF(Py_None), Py_None)) +#define SET_ITEM_STR(ATTR) \ + SET_ITEM(#ATTR, FROM_STRING(config->ATTR)) SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); @@ -457,13 +518,17 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) #ifdef MS_WINDOWS SET_ITEM_INT(legacy_windows_fs_encoding); #endif + SET_ITEM_INT(dev_mode); + SET_ITEM_STR(allocator); return 0; fail: return -1; +#undef FROM_STRING #undef SET_ITEM #undef SET_ITEM_INT +#undef SET_ITEM_STR } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index dec8904..c955a1d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -482,9 +482,9 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p, /* bpo-34008: For backward compatibility reasons, calling Py_Main() after Py_Initialize() ignores the new configuration. */ - if (core_config->allocator != NULL) { + if (core_config->preconfig.allocator != NULL) { const char *allocator = _PyMem_GetAllocatorsName(); - if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) { + if (allocator == NULL || strcmp(core_config->preconfig.allocator, allocator) != 0) { return _Py_INIT_USER_ERR("cannot modify memory allocator " "after first Py_Initialize()"); } @@ -521,8 +521,8 @@ pycore_init_runtime(const _PyCoreConfig *core_config) return err; } - if (core_config->allocator != NULL) { - if (_PyMem_SetupAllocators(core_config->allocator) < 0) { + if (core_config->preconfig.allocator != NULL) { + if (_PyMem_SetupAllocators(core_config->preconfig.allocator) < 0) { return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator"); } } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 50ba1a7..99fd460 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2180,7 +2180,7 @@ make_flags(void) SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); SetFlag(config->preconfig.isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); + PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode)); SetFlag(config->preconfig.utf8_mode); #undef SetFlag -- cgit v0.12