diff options
| -rw-r--r-- | Include/internal/pystate.h | 2 | ||||
| -rw-r--r-- | Include/pydebug.h | 1 | ||||
| -rw-r--r-- | Include/pystate.h | 13 | ||||
| -rw-r--r-- | Lib/test/test_tracemalloc.py | 39 | ||||
| -rw-r--r-- | Modules/_tracemalloc.c | 108 | ||||
| -rw-r--r-- | Modules/clinic/_tracemalloc.c.h | 8 | ||||
| -rw-r--r-- | Modules/faulthandler.c | 35 | ||||
| -rw-r--r-- | Modules/main.c | 440 | ||||
| -rw-r--r-- | Python/bootstrap_hash.c | 4 | ||||
| -rw-r--r-- | Python/import.c | 31 | ||||
| -rw-r--r-- | Python/pylifecycle.c | 161 | ||||
| -rw-r--r-- | Python/pystate.c | 8 |
12 files changed, 467 insertions, 383 deletions
diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 516f829..67b4a51 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -90,7 +90,7 @@ PyAPI_FUNC(_PyInitError) _PyRuntime_Initialize(void); /* Other */ -PyAPI_FUNC(void) _PyInterpreterState_Enable(_PyRuntimeState *); +PyAPI_FUNC(_PyInitError) _PyInterpreterState_Enable(_PyRuntimeState *); #ifdef __cplusplus } diff --git a/Include/pydebug.h b/Include/pydebug.h index 6e23a89..d3b9596 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -25,6 +25,7 @@ PyAPI_DATA(int) Py_HashRandomizationFlag; PyAPI_DATA(int) Py_IsolatedFlag; #ifdef MS_WINDOWS +PyAPI_DATA(int) Py_LegacyWindowsFSEncodingFlag; PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; #endif diff --git a/Include/pystate.h b/Include/pystate.h index 9381585..4401225 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -30,9 +30,20 @@ typedef struct { unsigned long hash_seed; int _disable_importlib; /* Needed by freeze_importlib */ char *allocator; + int faulthandler; + int tracemalloc; /* Number of saved frames, 0=don't trace */ + int importtime; /* -X importtime */ } _PyCoreConfig; -#define _PyCoreConfig_INIT {0, -1, 0, 0, NULL} +#define _PyCoreConfig_INIT \ + {.ignore_environment = 0, \ + .use_hash_seed = -1, \ + .hash_seed = 0, \ + ._disable_importlib = 0, \ + .allocator = NULL, \ + .faulthandler = 0, \ + .tracemalloc = 0, \ + .importtime = 0} /* Placeholders while working on the new configuration API * diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 780942a..533ba6d 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -829,16 +829,23 @@ class TestCommandLine(unittest.TestCase): stdout = stdout.rstrip() self.assertEqual(stdout, b'10') + def check_env_var_invalid(self, nframe): + with support.SuppressCrashReport(): + ok, stdout, stderr = assert_python_failure( + '-c', 'pass', + PYTHONTRACEMALLOC=str(nframe)) + + if b'ValueError: the number of frames must be in range' in stderr: + return + if b'PYTHONTRACEMALLOC: invalid number of frames' in stderr: + return + self.fail(f"unexpeced output: {stderr!a}") + + def test_env_var_invalid(self): for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): - with support.SuppressCrashReport(): - ok, stdout, stderr = assert_python_failure( - '-c', 'pass', - PYTHONTRACEMALLOC=str(nframe)) - self.assertIn(b'PYTHONTRACEMALLOC: invalid ' - b'number of frames', - stderr) + self.check_env_var_invalid(nframe) def test_sys_xoptions(self): for xoptions, nframe in ( @@ -852,15 +859,21 @@ class TestCommandLine(unittest.TestCase): stdout = stdout.rstrip() self.assertEqual(stdout, str(nframe).encode('ascii')) + def check_sys_xoptions_invalid(self, nframe): + args = ('-X', 'tracemalloc=%s' % nframe, '-c', 'pass') + with support.SuppressCrashReport(): + ok, stdout, stderr = assert_python_failure(*args) + + if b'ValueError: the number of frames must be in range' in stderr: + return + if b'-X tracemalloc=NFRAME: invalid number of frames' in stderr: + return + self.fail(f"unexpeced output: {stderr!a}") + def test_sys_xoptions_invalid(self): for nframe in (-1, 0, 2**30): with self.subTest(nframe=nframe): - with support.SuppressCrashReport(): - args = ('-X', 'tracemalloc=%s' % nframe, '-c', 'pass') - ok, stdout, stderr = assert_python_failure(*args) - self.assertIn(b'-X tracemalloc=NFRAME: invalid ' - b'number of frames', - stderr) + self.check_sys_xoptions_invalid(nframe) @unittest.skipIf(_testcapi is None, 'need _testcapi') def test_pymem_alloc0(self): diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index af2a2fa..e07022c 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1066,8 +1066,16 @@ tracemalloc_start(int max_nframe) PyMemAllocatorEx alloc; size_t size; - if (tracemalloc_init() < 0) + if (max_nframe < 1 || max_nframe > MAX_NFRAME) { + PyErr_Format(PyExc_ValueError, + "the number of frames must be in range [1; %i]", + (int)MAX_NFRAME); return -1; + } + + if (tracemalloc_init() < 0) { + return -1; + } if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ @@ -1500,7 +1508,7 @@ _PyMem_DumpTraceback(int fd, const void *ptr) /*[clinic input] _tracemalloc.start - nframe: Py_ssize_t = 1 + nframe: int = 1 / Start tracing Python memory allocations. @@ -1510,22 +1518,12 @@ trace to nframe. [clinic start generated code]*/ static PyObject * -_tracemalloc_start_impl(PyObject *module, Py_ssize_t nframe) -/*[clinic end generated code: output=0f558d2079511553 input=997841629cc441cb]*/ +_tracemalloc_start_impl(PyObject *module, int nframe) +/*[clinic end generated code: output=caae05c23c159d3c input=40d849b5b29d1933]*/ { - int nframe_int; - - if (nframe < 1 || nframe > MAX_NFRAME) { - PyErr_Format(PyExc_ValueError, - "the number of frames must be in range [1; %i]", - (int)MAX_NFRAME); + if (tracemalloc_start(nframe) < 0) { return NULL; } - nframe_int = Py_SAFE_DOWNCAST(nframe, Py_ssize_t, int); - - if (tracemalloc_start(nframe_int) < 0) - return NULL; - Py_RETURN_NONE; } @@ -1658,87 +1656,13 @@ PyInit__tracemalloc(void) } -static int -parse_sys_xoptions(PyObject *value) -{ - PyObject *valuelong; - long nframe; - - if (value == Py_True) - return 1; - - assert(PyUnicode_Check(value)); - if (PyUnicode_GetLength(value) == 0) - return -1; - - valuelong = PyLong_FromUnicodeObject(value, 10); - if (valuelong == NULL) - return -1; - - nframe = PyLong_AsLong(valuelong); - Py_DECREF(valuelong); - if (nframe == -1 && PyErr_Occurred()) - return -1; - - if (nframe < 1 || nframe > MAX_NFRAME) - return -1; - - return Py_SAFE_DOWNCAST(nframe, long, int); -} - - int -_PyTraceMalloc_Init(void) +_PyTraceMalloc_Init(int nframe) { - char *p; - int nframe; - assert(PyGILState_Check()); - - if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { - char *endptr = p; - long value; - - errno = 0; - value = strtol(p, &endptr, 10); - if (*endptr != '\0' - || value < 1 - || value > MAX_NFRAME - || errno == ERANGE) - { - Py_FatalError("PYTHONTRACEMALLOC: invalid number of frames"); - return -1; - } - - nframe = (int)value; - } - else { - PyObject *xoptions, *key, *value; - - xoptions = PySys_GetXOptions(); - if (xoptions == NULL) - return -1; - - key = PyUnicode_FromString("tracemalloc"); - if (key == NULL) - return -1; - - value = PyDict_GetItemWithError(xoptions, key); /* borrowed */ - Py_DECREF(key); - if (value == NULL) { - if (PyErr_Occurred()) - return -1; - - /* -X tracemalloc is not used */ - return 0; - } - - nframe = parse_sys_xoptions(value); - if (nframe < 0) { - Py_FatalError("-X tracemalloc=NFRAME: invalid number of frames"); - } + if (nframe == 0) { + return 0; } - return tracemalloc_start(nframe); } diff --git a/Modules/clinic/_tracemalloc.c.h b/Modules/clinic/_tracemalloc.c.h index df7b750..206dea7 100644 --- a/Modules/clinic/_tracemalloc.c.h +++ b/Modules/clinic/_tracemalloc.c.h @@ -87,15 +87,15 @@ PyDoc_STRVAR(_tracemalloc_start__doc__, {"start", (PyCFunction)_tracemalloc_start, METH_FASTCALL, _tracemalloc_start__doc__}, static PyObject * -_tracemalloc_start_impl(PyObject *module, Py_ssize_t nframe); +_tracemalloc_start_impl(PyObject *module, int nframe); static PyObject * _tracemalloc_start(PyObject *module, PyObject **args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t nframe = 1; + int nframe = 1; - if (!_PyArg_ParseStack(args, nargs, "|n:start", + if (!_PyArg_ParseStack(args, nargs, "|i:start", &nframe)) { goto exit; } @@ -185,4 +185,4 @@ _tracemalloc_get_traced_memory(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _tracemalloc_get_traced_memory_impl(module); } -/*[clinic end generated code: output=c9a0111391b3ec45 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=db4f909464c186e2 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index b6d5aee..0e85cce 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1299,36 +1299,8 @@ faulthandler_init_enable(void) return 0; } -/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable - is defined, or if sys._xoptions has a 'faulthandler' key. */ - -static int -faulthandler_init_parse(void) -{ - char *p = Py_GETENV("PYTHONFAULTHANDLER"); - if (p && *p != '\0') { - return 1; - } - - /* PYTHONFAULTHANDLER environment variable is missing - or an empty string */ - PyObject *xoptions = PySys_GetXOptions(); - if (xoptions == NULL) { - return -1; - } - - PyObject *key = PyUnicode_FromString("faulthandler"); - if (key == NULL) { - return -1; - } - - int has_key = PyDict_Contains(xoptions, key); - Py_DECREF(key); - return has_key; -} - _PyInitError -_PyFaulthandler_Init(void) +_PyFaulthandler_Init(int enable) { #ifdef HAVE_SIGALTSTACK int err; @@ -1357,11 +1329,6 @@ _PyFaulthandler_Init(void) PyThread_acquire_lock(thread.cancel_event, 1); #endif - int enable = faulthandler_init_parse(); - if (enable < 0) { - return _Py_INIT_ERR("failed to parse faulthandler env var and cmdline"); - } - if (enable) { if (faulthandler_init_enable() < 0) { return _Py_INIT_ERR("failed to enable faulthandler"); diff --git a/Modules/main.c b/Modules/main.c index 6391ba7..95076c6 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -111,7 +111,7 @@ static const char usage_6[] = " locale coercion and locale compatibility warnings on stderr.\n"; static void -usage(int error, const wchar_t* program) +pymain_usage(int error, const wchar_t* program) { FILE *f = error ? stderr : stdout; @@ -128,6 +128,20 @@ usage(int error, const wchar_t* program) } } + +static char* +pymain_get_env_var(const char *name) +{ + char *var = Py_GETENV(name); + if (var && var[0] != '\0') { + return var; + } + else { + return NULL; + } +} + + static void pymain_run_statup(PyCompilerFlags *cf) { @@ -444,7 +458,7 @@ pymain_free_impl(_PyMain *pymain) static void pymain_free(_PyMain *pymain) { - /* Call pymain_free() with the memory allocator used by pymain_init() */ + /* Force malloc() memory allocator */ PyMemAllocatorEx old_alloc, raw_alloc; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyMem_GetDefaultRawAllocator(&raw_alloc); @@ -488,11 +502,12 @@ error: static wchar_t* -pymain_strdup(wchar_t *str) +pymain_strdup(_PyMain *pymain, wchar_t *str) { size_t len = wcslen(str) + 1; /* +1 for NUL character */ wchar_t *str2 = PyMem_RawMalloc(sizeof(wchar_t) * len); if (str2 == NULL) { + pymain->err = INIT_NO_MEMORY(); return NULL; } memcpy(str2, str, len * sizeof(wchar_t)); @@ -501,9 +516,9 @@ pymain_strdup(wchar_t *str) static int -pymain_optlist_append(_Py_OptList *list, wchar_t *str) +pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str) { - wchar_t *str2 = pymain_strdup(str); + wchar_t *str2 = pymain_strdup(pymain, str); if (str2 == NULL) { return -1; } @@ -512,6 +527,7 @@ pymain_optlist_append(_Py_OptList *list, wchar_t *str) wchar_t **options2 = (wchar_t **)PyMem_RawRealloc(list->options, size); if (options2 == NULL) { PyMem_RawFree(str2); + pymain->err = INIT_NO_MEMORY(); return -1; } options2[list->len] = str2; @@ -526,7 +542,7 @@ pymain_optlist_append(_Py_OptList *list, wchar_t *str) Return 1 if parsing failed. Set pymain->err and return -1 on other errors. */ static int -pymain_parse_cmdline(_PyMain *pymain) +pymain_parse_cmdline_impl(_PyMain *pymain) { _Py_CommandLineDetails *cmdline = &pymain->cmdline; @@ -544,7 +560,8 @@ pymain_parse_cmdline(_PyMain *pymain) size_t len = wcslen(_PyOS_optarg) + 1 + 1; wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); if (command == NULL) { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } memcpy(command, _PyOS_optarg, len * sizeof(wchar_t)); command[len - 2] = '\n'; @@ -629,16 +646,16 @@ pymain_parse_cmdline(_PyMain *pymain) break; case 'W': - if (pymain_optlist_append(&cmdline->warning_options, + if (pymain_optlist_append(pymain, &cmdline->warning_options, _PyOS_optarg) < 0) { - goto out_of_memory; + return -1; } break; case 'X': - if (pymain_optlist_append(&cmdline->xoptions, + if (pymain_optlist_append(pymain, &cmdline->xoptions, _PyOS_optarg) < 0) { - goto out_of_memory; + return -1; } break; @@ -666,10 +683,6 @@ pymain_parse_cmdline(_PyMain *pymain) } return 0; - -out_of_memory: - pymain->err = INIT_NO_MEMORY(); - return -1; } @@ -753,17 +766,18 @@ pymain_warnings_envvar(_PyMain *pymain) buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t)); if (buf == NULL) { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } wcscpy(buf, wp); for (warning = wcstok_s(buf, L",", &context); warning != NULL; warning = wcstok_s(NULL, L",", &context)) { - if (pymain_optlist_append(&pymain->env_warning_options, + if (pymain_optlist_append(pymain, &pymain->env_warning_options, warning) < 0) { PyMem_RawFree(buf); - goto out_of_memory; + return -1; } } PyMem_RawFree(buf); @@ -778,7 +792,8 @@ pymain_warnings_envvar(_PyMain *pymain) C89 wcstok */ buf = (char *)PyMem_RawMalloc(strlen(p) + 1); if (buf == NULL) { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } strcpy(buf, p); oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL)); @@ -793,13 +808,14 @@ pymain_warnings_envvar(_PyMain *pymain) return -1; } else { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } } - if (pymain_optlist_append(&pymain->env_warning_options, + if (pymain_optlist_append(pymain, &pymain->env_warning_options, warning) < 0) { PyMem_RawFree(warning); - goto out_of_memory; + return -1; } PyMem_RawFree(warning); } @@ -809,10 +825,6 @@ pymain_warnings_envvar(_PyMain *pymain) } #endif return 0; - -out_of_memory: - pymain->err = INIT_NO_MEMORY(); - return -1; } @@ -882,7 +894,8 @@ pymain_get_program_name(_PyMain *pymain) buffer = PyMem_RawMalloc(len * sizeof(wchar_t)); if (buffer == NULL) { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } mbstowcs(buffer, p, len); @@ -904,7 +917,8 @@ pymain_get_program_name(_PyMain *pymain) return -1; } else { - goto out_of_memory; + pymain->err = INIT_NO_MEMORY(); + return -1; } } pymain->program_name = wbuf; @@ -915,16 +929,12 @@ pymain_get_program_name(_PyMain *pymain) if (pymain->program_name == NULL) { /* Use argv[0] by default */ - pymain->program_name = pymain_strdup(pymain->argv[0]); + pymain->program_name = pymain_strdup(pymain, pymain->argv[0]); if (pymain->program_name == NULL) { - goto out_of_memory; + return -1; } } return 0; - -out_of_memory: - pymain->err = INIT_NO_MEMORY(); - return -1; } @@ -982,7 +992,7 @@ pymain_header(_PyMain *pymain) static void -pymain_init_argv(_PyMain *pymain) +pymain_set_argv(_PyMain *pymain) { _Py_CommandLineDetails *cmdline = &pymain->cmdline; @@ -999,10 +1009,6 @@ pymain_init_argv(_PyMain *pymain) pymain->argv[_PyOS_optind] = L"-m"; } - if (cmdline->filename != NULL) { - pymain->main_importer_path = pymain_get_importer(cmdline->filename); - } - int update_path; if (pymain->main_importer_path != NULL) { /* Let pymain_run_main_from_importer() adjust sys.path[0] later */ @@ -1039,28 +1045,6 @@ pymain_set_global_config(_PyMain *pymain) } -/* Propagate options parsed from the command line and environment variables - to the Python runtime. - - Return 0 on success, or set pymain->err and return -1 on error. */ -static int -pymain_configure_pyruntime(_PyMain *pymain) -{ - Py_SetProgramName(pymain->program_name); - /* Don't free program_name here: the argument to Py_SetProgramName - must remain valid until Py_FinalizeEx is called. The string is freed - by pymain_free(). */ - - if (pymain_add_xoptions(pymain)) { - return -1; - } - if (pymain_add_warnings_options(pymain)) { - return -1; - } - return 0; -} - - static void pymain_import_readline(_PyMain *pymain) { @@ -1135,7 +1119,7 @@ pymain_open_filename(_PyMain *pymain) static void -pymain_run(_PyMain *pymain) +pymain_run_filename(_PyMain *pymain) { _Py_CommandLineDetails *cmdline = &pymain->cmdline; @@ -1169,24 +1153,22 @@ pymain_run(_PyMain *pymain) static void pymain_repl(_PyMain *pymain) { - char *p; - /* Check this environment variable at the end, to give programs the - * opportunity to set it from Python. - */ - if (!Py_InspectFlag && - (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') - { + opportunity to set it from Python. */ + if (!Py_InspectFlag && pymain_get_env_var("PYTHONINSPECT")) { Py_InspectFlag = 1; } - if (Py_InspectFlag && pymain->stdin_is_interactive && pymain->run_code) { - Py_InspectFlag = 0; - pymain_run_interactive_hook(); - /* XXX */ - int res = PyRun_AnyFileFlags(stdin, "<stdin>", &pymain->cf); - pymain->status = (res != 0); + if (!(Py_InspectFlag && pymain->stdin_is_interactive + && pymain->run_code)) { + return; } + + Py_InspectFlag = 0; + pymain_run_interactive_hook(); + /* XXX */ + int res = PyRun_AnyFileFlags(stdin, "<stdin>", &pymain->cf); + pymain->status = (res != 0); } @@ -1197,22 +1179,22 @@ pymain_repl(_PyMain *pymain) Return 0 on success. Set pymain->err and return -1 on failure. */ static int -pymain_init_cmdline(_PyMain *pymain) +pymain_parse_cmdline(_PyMain *pymain) { _Py_CommandLineDetails *cmdline = &pymain->cmdline; - int res = pymain_parse_cmdline(pymain); + int res = pymain_parse_cmdline_impl(pymain); if (res < 0) { return -1; } if (res) { - usage(1, pymain->argv[0]); + pymain_usage(1, pymain->argv[0]); pymain->status = 2; return 1; } if (cmdline->print_help) { - usage(0, pymain->argv[0]); + pymain_usage(0, pymain->argv[0]); pymain->status = 0; return 1; } @@ -1230,34 +1212,170 @@ pymain_init_cmdline(_PyMain *pymain) } -/* Initialize Py_Main(). - This code must not use Python runtime apart PyMem_Raw memory allocator. +static wchar_t* +pymain_get_xoption(_PyMain *pymain, wchar_t *name) +{ + _Py_OptList *list = &pymain->cmdline.xoptions; + for (size_t i=0; i < list->len; i++) { + wchar_t *option = list->options[i]; + size_t len; + wchar_t *sep = wcschr(option, L'='); + if (sep != NULL) { + len = (sep - option); + } + else { + len = wcslen(option); + } + if (wcsncmp(option, name, len) == 0 && name[len] == L'\0') { + return option; + } + } + return NULL; +} + - Return 0 on success. - Return 1 if Python is done and must exit. - Set pymain->err and return -1 on error. */ static int -pymain_init_impl(_PyMain *pymain) +pymain_str_to_int(char *str, int *result) { - _PyCoreConfig *core_config = &pymain->core_config; - core_config->_disable_importlib = 0; + errno = 0; + char *endptr = str; + long value = strtol(str, &endptr, 10); + if (*endptr != '\0' || errno == ERANGE) { + return -1; + } + if (value < INT_MIN || value > INT_MAX) { + return -1; + } - orig_argc = pymain->argc; /* For Py_GetArgcArgv() */ - orig_argv = pymain->argv; + *result = (int)value; + return 0; +} - /* Parse the command line */ - int res = pymain_init_cmdline(pymain); - if (res < 0) { + +static int +pymain_wstr_to_int(wchar_t *wstr, int *result) +{ + errno = 0; + wchar_t *endptr = wstr; + long value = wcstol(wstr, &endptr, 10); + if (*endptr != '\0' || errno == ERANGE) { return -1; } - if (res > 0) { - return 1; + if (value < INT_MIN || value > INT_MAX) { + return -1; } - pymain_set_global_config(pymain); - pymain_init_stdio(pymain); + *result = (int)value; + return 0; +} + + +static int +pymain_init_tracemalloc(_PyMain *pymain) +{ + int nframe; + int valid; + + char *env = pymain_get_env_var("PYTHONTRACEMALLOC"); + if (env) { + if (!pymain_str_to_int(env, &nframe)) { + valid = (nframe >= 1); + } + else { + valid = 0; + } + if (!valid) { + pymain->err = _Py_INIT_USER_ERR("PYTHONTRACEMALLOC: invalid " + "number of frames"); + return -1; + } + pymain->core_config.tracemalloc = nframe; + } + + wchar_t *xoption = pymain_get_xoption(pymain, L"tracemalloc"); + if (xoption) { + wchar_t *sep = wcschr(xoption, L'='); + if (sep) { + if (!pymain_wstr_to_int(sep + 1, &nframe)) { + valid = (nframe >= 1); + } + else { + valid = 0; + } + if (!valid) { + pymain->err = _Py_INIT_USER_ERR("-X tracemalloc=NFRAME: " + "invalid number of frames"); + return -1; + } + } + else { + /* -X tracemalloc behaves as -X tracemalloc=1 */ + nframe = 1; + } + pymain->core_config.tracemalloc = nframe; + } + return 0; +} + + +static void +pymain_set_flag_from_env(int *flag, const char *name) +{ + char *var = pymain_get_env_var(name); + if (!var) { + return; + } + int value; + if (pymain_str_to_int(var, &value) < 0 || value < 0) { + /* PYTHONDEBUG=text and PYTHONDEBUG=-2 behave as PYTHONDEBUG=1 */ + value = 1; + } + if (*flag < value) { + *flag = value; + } +} + + +static void +pymain_set_flags_from_env(_PyMain *pymain) +{ + pymain_set_flag_from_env(&Py_DebugFlag, + "PYTHONDEBUG"); + pymain_set_flag_from_env(&Py_VerboseFlag, + "PYTHONVERBOSE"); + pymain_set_flag_from_env(&Py_OptimizeFlag, + "PYTHONOPTIMIZE"); + pymain_set_flag_from_env(&Py_InspectFlag, + "PYTHONINSPECT"); + pymain_set_flag_from_env(&Py_DontWriteBytecodeFlag, + "PYTHONDONTWRITEBYTECODE"); + pymain_set_flag_from_env(&Py_NoUserSiteDirectory, + "PYTHONNOUSERSITE"); + pymain_set_flag_from_env(&Py_UnbufferedStdioFlag, + "PYTHONUNBUFFERED"); +#ifdef MS_WINDOWS + pymain_set_flag_from_env(&Py_LegacyWindowsFSEncodingFlag, + "PYTHONLEGACYWINDOWSFSENCODING"); + pymain_set_flag_from_env(&Py_LegacyWindowsStdioFlag, + "PYTHONLEGACYWINDOWSSTDIO"); +#endif +} + + +static int +pymain_parse_envvars(_PyMain *pymain) +{ + _PyCoreConfig *core_config = &pymain->core_config; /* Get environment variables */ + pymain_set_flags_from_env(pymain); + + /* The variable is only tested for existence here; + _Py_HashRandomization_Init will check its value further. */ + if (pymain_get_env_var("PYTHONHASHSEED")) { + Py_HashRandomizationFlag = 1; + } + if (pymain_warnings_envvar(pymain) < 0) { return -1; } @@ -1266,26 +1384,59 @@ pymain_init_impl(_PyMain *pymain) } core_config->allocator = Py_GETENV("PYTHONMALLOC"); + /* More complex options: env var and/or -X option */ + if (pymain_get_env_var("PYTHONFAULTHANDLER") + || pymain_get_xoption(pymain, L"faulthandler")) { + core_config->faulthandler = 1; + } + if (pymain_get_env_var("PYTHONPROFILEIMPORTTIME") + || pymain_get_xoption(pymain, L"importtime")) { + core_config->importtime = 1; + } + if (pymain_init_tracemalloc(pymain) < 0) { + return -1; + } return 0; } +/* Parse command line options and environment variables. + This code must not use Python runtime apart PyMem_Raw memory allocator. + + Return 0 on success. + Return 1 if Python is done and must exit. + Set pymain->err and return -1 on error. */ static int -pymain_init(_PyMain *pymain) +pymain_parse_cmdline_envvars_impl(_PyMain *pymain) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { + int res = pymain_parse_cmdline(pymain); + if (res < 0) { + return -1; + } + if (res > 0) { + return 1; + } + + pymain_set_global_config(pymain); + + if (pymain_parse_envvars(pymain) < 0) { return -1; } - /* Make sure that all memory allocated in pymain_init() is allocated - by malloc() */ + return 0; +} + + +static int +pymain_parse_cmdline_envvars(_PyMain *pymain) +{ + /* Force malloc() memory allocator */ PyMemAllocatorEx old_alloc, raw_alloc; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyMem_GetDefaultRawAllocator(&raw_alloc); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc); - int res = pymain_init_impl(pymain); + int res = pymain_parse_cmdline_envvars_impl(pymain); /* Restore the old memory allocator */ PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -1294,27 +1445,47 @@ pymain_init(_PyMain *pymain) } static int -pymain_core(_PyMain *pymain) +pymain_init_python(_PyMain *pymain) { - _Py_CommandLineDetails *cmdline = &pymain->cmdline; + pymain_init_stdio(pymain); pymain->err = _Py_InitializeCore(&pymain->core_config); if (_Py_INIT_FAILED(pymain->err)) { return -1; } - if (pymain_configure_pyruntime(pymain)) { + Py_SetProgramName(pymain->program_name); + /* Don't free program_name here: the argument to Py_SetProgramName + must remain valid until Py_FinalizeEx is called. The string is freed + by pymain_free(). */ + + if (pymain_add_xoptions(pymain)) { + return -1; + } + if (pymain_add_warnings_options(pymain)) { return -1; } if (pymain_init_main_interpreter(pymain)) { return -1; } + return 0; +} + + +static void +pymain_run_python(_PyMain *pymain) +{ + _Py_CommandLineDetails *cmdline = &pymain->cmdline; pymain_header(pymain); pymain_import_readline(pymain); - pymain_init_argv(pymain); + if (cmdline->filename != NULL) { + pymain->main_importer_path = pymain_get_importer(cmdline->filename); + } + + pymain_set_argv(pymain); if (cmdline->command) { pymain->status = pymain_run_command(cmdline->command, &pymain->cf); @@ -1323,16 +1494,57 @@ pymain_core(_PyMain *pymain) pymain->status = (pymain_run_module(cmdline->module, 1) != 0); } else { - pymain_run(pymain); + pymain_run_filename(pymain); } pymain_repl(pymain); +} + + +static int +pymain_init(_PyMain *pymain) +{ + pymain->err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(pymain->err)) { + return -1; + } + + pymain->core_config._disable_importlib = 0; + + orig_argc = pymain->argc; /* For Py_GetArgcArgv() */ + orig_argv = pymain->argv; + return 0; +} + + +static int +pymain_impl(_PyMain *pymain) +{ + int res = pymain_init(pymain); + if (res < 0) { + return -1; + } + + res = pymain_parse_cmdline_envvars(pymain); + if (res < 0) { + return -1; + } + if (res > 0) { + /* --help or --version command: we are done */ + return 0; + } + + res = pymain_init_python(pymain); + if (res < 0) { + return -1; + } + + pymain_run_python(pymain); if (Py_FinalizeEx() < 0) { /* Value unlikely to be confused with a non-error exit status or - other special meaning */ + other special meaning */ pymain->status = 120; } - return 0; } @@ -1345,17 +1557,9 @@ Py_Main(int argc, wchar_t **argv) pymain.argc = argc; pymain.argv = argv; - int res = pymain_init(&pymain); - if (res < 0) { + if (pymain_impl(&pymain) < 0) { _Py_FatalInitError(pymain.err); } - if (res == 0) { - res = pymain_core(&pymain); - if (res < 0) { - _Py_FatalInitError(pymain.err); - } - } - pymain_free(&pymain); return pymain.status; diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index e667fa9..610541d 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -594,8 +594,8 @@ init_hash_secret(int use_hash_seed, pyurandom() is non-blocking mode (blocking=0): see the PEP 524. */ res = pyurandom(secret, secret_size, 0, 0); if (res < 0) { - return _Py_INIT_ERR("failed to get random numbers " - "to initialize Python"); + return _Py_INIT_USER_ERR("failed to get random numbers " + "to initialize Python"); } } return _Py_INIT_OK(); diff --git a/Python/import.c b/Python/import.c index fe60844..fef6398 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1675,10 +1675,9 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } else { /* 1 -- true, 0 -- false, -1 -- not initialized */ - static int ximporttime = -1; + int importtime = interp->core_config.importtime; static int import_level; static _PyTime_t accumulated; - _Py_IDENTIFIER(importtime); _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -1687,32 +1686,14 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, * Anyway, importlib._find_and_load is much slower than * _PyDict_GetItemIdWithError(). */ - if (ximporttime < 0) { - const char *envoption = Py_GETENV("PYTHONPROFILEIMPORTTIME"); - if (envoption != NULL && *envoption != '\0') { - ximporttime = 1; - } - else { - PyObject *xoptions = PySys_GetXOptions(); - PyObject *value = NULL; - if (xoptions) { - value = _PyDict_GetItemIdWithError( - xoptions, &PyId_importtime); - } - if (value == NULL && PyErr_Occurred()) { - goto error; - } - if (value != NULL || Py_IsInitialized()) { - ximporttime = (value == Py_True); - } - } - if (ximporttime > 0) { + if (importtime) { + static int header = 1; + if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); + header = 0; } - } - if (ximporttime > 0) { import_level++; t1 = _PyTime_GetPerfCounter(); accumulated = 0; @@ -1731,7 +1712,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), mod != NULL); - if (ximporttime > 0) { + if (importtime) { _PyTime_t cum = _PyTime_GetPerfCounter() - t1; import_level--; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 66cc711..14fe75e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -56,7 +56,7 @@ extern grammar _PyParser_Grammar; /* From graminit.c */ static _PyInitError add_main_module(PyInterpreterState *interp); static _PyInitError initfsencoding(PyInterpreterState *interp); static _PyInitError initsite(void); -static int initstdio(void); +static _PyInitError init_sys_streams(void); static _PyInitError initsigs(void); static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); @@ -66,10 +66,10 @@ extern int _PyStructSequence_Init(void); extern void _PyUnicode_Fini(void); extern int _PyLong_Init(void); extern void PyLong_Fini(void); -extern _PyInitError _PyFaulthandler_Init(void); +extern _PyInitError _PyFaulthandler_Init(int enable); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); -extern int _PyTraceMalloc_Init(void); +extern int _PyTraceMalloc_Init(int enable); extern int _PyTraceMalloc_Fini(void); extern void _Py_ReadyTypes(void); @@ -219,20 +219,6 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors) */ -static void -set_flag(int *flag, const char *envs) -{ - /* Helper to set flag variables from environment variables: - * - uses the higher of the two values if they're both set - * - otherwise sets the flag to 1 - */ - int env = atoi(envs); - if (*flag < env) - *flag = env; - if (*flag < 1) - *flag = 1; -} - static char* get_codec_name(const char *encoding) { @@ -284,7 +270,6 @@ get_locale_encoding(void) #endif } -/* Return NULL on success, or return an error message on failure */ static _PyInitError initimport(PyInterpreterState *interp, PyObject *sysmod) { @@ -346,7 +331,6 @@ initimport(PyInterpreterState *interp, PyObject *sysmod) return _Py_INIT_OK(); } -/* Return NULL on success, or return an error message on failure */ static _PyInitError initexternalimport(PyInterpreterState *interp) { @@ -623,8 +607,6 @@ _Py_SetLocaleFromEnv(int category) * Any code invoked from this function should *not* assume it has access * to the Python C API (unless the API is explicitly listed as being * safe to call without calling Py_Initialize first) - * - * Return NULL on success, or return an error message on failure. */ /* TODO: Progressively move functionality from Py_BeginInitialization to @@ -637,7 +619,6 @@ _Py_InitializeCore(const _PyCoreConfig *config) PyInterpreterState *interp; PyThreadState *tstate; PyObject *bimod, *sysmod, *pstderr; - char *p; _PyCoreConfig core_config = _PyCoreConfig_INIT; _PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT; _PyInitError err; @@ -681,31 +662,6 @@ _Py_InitializeCore(const _PyCoreConfig *config) _emit_stderr_warning_for_legacy_locale(); #endif - if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0') - set_flag(&Py_DebugFlag, p); - if ((p = Py_GETENV("PYTHONVERBOSE")) && *p != '\0') - set_flag(&Py_VerboseFlag, p); - if ((p = Py_GETENV("PYTHONOPTIMIZE")) && *p != '\0') - set_flag(&Py_OptimizeFlag, p); - if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') - set_flag(&Py_InspectFlag, p); - if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0') - set_flag(&Py_DontWriteBytecodeFlag, p); - if ((p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0') - set_flag(&Py_NoUserSiteDirectory, p); - if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') - set_flag(&Py_UnbufferedStdioFlag, p); - /* The variable is only tested for existence here; - _Py_HashRandomization_Init will check its value further. */ - if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0') - set_flag(&Py_HashRandomizationFlag, p); -#ifdef MS_WINDOWS - if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0') - set_flag(&Py_LegacyWindowsFSEncodingFlag, p); - if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0') - set_flag(&Py_LegacyWindowsStdioFlag, p); -#endif - err = _Py_HashRandomization_Init(&core_config); if (_Py_INIT_FAILED(err)) { return err; @@ -716,7 +672,11 @@ _Py_InitializeCore(const _PyCoreConfig *config) Py_HashRandomizationFlag = 1; } - _PyInterpreterState_Enable(&_PyRuntime); + err = _PyInterpreterState_Enable(&_PyRuntime); + if (_Py_INIT_FAILED(err)) { + return err; + } + interp = PyInterpreterState_New(); if (interp == NULL) return _Py_INIT_ERR("can't make main interpreter"); @@ -834,8 +794,6 @@ _Py_InitializeCore(const _PyCoreConfig *config) * * More advanced selective initialization tricks are possible by calling * this function multiple times with various preconfigured settings. - * - * Return NULL on success, or return an error message on failure. */ _PyInitError @@ -858,8 +816,6 @@ _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config) * initialized or without a valid current thread state is a fatal error. * Other errors should be reported as normal Python exceptions with a * non-zero return code. - * - * Return NULL on success, or return an error message on failure. */ _PyInitError _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) @@ -907,13 +863,14 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) PySys_SetPath(Py_GetPath()); if (_PySys_EndInit(interp->sysdict) < 0) return _Py_INIT_ERR("can't finish initializing sys"); + err = initexternalimport(interp); if (_Py_INIT_FAILED(err)) { return err; } /* initialize the faulthandler module */ - err = _PyFaulthandler_Init(); + err = _PyFaulthandler_Init(interp->core_config.faulthandler); if (_Py_INIT_FAILED(err)) { return err; } @@ -930,15 +887,17 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config) } } - if (_PyTraceMalloc_Init() < 0) + if (_PyTraceMalloc_Init(interp->core_config.tracemalloc) < 0) return _Py_INIT_ERR("can't initialize tracemalloc"); err = add_main_module(interp); if (_Py_INIT_FAILED(err)) { return err; } - if (initstdio() < 0) { - return _Py_INIT_ERR("can't initialize sys standard streams"); + + err = init_sys_streams(); + if (_Py_INIT_FAILED(err)) { + return err; } /* Initialize warnings. */ @@ -1302,29 +1261,32 @@ Py_Finalize(void) */ -PyThreadState * -Py_NewInterpreter(void) +static _PyInitError +new_interpreter(PyThreadState **tstate_p) { PyInterpreterState *interp; PyThreadState *tstate, *save_tstate; PyObject *bimod, *sysmod; - _PyInitError err = _Py_INIT_OK(); - if (!_PyRuntime.initialized) - Py_FatalError("Py_NewInterpreter: call Py_Initialize first"); + if (!_PyRuntime.initialized) { + return _Py_INIT_ERR("Py_Initialize must be called first"); + } /* Issue #10915, #15751: The GIL API doesn't work with multiple interpreters: disable PyGILState_Check(). */ _PyGILState_check_enabled = 0; interp = PyInterpreterState_New(); - if (interp == NULL) - return NULL; + if (interp == NULL) { + *tstate_p = NULL; + return _Py_INIT_OK(); + } tstate = PyThreadState_New(interp); if (tstate == NULL) { PyInterpreterState_Delete(interp); - return NULL; + *tstate_p = NULL; + return _Py_INIT_OK(); } save_tstate = PyThreadState_Swap(tstate); @@ -1343,8 +1305,9 @@ Py_NewInterpreter(void) /* XXX The following is lax in error checking */ PyObject *modules = PyDict_New(); - if (modules == NULL) - Py_FatalError("Py_NewInterpreter: can't make modules dictionary"); + if (modules == NULL) { + return _Py_INIT_ERR("can't make modules dictionary"); + } interp->modules = modules; sysmod = _PyImport_FindBuiltin("sys", modules); @@ -1371,59 +1334,62 @@ Py_NewInterpreter(void) if (bimod != NULL && sysmod != NULL) { PyObject *pstderr; + _PyInitError err; /* Set up a preliminary stderr printer until we have enough infrastructure for the io module in place. */ pstderr = PyFile_NewStdPrinter(fileno(stderr)); - if (pstderr == NULL) - Py_FatalError("Py_NewInterpreter: can't set preliminary stderr"); + if (pstderr == NULL) { + return _Py_INIT_ERR("can't set preliminary stderr"); + } _PySys_SetObjectId(&PyId_stderr, pstderr); PySys_SetObject("__stderr__", pstderr); Py_DECREF(pstderr); err = _PyImportHooks_Init(); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } err = initimport(interp, sysmod); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } err = initexternalimport(interp); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } err = initfsencoding(interp); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } - if (initstdio() < 0) { - err = _Py_INIT_ERR("can't initialize sys standard streams"); - goto init_failed; + err = init_sys_streams(); + if (_Py_INIT_FAILED(err)) { + return err; } err = add_main_module(interp); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } if (!Py_NoSiteFlag) { err = initsite(); if (_Py_INIT_FAILED(err)) { - goto init_failed; + return err; } } } - if (!PyErr_Occurred()) - return tstate; + if (PyErr_Occurred()) { + goto handle_error; + } -init_failed: - _Py_FatalInitError(err); + *tstate_p = tstate; + return _Py_INIT_OK(); handle_error: /* Oops, it didn't work. Undo it all. */ @@ -1434,7 +1400,20 @@ handle_error: PyThreadState_Delete(tstate); PyInterpreterState_Delete(interp); - return NULL; + *tstate_p = NULL; + return _Py_INIT_OK(); +} + +PyThreadState * +Py_NewInterpreter(void) +{ + PyThreadState *tstate; + _PyInitError err = new_interpreter(&tstate); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + return tstate; + } /* Delete an interpreter and its last thread. This requires that the @@ -1770,16 +1749,17 @@ error: } /* Initialize sys.stdin, stdout, stderr and builtins.open */ -static int -initstdio(void) +static _PyInitError +init_sys_streams(void) { PyObject *iomod = NULL, *wrapper; PyObject *bimod = NULL; PyObject *m; PyObject *std = NULL; - int status = 0, fd; + int fd; PyObject * encoding_attr; char *pythonioencoding = NULL, *encoding, *errors; + _PyInitError res = _Py_INIT_OK(); /* Hack to avoid a nasty recursion issue when Python is invoked in verbose mode: pre-import the Latin-1 and UTF-8 codecs */ @@ -1893,11 +1873,12 @@ initstdio(void) Py_DECREF(std); #endif - if (0) { - error: - status = -1; - } + goto done; +error: + res = _Py_INIT_ERR("can't initialize sys standard streams"); + +done: /* We won't need them anymore. */ if (_Py_StandardStreamEncoding) { PyMem_RawFree(_Py_StandardStreamEncoding); @@ -1910,7 +1891,7 @@ initstdio(void) PyMem_Free(pythonioencoding); Py_XDECREF(bimod); Py_XDECREF(iomod); - return status; + return res; } diff --git a/Python/pystate.c b/Python/pystate.c index 30f214e..4544de9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -76,7 +76,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) static void _PyGILState_NoteThreadState(PyThreadState* tstate); -void +_PyInitError _PyInterpreterState_Enable(_PyRuntimeState *runtime) { runtime->interpreters.next_id = 0; @@ -85,9 +85,11 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) initialized here. */ if (runtime->interpreters.mutex == NULL) { runtime->interpreters.mutex = PyThread_allocate_lock(); - if (runtime->interpreters.mutex == NULL) - Py_FatalError("Can't initialize threads for interpreter"); + if (runtime->interpreters.mutex == NULL) { + return _Py_INIT_ERR("Can't initialize threads for interpreter"); + } } + return _Py_INIT_OK(); } PyInterpreterState * |