From ccb0442a338066bf40fe417455e5a374e5238afb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 Nov 2017 03:20:31 -0800 Subject: bpo-32043: New "developer mode": "-X dev" option (#4413) Add a new "developer mode": new "-X dev" command line option to enable debug checks at runtime. Changes: * Add unit tests for -X dev * test_cmd_line: replace test.support with support. * Fix _PyRuntimeState_Fini(): Use the same memory allocator than _PyRuntimeState_Init(). * Fix _PyMem_GetDefaultRawAllocator() --- Doc/using/cmdline.rst | 14 ++++- Doc/whatsnew/3.7.rst | 13 ++++ Lib/test/test_cmd_line.py | 69 +++++++++++++++++----- .../2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst | 2 + Modules/main.c | 10 ++++ Objects/obmalloc.c | 19 +++--- Python/pystate.c | 8 +++ 7 files changed, 110 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9100494..01869d1 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -413,6 +413,17 @@ Miscellaneous options nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. + * ``-X dev`` enables the "developer mode": enable debug checks at runtime. + In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 + -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC` + environment variable is not set in practice. Developer mode: + + * Add ``default`` warnings option. For example, display + :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings. + * Install debug hooks on memory allocators as if :envvar:`PYTHONMALLOC` + is set to ``debug``. + * Enable the :mod:`faulthandler` module to dump the Python traceback + on a crash. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -430,7 +441,8 @@ Miscellaneous options The ``-X showalloccount`` option. .. versionadded:: 3.7 - The ``-X importtime`` and :envvar:`PYTHONPROFILEIMPORTTIME` options. + The ``-X importtime``, ``-X dev`` and :envvar:`PYTHONPROFILEIMPORTTIME` + options. Options you shouldn't use diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 9d63540..bb75939 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -185,6 +185,19 @@ resolution on Linux and Windows. PEP written and implemented by Victor Stinner +New Developer Mode: -X dev +-------------------------- + +Add a new "developer mode": ``-X dev`` command line option to enable debug +checks at runtime. + +In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W +default -X faulthandler ...``, except that the PYTHONMALLOC environment +variable is not set in practice. + +See :option:`-X` ``dev`` for the details. + + Other Language Changes ====================== diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1b584eb..3dbe75f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,14 +2,15 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution -import test.support, unittest import os -import sys import subprocess +import sys import tempfile -from test.support import script_helper, is_android +import unittest +from test import support from test.support.script_helper import ( - spawn_python, kill_python, assert_python_ok, assert_python_failure + spawn_python, kill_python, assert_python_ok, assert_python_failure, + interpreter_requires_environment ) # XXX (ncoghlan): Move to script_helper and make consistent with run_python @@ -132,11 +133,11 @@ class CmdLineTest(unittest.TestCase): # All good if execution is successful assert_python_ok('-c', 'pass') - @unittest.skipUnless(test.support.FS_NONASCII, 'need support.FS_NONASCII') + @unittest.skipUnless(support.FS_NONASCII, 'need support.FS_NONASCII') def test_non_ascii(self): # Test handling of non-ascii data command = ("assert(ord(%r) == %s)" - % (test.support.FS_NONASCII, ord(test.support.FS_NONASCII))) + % (support.FS_NONASCII, ord(support.FS_NONASCII))) assert_python_ok('-c', command) # On Windows, pass bytes to subprocess doesn't test how Python decodes the @@ -179,7 +180,7 @@ class CmdLineTest(unittest.TestCase): raise AssertionError("%a doesn't start with %a" % (stdout, pattern)) @unittest.skipUnless((sys.platform == 'darwin' or - is_android), 'test specific to Mac OS X and Android') + support.is_android), 'test specific to Mac OS X and Android') def test_osx_android_utf8(self): def check_output(text): decoded = text.decode('utf-8', 'surrogateescape') @@ -385,7 +386,7 @@ class CmdLineTest(unittest.TestCase): stderr=subprocess.PIPE, preexec_fn=preexec) out, err = p.communicate() - self.assertEqual(test.support.strip_python_stderr(err), b'') + self.assertEqual(support.strip_python_stderr(err), b'') self.assertEqual(p.returncode, 42) def test_no_stdin(self): @@ -433,8 +434,8 @@ class CmdLineTest(unittest.TestCase): # Issue #15001: PyRun_SimpleFileExFlags() did crash because it kept a # borrowed reference to the dict of __main__ module and later modify # the dict whereas the module was destroyed - filename = test.support.TESTFN - self.addCleanup(test.support.unlink, filename) + filename = support.TESTFN + self.addCleanup(support.unlink, filename) with open(filename, "w") as script: print("import sys", file=script) print("del sys.modules['__main__']", file=script) @@ -458,7 +459,7 @@ class CmdLineTest(unittest.TestCase): self.assertEqual(err.splitlines().count(b'Unknown option: -a'), 1) self.assertEqual(b'', out) - @unittest.skipIf(script_helper.interpreter_requires_environment(), + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') def test_isolatedmode(self): self.verify_valid_flag('-I') @@ -469,7 +470,7 @@ class CmdLineTest(unittest.TestCase): # dummyvar to prevent extraneous -E dummyvar="") self.assertEqual(out.strip(), b'1 1 1') - with test.support.temp_cwd() as tmpdir: + with support.temp_cwd() as tmpdir: fake = os.path.join(tmpdir, "uuid.py") main = os.path.join(tmpdir, "main.py") with open(fake, "w") as f: @@ -506,6 +507,46 @@ class CmdLineTest(unittest.TestCase): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) + def run_xdev(self, code, check_exitcode=True): + env = dict(os.environ) + env.pop('PYTHONWARNINGS', None) + # Force malloc() to disable the debug hooks which are enabled + # by default for Python compiled in debug mode + env['PYTHONMALLOC'] = 'malloc' + + args = (sys.executable, '-X', 'dev', '-c', code) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + if check_exitcode: + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() + + def test_xdev(self): + out = self.run_xdev("import sys; print(sys.warnoptions)") + self.assertEqual(out, "['default']") + + try: + import _testcapi + except ImportError: + pass + else: + code = "import _testcapi; _testcapi.pymem_api_misuse()" + with support.SuppressCrashReport(): + out = self.run_xdev(code, check_exitcode=False) + self.assertIn("Debug memory block at address p=", out) + + try: + import faulthandler + except ImportError: + pass + else: + code = "import faulthandler; print(faulthandler.is_enabled())" + out = self.run_xdev(code) + self.assertEqual(out, "True") + class IgnoreEnvironmentTest(unittest.TestCase): def run_ignoring_vars(self, predicate, **env_vars): @@ -541,8 +582,8 @@ class IgnoreEnvironmentTest(unittest.TestCase): def test_main(): - test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest) - test.support.reap_children() + support.run_unittest(CmdLineTest, IgnoreEnvironmentTest) + support.reap_children() if __name__ == "__main__": test_main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst new file mode 100644 index 0000000..21d59e0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-16-03-44-08.bpo-32043.AAzwpZ.rst @@ -0,0 +1,2 @@ +Add a new "developer mode": new "-X dev" command line option to enable debug +checks at runtime. diff --git a/Modules/main.c b/Modules/main.c index 95076c6..203abf4 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1396,6 +1396,16 @@ pymain_parse_envvars(_PyMain *pymain) if (pymain_init_tracemalloc(pymain) < 0) { return -1; } + if (pymain_get_xoption(pymain, L"dev")) { + /* "python3 -X dev ..." behaves + as "PYTHONMALLOC=debug python3 -Wd -X faulthandler ..." */ + core_config->allocator = "debug"; + if (pymain_optlist_append(pymain, &pymain->cmdline.warning_options, + L"default") < 0) { + return -1; + } + core_config->faulthandler = 1; + } return 0; } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 699cce9..7c6973e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -190,8 +190,14 @@ static struct { void _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p) { - PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; - *alloc_p = alloc; + PyMemAllocatorEx pymem_raw = { +#ifdef Py_DEBUG + &_PyMem_Debug.raw, PYRAWDBG_FUNCS +#else + NULL, PYRAW_FUNCS +#endif + }; + *alloc_p = pymem_raw; } int @@ -274,13 +280,6 @@ _PyObject_Initialize(struct _pyobj_runtime_state *state) void _PyMem_Initialize(struct _pymem_runtime_state *state) { - PyMemAllocatorEx pymem_raw = { -#ifdef Py_DEBUG - &_PyMem_Debug.raw, PYRAWDBG_FUNCS -#else - NULL, PYRAW_FUNCS -#endif - }; PyMemAllocatorEx pymem = { #ifdef Py_DEBUG &_PyMem_Debug.mem, PYDBG_FUNCS @@ -296,7 +295,7 @@ _PyMem_Initialize(struct _pymem_runtime_state *state) #endif }; - state->allocators.raw = pymem_raw; + _PyMem_GetDefaultRawAllocator(&state->allocators.raw); state->allocators.mem = pymem; state->allocators.obj = pyobject; diff --git a/Python/pystate.c b/Python/pystate.c index 4544de9..807ac4e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -64,10 +64,18 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) void _PyRuntimeState_Fini(_PyRuntimeState *runtime) { + /* Use the same memory allocator than _PyRuntimeState_Init() */ + PyMemAllocatorEx old_alloc, raw_alloc; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyMem_GetDefaultRawAllocator(&raw_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &raw_alloc); + if (runtime->interpreters.mutex != NULL) { PyThread_free_lock(runtime->interpreters.mutex); runtime->interpreters.mutex = NULL; } + + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } #define HEAD_LOCK() PyThread_acquire_lock(_PyRuntime.interpreters.mutex, \ -- cgit v0.12