From 09f3a8a1249308a104a89041d82fe99e6c087043 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Nov 2017 17:32:40 -0800 Subject: bpo-32089: Fix warnings filters in dev mode (#4482) The developer mode (-X dev) now creates all default warnings filters to order filters in the correct order to always show ResourceWarning and make BytesWarning depend on the -b option. Write a functional test to make sure that ResourceWarning is logged twice at the same location in the developer mode. Add a new 'dev_mode' field to _PyCoreConfig. --- Doc/using/cmdline.rst | 12 +++++------ Include/pystate.h | 4 +++- Lib/subprocess.py | 8 ++------ Lib/test/test_cmd_line.py | 52 +++++++++++++++++++++++++++++++++++++++++------ Lib/warnings.py | 14 ++++++++++--- Modules/main.c | 9 ++------ Python/_warnings.c | 37 ++++++++++++++++++++++++--------- 7 files changed, 97 insertions(+), 39 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b226930..8e186b3 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -412,13 +412,13 @@ Miscellaneous options application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. * ``-X dev`` enables the "developer mode": enable debug checks at runtime. - In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug PYTHONASYNCIODEBUG=1 python3 - -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC` - and :envvar:`PYTHONASYNCIODEBUG` environment variables are not set in - practice. Developer mode: + Developer mode: - * Add ``default`` warnings option. For example, display - :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings. + * Warning filters: add a filter to display all warnings (``"default"`` + action), except of :exc:`BytesWarning` which still depends on the + :option:`-b` option, and use ``"always"`` action for + :exc:`ResourceWarning` warnings. For example, display + :exc:`DeprecationWarning` warnings. * Install debug hooks on memory allocators: see the :c:func:`PyMem_SetupDebugHooks` C function. * Enable the :mod:`faulthandler` module to dump the Python traceback diff --git a/Include/pystate.h b/Include/pystate.h index 4401225..2081ff5 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -33,6 +33,7 @@ typedef struct { int faulthandler; int tracemalloc; /* Number of saved frames, 0=don't trace */ int importtime; /* -X importtime */ + int dev_mode; /* -X dev */ } _PyCoreConfig; #define _PyCoreConfig_INIT \ @@ -43,7 +44,8 @@ typedef struct { .allocator = NULL, \ .faulthandler = 0, \ .tracemalloc = 0, \ - .importtime = 0} + .importtime = 0, \ + .dev_mode = 0} /* Placeholders while working on the new configuration API * diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 97b4493..35bfddd 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -262,15 +262,11 @@ def _args_from_interpreter_flags(): args.append('-' + opt * v) # -W options - warnoptions = sys.warnoptions - xoptions = getattr(sys, '_xoptions', {}) - if 'dev' in xoptions and warnoptions and warnoptions[-1] == 'default': - # special case: -X dev adds 'default' to sys.warnoptions - warnoptions = warnoptions[:-1] - for opt in warnoptions: + for opt in sys.warnoptions: args.append('-W' + opt) # -X options + xoptions = getattr(sys, '_xoptions', {}) if 'dev' in xoptions: args.extend(('-X', 'dev')) for opt in ('faulthandler', 'tracemalloc', 'importtime', diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3dbe75f..75f7d00 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -507,14 +507,14 @@ class CmdLineTest(unittest.TestCase): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) - def run_xdev(self, code, check_exitcode=True): + def run_xdev(self, *args, check_exitcode=True): env = dict(os.environ) env.pop('PYTHONWARNINGS', None) # Force malloc() to disable the debug hooks which are enabled # by default for Python compiled in debug mode env['PYTHONMALLOC'] = 'malloc' - args = (sys.executable, '-X', 'dev', '-c', code) + args = (sys.executable, '-X', 'dev', *args) proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -525,8 +525,34 @@ class CmdLineTest(unittest.TestCase): return proc.stdout.rstrip() def test_xdev(self): - out = self.run_xdev("import sys; print(sys.warnoptions)") - self.assertEqual(out, "['default']") + code = ("import sys, warnings; " + "print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + + out = self.run_xdev("-c", code) + self.assertEqual(out, + "ignore::BytesWarning " + "always::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-b", "-c", code) + self.assertEqual(out, + "default::BytesWarning " + "always::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-bb", "-c", code) + self.assertEqual(out, + "error::BytesWarning " + "always::ResourceWarning " + "default::Warning") + + out = self.run_xdev("-Werror", "-c", code) + self.assertEqual(out, + "error::Warning " + "ignore::BytesWarning " + "always::ResourceWarning " + "default::Warning") try: import _testcapi @@ -535,7 +561,7 @@ class CmdLineTest(unittest.TestCase): else: code = "import _testcapi; _testcapi.pymem_api_misuse()" with support.SuppressCrashReport(): - out = self.run_xdev(code, check_exitcode=False) + out = self.run_xdev("-c", code, check_exitcode=False) self.assertIn("Debug memory block at address p=", out) try: @@ -544,9 +570,23 @@ class CmdLineTest(unittest.TestCase): pass else: code = "import faulthandler; print(faulthandler.is_enabled())" - out = self.run_xdev(code) + out = self.run_xdev("-c", code) self.assertEqual(out, "True") + # Make sure that ResourceWarning emitted twice at the same line number + # is logged twice + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w", encoding="utf8") as fp: + print("def func(): open(__file__)", file=fp) + print("func()", file=fp) + print("func()", file=fp) + fp.flush() + + out = self.run_xdev(filename) + self.assertEqual(out.count(':1: ResourceWarning: '), 2, out) + + class IgnoreEnvironmentTest(unittest.TestCase): def run_ignoring_vars(self, predicate, **env_vars): diff --git a/Lib/warnings.py b/Lib/warnings.py index 48d5e16..b2605f8 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -486,7 +486,6 @@ class catch_warnings(object): # - a compiled regex that must match the module that is being warned # - a line number for the line being warning, or 0 to mean any line # If either if the compiled regexs are None, match anything. -_warnings_defaults = False try: from _warnings import (filters, _defaultaction, _onceregistry, warn, warn_explicit, _filters_mutated) @@ -504,12 +503,16 @@ except ImportError: global _filters_version _filters_version += 1 + _warnings_defaults = False + # Module initialization _processoptions(sys.warnoptions) if not _warnings_defaults: + dev_mode = ('dev' in getattr(sys, '_xoptions', {})) py_debug = hasattr(sys, 'gettotalrefcount') - if not py_debug: + + if not(dev_mode or py_debug): silence = [ImportWarning, PendingDeprecationWarning] silence.append(DeprecationWarning) for cls in silence: @@ -525,10 +528,15 @@ if not _warnings_defaults: simplefilter(bytes_action, category=BytesWarning, append=1) # resource usage warnings are enabled by default in pydebug mode - if py_debug: + if dev_mode or py_debug: resource_action = "always" else: resource_action = "ignore" simplefilter(resource_action, category=ResourceWarning, append=1) + if dev_mode: + simplefilter("default", category=Warning, append=1) + + del py_debug, dev_mode + del _warnings_defaults diff --git a/Modules/main.c b/Modules/main.c index 203abf4..70c1c3d 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1397,14 +1397,9 @@ pymain_parse_envvars(_PyMain *pymain) return -1; } if (pymain_get_xoption(pymain, L"dev")) { - /* "python3 -X dev ..." behaves - as "PYTHONMALLOC=debug python3 -Wd -X faulthandler ..." */ - core_config->allocator = "debug"; - if (pymain_optlist_append(pymain, &pymain->cmdline.warning_options, - L"default") < 0) { - return -1; - } + core_config->dev_mode = 1; core_config->faulthandler = 1; + core_config->allocator = "debug"; } return 0; } diff --git a/Python/_warnings.c b/Python/_warnings.c index 8cfae76..f2110ed 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1196,11 +1196,19 @@ create_filter(PyObject *category, const char *action) static PyObject * init_filters(void) { + PyInterpreterState *interp = PyThreadState_GET()->interp; + int dev_mode = interp->core_config.dev_mode; + + Py_ssize_t count = 2; + if (dev_mode) { + count++; + } #ifndef Py_DEBUG - PyObject *filters = PyList_New(5); -#else - PyObject *filters = PyList_New(2); + if (!dev_mode) { + count += 3; + } #endif + PyObject *filters = PyList_New(count); unsigned int pos = 0; /* Post-incremented in each use. */ unsigned int x; const char *bytes_action, *resource_action; @@ -1209,12 +1217,14 @@ init_filters(void) return NULL; #ifndef Py_DEBUG - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, "ignore")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, "ignore")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, "ignore")); + if (!dev_mode) { + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, "ignore")); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, "ignore")); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, "ignore")); + } #endif if (Py_BytesWarningFlag > 1) @@ -1225,14 +1235,21 @@ init_filters(void) bytes_action = "ignore"; PyList_SET_ITEM(filters, pos++, create_filter(PyExc_BytesWarning, bytes_action)); + /* resource usage warnings are enabled by default in pydebug mode */ #ifdef Py_DEBUG resource_action = "always"; #else - resource_action = "ignore"; + resource_action = (dev_mode ? "always" : "ignore"); #endif PyList_SET_ITEM(filters, pos++, create_filter(PyExc_ResourceWarning, resource_action)); + + if (dev_mode) { + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_Warning, "default")); + } + for (x = 0; x < pos; x += 1) { if (PyList_GET_ITEM(filters, x) == NULL) { Py_DECREF(filters); -- cgit v0.12