On ResourceWarning, log traceback where the object was allocated

Issue #26567:

* Add a new function PyErr_ResourceWarning() function to pass the destroyed
  object
* Add a source attribute to warnings.WarningMessage
* Add warnings._showwarnmsg() which uses tracemalloc to get the traceback where
  source object was allocated.

11 files changed, 175 insertions, 40 deletions

diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 1e708a8..57f36ac 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -334,6 +334,14 @@ an error value).
    .. versionadded:: 3.2
 
 
+.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...)
+
+   Function similar to :c:func:`PyErr_WarnFormat`, but *category* is
+   :exc:`ResourceWarning` and pass *source* to :func:`warnings.WarningMessage`.
+
+   .. versionadded:: 3.6
+
+
 Querying the error indicator
 ============================
 
diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst
index 8a538ad..4ce88ab 100644
--- a/Doc/library/warnings.rst
+++ b/Doc/library/warnings.rst
@@ -319,7 +319,7 @@ Available Functions
    of the warning message).
 
 
-.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None)
+.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None)
 
    This is a low-level interface to the functionality of :func:`warn`, passing in
    explicitly the message, category, filename and line number, and optionally the
@@ -335,6 +335,12 @@ Available Functions
    source for modules found in zipfiles or other non-filesystem import
    sources).
 
+   *source*, if supplied, is the destroyed object which emitted a
+   :exc:`ResourceWarning`.
+
+   .. versionchanged:: 3.6
+      Add the *source* parameter.
+
 
 .. function:: showwarning(message, category, filename, lineno, file=None, line=None)
 
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
index cc63589..b709917 100644
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -258,6 +258,40 @@ urllib.robotparser
 (Contributed by Nikolay Bogoychev in :issue:`16099`.)
 
 
+warnings
+--------
+
+A new optional *source* parameter has been added to the
+:func:`warnings.warn_explicit` function: the destroyed object which emitted a
+:exc:`ResourceWarning`. A *source* attribute has also been added to
+:class:`warnings.WarningMessage` (contributed by Victor Stinner in
+:issue:`26568` and :issue:`26567`).
+
+When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` is now
+used to try to retrieve the traceback where the detroyed object was allocated.
+
+Example with the script ``example.py``::
+
+    def func():
+        f = open(__file__)
+        f = None
+
+    func()
+
+Output of the command ``python3.6 -Wd -X tracemalloc=5 example.py``::
+
+    example.py:3: ResourceWarning: unclosed file <...>
+      f = None
+    Object allocated at (most recent call first):
+      File "example.py", lineno 2
+        f = open(__file__)
+      File "example.py", lineno 5
+        func()
+
+The "Object allocated at" traceback is new and only displayed if
+:mod:`tracemalloc` is tracing Python memory allocations.
+
+
 zipfile
 -------
 
diff --git a/Include/warnings.h b/Include/warnings.h
index effb9fad..c1c6992 100644
--- a/Include/warnings.h
+++ b/Include/warnings.h
@@ -17,6 +17,13 @@ PyAPI_FUNC(int) PyErr_WarnFormat(
     Py_ssize_t stack_level,
     const char *format,         /* ASCII-encoded string  */
     ...);
+
+/* Emit a ResourceWarning warning */
+PyAPI_FUNC(int) PyErr_ResourceWarning(
+    PyObject *source,
+    Py_ssize_t stack_level,
+    const char *format,         /* ASCII-encoded string  */
+    ...);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyErr_WarnExplicitObject(
     PyObject *category,
diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py
index 70eae4c..a1b3dba 100644
--- a/Lib/test/test_warnings/__init__.py
+++ b/Lib/test/test_warnings/__init__.py
@@ -2,7 +2,10 @@ from contextlib import contextmanager
 import linecache
 import os
 from io import StringIO
+import re
 import sys
+import tempfile
+import textwrap
 import unittest
 from test import support
 from test.support.script_helper import assert_python_ok, assert_python_failure
@@ -763,12 +766,39 @@ class WarningsDisplayTests(BaseTest):
                                 file_object, expected_file_line)
         self.assertEqual(expect, file_object.getvalue())
 
+
 class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
     module = c_warnings
 
 class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase):
     module = py_warnings
 
+    def test_tracemalloc(self):
+        with tempfile.NamedTemporaryFile("w", suffix=".py") as tmpfile:
+            tmpfile.write(textwrap.dedent("""
+                def func():
+                    f = open(__file__)
+                    # Emit ResourceWarning
+                    f = None
+
+                func()
+            """))
+            tmpfile.flush()
+            fname = tmpfile.name
+            res = assert_python_ok('-Wd', '-X', 'tracemalloc=2', fname)
+        stderr = res.err.decode('ascii', 'replace')
+        stderr = re.sub('<.*>', '<...>', stderr)
+        expected = textwrap.dedent(f'''
+            {fname}:5: ResourceWarning: unclosed file <...>
+              f = None
+            Object allocated at (most recent call first):
+              File "{fname}", lineno 3
+                f = open(__file__)
+              File "{fname}", lineno 7
+                func()
+        ''').strip()
+        self.assertEqual(stderr, expected)
+
 
 class CatchWarningTests(BaseTest):
 
diff --git a/Lib/warnings.py b/Lib/warnings.py
index f54726a..1566065 100644
--- a/Lib/warnings.py
+++ b/Lib/warnings.py
@@ -2,6 +2,7 @@
 
 import sys
 
+
 __all__ = ["warn", "warn_explicit", "showwarning",
            "formatwarning", "filterwarnings", "simplefilter",
            "resetwarnings", "catch_warnings"]
@@ -66,6 +67,18 @@ def _formatwarnmsg(msg):
     if line:
         line = line.strip()
         s += "  %s\n" % line
+    if msg.source is not None:
+        import tracemalloc
+        tb = tracemalloc.get_object_traceback(msg.source)
+        if tb is not None:
+            s += 'Object allocated at (most recent call first):\n'
+            for frame in tb:
+                s += ('  File "%s", lineno %s\n'
+                      % (frame.filename, frame.lineno))
+                line = linecache.getline(frame.filename, frame.lineno)
+                if line:
+                    line = line.strip()
+                    s += '    %s\n' % line
     return s
 
 def filterwarnings(action, message="", category=Warning, module="", lineno=0,
@@ -267,7 +280,8 @@ def warn(message, category=None, stacklevel=1):
                   globals)
 
 def warn_explicit(message, category, filename, lineno,
-                  module=None, registry=None, module_globals=None):
+                  module=None, registry=None, module_globals=None,
+                  source=None):
     lineno = int(lineno)
     if module is None:
         module = filename or "<unknown>"
@@ -333,17 +347,17 @@ def warn_explicit(message, category, filename, lineno,
               "Unrecognized action (%r) in warnings.filters:\n %s" %
               (action, item))
     # Print message and context
-    msg = WarningMessage(message, category, filename, lineno)
+    msg = WarningMessage(message, category, filename, lineno, source)
     _showwarnmsg(msg)
 
 
 class WarningMessage(object):
 
     _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
-                        "line")
+                        "line", "source")
 
     def __init__(self, message, category, filename, lineno, file=None,
-                    line=None):
+                 line=None, source=None):
         local_values = locals()
         for attr in self._WARNING_DETAILS:
             setattr(self, attr, local_values[attr])
diff --git a/Misc/NEWS b/Misc/NEWS
index cef2b81..832d1ac 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -226,6 +226,11 @@ Core and Builtins
 Library
 -------
 
+- Issue #26567: Add a new function :c:func:`PyErr_ResourceWarning` function to
+  pass the destroyed object. Add a *source* attribute to
+  :class:`warnings.WarningMessage`. Add warnings._showwarnmsg() which uses
+  tracemalloc to get the traceback where source object was allocated.
+
 - Issue #26313: ssl.py _load_windows_store_certs fails if windows cert store
   is empty. Patch by Baji.
 
diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c
index 8bf3922..a02a9c1 100644
--- a/Modules/_io/fileio.c
+++ b/Modules/_io/fileio.c
@@ -92,8 +92,7 @@ fileio_dealloc_warn(fileio *self, PyObject *source)
     if (self->fd >= 0 && self->closefd) {
         PyObject *exc, *val, *tb;
         PyErr_Fetch(&exc, &val, &tb);
-        if (PyErr_WarnFormat(PyExc_ResourceWarning, 1,
-                             "unclosed file %R", source)) {
+        if (PyErr_ResourceWarning(source, 1, "unclosed file %R", source)) {
             /* Spurious errors can appear at shutdown */
             if (PyErr_ExceptionMatches(PyExc_Warning))
                 PyErr_WriteUnraisable((PyObject *) self);
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 65b20be..3f22d14 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -12111,8 +12111,8 @@ ScandirIterator_dealloc(ScandirIterator *iterator)
          */
         ++Py_REFCNT(iterator);
         PyErr_Fetch(&exc, &val, &tb);
-        if (PyErr_WarnFormat(PyExc_ResourceWarning, 1,
-                             "unclosed scandir iterator %R", iterator)) {
+        if (PyErr_ResourceWarning((PyObject *)iterator, 1,
+                                  "unclosed scandir iterator %R", iterator)) {
             /* Spurious errors can appear at shutdown */
             if (PyErr_ExceptionMatches(PyExc_Warning))
                 PyErr_WriteUnraisable((PyObject *) iterator);
diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c
index 77a6b31..657b04b 100644
--- a/Modules/socketmodule.c
+++ b/Modules/socketmodule.c
@@ -4170,8 +4170,7 @@ sock_dealloc(PySocketSockObject *s)
         Py_ssize_t old_refcount = Py_REFCNT(s);
         ++Py_REFCNT(s);
         PyErr_Fetch(&exc, &val, &tb);
-        if (PyErr_WarnFormat(PyExc_ResourceWarning, 1,
-                             "unclosed %R", s))
+        if (PyErr_ResourceWarning(s, 1, "unclosed %R", s))
             /* Spurious errors can appear at shutdown */
             if (PyErr_ExceptionMatches(PyExc_Warning))
                 PyErr_WriteUnraisable((PyObject *) s);
diff --git a/Python/_warnings.c b/Python/_warnings.c
index a8c3703..25299fb 100644
--- a/Python/_warnings.c
+++ b/Python/_warnings.c
@@ -287,8 +287,8 @@ update_registry(PyObject *registry, PyObject *text, PyObject *category,
 }
 
 static void
-show_warning(PyObject *filename, int lineno, PyObject *text, PyObject
-                *category, PyObject *sourceline)
+show_warning(PyObject *filename, int lineno, PyObject *text,
+             PyObject *category, PyObject *sourceline)
 {
     PyObject *f_stderr;
     PyObject *name;
@@ -362,7 +362,7 @@ error:
 static int
 call_show_warning(PyObject *category, PyObject *text, PyObject *message,
                   PyObject *filename, int lineno, PyObject *lineno_obj,
-                  PyObject *sourceline)
+                  PyObject *sourceline, PyObject *source)
 {
     PyObject *show_fn, *msg, *res, *warnmsg_cls = NULL;
 
@@ -388,7 +388,7 @@ call_show_warning(PyObject *category, PyObject *text, PyObject *message,
     }
 
     msg = PyObject_CallFunctionObjArgs(warnmsg_cls, message, category,
-            filename, lineno_obj,
+            filename, lineno_obj, Py_None, Py_None, source,
             NULL);
     Py_DECREF(warnmsg_cls);
     if (msg == NULL)
@@ -412,7 +412,8 @@ error:
 static PyObject *
 warn_explicit(PyObject *category, PyObject *message,
               PyObject *filename, int lineno,
-              PyObject *module, PyObject *registry, PyObject *sourceline)
+              PyObject *module, PyObject *registry, PyObject *sourceline,
+              PyObject *source)
 {
     PyObject *key = NULL, *text = NULL, *result = NULL, *lineno_obj = NULL;
     PyObject *item = NULL;
@@ -521,7 +522,7 @@ warn_explicit(PyObject *category, PyObject *message,
         goto return_none;
     if (rc == 0) {
         if (call_show_warning(category, text, message, filename, lineno,
-                              lineno_obj, sourceline) < 0)
+                              lineno_obj, sourceline, source) < 0)
             goto cleanup;
     }
     else /* if (rc == -1) */
@@ -766,7 +767,8 @@ get_category(PyObject *message, PyObject *category)
 }
 
 static PyObject *
-do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level)
+do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level,
+        PyObject *source)
 {
     PyObject *filename, *module, *registry, *res;
     int lineno;
@@ -775,7 +777,7 @@ do_warn(PyObject *message, PyObject *category, Py_ssize_t stack_level)
         return NULL;
 
     res = warn_explicit(category, message, filename, lineno, module, registry,
-                        NULL);
+                        NULL, source);
     Py_DECREF(filename);
     Py_DECREF(registry);
     Py_DECREF(module);
@@ -796,14 +798,15 @@ warnings_warn(PyObject *self, PyObject *args, PyObject *kwds)
     category = get_category(message, category);
     if (category == NULL)
         return NULL;
-    return do_warn(message, category, stack_level);
+    return do_warn(message, category, stack_level, NULL);
 }
 
 static PyObject *
 warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds)
 {
     static char *kwd_list[] = {"message", "category", "filename", "lineno",
-                                "module", "registry", "module_globals", 0};
+                                "module", "registry", "module_globals",
+                                "source", 0};
     PyObject *message;
     PyObject *category;
     PyObject *filename;
@@ -811,10 +814,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds)
     PyObject *module = NULL;
     PyObject *registry = NULL;
     PyObject *module_globals = NULL;
+    PyObject *sourceobj = NULL;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOO:warn_explicit",
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOUi|OOOO:warn_explicit",
                 kwd_list, &message, &category, &filename, &lineno, &module,
-                &registry, &module_globals))
+                &registry, &module_globals, &sourceobj))
         return NULL;
 
     if (module_globals) {
@@ -870,14 +874,14 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds)
 
         /* Handle the warning. */
         returned = warn_explicit(category, message, filename, lineno, module,
-                                 registry, source_line);
+                                 registry, source_line, sourceobj);
         Py_DECREF(source_list);
         return returned;
     }
 
  standard_call:
     return warn_explicit(category, message, filename, lineno, module,
-                         registry, NULL);
+                         registry, NULL, sourceobj);
 }
 
 static PyObject *
@@ -892,14 +896,14 @@ warnings_filters_mutated(PyObject *self, PyObject *args)
 
 static int
 warn_unicode(PyObject *category, PyObject *message,
-             Py_ssize_t stack_level)
+             Py_ssize_t stack_level, PyObject *source)
 {
     PyObject *res;
 
     if (category == NULL)
         category = PyExc_RuntimeWarning;
 
-    res = do_warn(message, category, stack_level);
+    res = do_warn(message, category, stack_level, source);
     if (res == NULL)
         return -1;
     Py_DECREF(res);
@@ -907,12 +911,28 @@ warn_unicode(PyObject *category, PyObject *message,
     return 0;
 }
 
+static int
+_PyErr_WarnFormatV(PyObject *source,
+                   PyObject *category, Py_ssize_t stack_level,
+                   const char *format, va_list vargs)
+{
+    PyObject *message;
+    int res;
+
+    message = PyUnicode_FromFormatV(format, vargs);
+    if (message == NULL)
+        return -1;
+
+    res = warn_unicode(category, message, stack_level, source);
+    Py_DECREF(message);
+    return res;
+}
+
 int
 PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
                  const char *format, ...)
 {
-    int ret;
-    PyObject *message;
+    int res;
     va_list vargs;
 
 #ifdef HAVE_STDARG_PROTOTYPES
@@ -920,25 +940,38 @@ PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level,
 #else
     va_start(vargs);
 #endif
-    message = PyUnicode_FromFormatV(format, vargs);
-    if (message != NULL) {
-        ret = warn_unicode(category, message, stack_level);
-        Py_DECREF(message);
-    }
-    else
-        ret = -1;
+    res = _PyErr_WarnFormatV(NULL, category, stack_level, format, vargs);
     va_end(vargs);
-    return ret;
+    return res;
 }
 
 int
+PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level,
+                      const char *format, ...)
+{
+    int res;
+    va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    res = _PyErr_WarnFormatV(source, PyExc_ResourceWarning,
+                             stack_level, format, vargs);
+    va_end(vargs);
+    return res;
+}
+
+
+int
 PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level)
 {
     int ret;
     PyObject *message = PyUnicode_FromString(text);
     if (message == NULL)
         return -1;
-    ret = warn_unicode(category, message, stack_level);
+    ret = warn_unicode(category, message, stack_level, NULL);
     Py_DECREF(message);
     return ret;
 }
@@ -964,7 +997,7 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message,
     if (category == NULL)
         category = PyExc_RuntimeWarning;
     res = warn_explicit(category, message, filename, lineno,
-                        module, registry, NULL);
+                        module, registry, NULL, NULL);
     if (res == NULL)
         return -1;
     Py_DECREF(res);
@@ -1028,7 +1061,7 @@ PyErr_WarnExplicitFormat(PyObject *category,
     if (message != NULL) {
         PyObject *res;
         res = warn_explicit(category, message, filename, lineno,
-                            module, registry, NULL);
+                            module, registry, NULL, NULL);
         Py_DECREF(message);
         if (res != NULL) {
             Py_DECREF(res);