From 0611c26a58a937dace420691e797fbea21db619b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2016 22:22:13 +0100 Subject: On memory error, dump the memory block traceback Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a memory block was allocated on memory block. Use the tracemalloc module to get the traceback. --- Doc/c-api/memory.rst | 7 +++++ Doc/whatsnew/3.6.rst | 43 +++++++++++++++++++++++++++++- Misc/NEWS | 4 +++ Modules/_tracemalloc.c | 67 +++++++++++++++++++++++++++++++++++++++-------- Modules/hashtable.c | 6 ++--- Objects/bytearrayobject.c | 1 + Objects/obmalloc.c | 9 +++++++ Parser/pgenmain.c | 8 +++--- 8 files changed, 126 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 25867d9..843ccac 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -349,12 +349,19 @@ Customize Memory Allocators allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex: :c:func:`PyObject_Malloc`) are called + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + These hooks are installed by default if Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. .. versionchanged:: 3.6 This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. .. _pymalloc: diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index b644a5c..443e46a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -129,7 +129,48 @@ the C library for all Python memory allocations using ``PYTHONMALLOC=malloc``. It helps to use external memory debuggers like Valgrind on a Python compiled in release mode. -(Contributed by Victor Stinner in :issue:`26516`.) +On error, the debug hooks on Python memory allocators now use the +:mod:`tracemalloc` module to get the traceback where a memory block was +allocated. + +Example of fatal error on buffer overflow using +``python3.6 -X tracemalloc=5`` (store 5 frames in traces):: + + Debug memory block at address p=0x7fbcd41666f8: API 'o' + 4 bytes originally requested + The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected. + The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb): + at tail+0: 0x02 *** OUCH + at tail+1: 0xfb + at tail+2: 0xfb + at tail+3: 0xfb + at tail+4: 0xfb + at tail+5: 0xfb + at tail+6: 0xfb + at tail+7: 0xfb + The block was made by call #1233329 to debug malloc/realloc. + Data at p: 1a 2b 30 00 + + Memory block allocated at (most recent call first): + File "test/test_bytes.py", line 323 + File "unittest/case.py", line 600 + File "unittest/case.py", line 648 + File "unittest/suite.py", line 122 + File "unittest/suite.py", line 84 + + Fatal Python error: bad trailing pad byte + + Current thread 0x00007fbcdbd32700 (most recent call first): + File "test/test_bytes.py", line 323 in test_hex + File "unittest/case.py", line 600 in run + File "unittest/case.py", line 648 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + File "unittest/suite.py", line 122 in run + File "unittest/suite.py", line 84 in __call__ + ... + +(Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.) Other Language Changes diff --git a/Misc/NEWS b/Misc/NEWS index 45769ba..cbb1a51 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26564: On error, the debug hooks on Python memory allocators now use + the :mod:`tracemalloc` module to get the traceback where a memory block was + allocated. + - Issue #26558: The debug hooks on Python memory allocator :c:func:`PyObject_Malloc` now detect when functions are called without holding the GIL. diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 1940af4..5752904 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1161,6 +1161,25 @@ finally: return get_traces.list; } +static traceback_t* +tracemalloc_get_traceback(const void *ptr) +{ + trace_t trace; + int found; + + if (!tracemalloc_config.tracing) + return NULL; + + TABLES_LOCK(); + found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (!found) + return NULL; + + return trace.traceback; +} + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1175,11 +1194,7 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) { PyTypeObject *type; void *ptr; - trace_t trace; - int found; - - if (!tracemalloc_config.tracing) - Py_RETURN_NONE; + traceback_t *traceback; type = Py_TYPE(obj); if (PyType_IS_GC(type)) @@ -1187,16 +1202,46 @@ py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) else ptr = (void *)obj; - TABLES_LOCK(); - found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace); - TABLES_UNLOCK(); - - if (!found) + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) Py_RETURN_NONE; - return traceback_to_pyobject(trace.traceback, NULL); + return traceback_to_pyobject(traceback, NULL); +} + +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + +static void +_PyMem_DumpFrame(int fd, frame_t * frame) +{ + PUTS(fd, " File \""); + _Py_DumpASCII(fd, frame->filename); + PUTS(fd, "\", line "); + _Py_DumpDecimal(fd, frame->lineno); + PUTS(fd, "\n"); } +/* Dump the traceback where a memory block was allocated into file descriptor + fd. The function may block on TABLES_LOCK() but it is unlikely. */ +void +_PyMem_DumpTraceback(int fd, const void *ptr) +{ + traceback_t *traceback; + int i; + + traceback = tracemalloc_get_traceback(ptr); + if (traceback == NULL) + return; + + PUTS(fd, "Memory block allocated at (most recent call first):\n"); + for (i=0; i < traceback->nframe; i++) { + _PyMem_DumpFrame(fd, &traceback->frames[i]); + } + PUTS(fd, "\n"); +} + +#undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n" diff --git a/Modules/hashtable.c b/Modules/hashtable.c index 133f313..7de154b 100644 --- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -486,9 +486,9 @@ _Py_hashtable_copy(_Py_hashtable_t *src) void *data, *new_data; dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, - src->hash_func, src->compare_func, - src->copy_data_func, src->free_data_func, - src->get_data_size_func, &src->alloc); + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func, &src->alloc); if (dst == NULL) return NULL; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 9e8ba39..67ae7d9 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,6 +2820,7 @@ bytearray_hex(PyBytesObject *self) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); + PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f526b1f..8812f59 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,10 @@ #include "Python.h" + +/* Defined in tracemalloc.c */ +extern void _PyMem_DumpTraceback(int fd, const void *ptr); + + /* Python's malloc wrappers (see pymem.h) */ /* @@ -2202,6 +2207,10 @@ _PyObject_DebugDumpAddress(const void *p) } fputc('\n', stderr); } + fputc('\n', stderr); + + fflush(stderr); + _PyMem_DumpTraceback(fileno(stderr), p); } diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c index 671a0cb..d5a13fe 100644 --- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -38,11 +38,11 @@ Py_Exit(int sts) } #ifdef WITH_THREAD -/* Needed by obmalloc.c */ +/* Functions needed by obmalloc.c */ int PyGILState_Check(void) -{ - return 1; -} +{ return 1; } +void _PyMem_DumpTraceback(int fd, const void *ptr) +{} #endif int -- cgit v0.12