From 4c409beb4c360a73d054f37807d3daad58d1b567 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 11 Apr 2019 13:01:15 +0200 Subject: bpo-36389: Change PyMem_SetupDebugHooks() constants (GH-12782) Modify CLEANBYTE, DEADDYTE and FORBIDDENBYTE constants: use 0xCD, 0xDD and 0xFD, rather than 0xCB, 0xBB and 0xFB, to use the same byte patterns than Windows CRT debug malloc() and free(). --- Doc/c-api/memory.rst | 11 +++++++++-- Include/internal/pycore_pymem.h | 19 +++++++++---------- Lib/test/test_capi.py | 8 ++++---- .../C API/2019-04-11-12-20-35.bpo-36389.P9QFoP.rst | 5 +++++ Objects/object.c | 9 ++++----- Objects/obmalloc.c | 9 ++++++--- 6 files changed, 37 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-04-11-12-20-35.bpo-36389.P9QFoP.rst diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index b79b7e4..65a2076 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -440,8 +440,9 @@ Customize Memory Allocators Setup hooks to detect bugs in the Python memory allocator functions. - Newly allocated memory is filled with the byte ``0xCB``, freed memory is - filled with the byte ``0xDB``. + Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), + freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks + are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). Runtime checks: @@ -471,6 +472,12 @@ Customize Memory Allocators if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are called. + .. versionchanged:: 3.8.0 + Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and + ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` + and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` + and ``free()``. + .. _pymalloc: diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 78d457d..8da1bd9 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -160,21 +160,20 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( pointer value is checked. The heuristic relies on the debug hooks on Python memory allocators which - fills newly allocated memory with CLEANBYTE (0xCB) and newly freed memory - with DEADBYTE (0xDB). Detect also "untouchable bytes" marked - with FORBIDDENBYTE (0xFB). */ + fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory + with DEADBYTE (0xDD). Detect also "untouchable bytes" marked + with FORBIDDENBYTE (0xFD). */ static inline int _PyMem_IsPtrFreed(void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == (uintptr_t)0xCBCBCBCBCBCBCBCB - || value == (uintptr_t)0xDBDBDBDBDBDBDBDB - || value == (uintptr_t)0xFBFBFBFBFBFBFBFB - ); + return (value == (uintptr_t)0xCDCDCDCDCDCDCDCD + || value == (uintptr_t)0xDDDDDDDDDDDDDDDD + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); #elif SIZEOF_VOID_P == 4 - return (value == (uintptr_t)0xCBCBCBCB - || value == (uintptr_t)0xDBDBDBDB - || value == (uintptr_t)0xFBFBFBFB); + return (value == (uintptr_t)0xCDCDCDCD + || value == (uintptr_t)0xDDDDDDDD + || value == (uintptr_t)0xFDFDFDFD); #else # error "unknown pointer size" #endif diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 3cd39d4..33c98ac 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -480,11 +480,11 @@ class PyMemDebugTests(unittest.TestCase): r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" r" at tail\+0: 0x78 \*\*\* OUCH\n" - r" at tail\+1: 0xfb\n" - r" at tail\+2: 0xfb\n" + r" at tail\+1: 0xfd\n" + r" at tail\+2: 0xfd\n" r" .*\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: cb cb cb .*\n" + r" Data at p: cd cd cd .*\n" r"\n" r"Enable tracemalloc to get the memory block allocation traceback\n" r"\n" @@ -500,7 +500,7 @@ class PyMemDebugTests(unittest.TestCase): r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" - r" Data at p: cb cb cb .*\n" + r" Data at p: cd cd cd .*\n" r"\n" r"Enable tracemalloc to get the memory block allocation traceback\n" r"\n" diff --git a/Misc/NEWS.d/next/C API/2019-04-11-12-20-35.bpo-36389.P9QFoP.rst b/Misc/NEWS.d/next/C API/2019-04-11-12-20-35.bpo-36389.P9QFoP.rst new file mode 100644 index 0000000..f2b507a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-04-11-12-20-35.bpo-36389.P9QFoP.rst @@ -0,0 +1,5 @@ +Change the value of ``CLEANBYTE``, ``DEADDYTE`` and ``FORBIDDENBYTE`` internal +constants used by debug hooks on Python memory allocators +(:c:func:`PyMem_SetupDebugHooks` function). Byte patterns ``0xCB``, ``0xDB`` +and ``0xFB`` have been replaced with ``0xCD``, ``0xDD`` and ``0xFD`` to use the +same values than Windows CRT debug ``malloc()`` and ``free()``. diff --git a/Objects/object.c b/Objects/object.c index c9aa479..3fad73c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -415,13 +415,12 @@ _Py_BreakPoint(void) } -/* Heuristic checking if the object memory has been deallocated. - Rely on the debug hooks on Python memory allocators which fills the memory - with DEADBYTE (0xDB) when memory is deallocated. +/* Heuristic checking if the object memory is uninitialized or deallocated. + Rely on the debug hooks on Python memory allocators: + see _PyMem_IsPtrFreed(). The function can be used to prevent segmentation fault on dereferencing - pointers like 0xdbdbdbdbdbdbdbdb. Such pointer is very unlikely to be mapped - in memory. */ + pointers like 0xDDDDDDDDDDDDDDDD. */ int _PyObject_IsFreed(PyObject *op) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index e919fad..be43c7a 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1915,13 +1915,16 @@ _Py_GetAllocatedBlocks(void) /* Special bytes broadcast into debug memory blocks at appropriate times. * Strings of these are unlikely to be valid addresses, floats, ints or * 7-bit ASCII. If modified, _PyMem_IsPtrFreed() should be updated as well. + * + * Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and + * 0xFD to use the same values than Windows CRT debug malloc() and free(). */ #undef CLEANBYTE #undef DEADBYTE #undef FORBIDDENBYTE -#define CLEANBYTE 0xCB /* clean (newly allocated) memory */ -#define DEADBYTE 0xDB /* dead (newly freed) memory */ -#define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ +#define CLEANBYTE 0xCD /* clean (newly allocated) memory */ +#define DEADBYTE 0xDD /* dead (newly freed) memory */ +#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */ static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ -- cgit v0.12