diff options
Diffstat (limited to 'Objects/obmalloc.c')
| -rw-r--r-- | Objects/obmalloc.c | 2532 |
1 files changed, 890 insertions, 1642 deletions
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 722e91e..2067cf5 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,735 +1,47 @@ #include "Python.h" -#include "pycore_pymem.h" - -#include <stdbool.h> - - -/* Defined in tracemalloc.c */ -extern void _PyMem_DumpTraceback(int fd, const void *ptr); - - -/* Python's malloc wrappers (see pymem.h) */ - -#undef uint -#define uint unsigned int /* assuming >= 16 bits */ - -/* Forward declaration */ -static void* _PyMem_DebugRawMalloc(void *ctx, size_t size); -static void* _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize); -static void* _PyMem_DebugRawRealloc(void *ctx, void *ptr, size_t size); -static void _PyMem_DebugRawFree(void *ctx, void *ptr); - -static void* _PyMem_DebugMalloc(void *ctx, size_t size); -static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); -static void* _PyMem_DebugRealloc(void *ctx, void *ptr, size_t size); -static void _PyMem_DebugFree(void *ctx, void *p); - -static void _PyObject_DebugDumpAddress(const void *p); -static void _PyMem_DebugCheckAddress(char api_id, const void *p); - -static void _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain); #if defined(__has_feature) /* Clang */ -# if __has_feature(address_sanitizer) /* is ASAN enabled? */ -# define _Py_NO_ADDRESS_SAFETY_ANALYSIS \ - __attribute__((no_address_safety_analysis)) -# endif -# if __has_feature(thread_sanitizer) /* is TSAN enabled? */ -# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) -# endif -# if __has_feature(memory_sanitizer) /* is MSAN enabled? */ -# define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) -# endif -#elif defined(__GNUC__) -# if defined(__SANITIZE_ADDRESS__) /* GCC 4.8+, is ASAN enabled? */ -# define _Py_NO_ADDRESS_SAFETY_ANALYSIS \ - __attribute__((no_address_safety_analysis)) -# endif - // TSAN is supported since GCC 5.1, but __SANITIZE_THREAD__ macro - // is provided only since GCC 7. -# if __GNUC__ > 5 || (__GNUC__ == 5 && __GNUC_MINOR__ >= 1) -# define _Py_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) -# endif -#endif - -#ifndef _Py_NO_ADDRESS_SAFETY_ANALYSIS -# define _Py_NO_ADDRESS_SAFETY_ANALYSIS -#endif -#ifndef _Py_NO_SANITIZE_THREAD -# define _Py_NO_SANITIZE_THREAD -#endif -#ifndef _Py_NO_SANITIZE_MEMORY -# define _Py_NO_SANITIZE_MEMORY -#endif - -#ifdef WITH_PYMALLOC - -#ifdef MS_WINDOWS -# include <windows.h> -#elif defined(HAVE_MMAP) -# include <sys/mman.h> -# ifdef MAP_ANONYMOUS -# define ARENAS_USE_MMAP -# endif -#endif - -/* Forward declaration */ -static void* _PyObject_Malloc(void *ctx, size_t size); -static void* _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize); -static void _PyObject_Free(void *ctx, void *p); -static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); -#endif - - -/* bpo-35053: Declare tracemalloc configuration here rather than - Modules/_tracemalloc.c because _tracemalloc can be compiled as dynamic - library, whereas _Py_NewReference() requires it. */ -struct _PyTraceMalloc_Config _Py_tracemalloc_config = _PyTraceMalloc_Config_INIT; - - -static void * -_PyMem_RawMalloc(void *ctx, size_t size) -{ - /* PyMem_RawMalloc(0) means malloc(1). Some systems would return NULL - for malloc(0), which would be treated as an error. Some platforms would - return a pointer with no memory behind it, which would break pymalloc. - To solve these problems, allocate an extra byte. */ - if (size == 0) - size = 1; - return malloc(size); -} - -static void * -_PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize) -{ - /* PyMem_RawCalloc(0, 0) means calloc(1, 1). Some systems would return NULL - for calloc(0, 0), which would be treated as an error. Some platforms - would return a pointer with no memory behind it, which would break - pymalloc. To solve these problems, allocate an extra byte. */ - if (nelem == 0 || elsize == 0) { - nelem = 1; - elsize = 1; - } - return calloc(nelem, elsize); -} - -static void * -_PyMem_RawRealloc(void *ctx, void *ptr, size_t size) -{ - if (size == 0) - size = 1; - return realloc(ptr, size); -} - -static void -_PyMem_RawFree(void *ctx, void *ptr) -{ - free(ptr); -} - - -#ifdef MS_WINDOWS -static void * -_PyObject_ArenaVirtualAlloc(void *ctx, size_t size) -{ - return VirtualAlloc(NULL, size, - MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); -} - -static void -_PyObject_ArenaVirtualFree(void *ctx, void *ptr, size_t size) -{ - VirtualFree(ptr, 0, MEM_RELEASE); -} - -#elif defined(ARENAS_USE_MMAP) -static void * -_PyObject_ArenaMmap(void *ctx, size_t size) -{ - void *ptr; - ptr = mmap(NULL, size, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (ptr == MAP_FAILED) - return NULL; - assert(ptr != NULL); - return ptr; -} - -static void -_PyObject_ArenaMunmap(void *ctx, void *ptr, size_t size) -{ - munmap(ptr, size); -} - + #if __has_feature(address_sanitizer) /* is ASAN enabled? */ + #define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS \ + __attribute__((no_address_safety_analysis)) \ + __attribute__ ((noinline)) + #else + #define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS + #endif #else -static void * -_PyObject_ArenaMalloc(void *ctx, size_t size) -{ - return malloc(size); -} - -static void -_PyObject_ArenaFree(void *ctx, void *ptr, size_t size) -{ - free(ptr); -} + #if defined(__SANITIZE_ADDRESS__) /* GCC 4.8.x, is ASAN enabled? */ + #define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS \ + __attribute__((no_address_safety_analysis)) \ + __attribute__ ((noinline)) + #else + #define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS + #endif #endif -#define MALLOC_ALLOC {NULL, _PyMem_RawMalloc, _PyMem_RawCalloc, _PyMem_RawRealloc, _PyMem_RawFree} #ifdef WITH_PYMALLOC -# define PYMALLOC_ALLOC {NULL, _PyObject_Malloc, _PyObject_Calloc, _PyObject_Realloc, _PyObject_Free} -#endif - -#define PYRAW_ALLOC MALLOC_ALLOC -#ifdef WITH_PYMALLOC -# define PYOBJ_ALLOC PYMALLOC_ALLOC -#else -# define PYOBJ_ALLOC MALLOC_ALLOC -#endif -#define PYMEM_ALLOC PYOBJ_ALLOC - -typedef struct { - /* We tag each block with an API ID in order to tag API violations */ - char api_id; - PyMemAllocatorEx alloc; -} debug_alloc_api_t; -static struct { - debug_alloc_api_t raw; - debug_alloc_api_t mem; - debug_alloc_api_t obj; -} _PyMem_Debug = { - {'r', PYRAW_ALLOC}, - {'m', PYMEM_ALLOC}, - {'o', PYOBJ_ALLOC} - }; - -#define PYDBGRAW_ALLOC \ - {&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} -#define PYDBGMEM_ALLOC \ - {&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} -#define PYDBGOBJ_ALLOC \ - {&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} - -#ifdef Py_DEBUG -static PyMemAllocatorEx _PyMem_Raw = PYDBGRAW_ALLOC; -static PyMemAllocatorEx _PyMem = PYDBGMEM_ALLOC; -static PyMemAllocatorEx _PyObject = PYDBGOBJ_ALLOC; -#else -static PyMemAllocatorEx _PyMem_Raw = PYRAW_ALLOC; -static PyMemAllocatorEx _PyMem = PYMEM_ALLOC; -static PyMemAllocatorEx _PyObject = PYOBJ_ALLOC; -#endif - -static int -pymem_set_default_allocator(PyMemAllocatorDomain domain, int debug, - PyMemAllocatorEx *old_alloc) -{ - if (old_alloc != NULL) { - PyMem_GetAllocator(domain, old_alloc); - } - - - PyMemAllocatorEx new_alloc; - switch(domain) - { - case PYMEM_DOMAIN_RAW: - new_alloc = (PyMemAllocatorEx)PYRAW_ALLOC; - break; - case PYMEM_DOMAIN_MEM: - new_alloc = (PyMemAllocatorEx)PYMEM_ALLOC; - break; - case PYMEM_DOMAIN_OBJ: - new_alloc = (PyMemAllocatorEx)PYOBJ_ALLOC; - break; - default: - /* unknown domain */ - return -1; - } - PyMem_SetAllocator(domain, &new_alloc); - if (debug) { - _PyMem_SetupDebugHooksDomain(domain); - } - return 0; -} - - -int -_PyMem_SetDefaultAllocator(PyMemAllocatorDomain domain, - PyMemAllocatorEx *old_alloc) -{ -#ifdef Py_DEBUG - const int debug = 1; -#else - const int debug = 0; +#ifdef HAVE_MMAP + #include <sys/mman.h> + #ifdef MAP_ANONYMOUS + #define ARENAS_USE_MMAP + #endif #endif - return pymem_set_default_allocator(domain, debug, old_alloc); -} - - -int -_PyMem_GetAllocatorName(const char *name, PyMemAllocatorName *allocator) -{ - if (name == NULL || *name == '\0') { - /* PYTHONMALLOC is empty or is not set or ignored (-E/-I command line - nameions): use default memory allocators */ - *allocator = PYMEM_ALLOCATOR_DEFAULT; - } - else if (strcmp(name, "default") == 0) { - *allocator = PYMEM_ALLOCATOR_DEFAULT; - } - else if (strcmp(name, "debug") == 0) { - *allocator = PYMEM_ALLOCATOR_DEBUG; - } -#ifdef WITH_PYMALLOC - else if (strcmp(name, "pymalloc") == 0) { - *allocator = PYMEM_ALLOCATOR_PYMALLOC; - } - else if (strcmp(name, "pymalloc_debug") == 0) { - *allocator = PYMEM_ALLOCATOR_PYMALLOC_DEBUG; - } -#endif - else if (strcmp(name, "malloc") == 0) { - *allocator = PYMEM_ALLOCATOR_MALLOC; - } - else if (strcmp(name, "malloc_debug") == 0) { - *allocator = PYMEM_ALLOCATOR_MALLOC_DEBUG; - } - else { - /* unknown allocator */ - return -1; - } - return 0; -} - - -int -_PyMem_SetupAllocators(PyMemAllocatorName allocator) -{ - switch (allocator) { - case PYMEM_ALLOCATOR_NOT_SET: - /* do nothing */ - break; - - case PYMEM_ALLOCATOR_DEFAULT: - (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, NULL); - (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_MEM, NULL); - (void)_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_OBJ, NULL); - break; - - case PYMEM_ALLOCATOR_DEBUG: - (void)pymem_set_default_allocator(PYMEM_DOMAIN_RAW, 1, NULL); - (void)pymem_set_default_allocator(PYMEM_DOMAIN_MEM, 1, NULL); - (void)pymem_set_default_allocator(PYMEM_DOMAIN_OBJ, 1, NULL); - break; - -#ifdef WITH_PYMALLOC - case PYMEM_ALLOCATOR_PYMALLOC: - case PYMEM_ALLOCATOR_PYMALLOC_DEBUG: - { - PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); - - PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC; - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &pymalloc); - PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &pymalloc); - - if (allocator == PYMEM_ALLOCATOR_PYMALLOC_DEBUG) { - PyMem_SetupDebugHooks(); - } - break; - } -#endif - - case PYMEM_ALLOCATOR_MALLOC: - case PYMEM_ALLOCATOR_MALLOC_DEBUG: - { - PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &malloc_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &malloc_alloc); - PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &malloc_alloc); - - if (allocator == PYMEM_ALLOCATOR_MALLOC_DEBUG) { - PyMem_SetupDebugHooks(); - } - break; - } - - default: - /* unknown allocator */ - return -1; - } - return 0; -} - - -static int -pymemallocator_eq(PyMemAllocatorEx *a, PyMemAllocatorEx *b) -{ - return (memcmp(a, b, sizeof(PyMemAllocatorEx)) == 0); -} - - -const char* -_PyMem_GetCurrentAllocatorName(void) -{ - PyMemAllocatorEx malloc_alloc = MALLOC_ALLOC; -#ifdef WITH_PYMALLOC - PyMemAllocatorEx pymalloc = PYMALLOC_ALLOC; -#endif - - if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && - pymemallocator_eq(&_PyMem, &malloc_alloc) && - pymemallocator_eq(&_PyObject, &malloc_alloc)) - { - return "malloc"; - } -#ifdef WITH_PYMALLOC - if (pymemallocator_eq(&_PyMem_Raw, &malloc_alloc) && - pymemallocator_eq(&_PyMem, &pymalloc) && - pymemallocator_eq(&_PyObject, &pymalloc)) - { - return "pymalloc"; - } -#endif - - PyMemAllocatorEx dbg_raw = PYDBGRAW_ALLOC; - PyMemAllocatorEx dbg_mem = PYDBGMEM_ALLOC; - PyMemAllocatorEx dbg_obj = PYDBGOBJ_ALLOC; - - if (pymemallocator_eq(&_PyMem_Raw, &dbg_raw) && - pymemallocator_eq(&_PyMem, &dbg_mem) && - pymemallocator_eq(&_PyObject, &dbg_obj)) - { - /* Debug hooks installed */ - if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) && - pymemallocator_eq(&_PyMem_Debug.mem.alloc, &malloc_alloc) && - pymemallocator_eq(&_PyMem_Debug.obj.alloc, &malloc_alloc)) - { - return "malloc_debug"; - } -#ifdef WITH_PYMALLOC - if (pymemallocator_eq(&_PyMem_Debug.raw.alloc, &malloc_alloc) && - pymemallocator_eq(&_PyMem_Debug.mem.alloc, &pymalloc) && - pymemallocator_eq(&_PyMem_Debug.obj.alloc, &pymalloc)) - { - return "pymalloc_debug"; - } -#endif - } - return NULL; -} - - -#undef MALLOC_ALLOC -#undef PYMALLOC_ALLOC -#undef PYRAW_ALLOC -#undef PYMEM_ALLOC -#undef PYOBJ_ALLOC -#undef PYDBGRAW_ALLOC -#undef PYDBGMEM_ALLOC -#undef PYDBGOBJ_ALLOC - - -static PyObjectArenaAllocator _PyObject_Arena = {NULL, -#ifdef MS_WINDOWS - _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree -#elif defined(ARENAS_USE_MMAP) - _PyObject_ArenaMmap, _PyObject_ArenaMunmap -#else - _PyObject_ArenaMalloc, _PyObject_ArenaFree -#endif - }; - -#ifdef WITH_PYMALLOC -static int -_PyMem_DebugEnabled(void) -{ - return (_PyObject.malloc == _PyMem_DebugMalloc); -} - -static int -_PyMem_PymallocEnabled(void) -{ - if (_PyMem_DebugEnabled()) { - return (_PyMem_Debug.obj.alloc.malloc == _PyObject_Malloc); - } - else { - return (_PyObject.malloc == _PyObject_Malloc); - } -} -#endif - - -static void -_PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) -{ - PyMemAllocatorEx alloc; - - if (domain == PYMEM_DOMAIN_RAW) { - if (_PyMem_Raw.malloc == _PyMem_DebugRawMalloc) { - return; - } - - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); - alloc.ctx = &_PyMem_Debug.raw; - alloc.malloc = _PyMem_DebugRawMalloc; - alloc.calloc = _PyMem_DebugRawCalloc; - alloc.realloc = _PyMem_DebugRawRealloc; - alloc.free = _PyMem_DebugRawFree; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); - } - else if (domain == PYMEM_DOMAIN_MEM) { - if (_PyMem.malloc == _PyMem_DebugMalloc) { - return; - } - - PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); - alloc.ctx = &_PyMem_Debug.mem; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; - PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); - } - else if (domain == PYMEM_DOMAIN_OBJ) { - if (_PyObject.malloc == _PyMem_DebugMalloc) { - return; - } - - PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); - alloc.ctx = &_PyMem_Debug.obj; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; - PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); - } -} - - -void -PyMem_SetupDebugHooks(void) -{ - _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_RAW); - _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_MEM); - _PyMem_SetupDebugHooksDomain(PYMEM_DOMAIN_OBJ); -} - -void -PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) -{ - switch(domain) - { - case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break; - case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break; - case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break; - default: - /* unknown domain: set all attributes to NULL */ - allocator->ctx = NULL; - allocator->malloc = NULL; - allocator->calloc = NULL; - allocator->realloc = NULL; - allocator->free = NULL; - } -} - -void -PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) -{ - switch(domain) - { - case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break; - case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break; - case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break; - /* ignore unknown domain */ - } -} - -void -PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) -{ - *allocator = _PyObject_Arena; -} - -void -PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) -{ - _PyObject_Arena = *allocator; -} - -void * -PyMem_RawMalloc(size_t size) -{ - /* - * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. - * Most python internals blindly use a signed Py_ssize_t to track - * things without checking for overflows or negatives. - * As size_t is unsigned, checking for size < 0 is not required. - */ - if (size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size); -} - -void * -PyMem_RawCalloc(size_t nelem, size_t elsize) -{ - /* see PyMem_RawMalloc() */ - if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) - return NULL; - return _PyMem_Raw.calloc(_PyMem_Raw.ctx, nelem, elsize); -} - -void* -PyMem_RawRealloc(void *ptr, size_t new_size) -{ - /* see PyMem_RawMalloc() */ - if (new_size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); -} - -void PyMem_RawFree(void *ptr) -{ - _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); -} - - -void * -PyMem_Malloc(size_t size) -{ - /* see PyMem_RawMalloc() */ - if (size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyMem.malloc(_PyMem.ctx, size); -} - -void * -PyMem_Calloc(size_t nelem, size_t elsize) -{ - /* see PyMem_RawMalloc() */ - if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) - return NULL; - return _PyMem.calloc(_PyMem.ctx, nelem, elsize); -} - -void * -PyMem_Realloc(void *ptr, size_t new_size) -{ - /* see PyMem_RawMalloc() */ - if (new_size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyMem.realloc(_PyMem.ctx, ptr, new_size); -} - -void -PyMem_Free(void *ptr) -{ - _PyMem.free(_PyMem.ctx, ptr); -} - - -wchar_t* -_PyMem_RawWcsdup(const wchar_t *str) -{ - assert(str != NULL); - - size_t len = wcslen(str); - if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { - return NULL; - } - - size_t size = (len + 1) * sizeof(wchar_t); - wchar_t *str2 = PyMem_RawMalloc(size); - if (str2 == NULL) { - return NULL; - } - - memcpy(str2, str, size); - return str2; -} - -char * -_PyMem_RawStrdup(const char *str) -{ - assert(str != NULL); - size_t size = strlen(str) + 1; - char *copy = PyMem_RawMalloc(size); - if (copy == NULL) { - return NULL; - } - memcpy(copy, str, size); - return copy; -} - -char * -_PyMem_Strdup(const char *str) -{ - assert(str != NULL); - size_t size = strlen(str) + 1; - char *copy = PyMem_Malloc(size); - if (copy == NULL) { - return NULL; - } - memcpy(copy, str, size); - return copy; -} - -void * -PyObject_Malloc(size_t size) -{ - /* see PyMem_RawMalloc() */ - if (size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyObject.malloc(_PyObject.ctx, size); -} - -void * -PyObject_Calloc(size_t nelem, size_t elsize) -{ - /* see PyMem_RawMalloc() */ - if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) - return NULL; - return _PyObject.calloc(_PyObject.ctx, nelem, elsize); -} - -void * -PyObject_Realloc(void *ptr, size_t new_size) -{ - /* see PyMem_RawMalloc() */ - if (new_size > (size_t)PY_SSIZE_T_MAX) - return NULL; - return _PyObject.realloc(_PyObject.ctx, ptr, new_size); -} - -void -PyObject_Free(void *ptr) -{ - _PyObject.free(_PyObject.ctx, ptr); -} +#ifdef WITH_VALGRIND +#include <valgrind/valgrind.h> /* If we're using GCC, use __builtin_expect() to reduce overhead of the valgrind checks */ #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__) # define UNLIKELY(value) __builtin_expect((value), 0) -# define LIKELY(value) __builtin_expect((value), 1) #else # define UNLIKELY(value) (value) -# define LIKELY(value) (value) #endif -#ifdef WITH_PYMALLOC - -#ifdef WITH_VALGRIND -#include <valgrind/valgrind.h> - /* -1 indicates that we haven't checked that we're running on valgrind yet. */ static int running_on_valgrind = -1; #endif - /* An object allocator for Python. Here is an introduction to the layers of the Python memory architecture, @@ -850,6 +162,7 @@ static int running_on_valgrind = -1; #define ALIGNMENT 8 /* must be 2^N */ #define ALIGNMENT_SHIFT 3 #endif +#define ALIGNMENT_MASK (ALIGNMENT - 1) /* Return the number of bytes in size class I, as a uint. */ #define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) @@ -859,13 +172,13 @@ static int running_on_valgrind = -1; * small enough in order to use preallocated memory pools. You can tune * this value according to your application behaviour and memory needs. * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * * The following invariants must hold: * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT * + * Note: a size threshold of 512 guarantees that newly created dictionaries + * will be allocated from preallocated memory pools on 64-bit. + * * Although not required, for better performance and space efficiency, * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. */ @@ -907,7 +220,7 @@ static int running_on_valgrind = -1; * Arenas are allocated with mmap() on systems supporting anonymous memory * mappings to reduce heap fragmentation. */ -#define ARENA_SIZE (256 << 10) /* 256KB */ +#define ARENA_SIZE (256 << 10) /* 256KiB */ #ifdef WITH_MEMORY_LIMITS #define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) @@ -920,19 +233,54 @@ static int running_on_valgrind = -1; #define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ #define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK -#define MAX_POOLS_IN_ARENA (ARENA_SIZE / POOL_SIZE) -#if MAX_POOLS_IN_ARENA * POOL_SIZE != ARENA_SIZE -# error "arena size not an exact multiple of pool size" -#endif - /* * -- End of tunable settings section -- */ /*==========================================================================*/ +/* + * Locking + * + * To reduce lock contention, it would probably be better to refine the + * crude function locking with per size class locking. I'm not positive + * however, whether it's worth switching to such locking policy because + * of the performance penalty it might introduce. + * + * The following macros describe the simplest (should also be the fastest) + * lock object on a particular platform and the init/fini/lock/unlock + * operations on it. The locks defined here are not expected to be recursive + * because it is assumed that they will always be called in the order: + * INIT, [LOCK, UNLOCK]*, FINI. + */ + +/* + * Python's threads are serialized, so object malloc locking is disabled. + */ +#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ +#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ +#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ +#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ +#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ + +/* + * Basic types + * I don't care if these are defined in <sys/types.h> or elsewhere. Axiom. + */ +#undef uchar +#define uchar unsigned char /* assuming == 8 bits */ + +#undef uint +#define uint unsigned int /* assuming >= 16 bits */ + +#undef ulong +#define ulong unsigned long /* assuming >= 32 bits */ + +#undef uptr +#define uptr Py_uintptr_t + /* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t block; +typedef uchar block; /* Pool for small blocks. */ struct pool_header { @@ -956,7 +304,7 @@ struct arena_object { * here to mark an arena_object that doesn't correspond to an * allocated arena. */ - uintptr_t address; + uptr address; /* Pool-aligned pointer to the next pool to be carved off. */ block* pool_address; @@ -990,12 +338,14 @@ struct arena_object { struct arena_object* prevarena; }; -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) +#undef ROUNDUP +#define ROUNDUP(x) (((x) + ALIGNMENT_MASK) & ~ALIGNMENT_MASK) +#define POOL_OVERHEAD ROUNDUP(sizeof(struct pool_header)) #define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ /* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) +#define POOL_ADDR(P) ((poolp)((uptr)(P) & ~(uptr)POOL_SIZE_MASK)) /* Return total number of blocks in pool of size index I, as a uint. */ #define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) @@ -1003,6 +353,15 @@ struct arena_object { /*==========================================================================*/ /* + * This malloc lock + */ +SIMPLELOCK_DECL(_malloc_lock) +#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) +#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) +#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) +#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) + +/* * Pool table -- headed, circular, doubly-linked lists of partially used pools. This is involved. For an index i, usedpools[i+i] is the header for a list of @@ -1098,7 +457,7 @@ on that C doesn't insert any padding anywhere in a pool_header at or before the prevpool member. **************************************************************************** */ -#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *))) +#define PTA(x) ((poolp )((uchar *)&(usedpools[2*(x)]) - 2*sizeof(block *))) #define PT(x) PTA(x), PTA(x) static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { @@ -1162,18 +521,6 @@ usable_arenas Note that an arena_object associated with an arena all of whose pools are currently in use isn't on either list. - -Changed in Python 3.8: keeping usable_arenas sorted by number of free pools -used to be done by one-at-a-time linear search when an arena's number of -free pools changed. That could, overall, consume time quadratic in the -number of arenas. That didn't really matter when there were only a few -hundred arenas (typical!), but could be a timing disaster when there were -hundreds of thousands. See bpo-37029. - -Now we have a vector of "search fingers" to eliminate the need to search: -nfp2lasta[nfp] returns the last ("rightmost") arena in usable_arenas -with nfp free pools. This is NULL if and only if there is no arena with -nfp free pools in usable_arenas. */ /* Array of objects used to track chunks of memory (arenas). */ @@ -1191,9 +538,6 @@ static struct arena_object* unused_arena_objects = NULL; */ static struct arena_object* usable_arenas = NULL; -/* nfp2lasta[nfp] is the last arena in usable_arenas with nfp free pools */ -static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; - /* How many arena_objects do we initially allocate? * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the * `arenas` vector. @@ -1203,36 +547,12 @@ static struct arena_object* nfp2lasta[MAX_POOLS_IN_ARENA + 1] = { NULL }; /* Number of arenas allocated that haven't been free()'d. */ static size_t narenas_currently_allocated = 0; +#ifdef PYMALLOC_DEBUG /* Total number of times malloc() called to allocate an arena. */ static size_t ntimes_arena_allocated = 0; /* High water mark (max value ever seen) for narenas_currently_allocated. */ static size_t narenas_highwater = 0; - -static Py_ssize_t raw_allocated_blocks; - -Py_ssize_t -_Py_GetAllocatedBlocks(void) -{ - Py_ssize_t n = raw_allocated_blocks; - /* add up allocated blocks for used pools */ - for (uint i = 0; i < maxarenas; ++i) { - /* Skip arenas which are not allocated. */ - if (arenas[i].address == 0) { - continue; - } - - uintptr_t base = (uintptr_t)_Py_ALIGN_UP(arenas[i].address, POOL_SIZE); - - /* visit every pool in the arena */ - assert(base <= (uintptr_t) arenas[i].pool_address); - for (; base < (uintptr_t) arenas[i].pool_address; base += POOL_SIZE) { - poolp p = (poolp)base; - n += p->ref.count; - } - } - return n; -} - +#endif /* Allocate a new arena. If we run out of memory, return NULL. Else * allocate a new arena, and return the address of an arena_object @@ -1245,15 +565,12 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; - static int debug_stats = -1; - - if (debug_stats == -1) { - const char *opt = Py_GETENV("PYTHONMALLOCSTATS"); - debug_stats = (opt != NULL && *opt != '\0'); - } - if (debug_stats) - _PyObject_DebugMallocStats(stderr); + int err; +#ifdef PYMALLOC_DEBUG + if (Py_GETENV("PYTHONMALLOCSTATS")) + _PyObject_DebugMallocStats(); +#endif if (unused_arena_objects == NULL) { uint i; uint numarenas; @@ -1266,11 +583,11 @@ new_arena(void) if (numarenas <= maxarenas) return NULL; /* overflow */ #if SIZEOF_SIZE_T <= SIZEOF_INT - if (numarenas > SIZE_MAX / sizeof(*arenas)) + if (numarenas > PY_SIZE_MAX / sizeof(*arenas)) return NULL; /* overflow */ #endif nbytes = numarenas * sizeof(*arenas); - arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); + arenaobj = (struct arena_object *)realloc(arenas, nbytes); if (arenaobj == NULL) return NULL; arenas = arenaobj; @@ -1301,8 +618,15 @@ new_arena(void) arenaobj = unused_arena_objects; unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); - if (address == NULL) { +#ifdef ARENAS_USE_MMAP + address = mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + err = (address == MAP_FAILED); +#else + address = malloc(ARENA_SIZE); + err = (address == 0); +#endif + if (err) { /* The allocation failed: return NULL after putting the * arenaobj back. */ @@ -1310,17 +634,20 @@ new_arena(void) unused_arena_objects = arenaobj; return NULL; } - arenaobj->address = (uintptr_t)address; + arenaobj->address = (uptr)address; ++narenas_currently_allocated; +#ifdef PYMALLOC_DEBUG ++ntimes_arena_allocated; if (narenas_currently_allocated > narenas_highwater) narenas_highwater = narenas_currently_allocated; +#endif arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ arenaobj->pool_address = (block*)arenaobj->address; - arenaobj->nfreepools = MAX_POOLS_IN_ARENA; + arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; + assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); excess = (uint)(arenaobj->address & POOL_SIZE_MASK); if (excess != 0) { --arenaobj->nfreepools; @@ -1331,15 +658,14 @@ new_arena(void) return arenaobj; } - /* -address_in_range(P, POOL) +Py_ADDRESS_IN_RANGE(P, POOL) Return true if and only if P is an address that was allocated by pymalloc. POOL must be the pool address associated with P, i.e., POOL = POOL_ADDR(P) (the caller is asked to compute this because the macro expands POOL more than once, and for efficiency it's best for the caller to assign POOL_ADDR(P) to a -variable and pass the latter to the macro; because address_in_range is +variable and pass the latter to the macro; because Py_ADDRESS_IN_RANGE is called on every alloc/realloc/free, micro-efficiency is important here). Tricky: Let B be the arena base address associated with the pool, B = @@ -1364,7 +690,7 @@ arenas[(POOL)->arenaindex]. Suppose obmalloc controls P. Then (barring wild stores, etc), POOL is the correct address of P's pool, AO.address is the correct base address of the pool's arena, and P must be within ARENA_SIZE of AO.address. In addition, AO.address is not 0 (no arena can start at address 0 -(NULL)). Therefore address_in_range correctly reports that obmalloc +(NULL)). Therefore Py_ADDRESS_IN_RANGE correctly reports that obmalloc controls P. Now suppose obmalloc does not control P (e.g., P was obtained via a direct @@ -1405,128 +731,234 @@ that this test determines whether an arbitrary address is controlled by obmalloc in a small constant time, independent of the number of arenas obmalloc controls. Since this test is needed at every entry point, it's extremely desirable that it be this fast. + +Since Py_ADDRESS_IN_RANGE may be reading from memory which was not allocated +by Python, it is important that (POOL)->arenaindex is read only once, as +another thread may be concurrently modifying the value without holding the +GIL. To accomplish this, the arenaindex_temp variable is used to store +(POOL)->arenaindex for the duration of the Py_ADDRESS_IN_RANGE macro's +execution. The caller of the macro is responsible for declaring this +variable. */ +#define Py_ADDRESS_IN_RANGE(P, POOL) \ + ((arenaindex_temp = (POOL)->arenaindex) < maxarenas && \ + (uptr)(P) - arenas[arenaindex_temp].address < (uptr)ARENA_SIZE && \ + arenas[arenaindex_temp].address != 0) -static bool _Py_NO_ADDRESS_SAFETY_ANALYSIS - _Py_NO_SANITIZE_THREAD - _Py_NO_SANITIZE_MEMORY -address_in_range(void *p, poolp pool) -{ - // Since address_in_range may be reading from memory which was not allocated - // by Python, it is important that pool->arenaindex is read only once, as - // another thread may be concurrently modifying the value without holding - // the GIL. The following dance forces the compiler to read pool->arenaindex - // only once. - uint arenaindex = *((volatile uint *)&pool->arenaindex); - return arenaindex < maxarenas && - (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE && - arenas[arenaindex].address != 0; -} +/* This is only useful when running memory debuggers such as + * Purify or Valgrind. Uncomment to use. + * +#define Py_USING_MEMORY_DEBUGGER + */ -/*==========================================================================*/ +#ifdef Py_USING_MEMORY_DEBUGGER -// Called when freelist is exhausted. Extend the freelist if there is -// space for a block. Otherwise, remove this pool from usedpools. -static void -pymalloc_pool_extend(poolp pool, uint size) -{ - if (UNLIKELY(pool->nextoffset <= pool->maxnextoffset)) { - /* There is room for another block. */ - pool->freeblock = (block*)pool + pool->nextoffset; - pool->nextoffset += INDEX2SIZE(size); - *(block **)(pool->freeblock) = NULL; - return; - } +/* Py_ADDRESS_IN_RANGE may access uninitialized memory by design + * This leads to thousands of spurious warnings when using + * Purify or Valgrind. By making a function, we can easily + * suppress the uninitialized memory reads in this one function. + * So we won't ignore real errors elsewhere. + * + * Disable the macro and use a function. + */ - /* Pool is full, unlink from used pools. */ - poolp next; - next = pool->nextpool; - pool = pool->prevpool; - next->prevpool = pool; - pool->nextpool = next; -} +#undef Py_ADDRESS_IN_RANGE + +#if defined(__GNUC__) && ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) || \ + (__GNUC__ >= 4)) +#define Py_NO_INLINE __attribute__((__noinline__)) +#else +#define Py_NO_INLINE +#endif -/* called when pymalloc_alloc can not allocate a block from usedpool. - * This function takes new pool and allocate a block from it. +/* Don't make static, to try to ensure this isn't inlined. */ +int Py_ADDRESS_IN_RANGE(void *P, poolp pool) Py_NO_INLINE; +#undef Py_NO_INLINE +#endif + +/*==========================================================================*/ + +/* malloc. Note that nbytes==0 tries to return a non-NULL pointer, distinct + * from all other currently live pointers. This may not be possible. + */ + +/* + * The basic blocks are ordered by decreasing execution frequency, + * which minimizes the number of jumps in the most common cases, + * improves branching prediction and instruction scheduling (small + * block allocations typically result in a couple of instructions). + * Unless the optimizer reorders everything, being too smart... */ -static void* -allocate_from_new_pool(uint size) + +#undef PyObject_Malloc +void * +PyObject_Malloc(size_t nbytes) { - /* There isn't a pool of the right size class immediately - * available: use a free pool. - */ - if (UNLIKELY(usable_arenas == NULL)) { - /* No arena has a free pool: allocate a new arena. */ -#ifdef WITH_MEMORY_LIMITS - if (narenas_currently_allocated >= MAX_ARENAS) { - return NULL; - } + block *bp; + poolp pool; + poolp next; + uint size; + +#ifdef WITH_VALGRIND + if (UNLIKELY(running_on_valgrind == -1)) + running_on_valgrind = RUNNING_ON_VALGRIND; + if (UNLIKELY(running_on_valgrind)) + goto redirect; #endif - usable_arenas = new_arena(); - if (usable_arenas == NULL) { - return NULL; - } - usable_arenas->nextarena = usable_arenas->prevarena = NULL; - assert(nfp2lasta[usable_arenas->nfreepools] == NULL); - nfp2lasta[usable_arenas->nfreepools] = usable_arenas; - } - assert(usable_arenas->address != 0); - /* This arena already had the smallest nfreepools value, so decreasing - * nfreepools doesn't change that, and we don't need to rearrange the - * usable_arenas list. However, if the arena becomes wholly allocated, - * we need to remove its arena_object from usable_arenas. + /* + * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. + * Most python internals blindly use a signed Py_ssize_t to track + * things without checking for overflows or negatives. + * As size_t is unsigned, checking for nbytes < 0 is not required. */ - assert(usable_arenas->nfreepools > 0); - if (nfp2lasta[usable_arenas->nfreepools] == usable_arenas) { - /* It's the last of this size, so there won't be any. */ - nfp2lasta[usable_arenas->nfreepools] = NULL; - } - /* If any free pools will remain, it will be the new smallest. */ - if (usable_arenas->nfreepools > 1) { - assert(nfp2lasta[usable_arenas->nfreepools - 1] == NULL); - nfp2lasta[usable_arenas->nfreepools - 1] = usable_arenas; - } + if (nbytes > PY_SSIZE_T_MAX) + return NULL; - /* Try to get a cached free pool. */ - poolp pool = usable_arenas->freepools; - if (LIKELY(pool != NULL)) { - /* Unlink from cached pools. */ - usable_arenas->freepools = pool->nextpool; - usable_arenas->nfreepools--; - if (UNLIKELY(usable_arenas->nfreepools == 0)) { - /* Wholly allocated: remove. */ - assert(usable_arenas->freepools == NULL); - assert(usable_arenas->nextarena == NULL || - usable_arenas->nextarena->prevarena == - usable_arenas); - usable_arenas = usable_arenas->nextarena; - if (usable_arenas != NULL) { - usable_arenas->prevarena = NULL; - assert(usable_arenas->address != 0); + /* + * This implicitly redirects malloc(0). + */ + if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) { + LOCK(); + /* + * Most frequent paths first + */ + size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; + pool = usedpools[size + size]; + if (pool != pool->nextpool) { + /* + * There is a used pool for this size class. + * Pick up the head block of its free list. + */ + ++pool->ref.count; + bp = pool->freeblock; + assert(bp != NULL); + if ((pool->freeblock = *(block **)bp) != NULL) { + UNLOCK(); + return (void *)bp; + } + /* + * Reached the end of the free list, try to extend it. + */ + if (pool->nextoffset <= pool->maxnextoffset) { + /* There is room for another block. */ + pool->freeblock = (block*)pool + + pool->nextoffset; + pool->nextoffset += INDEX2SIZE(size); + *(block **)(pool->freeblock) = NULL; + UNLOCK(); + return (void *)bp; + } + /* Pool is full, unlink from used pools. */ + next = pool->nextpool; + pool = pool->prevpool; + next->prevpool = pool; + pool->nextpool = next; + UNLOCK(); + return (void *)bp; + } + + /* There isn't a pool of the right size class immediately + * available: use a free pool. + */ + if (usable_arenas == NULL) { + /* No arena has a free pool: allocate a new arena. */ +#ifdef WITH_MEMORY_LIMITS + if (narenas_currently_allocated >= MAX_ARENAS) { + UNLOCK(); + goto redirect; } +#endif + usable_arenas = new_arena(); + if (usable_arenas == NULL) { + UNLOCK(); + goto redirect; + } + usable_arenas->nextarena = + usable_arenas->prevarena = NULL; } - else { - /* nfreepools > 0: it must be that freepools - * isn't NULL, or that we haven't yet carved - * off all the arena's pools for the first - * time. + assert(usable_arenas->address != 0); + + /* Try to get a cached free pool. */ + pool = usable_arenas->freepools; + if (pool != NULL) { + /* Unlink from cached pools. */ + usable_arenas->freepools = pool->nextpool; + + /* This arena already had the smallest nfreepools + * value, so decreasing nfreepools doesn't change + * that, and we don't need to rearrange the + * usable_arenas list. However, if the arena has + * become wholly allocated, we need to remove its + * arena_object from usable_arenas. */ - assert(usable_arenas->freepools != NULL || - usable_arenas->pool_address <= - (block*)usable_arenas->address + - ARENA_SIZE - POOL_SIZE); + --usable_arenas->nfreepools; + if (usable_arenas->nfreepools == 0) { + /* Wholly allocated: remove. */ + assert(usable_arenas->freepools == NULL); + assert(usable_arenas->nextarena == NULL || + usable_arenas->nextarena->prevarena == + usable_arenas); + + usable_arenas = usable_arenas->nextarena; + if (usable_arenas != NULL) { + usable_arenas->prevarena = NULL; + assert(usable_arenas->address != 0); + } + } + else { + /* nfreepools > 0: it must be that freepools + * isn't NULL, or that we haven't yet carved + * off all the arena's pools for the first + * time. + */ + assert(usable_arenas->freepools != NULL || + usable_arenas->pool_address <= + (block*)usable_arenas->address + + ARENA_SIZE - POOL_SIZE); + } + init_pool: + /* Frontlink to used pools. */ + next = usedpools[size + size]; /* == prev */ + pool->nextpool = next; + pool->prevpool = next; + next->nextpool = pool; + next->prevpool = pool; + pool->ref.count = 1; + if (pool->szidx == size) { + /* Luckily, this pool last contained blocks + * of the same size class, so its header + * and free list are already initialized. + */ + bp = pool->freeblock; + pool->freeblock = *(block **)bp; + UNLOCK(); + return (void *)bp; + } + /* + * Initialize the pool header, set up the free list to + * contain just the second block, and return the first + * block. + */ + pool->szidx = size; + size = INDEX2SIZE(size); + bp = (block *)pool + POOL_OVERHEAD; + pool->nextoffset = POOL_OVERHEAD + (size << 1); + pool->maxnextoffset = POOL_SIZE - size; + pool->freeblock = bp + size; + *(block **)(pool->freeblock) = NULL; + UNLOCK(); + return (void *)bp; } - } - else { + /* Carve off a new pool. */ assert(usable_arenas->nfreepools > 0); assert(usable_arenas->freepools == NULL); pool = (poolp)usable_arenas->pool_address; assert((block*)pool <= (block*)usable_arenas->address + - ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(usable_arenas - arenas); + ARENA_SIZE - POOL_SIZE); + pool->arenaindex = usable_arenas - arenas; assert(&arenas[pool->arenaindex] == usable_arenas); pool->szidx = DUMMY_SIZE_IDX; usable_arenas->pool_address += POOL_SIZE; @@ -1543,472 +975,336 @@ allocate_from_new_pool(uint size) assert(usable_arenas->address != 0); } } - } - /* Frontlink to used pools. */ - block *bp; - poolp next = usedpools[size + size]; /* == prev */ - pool->nextpool = next; - pool->prevpool = next; - next->nextpool = pool; - next->prevpool = pool; - pool->ref.count = 1; - if (pool->szidx == size) { - /* Luckily, this pool last contained blocks - * of the same size class, so its header - * and free list are already initialized. - */ - bp = pool->freeblock; - assert(bp != NULL); - pool->freeblock = *(block **)bp; - return bp; + goto init_pool; } - /* - * Initialize the pool header, set up the free list to - * contain just the second block, and return the first - * block. + + /* The small block allocator ends here. */ + +redirect: + /* Redirect the original request to the underlying (libc) allocator. + * We jump here on bigger requests, on error in the code above (as a + * last chance to serve the request) or when the max memory limit + * has been reached. */ - pool->szidx = size; - size = INDEX2SIZE(size); - bp = (block *)pool + POOL_OVERHEAD; - pool->nextoffset = POOL_OVERHEAD + (size << 1); - pool->maxnextoffset = POOL_SIZE - size; - pool->freeblock = bp + size; - *(block **)(pool->freeblock) = NULL; - return bp; + if (nbytes == 0) + nbytes = 1; + return (void *)malloc(nbytes); } -/* pymalloc allocator - - Return a pointer to newly allocated memory if pymalloc allocated memory. +/* free */ - Return NULL if pymalloc failed to allocate the memory block: on bigger - requests, on error in the code below (as a last chance to serve the request) - or when the max memory limit has been reached. -*/ -static inline void* -pymalloc_alloc(void *ctx, size_t nbytes) +#undef PyObject_Free +ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS +void +PyObject_Free(void *p) { -#ifdef WITH_VALGRIND - if (UNLIKELY(running_on_valgrind == -1)) { - running_on_valgrind = RUNNING_ON_VALGRIND; - } - if (UNLIKELY(running_on_valgrind)) { - return NULL; - } + poolp pool; + block *lastfree; + poolp next, prev; + uint size; +#ifndef Py_USING_MEMORY_DEBUGGER + uint arenaindex_temp; #endif - if (UNLIKELY(nbytes == 0)) { - return NULL; - } - if (UNLIKELY(nbytes > SMALL_REQUEST_THRESHOLD)) { - return NULL; - } - - uint size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - poolp pool = usedpools[size + size]; - block *bp; + if (p == NULL) /* free(NULL) has no effect */ + return; - if (LIKELY(pool != pool->nextpool)) { - /* - * There is a used pool for this size class. - * Pick up the head block of its free list. - */ - ++pool->ref.count; - bp = pool->freeblock; - assert(bp != NULL); +#ifdef WITH_VALGRIND + if (UNLIKELY(running_on_valgrind > 0)) + goto redirect; +#endif - if (UNLIKELY((pool->freeblock = *(block **)bp) == NULL)) { - // Reached the end of the free list, try to extend it. - pymalloc_pool_extend(pool, size); - } - } - else { - /* There isn't a pool of the right size class immediately - * available: use a free pool. + pool = POOL_ADDR(p); + if (Py_ADDRESS_IN_RANGE(p, pool)) { + /* We allocated this address. */ + LOCK(); + /* Link p to the start of the pool's freeblock list. Since + * the pool had at least the p block outstanding, the pool + * wasn't empty (so it's already in a usedpools[] list, or + * was full and is in no list -- it's not in the freeblocks + * list in any case). */ - bp = allocate_from_new_pool(size); - } - - return (void *)bp; -} - - -static void * -_PyObject_Malloc(void *ctx, size_t nbytes) -{ - void* ptr = pymalloc_alloc(ctx, nbytes); - if (LIKELY(ptr != NULL)) { - return ptr; - } - - ptr = PyMem_RawMalloc(nbytes); - if (ptr != NULL) { - raw_allocated_blocks++; - } - return ptr; -} - - -static void * -_PyObject_Calloc(void *ctx, size_t nelem, size_t elsize) -{ - assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize); - size_t nbytes = nelem * elsize; - - void* ptr = pymalloc_alloc(ctx, nbytes); - if (LIKELY(ptr != NULL)) { - memset(ptr, 0, nbytes); - return ptr; - } - - ptr = PyMem_RawCalloc(nelem, elsize); - if (ptr != NULL) { - raw_allocated_blocks++; - } - return ptr; -} - - -static void -insert_to_usedpool(poolp pool) -{ - assert(pool->ref.count > 0); /* else the pool is empty */ - - uint size = pool->szidx; - poolp next = usedpools[size + size]; - poolp prev = next->prevpool; - - /* insert pool before next: prev <-> pool <-> next */ - pool->nextpool = next; - pool->prevpool = prev; - next->prevpool = pool; - prev->nextpool = pool; -} - -static void -insert_to_freepool(poolp pool) -{ - poolp next = pool->nextpool; - poolp prev = pool->prevpool; - next->prevpool = prev; - prev->nextpool = next; + assert(pool->ref.count > 0); /* else it was empty */ + *(block **)p = lastfree = pool->freeblock; + pool->freeblock = (block *)p; + if (lastfree) { + struct arena_object* ao; + uint nf; /* ao->nfreepools */ + + /* freeblock wasn't NULL, so the pool wasn't full, + * and the pool is in a usedpools[] list. + */ + if (--pool->ref.count != 0) { + /* pool isn't empty: leave it in usedpools */ + UNLOCK(); + return; + } + /* Pool is now empty: unlink from usedpools, and + * link to the front of freepools. This ensures that + * previously freed pools will be allocated later + * (being not referenced, they are perhaps paged out). + */ + next = pool->nextpool; + prev = pool->prevpool; + next->prevpool = prev; + prev->nextpool = next; - /* Link the pool to freepools. This is a singly-linked - * list, and pool->prevpool isn't used there. - */ - struct arena_object *ao = &arenas[pool->arenaindex]; - pool->nextpool = ao->freepools; - ao->freepools = pool; - uint nf = ao->nfreepools; - /* If this is the rightmost arena with this number of free pools, - * nfp2lasta[nf] needs to change. Caution: if nf is 0, there - * are no arenas in usable_arenas with that value. - */ - struct arena_object* lastnf = nfp2lasta[nf]; - assert((nf == 0 && lastnf == NULL) || - (nf > 0 && - lastnf != NULL && - lastnf->nfreepools == nf && - (lastnf->nextarena == NULL || - nf < lastnf->nextarena->nfreepools))); - if (lastnf == ao) { /* it is the rightmost */ - struct arena_object* p = ao->prevarena; - nfp2lasta[nf] = (p != NULL && p->nfreepools == nf) ? p : NULL; - } - ao->nfreepools = ++nf; - - /* All the rest is arena management. We just freed - * a pool, and there are 4 cases for arena mgmt: - * 1. If all the pools are free, return the arena to - * the system free(). Except if this is the last - * arena in the list, keep it to avoid thrashing: - * keeping one wholly free arena in the list avoids - * pathological cases where a simple loop would - * otherwise provoke needing to allocate and free an - * arena on every iteration. See bpo-37257. - * 2. If this is the only free pool in the arena, - * add the arena back to the `usable_arenas` list. - * 3. If the "next" arena has a smaller count of free - * pools, we have to "slide this arena right" to - * restore that usable_arenas is sorted in order of - * nfreepools. - * 4. Else there's nothing more to do. - */ - if (nf == ao->ntotalpools && ao->nextarena != NULL) { - /* Case 1. First unlink ao from usable_arenas. - */ - assert(ao->prevarena == NULL || - ao->prevarena->address != 0); - assert(ao ->nextarena == NULL || - ao->nextarena->address != 0); + /* Link the pool to freepools. This is a singly-linked + * list, and pool->prevpool isn't used there. + */ + ao = &arenas[pool->arenaindex]; + pool->nextpool = ao->freepools; + ao->freepools = pool; + nf = ++ao->nfreepools; + + /* All the rest is arena management. We just freed + * a pool, and there are 4 cases for arena mgmt: + * 1. If all the pools are free, return the arena to + * the system free(). + * 2. If this is the only free pool in the arena, + * add the arena back to the `usable_arenas` list. + * 3. If the "next" arena has a smaller count of free + * pools, we have to "slide this arena right" to + * restore that usable_arenas is sorted in order of + * nfreepools. + * 4. Else there's nothing more to do. + */ + if (nf == ao->ntotalpools) { + /* Case 1. First unlink ao from usable_arenas. + */ + assert(ao->prevarena == NULL || + ao->prevarena->address != 0); + assert(ao ->nextarena == NULL || + ao->nextarena->address != 0); + + /* Fix the pointer in the prevarena, or the + * usable_arenas pointer. + */ + if (ao->prevarena == NULL) { + usable_arenas = ao->nextarena; + assert(usable_arenas == NULL || + usable_arenas->address != 0); + } + else { + assert(ao->prevarena->nextarena == ao); + ao->prevarena->nextarena = + ao->nextarena; + } + /* Fix the pointer in the nextarena. */ + if (ao->nextarena != NULL) { + assert(ao->nextarena->prevarena == ao); + ao->nextarena->prevarena = + ao->prevarena; + } + /* Record that this arena_object slot is + * available to be reused. + */ + ao->nextarena = unused_arena_objects; + unused_arena_objects = ao; + + /* Free the entire arena. */ +#ifdef ARENAS_USE_MMAP + munmap((void *)ao->address, ARENA_SIZE); +#else + free((void *)ao->address); +#endif + ao->address = 0; /* mark unassociated */ + --narenas_currently_allocated; - /* Fix the pointer in the prevarena, or the - * usable_arenas pointer. - */ - if (ao->prevarena == NULL) { - usable_arenas = ao->nextarena; - assert(usable_arenas == NULL || - usable_arenas->address != 0); - } - else { - assert(ao->prevarena->nextarena == ao); - ao->prevarena->nextarena = - ao->nextarena; - } - /* Fix the pointer in the nextarena. */ - if (ao->nextarena != NULL) { - assert(ao->nextarena->prevarena == ao); - ao->nextarena->prevarena = - ao->prevarena; - } - /* Record that this arena_object slot is - * available to be reused. - */ - ao->nextarena = unused_arena_objects; - unused_arena_objects = ao; + UNLOCK(); + return; + } + if (nf == 1) { + /* Case 2. Put ao at the head of + * usable_arenas. Note that because + * ao->nfreepools was 0 before, ao isn't + * currently on the usable_arenas list. + */ + ao->nextarena = usable_arenas; + ao->prevarena = NULL; + if (usable_arenas) + usable_arenas->prevarena = ao; + usable_arenas = ao; + assert(usable_arenas->address != 0); - /* Free the entire arena. */ - _PyObject_Arena.free(_PyObject_Arena.ctx, - (void *)ao->address, ARENA_SIZE); - ao->address = 0; /* mark unassociated */ - --narenas_currently_allocated; + UNLOCK(); + return; + } + /* If this arena is now out of order, we need to keep + * the list sorted. The list is kept sorted so that + * the "most full" arenas are used first, which allows + * the nearly empty arenas to be completely freed. In + * a few un-scientific tests, it seems like this + * approach allowed a lot more memory to be freed. + */ + if (ao->nextarena == NULL || + nf <= ao->nextarena->nfreepools) { + /* Case 4. Nothing to do. */ + UNLOCK(); + return; + } + /* Case 3: We have to move the arena towards the end + * of the list, because it has more free pools than + * the arena to its right. + * First unlink ao from usable_arenas. + */ + if (ao->prevarena != NULL) { + /* ao isn't at the head of the list */ + assert(ao->prevarena->nextarena == ao); + ao->prevarena->nextarena = ao->nextarena; + } + else { + /* ao is at the head of the list */ + assert(usable_arenas == ao); + usable_arenas = ao->nextarena; + } + ao->nextarena->prevarena = ao->prevarena; - return; - } + /* Locate the new insertion point by iterating over + * the list, using our nextarena pointer. + */ + while (ao->nextarena != NULL && + nf > ao->nextarena->nfreepools) { + ao->prevarena = ao->nextarena; + ao->nextarena = ao->nextarena->nextarena; + } - if (nf == 1) { - /* Case 2. Put ao at the head of - * usable_arenas. Note that because - * ao->nfreepools was 0 before, ao isn't - * currently on the usable_arenas list. - */ - ao->nextarena = usable_arenas; - ao->prevarena = NULL; - if (usable_arenas) - usable_arenas->prevarena = ao; - usable_arenas = ao; - assert(usable_arenas->address != 0); - if (nfp2lasta[1] == NULL) { - nfp2lasta[1] = ao; + /* Insert ao at this point. */ + assert(ao->nextarena == NULL || + ao->prevarena == ao->nextarena->prevarena); + assert(ao->prevarena->nextarena == ao->nextarena); + + ao->prevarena->nextarena = ao; + if (ao->nextarena != NULL) + ao->nextarena->prevarena = ao; + + /* Verify that the swaps worked. */ + assert(ao->nextarena == NULL || + nf <= ao->nextarena->nfreepools); + assert(ao->prevarena == NULL || + nf > ao->prevarena->nfreepools); + assert(ao->nextarena == NULL || + ao->nextarena->prevarena == ao); + assert((usable_arenas == ao && + ao->prevarena == NULL) || + ao->prevarena->nextarena == ao); + + UNLOCK(); + return; } - - return; - } - - /* If this arena is now out of order, we need to keep - * the list sorted. The list is kept sorted so that - * the "most full" arenas are used first, which allows - * the nearly empty arenas to be completely freed. In - * a few un-scientific tests, it seems like this - * approach allowed a lot more memory to be freed. - */ - /* If this is the only arena with nf, record that. */ - if (nfp2lasta[nf] == NULL) { - nfp2lasta[nf] = ao; - } /* else the rightmost with nf doesn't change */ - /* If this was the rightmost of the old size, it remains in place. */ - if (ao == lastnf) { - /* Case 4. Nothing to do. */ - return; - } - /* If ao were the only arena in the list, the last block would have - * gotten us out. - */ - assert(ao->nextarena != NULL); - - /* Case 3: We have to move the arena towards the end of the list, - * because it has more free pools than the arena to its right. It needs - * to move to follow lastnf. - * First unlink ao from usable_arenas. - */ - if (ao->prevarena != NULL) { - /* ao isn't at the head of the list */ - assert(ao->prevarena->nextarena == ao); - ao->prevarena->nextarena = ao->nextarena; - } - else { - /* ao is at the head of the list */ - assert(usable_arenas == ao); - usable_arenas = ao->nextarena; - } - ao->nextarena->prevarena = ao->prevarena; - /* And insert after lastnf. */ - ao->prevarena = lastnf; - ao->nextarena = lastnf->nextarena; - if (ao->nextarena != NULL) { - ao->nextarena->prevarena = ao; - } - lastnf->nextarena = ao; - /* Verify that the swaps worked. */ - assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools); - assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools); - assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); - assert((usable_arenas == ao && ao->prevarena == NULL) - || ao->prevarena->nextarena == ao); -} - -/* Free a memory block allocated by pymalloc_alloc(). - Return 1 if it was freed. - Return 0 if the block was not allocated by pymalloc_alloc(). */ -static inline int -pymalloc_free(void *ctx, void *p) -{ - assert(p != NULL); - -#ifdef WITH_VALGRIND - if (UNLIKELY(running_on_valgrind > 0)) { - return 0; - } -#endif - - poolp pool = POOL_ADDR(p); - if (UNLIKELY(!address_in_range(p, pool))) { - return 0; - } - /* We allocated this address. */ - - /* Link p to the start of the pool's freeblock list. Since - * the pool had at least the p block outstanding, the pool - * wasn't empty (so it's already in a usedpools[] list, or - * was full and is in no list -- it's not in the freeblocks - * list in any case). - */ - assert(pool->ref.count > 0); /* else it was empty */ - block *lastfree = pool->freeblock; - *(block **)p = lastfree; - pool->freeblock = (block *)p; - pool->ref.count--; - - if (UNLIKELY(lastfree == NULL)) { /* Pool was full, so doesn't currently live in any list: * link it to the front of the appropriate usedpools[] list. * This mimics LRU pool usage for new allocations and * targets optimal filling when several pools contain * blocks of the same size class. */ - insert_to_usedpool(pool); - return 1; - } - - /* freeblock wasn't NULL, so the pool wasn't full, - * and the pool is in a usedpools[] list. - */ - if (LIKELY(pool->ref.count != 0)) { - /* pool isn't empty: leave it in usedpools */ - return 1; - } - - /* Pool is now empty: unlink from usedpools, and - * link to the front of freepools. This ensures that - * previously freed pools will be allocated later - * (being not referenced, they are perhaps paged out). - */ - insert_to_freepool(pool); - return 1; -} - - -static void -_PyObject_Free(void *ctx, void *p) -{ - /* PyObject_Free(NULL) has no effect */ - if (p == NULL) { + --pool->ref.count; + assert(pool->ref.count > 0); /* else the pool is empty */ + size = pool->szidx; + next = usedpools[size + size]; + prev = next->prevpool; + /* insert pool before next: prev <-> pool <-> next */ + pool->nextpool = next; + pool->prevpool = prev; + next->prevpool = pool; + prev->nextpool = pool; + UNLOCK(); return; } - if (UNLIKELY(!pymalloc_free(ctx, p))) { - /* pymalloc didn't allocate this address */ - PyMem_RawFree(p); - raw_allocated_blocks--; - } +#ifdef WITH_VALGRIND +redirect: +#endif + /* We didn't allocate this address. */ + free(p); } +/* realloc. If p is NULL, this acts like malloc(nbytes). Else if nbytes==0, + * then as the Python docs promise, we do not treat this like free(p), and + * return a non-NULL result. + */ -/* pymalloc realloc. - - If nbytes==0, then as the Python docs promise, we do not treat this like - free(p), and return a non-NULL result. - - Return 1 if pymalloc reallocated memory and wrote the new pointer into - newptr_p. - - Return 0 if pymalloc didn't allocated p. */ -static int -pymalloc_realloc(void *ctx, void **newptr_p, void *p, size_t nbytes) +#undef PyObject_Realloc +ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS +void * +PyObject_Realloc(void *p, size_t nbytes) { void *bp; poolp pool; size_t size; +#ifndef Py_USING_MEMORY_DEBUGGER + uint arenaindex_temp; +#endif - assert(p != NULL); + if (p == NULL) + return PyObject_Malloc(nbytes); + + /* + * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. + * Most python internals blindly use a signed Py_ssize_t to track + * things without checking for overflows or negatives. + * As size_t is unsigned, checking for nbytes < 0 is not required. + */ + if (nbytes > PY_SSIZE_T_MAX) + return NULL; #ifdef WITH_VALGRIND /* Treat running_on_valgrind == -1 the same as 0 */ - if (UNLIKELY(running_on_valgrind > 0)) { - return 0; - } + if (UNLIKELY(running_on_valgrind > 0)) + goto redirect; #endif pool = POOL_ADDR(p); - if (!address_in_range(p, pool)) { - /* pymalloc is not managing this block. - - If nbytes <= SMALL_REQUEST_THRESHOLD, it's tempting to try to take - over this block. However, if we do, we need to copy the valid data - from the C-managed block to one of our blocks, and there's no - portable way to know how much of the memory space starting at p is - valid. - - As bug 1185883 pointed out the hard way, it's possible that the - C-managed block is "at the end" of allocated VM space, so that a - memory fault can occur if we try to copy nbytes bytes starting at p. - Instead we punt: let C continue to manage this block. */ - return 0; - } - - /* pymalloc is in charge of this block */ - size = INDEX2SIZE(pool->szidx); - if (nbytes <= size) { - /* The block is staying the same or shrinking. - - If it's shrinking, there's a tradeoff: it costs cycles to copy the - block to a smaller size class, but it wastes memory not to copy it. - - The compromise here is to copy on shrink only if at least 25% of - size can be shaved off. */ - if (4 * nbytes > 3 * size) { - /* It's the same, or shrinking and new/old > 3/4. */ - *newptr_p = p; - return 1; + if (Py_ADDRESS_IN_RANGE(p, pool)) { + /* We're in charge of this block */ + size = INDEX2SIZE(pool->szidx); + if (nbytes <= size) { + /* The block is staying the same or shrinking. If + * it's shrinking, there's a tradeoff: it costs + * cycles to copy the block to a smaller size class, + * but it wastes memory not to copy it. The + * compromise here is to copy on shrink only if at + * least 25% of size can be shaved off. + */ + if (4 * nbytes > 3 * size) { + /* It's the same, + * or shrinking and new/old > 3/4. + */ + return p; + } + size = nbytes; } - size = nbytes; - } - - bp = _PyObject_Malloc(ctx, nbytes); - if (bp != NULL) { - memcpy(bp, p, size); - _PyObject_Free(ctx, p); - } - *newptr_p = bp; - return 1; -} - - -static void * -_PyObject_Realloc(void *ctx, void *ptr, size_t nbytes) -{ - void *ptr2; - - if (ptr == NULL) { - return _PyObject_Malloc(ctx, nbytes); - } - - if (pymalloc_realloc(ctx, &ptr2, ptr, nbytes)) { - return ptr2; + bp = PyObject_Malloc(nbytes); + if (bp != NULL) { + memcpy(bp, p, size); + PyObject_Free(p); + } + return bp; } - - return PyMem_RawRealloc(ptr, nbytes); +#ifdef WITH_VALGRIND + redirect: +#endif + /* We're not managing this block. If nbytes <= + * SMALL_REQUEST_THRESHOLD, it's tempting to try to take over this + * block. However, if we do, we need to copy the valid data from + * the C-managed block to one of our blocks, and there's no portable + * way to know how much of the memory space starting at p is valid. + * As bug 1185883 pointed out the hard way, it's possible that the + * C-managed block is "at the end" of allocated VM space, so that + * a memory fault can occur if we try to copy nbytes bytes starting + * at p. Instead we punt: let C continue to manage this block. + */ + if (nbytes) + return realloc(p, nbytes); + /* C doesn't define the result of realloc(p, 0) (it may or may not + * return NULL then), but Python's docs promise that nbytes==0 never + * returns NULL. We don't pass 0 to realloc(), to avoid that endcase + * to begin with. Even then, we can't be sure that realloc() won't + * return NULL. + */ + bp = realloc(p, 1); + return bp ? bp : p; } #else /* ! WITH_PYMALLOC */ @@ -2017,24 +1313,46 @@ _PyObject_Realloc(void *ctx, void *ptr, size_t nbytes) /* pymalloc not enabled: Redirect the entry points to malloc. These will * only be used by extensions that are compiled with pymalloc enabled. */ -Py_ssize_t -_Py_GetAllocatedBlocks(void) +void * +PyObject_Malloc(size_t n) { - return 0; + return PyMem_MALLOC(n); } -#endif /* WITH_PYMALLOC */ +void * +PyObject_Realloc(void *p, size_t n) +{ + return PyMem_REALLOC(p, n); +} +void +PyObject_Free(void *p) +{ + PyMem_FREE(p); +} +#endif /* WITH_PYMALLOC */ +#ifdef PYMALLOC_DEBUG /*==========================================================================*/ /* A x-platform debugging allocator. This doesn't manage memory directly, * it wraps a real allocator, adding extra debugging info to the memory blocks. */ -/* Uncomment this define to add the "serialno" field */ -/* #define PYMEM_DEBUG_SERIALNO */ +/* Special bytes broadcast into debug memory blocks at appropriate times. + * Strings of these are unlikely to be valid addresses, floats, ints or + * 7-bit ASCII. + */ +#undef CLEANBYTE +#undef DEADBYTE +#undef FORBIDDENBYTE +#define CLEANBYTE 0xCB /* clean (newly allocated) memory */ +#define DEADBYTE 0xDB /* dead (newly freed) memory */ +#define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ + +/* We tag each block with an API ID in order to tag API violations */ +#define _PYMALLOC_MEM_ID 'm' /* the PyMem_Malloc() API */ +#define _PYMALLOC_OBJ_ID 'o' /* The PyObject_Malloc() API */ -#ifdef PYMEM_DEBUG_SERIALNO static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ /* serialno is always incremented via calling this routine. The point is @@ -2045,21 +1363,14 @@ bumpserialno(void) { ++serialno; } -#endif #define SST SIZEOF_SIZE_T -#ifdef PYMEM_DEBUG_SERIALNO -# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST -#else -# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST -#endif - /* Read sizeof(size_t) bytes at p as a big-endian size_t. */ static size_t read_size_t(const void *p) { - const uint8_t *q = (const uint8_t *)p; + const uchar *q = (const uchar *)p; size_t result = *q++; int i; @@ -2074,282 +1385,261 @@ read_size_t(const void *p) static void write_size_t(void *p, size_t n) { - uint8_t *q = (uint8_t *)p + SST - 1; + uchar *q = (uchar *)p + SST - 1; int i; for (i = SST; --i >= 0; --q) { - *q = (uint8_t)(n & 0xff); + *q = (uchar)(n & 0xff); n >>= 8; } } -/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and +#ifdef Py_DEBUG +/* Is target in the list? The list is traversed via the nextpool pointers. + * The list may be NULL-terminated, or circular. Return 1 if target is in + * list, else 0. + */ +static int +pool_is_in_list(const poolp target, poolp list) +{ + poolp origlist = list; + assert(target != NULL); + if (list == NULL) + return 0; + do { + if (target == list) + return 1; + list = list->nextpool; + } while (list != NULL && list != origlist); + return 0; +} + +#else +#define pool_is_in_list(X, Y) 1 + +#endif /* Py_DEBUG */ + +static void * +_PyMem_Malloc(size_t nbytes) +{ + if (nbytes > (size_t)PY_SSIZE_T_MAX) { + return NULL; + } + if (nbytes == 0) { + nbytes = 1; + } + return malloc(nbytes); +} + +static void * +_PyMem_Realloc(void *p, size_t nbytes) +{ + if (nbytes > (size_t)PY_SSIZE_T_MAX) { + return NULL; + } + if (nbytes == 0) { + nbytes = 1; + } + return realloc(p, nbytes); +} + + +static void +_PyMem_Free(void *p) +{ + free(p); +} + + +/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: p[0: S] Number of bytes originally asked for. This is a size_t, big-endian (easier to read in a memory dump). -p[S] - API ID. See PEP 445. This is a character, but seems undocumented. -p[S+1: 2*S] - Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. +p[S: 2*S] + Copies of FORBIDDENBYTE. Used to catch under- writes and reads. p[2*S: 2*S+n] - The requested memory, filled with copies of PYMEM_CLEANBYTE. + The requested memory, filled with copies of CLEANBYTE. Used to catch reference to uninitialized memory. &p[2*S] is returned. Note that this is 8-byte aligned if pymalloc handled the request itself. p[2*S+n: 2*S+n+S] - Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads. + Copies of FORBIDDENBYTE. Used to catch over- writes and reads. p[2*S+n+S: 2*S+n+2*S] - A serial number, incremented by 1 on each call to _PyMem_DebugMalloc - and _PyMem_DebugRealloc. + A serial number, incremented by 1 on each call to _PyObject_DebugMalloc + and _PyObject_DebugRealloc. This is a big-endian size_t. If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. - -If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks -for 3 * S extra bytes, and omits the last serialno field. */ -static void * -_PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) +/* debug replacements for the PyMem_* memory API */ +void * +_PyMem_DebugMalloc(size_t nbytes) { - debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; - uint8_t *p; /* base address of malloc'ed pad block */ - uint8_t *data; /* p + 2*SST == pointer to data bytes */ - uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ - size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */ - - if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { - /* integer overflow: can't represent total as a Py_ssize_t */ - return NULL; - } - total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; + return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes); +} +void * +_PyMem_DebugRealloc(void *p, size_t nbytes) +{ + return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes); +} +void +_PyMem_DebugFree(void *p) +{ + _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p); +} + +/* debug replacements for the PyObject_* memory API */ +void * +_PyObject_DebugMalloc(size_t nbytes) +{ + return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes); +} +void * +_PyObject_DebugRealloc(void *p, size_t nbytes) +{ + return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes); +} +void +_PyObject_DebugFree(void *p) +{ + _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p); +} +void +_PyObject_DebugCheckAddress(const void *p) +{ + _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p); +} - /* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN] - ^--- p ^--- data ^--- tail - S: nbytes stored as size_t - I: API identifier (1 byte) - F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after) - C: Clean bytes used later to store actual data - N: Serial number stored as size_t - If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field - is omitted. */ +/* generic debug memory api, with an "id" to identify the API in use */ +void * +_PyObject_DebugMallocApi(char api, size_t nbytes) +{ + uchar *p; /* base address of malloc'ed block */ + uchar *tail; /* p + 2*SST + nbytes == pointer to tail pad bytes */ + size_t total; /* nbytes + 4*SST */ + + bumpserialno(); + total = nbytes + 4*SST; + if (total < nbytes) + /* overflow: can't represent total as a size_t */ + return NULL; - if (use_calloc) { - p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total); + if (api == _PYMALLOC_OBJ_ID) { + p = (uchar *)PyObject_Malloc(total); } else { - p = (uint8_t *)api->alloc.malloc(api->alloc.ctx, total); + p = (uchar *)_PyMem_Malloc(total); } - if (p == NULL) { + if (p == NULL) return NULL; - } - data = p + 2*SST; -#ifdef PYMEM_DEBUG_SERIALNO - bumpserialno(); -#endif - - /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ + /* at p, write size (SST bytes), api (1 byte), pad (SST-1 bytes) */ write_size_t(p, nbytes); - p[SST] = (uint8_t)api->api_id; - memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); + p[SST] = (uchar)api; + memset(p + SST + 1 , FORBIDDENBYTE, SST-1); - if (nbytes > 0 && !use_calloc) { - memset(data, PYMEM_CLEANBYTE, nbytes); - } + if (nbytes > 0) + memset(p + 2*SST, CLEANBYTE, nbytes); /* at tail, write pad (SST bytes) and serialno (SST bytes) */ - tail = data + nbytes; - memset(tail, PYMEM_FORBIDDENBYTE, SST); -#ifdef PYMEM_DEBUG_SERIALNO + tail = p + 2*SST + nbytes; + memset(tail, FORBIDDENBYTE, SST); write_size_t(tail + SST, serialno); -#endif - - return data; -} -static void * -_PyMem_DebugRawMalloc(void *ctx, size_t nbytes) -{ - return _PyMem_DebugRawAlloc(0, ctx, nbytes); -} - -static void * -_PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize) -{ - size_t nbytes; - assert(elsize == 0 || nelem <= (size_t)PY_SSIZE_T_MAX / elsize); - nbytes = nelem * elsize; - return _PyMem_DebugRawAlloc(1, ctx, nbytes); + return p + 2*SST; } - /* The debug free first checks the 2*SST bytes on each end for sanity (in particular, that the FORBIDDENBYTEs with the api ID are still intact). - Then fills the original bytes with PYMEM_DEADBYTE. + Then fills the original bytes with DEADBYTE. Then calls the underlying free. */ -static void -_PyMem_DebugRawFree(void *ctx, void *p) +void +_PyObject_DebugFreeApi(char api, void *p) { - /* PyMem_Free(NULL) has no effect */ - if (p == NULL) { - return; - } - - debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; - uint8_t *q = (uint8_t *)p - 2*SST; /* address returned from malloc */ + uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */ size_t nbytes; - _PyMem_DebugCheckAddress(api->api_id, p); + if (p == NULL) + return; + _PyObject_DebugCheckAddressApi(api, p); nbytes = read_size_t(q); - nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, PYMEM_DEADBYTE, nbytes); - api->alloc.free(api->alloc.ctx, q); + nbytes += 4*SST; + if (nbytes > 0) + memset(q, DEADBYTE, nbytes); + if (api == _PYMALLOC_OBJ_ID) { + PyObject_Free(q); + } + else { + _PyMem_Free(q); + } } - -static void * -_PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) +void * +_PyObject_DebugReallocApi(char api, void *p, size_t nbytes) { - if (p == NULL) { - return _PyMem_DebugRawAlloc(0, ctx, nbytes); - } - - debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; - uint8_t *head; /* base address of malloc'ed pad block */ - uint8_t *data; /* pointer to data bytes */ - uint8_t *r; - uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ - size_t total; /* 2 * SST + nbytes + 2 * SST */ + uchar *q = (uchar *)p; + uchar *tail; + size_t total; /* nbytes + 4*SST */ size_t original_nbytes; -#define ERASED_SIZE 64 - uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ + int i; - _PyMem_DebugCheckAddress(api->api_id, p); + if (p == NULL) + return _PyObject_DebugMallocApi(api, nbytes); - data = (uint8_t *)p; - head = data - 2*SST; - original_nbytes = read_size_t(head); - if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { - /* integer overflow: can't represent total as a Py_ssize_t */ + _PyObject_DebugCheckAddressApi(api, p); + bumpserialno(); + original_nbytes = read_size_t(q - 2*SST); + total = nbytes + 4*SST; + if (total < nbytes) + /* overflow: can't represent total as a size_t */ return NULL; - } - total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; - tail = data + original_nbytes; -#ifdef PYMEM_DEBUG_SERIALNO - size_t block_serialno = read_size_t(tail + SST); -#endif - /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and - ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. - */ - if (original_nbytes <= sizeof(save)) { - memcpy(save, data, original_nbytes); - memset(data - 2 * SST, PYMEM_DEADBYTE, - original_nbytes + PYMEM_DEBUG_EXTRA_BYTES); - } - else { - memcpy(save, data, ERASED_SIZE); - memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST); - memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE); - memset(tail - ERASED_SIZE, PYMEM_DEADBYTE, - ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); + if (nbytes <= original_nbytes) { + /* shrinking: mark old extra memory dead */ + memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST); } - /* Resize and add decorations. */ - r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); - if (r == NULL) { - /* if realloc() failed: rewrite header and footer which have - just been erased */ - nbytes = original_nbytes; + /* Resize and add decorations. We may get a new pointer here, in which + * case we didn't get the chance to mark the old memory with DEADBYTE, + * but we live with that. + */ + if (api == _PYMALLOC_OBJ_ID) { + q = (uchar *)PyObject_Realloc(q - 2*SST, total); } else { - head = r; -#ifdef PYMEM_DEBUG_SERIALNO - bumpserialno(); - block_serialno = serialno; -#endif + q = (uchar *)_PyMem_Realloc(q - 2*SST, total); } - data = head + 2*SST; - - write_size_t(head, nbytes); - head[SST] = (uint8_t)api->api_id; - memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1); - - tail = data + nbytes; - memset(tail, PYMEM_FORBIDDENBYTE, SST); -#ifdef PYMEM_DEBUG_SERIALNO - write_size_t(tail + SST, block_serialno); -#endif - - /* Restore saved bytes. */ - if (original_nbytes <= sizeof(save)) { - memcpy(data, save, Py_MIN(nbytes, original_nbytes)); - } - else { - size_t i = original_nbytes - ERASED_SIZE; - memcpy(data, save, Py_MIN(nbytes, ERASED_SIZE)); - if (nbytes > i) { - memcpy(data + i, &save[ERASED_SIZE], - Py_MIN(nbytes - i, ERASED_SIZE)); + if (q == NULL) { + if (nbytes <= original_nbytes) { + /* bpo-31626: the memset() above expects that realloc never fails + on shrinking a memory block. */ + Py_FatalError("Shrinking reallocation failed"); } - } - - if (r == NULL) { return NULL; } + write_size_t(q, nbytes); + assert(q[SST] == (uchar)api); + for (i = 1; i < SST; ++i) + assert(q[SST + i] == FORBIDDENBYTE); + q += 2*SST; + tail = q + nbytes; + memset(tail, FORBIDDENBYTE, SST); + write_size_t(tail + SST, serialno); + if (nbytes > original_nbytes) { - /* growing: mark new extra memory clean */ - memset(data + original_nbytes, PYMEM_CLEANBYTE, + /* growing: mark new extra memory clean */ + memset(q + original_nbytes, CLEANBYTE, nbytes - original_nbytes); } - return data; -} - -static inline void -_PyMem_DebugCheckGIL(void) -{ - if (!PyGILState_Check()) { - Py_FatalError("Python memory allocator called " - "without holding the GIL"); - } -} - -static void * -_PyMem_DebugMalloc(void *ctx, size_t nbytes) -{ - _PyMem_DebugCheckGIL(); - return _PyMem_DebugRawMalloc(ctx, nbytes); -} - -static void * -_PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize) -{ - _PyMem_DebugCheckGIL(); - return _PyMem_DebugRawCalloc(ctx, nelem, elsize); -} - - -static void -_PyMem_DebugFree(void *ctx, void *ptr) -{ - _PyMem_DebugCheckGIL(); - _PyMem_DebugRawFree(ctx, ptr); -} - - -static void * -_PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) -{ - _PyMem_DebugCheckGIL(); - return _PyMem_DebugRawRealloc(ctx, ptr, nbytes); + return q; } /* Check the forbidden bytes on both ends of the memory allocated for p. @@ -2357,14 +1647,14 @@ _PyMem_DebugRealloc(void *ctx, void *ptr, size_t nbytes) * and call Py_FatalError to kill the program. * The API id, is also checked. */ -static void -_PyMem_DebugCheckAddress(char api, const void *p) + void +_PyObject_DebugCheckAddressApi(char api, const void *p) { - const uint8_t *q = (const uint8_t *)p; + const uchar *q = (const uchar *)p; char msgbuf[64]; - const char *msg; + char *msg; size_t nbytes; - const uint8_t *tail; + const uchar *tail; int i; char id; @@ -2377,7 +1667,7 @@ _PyMem_DebugCheckAddress(char api, const void *p) id = (char)q[-SST]; if (id != api) { msg = msgbuf; - snprintf(msgbuf, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api); + snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api); msgbuf[sizeof(msgbuf)-1] = 0; goto error; } @@ -2387,7 +1677,7 @@ _PyMem_DebugCheckAddress(char api, const void *p) * the tail could lead to a segfault then. */ for (i = SST-1; i >= 1; --i) { - if (*(q-i) != PYMEM_FORBIDDENBYTE) { + if (*(q-i) != FORBIDDENBYTE) { msg = "bad leading pad byte"; goto error; } @@ -2396,7 +1686,7 @@ _PyMem_DebugCheckAddress(char api, const void *p) nbytes = read_size_t(q - 2*SST); tail = q + nbytes; for (i = 0; i < SST; ++i) { - if (tail[i] != PYMEM_FORBIDDENBYTE) { + if (tail[i] != FORBIDDENBYTE) { msg = "bad trailing pad byte"; goto error; } @@ -2410,12 +1700,12 @@ error: } /* Display info to stderr about the memory block at p. */ -static void +void _PyObject_DebugDumpAddress(const void *p) { - const uint8_t *q = (const uint8_t *)p; - const uint8_t *tail; - size_t nbytes; + const uchar *q = (const uchar *)p; + const uchar *tail; + size_t nbytes, serial; int i; int ok; char id; @@ -2436,7 +1726,7 @@ _PyObject_DebugDumpAddress(const void *p) fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1); ok = 1; for (i = 1; i <= SST-1; ++i) { - if (*(q-i) != PYMEM_FORBIDDENBYTE) { + if (*(q-i) != FORBIDDENBYTE) { ok = 0; break; } @@ -2445,11 +1735,11 @@ _PyObject_DebugDumpAddress(const void *p) fputs("FORBIDDENBYTE, as expected.\n", stderr); else { fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", - PYMEM_FORBIDDENBYTE); + FORBIDDENBYTE); for (i = SST-1; i >= 1; --i) { - const uint8_t byte = *(q-i); + const uchar byte = *(q-i); fprintf(stderr, " at p-%d: 0x%02x", i, byte); - if (byte != PYMEM_FORBIDDENBYTE) + if (byte != FORBIDDENBYTE) fputs(" *** OUCH", stderr); fputc('\n', stderr); } @@ -2461,10 +1751,10 @@ _PyObject_DebugDumpAddress(const void *p) } tail = q + nbytes; - fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail); + fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, tail); ok = 1; for (i = 0; i < SST; ++i) { - if (tail[i] != PYMEM_FORBIDDENBYTE) { + if (tail[i] != FORBIDDENBYTE) { ok = 0; break; } @@ -2473,22 +1763,20 @@ _PyObject_DebugDumpAddress(const void *p) fputs("FORBIDDENBYTE, as expected.\n", stderr); else { fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", - PYMEM_FORBIDDENBYTE); + FORBIDDENBYTE); for (i = 0; i < SST; ++i) { - const uint8_t byte = tail[i]; + const uchar byte = tail[i]; fprintf(stderr, " at tail+%d: 0x%02x", i, byte); - if (byte != PYMEM_FORBIDDENBYTE) + if (byte != FORBIDDENBYTE) fputs(" *** OUCH", stderr); fputc('\n', stderr); } } -#ifdef PYMEM_DEBUG_SERIALNO - size_t serial = read_size_t(tail + SST); + serial = read_size_t(tail + SST); fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T "u to debug malloc/realloc.\n", serial); -#endif if (nbytes > 0) { i = 0; @@ -2512,24 +1800,19 @@ _PyObject_DebugDumpAddress(const void *p) } fputc('\n', stderr); } - fputc('\n', stderr); - - fflush(stderr); - _PyMem_DumpTraceback(fileno(stderr), p); } - static size_t -printone(FILE *out, const char* msg, size_t value) +printone(const char* msg, size_t value) { int i, k; char buf[100]; size_t origvalue = value; - fputs(msg, out); + fputs(msg, stderr); for (i = (int)strlen(msg); i < 35; ++i) - fputc(' ', out); - fputc('=', out); + fputc(' ', stderr); + fputc('=', stderr); /* Write the value with commas. */ i = 22; @@ -2550,63 +1833,18 @@ printone(FILE *out, const char* msg, size_t value) while (i >= 0) buf[i--] = ' '; - fputs(buf, out); + fputs(buf, stderr); return origvalue; } -void -_PyDebugAllocatorStats(FILE *out, - const char *block_name, int num_blocks, size_t sizeof_block) -{ - char buf1[128]; - char buf2[128]; - PyOS_snprintf(buf1, sizeof(buf1), - "%d %ss * %" PY_FORMAT_SIZE_T "d bytes each", - num_blocks, block_name, sizeof_block); - PyOS_snprintf(buf2, sizeof(buf2), - "%48s ", buf1); - (void)printone(out, buf2, num_blocks * sizeof_block); -} - - -#ifdef WITH_PYMALLOC - -#ifdef Py_DEBUG -/* Is target in the list? The list is traversed via the nextpool pointers. - * The list may be NULL-terminated, or circular. Return 1 if target is in - * list, else 0. - */ -static int -pool_is_in_list(const poolp target, poolp list) -{ - poolp origlist = list; - assert(target != NULL); - if (list == NULL) - return 0; - do { - if (target == list) - return 1; - list = list->nextpool; - } while (list != NULL && list != origlist); - return 0; -} -#endif - -/* Print summary info to "out" about the state of pymalloc's structures. +/* Print summary info to stderr about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. - * - * Return 0 if the memory debug hooks are not installed or no statistics was - * written into out, return 1 otherwise. */ -int -_PyObject_DebugMallocStats(FILE *out) +void +_PyObject_DebugMallocStats(void) { - if (!_PyMem_PymallocEnabled()) { - return 0; - } - uint i; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; /* # of pools, allocated blocks, and free blocks per class index */ @@ -2634,7 +1872,7 @@ _PyObject_DebugMallocStats(FILE *out) size_t total; char buf[128]; - fprintf(out, "Small block threshold = %d, in %u size classes.\n", + fprintf(stderr, "Small block threshold = %d, in %u size classes.\n", SMALL_REQUEST_THRESHOLD, numclasses); for (i = 0; i < numclasses; ++i) @@ -2646,35 +1884,34 @@ _PyObject_DebugMallocStats(FILE *out) */ for (i = 0; i < maxarenas; ++i) { uint j; - uintptr_t base = arenas[i].address; + uptr base = arenas[i].address; /* Skip arenas which are not allocated. */ - if (arenas[i].address == (uintptr_t)NULL) + if (arenas[i].address == (uptr)NULL) continue; narenas += 1; numfreepools += arenas[i].nfreepools; /* round up to pool alignment */ - if (base & (uintptr_t)POOL_SIZE_MASK) { + if (base & (uptr)POOL_SIZE_MASK) { arena_alignment += POOL_SIZE; - base &= ~(uintptr_t)POOL_SIZE_MASK; + base &= ~(uptr)POOL_SIZE_MASK; base += POOL_SIZE; } /* visit every pool in the arena */ - assert(base <= (uintptr_t) arenas[i].pool_address); - for (j = 0; base < (uintptr_t) arenas[i].pool_address; - ++j, base += POOL_SIZE) { + assert(base <= (uptr) arenas[i].pool_address); + for (j = 0; + base < (uptr) arenas[i].pool_address; + ++j, base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; uint freeblocks; if (p->ref.count == 0) { /* currently unused */ -#ifdef Py_DEBUG assert(pool_is_in_list(p, arenas[i].freepools)); -#endif continue; } ++numpools[sz]; @@ -2689,10 +1926,10 @@ _PyObject_DebugMallocStats(FILE *out) } assert(narenas == narenas_currently_allocated); - fputc('\n', out); + fputc('\n', stderr); fputs("class size num pools blocks in use avail blocks\n" "----- ---- --------- ------------- ------------\n", - out); + stderr); for (i = 0; i < numclasses; ++i) { size_t p = numpools[i]; @@ -2703,7 +1940,7 @@ _PyObject_DebugMallocStats(FILE *out) assert(b == 0 && f == 0); continue; } - fprintf(out, "%5u %6u " + fprintf(stderr, "%5u %6u " "%11" PY_FORMAT_SIZE_T "u " "%15" PY_FORMAT_SIZE_T "u " "%13" PY_FORMAT_SIZE_T "u\n", @@ -2713,36 +1950,47 @@ _PyObject_DebugMallocStats(FILE *out) pool_header_bytes += p * POOL_OVERHEAD; quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } - fputc('\n', out); -#ifdef PYMEM_DEBUG_SERIALNO - if (_PyMem_DebugEnabled()) { - (void)printone(out, "# times object malloc called", serialno); - } -#endif - (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", narenas_highwater); - (void)printone(out, "# arenas allocated current", narenas); + fputc('\n', stderr); + (void)printone("# times object malloc called", serialno); + + (void)printone("# arenas allocated total", ntimes_arena_allocated); + (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone("# arenas highwater mark", narenas_highwater); + (void)printone("# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", narenas, ARENA_SIZE); - (void)printone(out, buf, narenas * ARENA_SIZE); + (void)printone(buf, narenas * ARENA_SIZE); - fputc('\n', out); + fputc('\n', stderr); - total = printone(out, "# bytes in allocated blocks", allocated_bytes); - total += printone(out, "# bytes in available blocks", available_bytes); + total = printone("# bytes in allocated blocks", allocated_bytes); + total += printone("# bytes in available blocks", available_bytes); PyOS_snprintf(buf, sizeof(buf), "%u unused pools * %d bytes", numfreepools, POOL_SIZE); - total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); + total += printone(buf, (size_t)numfreepools * POOL_SIZE); - total += printone(out, "# bytes lost to pool headers", pool_header_bytes); - total += printone(out, "# bytes lost to quantization", quantization); - total += printone(out, "# bytes lost to arena alignment", arena_alignment); - (void)printone(out, "Total", total); - return 1; + total += printone("# bytes lost to pool headers", pool_header_bytes); + total += printone("# bytes lost to quantization", quantization); + total += printone("# bytes lost to arena alignment", arena_alignment); + (void)printone("Total", total); } -#endif /* #ifdef WITH_PYMALLOC */ +#endif /* PYMALLOC_DEBUG */ + +#ifdef Py_USING_MEMORY_DEBUGGER +/* Make this function last so gcc won't inline it since the definition is + * after the reference. + */ +int +Py_ADDRESS_IN_RANGE(void *P, poolp pool) +{ + uint arenaindex_temp = pool->arenaindex; + + return arenaindex_temp < maxarenas && + (uptr)P - arenas[arenaindex_temp].address < (uptr)ARENA_SIZE && + arenas[arenaindex_temp].address != 0; +} +#endif |