diff options
Diffstat (limited to 'Objects/obmalloc.c')
| -rw-r--r-- | Objects/obmalloc.c | 772 |
1 files changed, 158 insertions, 614 deletions
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 32e7ecb..3698cfc 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -178,7 +178,9 @@ static struct { #define PYDBG_FUNCS \ _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree -static PyMemAllocatorEx _PyMem_Raw = { + +#define _PyMem_Raw _PyRuntime.mem.allocators.raw +static const PyMemAllocatorEx _pymem_raw = { #ifdef Py_DEBUG &_PyMem_Debug.raw, PYRAWDBG_FUNCS #else @@ -186,7 +188,8 @@ static PyMemAllocatorEx _PyMem_Raw = { #endif }; -static PyMemAllocatorEx _PyMem = { +#define _PyMem _PyRuntime.mem.allocators.mem +static const PyMemAllocatorEx _pymem = { #ifdef Py_DEBUG &_PyMem_Debug.mem, PYDBG_FUNCS #else @@ -194,7 +197,8 @@ static PyMemAllocatorEx _PyMem = { #endif }; -static PyMemAllocatorEx _PyObject = { +#define _PyObject _PyRuntime.mem.allocators.obj +static const PyMemAllocatorEx _pyobject = { #ifdef Py_DEBUG &_PyMem_Debug.obj, PYDBG_FUNCS #else @@ -267,7 +271,7 @@ _PyMem_SetupAllocators(const char *opt) #undef PYRAWDBG_FUNCS #undef PYDBG_FUNCS -static PyObjectArenaAllocator _PyObject_Arena = {NULL, +static const PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree #elif defined(ARENAS_USE_MMAP) @@ -277,6 +281,34 @@ static PyObjectArenaAllocator _PyObject_Arena = {NULL, #endif }; +void +_PyObject_Initialize(struct _pyobj_runtime_state *state) +{ + state->allocator_arenas = _PyObject_Arena; +} + +void +_PyMem_Initialize(struct _pymem_runtime_state *state) +{ + state->allocators.raw = _pymem_raw; + state->allocators.mem = _pymem; + state->allocators.obj = _pyobject; + +#ifdef WITH_PYMALLOC + for (int i = 0; i < 8; i++) { + if (NB_SMALL_SIZE_CLASSES <= i * 8) + break; + for (int j = 0; j < 8; j++) { + int x = i * 8 + j; + poolp *addr = &(state->usedpools[2*(x)]); + poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *)); + state->usedpools[x * 2] = val; + state->usedpools[x * 2 + 1] = val; + }; + }; +#endif /* WITH_PYMALLOC */ +} + #ifdef WITH_PYMALLOC static int _PyMem_DebugEnabled(void) @@ -363,13 +395,13 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) { - *allocator = _PyObject_Arena; + *allocator = _PyRuntime.obj.allocator_arenas; } void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) { - _PyObject_Arena = *allocator; + _PyRuntime.obj.allocator_arenas = *allocator; } void * @@ -404,7 +436,8 @@ PyMem_RawRealloc(void *ptr, size_t new_size) return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); } -void PyMem_RawFree(void *ptr) +void +PyMem_RawFree(void *ptr) { _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); } @@ -521,497 +554,10 @@ PyObject_Free(void *ptr) static int running_on_valgrind = -1; #endif -/* An object allocator for Python. - - Here is an introduction to the layers of the Python memory architecture, - showing where the object allocator is actually used (layer +2), It is - called for every object allocation and deallocation (PyObject_New/Del), - unless the object-specific allocators implement a proprietary allocation - scheme (ex.: ints use a simple free list). This is also the place where - the cyclic garbage collector operates selectively on container objects. - - - Object-specific allocators - _____ ______ ______ ________ - [ int ] [ dict ] [ list ] ... [ string ] Python core | -+3 | <----- Object-specific memory -----> | <-- Non-object memory --> | - _______________________________ | | - [ Python's object allocator ] | | -+2 | ####### Object memory ####### | <------ Internal buffers ------> | - ______________________________________________________________ | - [ Python's raw memory allocator (PyMem_ API) ] | -+1 | <----- Python memory (under PyMem manager's control) ------> | | - __________________________________________________________________ - [ Underlying general-purpose allocator (ex: C library malloc) ] - 0 | <------ Virtual memory allocated for the python process -------> | - - ========================================================================= - _______________________________________________________________________ - [ OS-specific Virtual Memory Manager (VMM) ] --1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | - __________________________________ __________________________________ - [ ] [ ] --2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | - -*/ -/*==========================================================================*/ - -/* A fast, special-purpose memory allocator for small blocks, to be used - on top of a general-purpose malloc -- heavily based on previous art. */ - -/* Vladimir Marangozov -- August 2000 */ - -/* - * "Memory management is where the rubber meets the road -- if we do the wrong - * thing at any level, the results will not be good. And if we don't make the - * levels work well together, we are in serious trouble." (1) - * - * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, - * "Dynamic Storage Allocation: A Survey and Critical Review", - * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. - */ - -/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ - -/*==========================================================================*/ - -/* - * Allocation strategy abstract: - * - * For small requests, the allocator sub-allocates <Big> blocks of memory. - * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the - * system's allocator. - * - * Small requests are grouped in size classes spaced 8 bytes apart, due - * to the required valid alignment of the returned address. Requests of - * a particular size are serviced from memory pools of 4K (one VMM page). - * Pools are fragmented on demand and contain free lists of blocks of one - * particular size class. In other words, there is a fixed-size allocator - * for each size class. Free pools are shared by the different allocators - * thus minimizing the space reserved for a particular size class. - * - * This allocation strategy is a variant of what is known as "simple - * segregated storage based on array of free lists". The main drawback of - * simple segregated storage is that we might end up with lot of reserved - * memory for the different free lists, which degenerate in time. To avoid - * this, we partition each free list in pools and we share dynamically the - * reserved space between all free lists. This technique is quite efficient - * for memory intensive programs which allocate mainly small-sized blocks. - * - * For small requests we have the following table: - * - * Request in bytes Size of allocated block Size class idx - * ---------------------------------------------------------------- - * 1-8 8 0 - * 9-16 16 1 - * 17-24 24 2 - * 25-32 32 3 - * 33-40 40 4 - * 41-48 48 5 - * 49-56 56 6 - * 57-64 64 7 - * 65-72 72 8 - * ... ... ... - * 497-504 504 62 - * 505-512 512 63 - * - * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying - * allocator. - */ - -/*==========================================================================*/ - -/* - * -- Main tunable settings section -- - */ - -/* - * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). - * The alignment value is also used for grouping small requests in size - * classes spaced ALIGNMENT bytes apart. - * - * You shouldn't change this unless you know what you are doing. - */ -#define ALIGNMENT 8 /* must be 2^N */ -#define ALIGNMENT_SHIFT 3 - -/* Return the number of bytes in size class I, as a uint. */ -#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT) - -/* - * Max size threshold below which malloc requests are considered to be - * small enough in order to use preallocated memory pools. You can tune - * this value according to your application behaviour and memory needs. - * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * - * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 - * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT - * - * Although not required, for better performance and space efficiency, - * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. - */ -#define SMALL_REQUEST_THRESHOLD 512 -#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) - -/* - * The system's VMM page size can be obtained on most unices with a - * getpagesize() call or deduced from various header files. To make - * things simpler, we assume that it is 4K, which is OK for most systems. - * It is probably better if this is the native page size, but it doesn't - * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page - * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation - * violation fault. 4K is apparently OK for all the platforms that python - * currently targets. - */ -#define SYSTEM_PAGE_SIZE (4 * 1024) -#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) - -/* - * Maximum amount of memory managed by the allocator for small requests. - */ -#ifdef WITH_MEMORY_LIMITS -#ifndef SMALL_MEMORY_LIMIT -#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */ -#endif -#endif - -/* - * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned - * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc()/mmap() call). In no way this - * means that the memory arenas will be used entirely. A malloc(<Big>) is - * usually an address range reservation for <Big> bytes, unless all pages within - * this space are referenced subsequently. So malloc'ing big blocks and not - * using them does not mean "wasting memory". It's an addressable range - * wastage... - * - * Arenas are allocated with mmap() on systems supporting anonymous memory - * mappings to reduce heap fragmentation. - */ -#define ARENA_SIZE (256 << 10) /* 256KB */ - -#ifdef WITH_MEMORY_LIMITS -#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) -#endif - -/* - * Size of the pools used for small blocks. Should be a power of 2, - * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. - */ -#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ -#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK - -/* - * -- End of tunable settings section -- - */ - -/*==========================================================================*/ - -/* - * Locking - * - * To reduce lock contention, it would probably be better to refine the - * crude function locking with per size class locking. I'm not positive - * however, whether it's worth switching to such locking policy because - * of the performance penalty it might introduce. - * - * The following macros describe the simplest (should also be the fastest) - * lock object on a particular platform and the init/fini/lock/unlock - * operations on it. The locks defined here are not expected to be recursive - * because it is assumed that they will always be called in the order: - * INIT, [LOCK, UNLOCK]*, FINI. - */ - -/* - * Python's threads are serialized, so object malloc locking is disabled. - */ -#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ -#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ -#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ -#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ -#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ - -/* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t block; - -/* Pool for small blocks. */ -struct pool_header { - union { block *_padding; - uint count; } ref; /* number of allocated blocks */ - block *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ - uint arenaindex; /* index into arenas of base adr */ - uint szidx; /* block size class index */ - uint nextoffset; /* bytes to virgin block */ - uint maxnextoffset; /* largest valid nextoffset */ -}; - -typedef struct pool_header *poolp; - -/* Record keeping for arenas. */ -struct arena_object { - /* The address of the arena, as returned by malloc. Note that 0 - * will never be returned by a successful malloc, and is used - * here to mark an arena_object that doesn't correspond to an - * allocated arena. - */ - uintptr_t address; - - /* Pool-aligned pointer to the next pool to be carved off. */ - block* pool_address; - - /* The number of available pools in the arena: free pools + never- - * allocated pools. - */ - uint nfreepools; - - /* The total number of pools in the arena, whether or not available. */ - uint ntotalpools; - - /* Singly-linked list of available pools. */ - struct pool_header* freepools; - - /* Whenever this arena_object is not associated with an allocated - * arena, the nextarena member is used to link all unassociated - * arena_objects in the singly-linked `unused_arena_objects` list. - * The prevarena member is unused in this case. - * - * When this arena_object is associated with an allocated arena - * with at least one available pool, both members are used in the - * doubly-linked `usable_arenas` list, which is maintained in - * increasing order of `nfreepools` values. - * - * Else this arena_object is associated with an allocated arena - * all of whose pools are in use. `nextarena` and `prevarena` - * are both meaningless in this case. - */ - struct arena_object* nextarena; - struct arena_object* prevarena; -}; - -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) - -#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ - -/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) - -/* Return total number of blocks in pool of size index I, as a uint. */ -#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) - -/*==========================================================================*/ - -/* - * This malloc lock - */ -SIMPLELOCK_DECL(_malloc_lock) -#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) -#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) -#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) -#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) - -/* - * Pool table -- headed, circular, doubly-linked lists of partially used pools. - -This is involved. For an index i, usedpools[i+i] is the header for a list of -all partially used pools holding small blocks with "size class idx" i. So -usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size -16, and so on: index 2*i <-> blocks of size (i+1)<<ALIGNMENT_SHIFT. - -Pools are carved off an arena's highwater mark (an arena_object's pool_address -member) as needed. Once carved off, a pool is in one of three states forever -after: - -used == partially used, neither empty nor full - At least one block in the pool is currently allocated, and at least one - block in the pool is not currently allocated (note this implies a pool - has room for at least two blocks). - This is a pool's initial state, as a pool is created only when malloc - needs space. - The pool holds blocks of a fixed size, and is in the circular list headed - at usedpools[i] (see above). It's linked to the other used pools of the - same size class via the pool_header's nextpool and prevpool members. - If all but one block is currently allocated, a malloc can cause a - transition to the full state. If all but one block is not currently - allocated, a free can cause a transition to the empty state. - -full == all the pool's blocks are currently allocated - On transition to full, a pool is unlinked from its usedpools[] list. - It's not linked to from anything then anymore, and its nextpool and - prevpool members are meaningless until it transitions back to used. - A free of a block in a full pool puts the pool back in the used state. - Then it's linked in at the front of the appropriate usedpools[] list, so - that the next allocation for its size class will reuse the freed block. - -empty == all the pool's blocks are currently available for allocation - On transition to empty, a pool is unlinked from its usedpools[] list, - and linked to the front of its arena_object's singly-linked freepools list, - via its nextpool member. The prevpool member has no meaning in this case. - Empty pools have no inherent size class: the next time a malloc finds - an empty list in usedpools[], it takes the first pool off of freepools. - If the size class needed happens to be the same as the size class the pool - last had, some pool initialization can be skipped. - - -Block Management - -Blocks within pools are again carved out as needed. pool->freeblock points to -the start of a singly-linked list of free blocks within the pool. When a -block is freed, it's inserted at the front of its pool's freeblock list. Note -that the available blocks in a pool are *not* linked all together when a pool -is initialized. Instead only "the first two" (lowest addresses) blocks are -set up, returning the first such block, and setting pool->freeblock to a -one-block list holding the second such block. This is consistent with that -pymalloc strives at all levels (arena, pool, and block) never to touch a piece -of memory until it's actually needed. - -So long as a pool is in the used state, we're certain there *is* a block -available for allocating, and pool->freeblock is not NULL. If pool->freeblock -points to the end of the free list before we've carved the entire pool into -blocks, that means we simply haven't yet gotten to one of the higher-address -blocks. The offset from the pool_header to the start of "the next" virgin -block is stored in the pool_header nextoffset member, and the largest value -of nextoffset that makes sense is stored in the maxnextoffset member when a -pool is initialized. All the blocks in a pool have been passed out at least -once when and only when nextoffset > maxnextoffset. - - -Major obscurity: While the usedpools vector is declared to have poolp -entries, it doesn't really. It really contains two pointers per (conceptual) -poolp entry, the nextpool and prevpool members of a pool_header. The -excruciating initialization code below fools C so that - - usedpool[i+i] - -"acts like" a genuine poolp, but only so long as you only reference its -nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is -compensating for that a pool_header's nextpool and prevpool members -immediately follow a pool_header's first two members: - - union { block *_padding; - uint count; } ref; - block *freeblock; - -each of which consume sizeof(block *) bytes. So what usedpools[i+i] really -contains is a fudged-up pointer p such that *if* C believes it's a poolp -pointer, then p->nextpool and p->prevpool are both p (meaning that the headed -circular list is empty). - -It's unclear why the usedpools setup is so convoluted. It could be to -minimize the amount of cache required to hold this heavily-referenced table -(which only *needs* the two interpool pointer members of a pool_header). OTOH, -referencing code has to remember to "double the index" and doing so isn't -free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying -on that C doesn't insert any padding anywhere in a pool_header at or before -the prevpool member. -**************************************************************************** */ - -#define PTA(x) ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *))) -#define PT(x) PTA(x), PTA(x) - -static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = { - PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7) -#if NB_SMALL_SIZE_CLASSES > 8 - , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15) -#if NB_SMALL_SIZE_CLASSES > 16 - , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23) -#if NB_SMALL_SIZE_CLASSES > 24 - , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31) -#if NB_SMALL_SIZE_CLASSES > 32 - , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39) -#if NB_SMALL_SIZE_CLASSES > 40 - , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47) -#if NB_SMALL_SIZE_CLASSES > 48 - , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) -#if NB_SMALL_SIZE_CLASSES > 56 - , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) -#if NB_SMALL_SIZE_CLASSES > 64 -#error "NB_SMALL_SIZE_CLASSES should be less than 64" -#endif /* NB_SMALL_SIZE_CLASSES > 64 */ -#endif /* NB_SMALL_SIZE_CLASSES > 56 */ -#endif /* NB_SMALL_SIZE_CLASSES > 48 */ -#endif /* NB_SMALL_SIZE_CLASSES > 40 */ -#endif /* NB_SMALL_SIZE_CLASSES > 32 */ -#endif /* NB_SMALL_SIZE_CLASSES > 24 */ -#endif /* NB_SMALL_SIZE_CLASSES > 16 */ -#endif /* NB_SMALL_SIZE_CLASSES > 8 */ -}; - -/*========================================================================== -Arena management. - -`arenas` is a vector of arena_objects. It contains maxarenas entries, some of -which may not be currently used (== they're arena_objects that aren't -currently associated with an allocated arena). Note that arenas proper are -separately malloc'ed. - -Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, -we do try to free() arenas, and use some mild heuristic strategies to increase -the likelihood that arenas eventually can be freed. - -unused_arena_objects - - This is a singly-linked list of the arena_objects that are currently not - being used (no arena is associated with them). Objects are taken off the - head of the list in new_arena(), and are pushed on the head of the list in - PyObject_Free() when the arena is empty. Key invariant: an arena_object - is on this list if and only if its .address member is 0. - -usable_arenas - - This is a doubly-linked list of the arena_objects associated with arenas - that have pools available. These pools are either waiting to be reused, - or have not been used before. The list is sorted to have the most- - allocated arenas first (ascending order based on the nfreepools member). - This means that the next allocation will come from a heavily used arena, - which gives the nearly empty arenas a chance to be returned to the system. - In my unscientific tests this dramatically improved the number of arenas - that could be freed. - -Note that an arena_object associated with an arena all of whose pools are -currently in use isn't on either list. -*/ - -/* Array of objects used to track chunks of memory (arenas). */ -static struct arena_object* arenas = NULL; -/* Number of slots currently allocated in the `arenas` vector. */ -static uint maxarenas = 0; - -/* The head of the singly-linked, NULL-terminated list of available - * arena_objects. - */ -static struct arena_object* unused_arena_objects = NULL; - -/* The head of the doubly-linked, NULL-terminated at each end, list of - * arena_objects associated with arenas that have pools available. - */ -static struct arena_object* usable_arenas = NULL; - -/* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the - * `arenas` vector. - */ -#define INITIAL_ARENA_OBJECTS 16 - -/* Number of arenas allocated that haven't been free()'d. */ -static size_t narenas_currently_allocated = 0; - -/* Total number of times malloc() called to allocate an arena. */ -static size_t ntimes_arena_allocated = 0; -/* High water mark (max value ever seen) for narenas_currently_allocated. */ -static size_t narenas_highwater = 0; - -static Py_ssize_t _Py_AllocatedBlocks = 0; - Py_ssize_t _Py_GetAllocatedBlocks(void) { - return _Py_AllocatedBlocks; + return _PyRuntime.mem.num_allocated_blocks; } @@ -1035,7 +581,7 @@ new_arena(void) if (debug_stats) _PyObject_DebugMallocStats(stderr); - if (unused_arena_objects == NULL) { + if (_PyRuntime.mem.unused_arena_objects == NULL) { uint i; uint numarenas; size_t nbytes; @@ -1043,18 +589,18 @@ new_arena(void) /* Double the number of arena objects on each allocation. * Note that it's possible for `numarenas` to overflow. */ - numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS; - if (numarenas <= maxarenas) + numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS; + if (numarenas <= _PyRuntime.mem.maxarenas) return NULL; /* overflow */ #if SIZEOF_SIZE_T <= SIZEOF_INT - if (numarenas > SIZE_MAX / sizeof(*arenas)) + if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas)) return NULL; /* overflow */ #endif - nbytes = numarenas * sizeof(*arenas); - arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); + nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas); + arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes); if (arenaobj == NULL) return NULL; - arenas = arenaobj; + _PyRuntime.mem.arenas = arenaobj; /* We might need to fix pointers that were copied. However, * new_arena only gets called when all the pages in the @@ -1062,45 +608,45 @@ new_arena(void) * into the old array. Thus, we don't have to worry about * invalid pointers. Just to be sure, some asserts: */ - assert(usable_arenas == NULL); - assert(unused_arena_objects == NULL); + assert(_PyRuntime.mem.usable_arenas == NULL); + assert(_PyRuntime.mem.unused_arena_objects == NULL); /* Put the new arenas on the unused_arena_objects list. */ - for (i = maxarenas; i < numarenas; ++i) { - arenas[i].address = 0; /* mark as unassociated */ - arenas[i].nextarena = i < numarenas - 1 ? - &arenas[i+1] : NULL; + for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) { + _PyRuntime.mem.arenas[i].address = 0; /* mark as unassociated */ + _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ? + &_PyRuntime.mem.arenas[i+1] : NULL; } /* Update globals. */ - unused_arena_objects = &arenas[maxarenas]; - maxarenas = numarenas; + _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas]; + _PyRuntime.mem.maxarenas = numarenas; } /* Take the next available arena object off the head of the list. */ - assert(unused_arena_objects != NULL); - arenaobj = unused_arena_objects; - unused_arena_objects = arenaobj->nextarena; + assert(_PyRuntime.mem.unused_arena_objects != NULL); + arenaobj = _PyRuntime.mem.unused_arena_objects; + _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); - address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE); + address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE); if (address == NULL) { /* The allocation failed: return NULL after putting the * arenaobj back. */ - arenaobj->nextarena = unused_arena_objects; - unused_arena_objects = arenaobj; + arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects; + _PyRuntime.mem.unused_arena_objects = arenaobj; return NULL; } arenaobj->address = (uintptr_t)address; - ++narenas_currently_allocated; - ++ntimes_arena_allocated; - if (narenas_currently_allocated > narenas_highwater) - narenas_highwater = narenas_currently_allocated; + ++_PyRuntime.mem.narenas_currently_allocated; + ++_PyRuntime.mem.ntimes_arena_allocated; + if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater) + _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated; arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ - arenaobj->pool_address = (block*)arenaobj->address; + arenaobj->pool_address = (pyblock*)arenaobj->address; arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE; assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE); excess = (uint)(arenaobj->address & POOL_SIZE_MASK); @@ -1197,9 +743,9 @@ address_in_range(void *p, poolp pool) // the GIL. The following dance forces the compiler to read pool->arenaindex // only once. uint arenaindex = *((volatile uint *)&pool->arenaindex); - return arenaindex < maxarenas && - (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE && - arenas[arenaindex].address != 0; + return arenaindex < _PyRuntime.mem.maxarenas && + (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE && + _PyRuntime.mem.arenas[arenaindex].address != 0; } /*==========================================================================*/ @@ -1220,12 +766,12 @@ static void * _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) { size_t nbytes; - block *bp; + pyblock *bp; poolp pool; poolp next; uint size; - _Py_AllocatedBlocks++; + _PyRuntime.mem.num_allocated_blocks++; assert(elsize == 0 || nelem <= PY_SSIZE_T_MAX / elsize); nbytes = nelem * elsize; @@ -1246,7 +792,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) * Most frequent paths first */ size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT; - pool = usedpools[size + size]; + pool = _PyRuntime.mem.usedpools[size + size]; if (pool != pool->nextpool) { /* * There is a used pool for this size class. @@ -1255,7 +801,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) ++pool->ref.count; bp = pool->freeblock; assert(bp != NULL); - if ((pool->freeblock = *(block **)bp) != NULL) { + if ((pool->freeblock = *(pyblock **)bp) != NULL) { UNLOCK(); if (use_calloc) memset(bp, 0, nbytes); @@ -1266,10 +812,10 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) */ if (pool->nextoffset <= pool->maxnextoffset) { /* There is room for another block. */ - pool->freeblock = (block*)pool + + pool->freeblock = (pyblock*)pool + pool->nextoffset; pool->nextoffset += INDEX2SIZE(size); - *(block **)(pool->freeblock) = NULL; + *(pyblock **)(pool->freeblock) = NULL; UNLOCK(); if (use_calloc) memset(bp, 0, nbytes); @@ -1289,29 +835,29 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) /* There isn't a pool of the right size class immediately * available: use a free pool. */ - if (usable_arenas == NULL) { + if (_PyRuntime.mem.usable_arenas == NULL) { /* No arena has a free pool: allocate a new arena. */ #ifdef WITH_MEMORY_LIMITS - if (narenas_currently_allocated >= MAX_ARENAS) { + if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) { UNLOCK(); goto redirect; } #endif - usable_arenas = new_arena(); - if (usable_arenas == NULL) { + _PyRuntime.mem.usable_arenas = new_arena(); + if (_PyRuntime.mem.usable_arenas == NULL) { UNLOCK(); goto redirect; } - usable_arenas->nextarena = - usable_arenas->prevarena = NULL; + _PyRuntime.mem.usable_arenas->nextarena = + _PyRuntime.mem.usable_arenas->prevarena = NULL; } - assert(usable_arenas->address != 0); + assert(_PyRuntime.mem.usable_arenas->address != 0); /* Try to get a cached free pool. */ - pool = usable_arenas->freepools; + pool = _PyRuntime.mem.usable_arenas->freepools; if (pool != NULL) { /* Unlink from cached pools. */ - usable_arenas->freepools = pool->nextpool; + _PyRuntime.mem.usable_arenas->freepools = pool->nextpool; /* This arena already had the smallest nfreepools * value, so decreasing nfreepools doesn't change @@ -1320,18 +866,18 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) * become wholly allocated, we need to remove its * arena_object from usable_arenas. */ - --usable_arenas->nfreepools; - if (usable_arenas->nfreepools == 0) { + --_PyRuntime.mem.usable_arenas->nfreepools; + if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { /* Wholly allocated: remove. */ - assert(usable_arenas->freepools == NULL); - assert(usable_arenas->nextarena == NULL || - usable_arenas->nextarena->prevarena == - usable_arenas); - - usable_arenas = usable_arenas->nextarena; - if (usable_arenas != NULL) { - usable_arenas->prevarena = NULL; - assert(usable_arenas->address != 0); + assert(_PyRuntime.mem.usable_arenas->freepools == NULL); + assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || + _PyRuntime.mem.usable_arenas->nextarena->prevarena == + _PyRuntime.mem.usable_arenas); + + _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; + if (_PyRuntime.mem.usable_arenas != NULL) { + _PyRuntime.mem.usable_arenas->prevarena = NULL; + assert(_PyRuntime.mem.usable_arenas->address != 0); } } else { @@ -1340,14 +886,14 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) * off all the arena's pools for the first * time. */ - assert(usable_arenas->freepools != NULL || - usable_arenas->pool_address <= - (block*)usable_arenas->address + + assert(_PyRuntime.mem.usable_arenas->freepools != NULL || + _PyRuntime.mem.usable_arenas->pool_address <= + (pyblock*)_PyRuntime.mem.usable_arenas->address + ARENA_SIZE - POOL_SIZE); } init_pool: /* Frontlink to used pools. */ - next = usedpools[size + size]; /* == prev */ + next = _PyRuntime.mem.usedpools[size + size]; /* == prev */ pool->nextpool = next; pool->prevpool = next; next->nextpool = pool; @@ -1360,7 +906,7 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) */ bp = pool->freeblock; assert(bp != NULL); - pool->freeblock = *(block **)bp; + pool->freeblock = *(pyblock **)bp; UNLOCK(); if (use_calloc) memset(bp, 0, nbytes); @@ -1373,11 +919,11 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) */ pool->szidx = size; size = INDEX2SIZE(size); - bp = (block *)pool + POOL_OVERHEAD; + bp = (pyblock *)pool + POOL_OVERHEAD; pool->nextoffset = POOL_OVERHEAD + (size << 1); pool->maxnextoffset = POOL_SIZE - size; pool->freeblock = bp + size; - *(block **)(pool->freeblock) = NULL; + *(pyblock **)(pool->freeblock) = NULL; UNLOCK(); if (use_calloc) memset(bp, 0, nbytes); @@ -1385,26 +931,26 @@ _PyObject_Alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) } /* Carve off a new pool. */ - assert(usable_arenas->nfreepools > 0); - assert(usable_arenas->freepools == NULL); - pool = (poolp)usable_arenas->pool_address; - assert((block*)pool <= (block*)usable_arenas->address + - ARENA_SIZE - POOL_SIZE); - pool->arenaindex = (uint)(usable_arenas - arenas); - assert(&arenas[pool->arenaindex] == usable_arenas); + assert(_PyRuntime.mem.usable_arenas->nfreepools > 0); + assert(_PyRuntime.mem.usable_arenas->freepools == NULL); + pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address; + assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address + + ARENA_SIZE - POOL_SIZE); + pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas); + assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas); pool->szidx = DUMMY_SIZE_IDX; - usable_arenas->pool_address += POOL_SIZE; - --usable_arenas->nfreepools; + _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE; + --_PyRuntime.mem.usable_arenas->nfreepools; - if (usable_arenas->nfreepools == 0) { - assert(usable_arenas->nextarena == NULL || - usable_arenas->nextarena->prevarena == - usable_arenas); + if (_PyRuntime.mem.usable_arenas->nfreepools == 0) { + assert(_PyRuntime.mem.usable_arenas->nextarena == NULL || + _PyRuntime.mem.usable_arenas->nextarena->prevarena == + _PyRuntime.mem.usable_arenas); /* Unlink the arena: it is completely allocated. */ - usable_arenas = usable_arenas->nextarena; - if (usable_arenas != NULL) { - usable_arenas->prevarena = NULL; - assert(usable_arenas->address != 0); + _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena; + if (_PyRuntime.mem.usable_arenas != NULL) { + _PyRuntime.mem.usable_arenas->prevarena = NULL; + assert(_PyRuntime.mem.usable_arenas->address != 0); } } @@ -1426,7 +972,7 @@ redirect: else result = PyMem_RawMalloc(nbytes); if (!result) - _Py_AllocatedBlocks--; + _PyRuntime.mem.num_allocated_blocks--; return result; } } @@ -1449,14 +995,14 @@ static void _PyObject_Free(void *ctx, void *p) { poolp pool; - block *lastfree; + pyblock *lastfree; poolp next, prev; uint size; if (p == NULL) /* free(NULL) has no effect */ return; - _Py_AllocatedBlocks--; + _PyRuntime.mem.num_allocated_blocks--; #ifdef WITH_VALGRIND if (UNLIKELY(running_on_valgrind > 0)) @@ -1474,8 +1020,8 @@ _PyObject_Free(void *ctx, void *p) * list in any case). */ assert(pool->ref.count > 0); /* else it was empty */ - *(block **)p = lastfree = pool->freeblock; - pool->freeblock = (block *)p; + *(pyblock **)p = lastfree = pool->freeblock; + pool->freeblock = (pyblock *)p; if (lastfree) { struct arena_object* ao; uint nf; /* ao->nfreepools */ @@ -1501,7 +1047,7 @@ _PyObject_Free(void *ctx, void *p) /* Link the pool to freepools. This is a singly-linked * list, and pool->prevpool isn't used there. */ - ao = &arenas[pool->arenaindex]; + ao = &_PyRuntime.mem.arenas[pool->arenaindex]; pool->nextpool = ao->freepools; ao->freepools = pool; nf = ++ao->nfreepools; @@ -1530,9 +1076,9 @@ _PyObject_Free(void *ctx, void *p) * usable_arenas pointer. */ if (ao->prevarena == NULL) { - usable_arenas = ao->nextarena; - assert(usable_arenas == NULL || - usable_arenas->address != 0); + _PyRuntime.mem.usable_arenas = ao->nextarena; + assert(_PyRuntime.mem.usable_arenas == NULL || + _PyRuntime.mem.usable_arenas->address != 0); } else { assert(ao->prevarena->nextarena == ao); @@ -1548,14 +1094,14 @@ _PyObject_Free(void *ctx, void *p) /* Record that this arena_object slot is * available to be reused. */ - ao->nextarena = unused_arena_objects; - unused_arena_objects = ao; + ao->nextarena = _PyRuntime.mem.unused_arena_objects; + _PyRuntime.mem.unused_arena_objects = ao; /* Free the entire arena. */ - _PyObject_Arena.free(_PyObject_Arena.ctx, + _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx, (void *)ao->address, ARENA_SIZE); ao->address = 0; /* mark unassociated */ - --narenas_currently_allocated; + --_PyRuntime.mem.narenas_currently_allocated; UNLOCK(); return; @@ -1566,12 +1112,12 @@ _PyObject_Free(void *ctx, void *p) * ao->nfreepools was 0 before, ao isn't * currently on the usable_arenas list. */ - ao->nextarena = usable_arenas; + ao->nextarena = _PyRuntime.mem.usable_arenas; ao->prevarena = NULL; - if (usable_arenas) - usable_arenas->prevarena = ao; - usable_arenas = ao; - assert(usable_arenas->address != 0); + if (_PyRuntime.mem.usable_arenas) + _PyRuntime.mem.usable_arenas->prevarena = ao; + _PyRuntime.mem.usable_arenas = ao; + assert(_PyRuntime.mem.usable_arenas->address != 0); UNLOCK(); return; @@ -1601,8 +1147,8 @@ _PyObject_Free(void *ctx, void *p) } else { /* ao is at the head of the list */ - assert(usable_arenas == ao); - usable_arenas = ao->nextarena; + assert(_PyRuntime.mem.usable_arenas == ao); + _PyRuntime.mem.usable_arenas = ao->nextarena; } ao->nextarena->prevarena = ao->prevarena; @@ -1631,7 +1177,7 @@ _PyObject_Free(void *ctx, void *p) nf > ao->prevarena->nfreepools); assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao); - assert((usable_arenas == ao && + assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao); @@ -1647,7 +1193,7 @@ _PyObject_Free(void *ctx, void *p) --pool->ref.count; assert(pool->ref.count > 0); /* else the pool is empty */ size = pool->szidx; - next = usedpools[size + size]; + next = _PyRuntime.mem.usedpools[size + size]; prev = next->prevpool; /* insert pool before next: prev <-> pool <-> next */ pool->nextpool = next; @@ -1769,15 +1315,13 @@ _Py_GetAllocatedBlocks(void) #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ -static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ - /* serialno is always incremented via calling this routine. The point is * to supply a single place to set a breakpoint. */ static void bumpserialno(void) { - ++serialno; + ++_PyRuntime.mem.serialno; } #define SST SIZEOF_SIZE_T @@ -1868,7 +1412,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = p + 2*SST + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, serialno); + write_size_t(tail + SST, _PyRuntime.mem.serialno); return p + 2*SST; } @@ -1953,7 +1497,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) tail = q + nbytes; memset(tail, FORBIDDENBYTE, SST); - write_size_t(tail + SST, serialno); + write_size_t(tail + SST, _PyRuntime.mem.serialno); if (nbytes > original_nbytes) { /* growing: mark new extra memory clean */ @@ -2285,16 +1829,16 @@ _PyObject_DebugMallocStats(FILE *out) * to march over all the arenas. If we're lucky, most of the memory * will be living in full pools -- would be a shame to miss them. */ - for (i = 0; i < maxarenas; ++i) { + for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) { uint j; - uintptr_t base = arenas[i].address; + uintptr_t base = _PyRuntime.mem.arenas[i].address; /* Skip arenas which are not allocated. */ - if (arenas[i].address == (uintptr_t)NULL) + if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL) continue; narenas += 1; - numfreepools += arenas[i].nfreepools; + numfreepools += _PyRuntime.mem.arenas[i].nfreepools; /* round up to pool alignment */ if (base & (uintptr_t)POOL_SIZE_MASK) { @@ -2304,8 +1848,8 @@ _PyObject_DebugMallocStats(FILE *out) } /* visit every pool in the arena */ - assert(base <= (uintptr_t) arenas[i].pool_address); - for (j = 0; base < (uintptr_t) arenas[i].pool_address; + assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address); + for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address; ++j, base += POOL_SIZE) { poolp p = (poolp)base; const uint sz = p->szidx; @@ -2314,7 +1858,7 @@ _PyObject_DebugMallocStats(FILE *out) if (p->ref.count == 0) { /* currently unused */ #ifdef Py_DEBUG - assert(pool_is_in_list(p, arenas[i].freepools)); + assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools)); #endif continue; } @@ -2324,11 +1868,11 @@ _PyObject_DebugMallocStats(FILE *out) numfreeblocks[sz] += freeblocks; #ifdef Py_DEBUG if (freeblocks > 0) - assert(pool_is_in_list(p, usedpools[sz + sz])); + assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz])); #endif } } - assert(narenas == narenas_currently_allocated); + assert(narenas == _PyRuntime.mem.narenas_currently_allocated); fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" @@ -2356,10 +1900,10 @@ _PyObject_DebugMallocStats(FILE *out) } fputc('\n', out); if (_PyMem_DebugEnabled()) - (void)printone(out, "# times object malloc called", serialno); - (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", narenas_highwater); + (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno); + (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater); (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), |