bpo-32096: Remove obj and mem from _PyRuntime (#4532)

bpo-32096, bpo-30860:  Partially revert the commit
2ebc5ce42a8a9e047e790aefbf9a94811569b2b6:

* Move structures back from Include/internal/mem.h to
  Objects/obmalloc.c
* Remove _PyObject_Initialize() and _PyMem_Initialize()
* Remove Include/internal/pymalloc.h
* Add test_capi.test_pre_initialization_api():
   Make sure that it's possible to call Py_DecodeLocale(), and then call
   Py_SetProgramName() with the decoded string, before Py_Initialize().

PyMem_RawMalloc() and Py_DecodeLocale() can be called again before
_PyRuntimeState_Init().

Co-Authored-By: Eric Snow <ericsnowcurrently@gmail.com>

1 files changed, 647 insertions, 186 deletions

diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 7c6973e..96a451e 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1,6 +1,4 @@
 #include "Python.h"
-#include "internal/mem.h"
-#include "internal/pystate.h"
 
 #include <stdbool.h>
 
@@ -180,24 +178,39 @@ static struct {
 #define PYDBG_FUNCS \
     _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree
 
+static PyMemAllocatorEx _PyMem_Raw = {
+#ifdef Py_DEBUG
+    &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+#else
+    NULL, PYRAW_FUNCS
+#endif
+    };
 
-#define _PyMem_Raw _PyRuntime.mem.allocators.raw
-
-#define _PyMem _PyRuntime.mem.allocators.mem
+static PyMemAllocatorEx _PyMem = {
+#ifdef Py_DEBUG
+    &_PyMem_Debug.mem, PYDBG_FUNCS
+#else
+    NULL, PYMEM_FUNCS
+#endif
+    };
 
-#define _PyObject _PyRuntime.mem.allocators.obj
+static PyMemAllocatorEx _PyObject = {
+#ifdef Py_DEBUG
+    &_PyMem_Debug.obj, PYDBG_FUNCS
+#else
+    NULL, PYOBJ_FUNCS
+#endif
+    };
 
 void
 _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc_p)
 {
-    PyMemAllocatorEx pymem_raw = {
 #ifdef Py_DEBUG
-        &_PyMem_Debug.raw, PYRAWDBG_FUNCS
+    PyMemAllocatorEx alloc = {&_PyMem_Debug.raw, PYDBG_FUNCS};
 #else
-        NULL, PYRAW_FUNCS
+    PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS};
 #endif
-    };
-    *alloc_p = pymem_raw;
+    *alloc_p = alloc;
 }
 
 int
@@ -259,62 +272,22 @@ _PyMem_SetupAllocators(const char *opt)
     return 0;
 }
 
+#undef PYRAW_FUNCS
+#undef PYMEM_FUNCS
+#undef PYOBJ_FUNCS
+#undef PYRAWDBG_FUNCS
+#undef PYDBG_FUNCS
 
-void
-_PyObject_Initialize(struct _pyobj_runtime_state *state)
-{
-    PyObjectArenaAllocator _PyObject_Arena = {NULL,
+static PyObjectArenaAllocator _PyObject_Arena = {NULL,
 #ifdef MS_WINDOWS
-        _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
+    _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree
 #elif defined(ARENAS_USE_MMAP)
-        _PyObject_ArenaMmap, _PyObject_ArenaMunmap
+    _PyObject_ArenaMmap, _PyObject_ArenaMunmap
 #else
-        _PyObject_ArenaMalloc, _PyObject_ArenaFree
+    _PyObject_ArenaMalloc, _PyObject_ArenaFree
 #endif
     };
 
-    state->allocator_arenas = _PyObject_Arena;
-}
-
-
-void
-_PyMem_Initialize(struct _pymem_runtime_state *state)
-{
-    PyMemAllocatorEx pymem = {
-#ifdef Py_DEBUG
-        &_PyMem_Debug.mem, PYDBG_FUNCS
-#else
-        NULL, PYMEM_FUNCS
-#endif
-    };
-    PyMemAllocatorEx pyobject = {
-#ifdef Py_DEBUG
-        &_PyMem_Debug.obj, PYDBG_FUNCS
-#else
-        NULL, PYOBJ_FUNCS
-#endif
-    };
-
-    _PyMem_GetDefaultRawAllocator(&state->allocators.raw);
-    state->allocators.mem = pymem;
-    state->allocators.obj = pyobject;
-
-#ifdef WITH_PYMALLOC
-    Py_BUILD_ASSERT(NB_SMALL_SIZE_CLASSES == 64);
-
-    for (int i = 0; i < 8; i++) {
-        for (int j = 0; j < 8; j++) {
-            int x = i * 8 + j;
-            poolp *addr = &(state->usedpools[2*(x)]);
-            poolp val = (poolp)((uint8_t *)addr - 2*sizeof(pyblock *));
-            state->usedpools[x * 2] = val;
-            state->usedpools[x * 2 + 1] = val;
-        };
-    };
-#endif /* WITH_PYMALLOC */
-}
-
-
 #ifdef WITH_PYMALLOC
 static int
 _PyMem_DebugEnabled(void)
@@ -401,13 +374,13 @@ PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator)
 void
 PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator)
 {
-    *allocator = _PyRuntime.obj.allocator_arenas;
+    *allocator = _PyObject_Arena;
 }
 
 void
 PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator)
 {
-    _PyRuntime.obj.allocator_arenas = *allocator;
+    _PyObject_Arena = *allocator;
 }
 
 void *
@@ -442,8 +415,7 @@ PyMem_RawRealloc(void *ptr, size_t new_size)
     return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size);
 }
 
-void
-PyMem_RawFree(void *ptr)
+void PyMem_RawFree(void *ptr)
 {
     _PyMem_Raw.free(_PyMem_Raw.ctx, ptr);
 }
@@ -563,10 +535,497 @@ static int running_on_valgrind = -1;
 #endif
 
 
+/* An object allocator for Python.
+
+   Here is an introduction to the layers of the Python memory architecture,
+   showing where the object allocator is actually used (layer +2), It is
+   called for every object allocation and deallocation (PyObject_New/Del),
+   unless the object-specific allocators implement a proprietary allocation
+   scheme (ex.: ints use a simple free list). This is also the place where
+   the cyclic garbage collector operates selectively on container objects.
+
+
+    Object-specific allocators
+    _____   ______   ______       ________
+   [ int ] [ dict ] [ list ] ... [ string ]       Python core         |
++3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
+    _______________________________       |                           |
+   [   Python's object allocator   ]      |                           |
++2 | ####### Object memory ####### | <------ Internal buffers ------> |
+    ______________________________________________________________    |
+   [          Python's raw memory allocator (PyMem_ API)          ]   |
++1 | <----- Python memory (under PyMem manager's control) ------> |   |
+    __________________________________________________________________
+   [    Underlying general-purpose allocator (ex: C library malloc)   ]
+ 0 | <------ Virtual memory allocated for the python process -------> |
+
+   =========================================================================
+    _______________________________________________________________________
+   [                OS-specific Virtual Memory Manager (VMM)               ]
+-1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
+    __________________________________   __________________________________
+   [                                  ] [                                  ]
+-2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
+
+*/
+/*==========================================================================*/
+
+/* A fast, special-purpose memory allocator for small blocks, to be used
+   on top of a general-purpose malloc -- heavily based on previous art. */
+
+/* Vladimir Marangozov -- August 2000 */
+
+/*
+ * "Memory management is where the rubber meets the road -- if we do the wrong
+ * thing at any level, the results will not be good. And if we don't make the
+ * levels work well together, we are in serious trouble." (1)
+ *
+ * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
+ *    "Dynamic Storage Allocation: A Survey and Critical Review",
+ *    in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
+ */
+
+/* #undef WITH_MEMORY_LIMITS */         /* disable mem limit checks  */
+
+/*==========================================================================*/
+
+/*
+ * Allocation strategy abstract:
+ *
+ * For small requests, the allocator sub-allocates <Big> blocks of memory.
+ * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the
+ * system's allocator.
+ *
+ * Small requests are grouped in size classes spaced 8 bytes apart, due
+ * to the required valid alignment of the returned address. Requests of
+ * a particular size are serviced from memory pools of 4K (one VMM page).
+ * Pools are fragmented on demand and contain free lists of blocks of one
+ * particular size class. In other words, there is a fixed-size allocator
+ * for each size class. Free pools are shared by the different allocators
+ * thus minimizing the space reserved for a particular size class.
+ *
+ * This allocation strategy is a variant of what is known as "simple
+ * segregated storage based on array of free lists". The main drawback of
+ * simple segregated storage is that we might end up with lot of reserved
+ * memory for the different free lists, which degenerate in time. To avoid
+ * this, we partition each free list in pools and we share dynamically the
+ * reserved space between all free lists. This technique is quite efficient
+ * for memory intensive programs which allocate mainly small-sized blocks.
+ *
+ * For small requests we have the following table:
+ *
+ * Request in bytes     Size of allocated block      Size class idx
+ * ----------------------------------------------------------------
+ *        1-8                     8                       0
+ *        9-16                   16                       1
+ *       17-24                   24                       2
+ *       25-32                   32                       3
+ *       33-40                   40                       4
+ *       41-48                   48                       5
+ *       49-56                   56                       6
+ *       57-64                   64                       7
+ *       65-72                   72                       8
+ *        ...                   ...                     ...
+ *      497-504                 504                      62
+ *      505-512                 512                      63
+ *
+ *      0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying
+ *      allocator.
+ */
+
+/*==========================================================================*/
+
+/*
+ * -- Main tunable settings section --
+ */
+
+/*
+ * Alignment of addresses returned to the user. 8-bytes alignment works
+ * on most current architectures (with 32-bit or 64-bit address busses).
+ * The alignment value is also used for grouping small requests in size
+ * classes spaced ALIGNMENT bytes apart.
+ *
+ * You shouldn't change this unless you know what you are doing.
+ */
+#define ALIGNMENT               8               /* must be 2^N */
+#define ALIGNMENT_SHIFT         3
+
+/* Return the number of bytes in size class I, as a uint. */
+#define INDEX2SIZE(I) (((uint)(I) + 1) << ALIGNMENT_SHIFT)
+
+/*
+ * Max size threshold below which malloc requests are considered to be
+ * small enough in order to use preallocated memory pools. You can tune
+ * this value according to your application behaviour and memory needs.
+ *
+ * Note: a size threshold of 512 guarantees that newly created dictionaries
+ * will be allocated from preallocated memory pools on 64-bit.
+ *
+ * The following invariants must hold:
+ *      1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512
+ *      2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT
+ *
+ * Although not required, for better performance and space efficiency,
+ * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.
+ */
+#define SMALL_REQUEST_THRESHOLD 512
+#define NB_SMALL_SIZE_CLASSES   (SMALL_REQUEST_THRESHOLD / ALIGNMENT)
+
+/*
+ * The system's VMM page size can be obtained on most unices with a
+ * getpagesize() call or deduced from various header files. To make
+ * things simpler, we assume that it is 4K, which is OK for most systems.
+ * It is probably better if this is the native page size, but it doesn't
+ * have to be.  In theory, if SYSTEM_PAGE_SIZE is larger than the native page
+ * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation
+ * violation fault.  4K is apparently OK for all the platforms that python
+ * currently targets.
+ */
+#define SYSTEM_PAGE_SIZE        (4 * 1024)
+#define SYSTEM_PAGE_SIZE_MASK   (SYSTEM_PAGE_SIZE - 1)
+
+/*
+ * Maximum amount of memory managed by the allocator for small requests.
+ */
+#ifdef WITH_MEMORY_LIMITS
+#ifndef SMALL_MEMORY_LIMIT
+#define SMALL_MEMORY_LIMIT      (64 * 1024 * 1024)      /* 64 MB -- more? */
+#endif
+#endif
+
+/*
+ * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
+ * on a page boundary. This is a reserved virtual address space for the
+ * current process (obtained through a malloc()/mmap() call). In no way this
+ * means that the memory arenas will be used entirely. A malloc(<Big>) is
+ * usually an address range reservation for <Big> bytes, unless all pages within
+ * this space are referenced subsequently. So malloc'ing big blocks and not
+ * using them does not mean "wasting memory". It's an addressable range
+ * wastage...
+ *
+ * Arenas are allocated with mmap() on systems supporting anonymous memory
+ * mappings to reduce heap fragmentation.
+ */
+#define ARENA_SIZE              (256 << 10)     /* 256KB */
+
+#ifdef WITH_MEMORY_LIMITS
+#define MAX_ARENAS              (SMALL_MEMORY_LIMIT / ARENA_SIZE)
+#endif
+
+/*
+ * Size of the pools used for small blocks. Should be a power of 2,
+ * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
+ */
+#define POOL_SIZE               SYSTEM_PAGE_SIZE        /* must be 2^N */
+#define POOL_SIZE_MASK          SYSTEM_PAGE_SIZE_MASK
+
+/*
+ * -- End of tunable settings section --
+ */
+
+/*==========================================================================*/
+
+/*
+ * Locking
+ *
+ * To reduce lock contention, it would probably be better to refine the
+ * crude function locking with per size class locking. I'm not positive
+ * however, whether it's worth switching to such locking policy because
+ * of the performance penalty it might introduce.
+ *
+ * The following macros describe the simplest (should also be the fastest)
+ * lock object on a particular platform and the init/fini/lock/unlock
+ * operations on it. The locks defined here are not expected to be recursive
+ * because it is assumed that they will always be called in the order:
+ * INIT, [LOCK, UNLOCK]*, FINI.
+ */
+
+/*
+ * Python's threads are serialized, so object malloc locking is disabled.
+ */
+#define SIMPLELOCK_DECL(lock)   /* simple lock declaration              */
+#define SIMPLELOCK_INIT(lock)   /* allocate (if needed) and initialize  */
+#define SIMPLELOCK_FINI(lock)   /* free/destroy an existing lock        */
+#define SIMPLELOCK_LOCK(lock)   /* acquire released lock */
+#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */
+
+/* When you say memory, my mind reasons in terms of (pointers to) blocks */
+typedef uint8_t block;
+
+/* Pool for small blocks. */
+struct pool_header {
+    union { block *_padding;
+            uint count; } ref;          /* number of allocated blocks    */
+    block *freeblock;                   /* pool's free list head         */
+    struct pool_header *nextpool;       /* next pool of this size class  */
+    struct pool_header *prevpool;       /* previous pool       ""        */
+    uint arenaindex;                    /* index into arenas of base adr */
+    uint szidx;                         /* block size class index        */
+    uint nextoffset;                    /* bytes to virgin block         */
+    uint maxnextoffset;                 /* largest valid nextoffset      */
+};
+
+typedef struct pool_header *poolp;
+
+/* Record keeping for arenas. */
+struct arena_object {
+    /* The address of the arena, as returned by malloc.  Note that 0
+     * will never be returned by a successful malloc, and is used
+     * here to mark an arena_object that doesn't correspond to an
+     * allocated arena.
+     */
+    uintptr_t address;
+
+    /* Pool-aligned pointer to the next pool to be carved off. */
+    block* pool_address;
+
+    /* The number of available pools in the arena:  free pools + never-
+     * allocated pools.
+     */
+    uint nfreepools;
+
+    /* The total number of pools in the arena, whether or not available. */
+    uint ntotalpools;
+
+    /* Singly-linked list of available pools. */
+    struct pool_header* freepools;
+
+    /* Whenever this arena_object is not associated with an allocated
+     * arena, the nextarena member is used to link all unassociated
+     * arena_objects in the singly-linked `unused_arena_objects` list.
+     * The prevarena member is unused in this case.
+     *
+     * When this arena_object is associated with an allocated arena
+     * with at least one available pool, both members are used in the
+     * doubly-linked `usable_arenas` list, which is maintained in
+     * increasing order of `nfreepools` values.
+     *
+     * Else this arena_object is associated with an allocated arena
+     * all of whose pools are in use.  `nextarena` and `prevarena`
+     * are both meaningless in this case.
+     */
+    struct arena_object* nextarena;
+    struct arena_object* prevarena;
+};
+
+#define POOL_OVERHEAD   _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT)
+
+#define DUMMY_SIZE_IDX          0xffff  /* size class of newly cached pools */
+
+/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */
+#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE))
+
+/* Return total number of blocks in pool of size index I, as a uint. */
+#define NUMBLOCKS(I) ((uint)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I))
+
+/*==========================================================================*/
+
+/*
+ * This malloc lock
+ */
+SIMPLELOCK_DECL(_malloc_lock)
+#define LOCK()          SIMPLELOCK_LOCK(_malloc_lock)
+#define UNLOCK()        SIMPLELOCK_UNLOCK(_malloc_lock)
+#define LOCK_INIT()     SIMPLELOCK_INIT(_malloc_lock)
+#define LOCK_FINI()     SIMPLELOCK_FINI(_malloc_lock)
+
+/*
+ * Pool table -- headed, circular, doubly-linked lists of partially used pools.
+
+This is involved.  For an index i, usedpools[i+i] is the header for a list of
+all partially used pools holding small blocks with "size class idx" i. So
+usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size
+16, and so on:  index 2*i <-> blocks of size (i+1)<<ALIGNMENT_SHIFT.
+
+Pools are carved off an arena's highwater mark (an arena_object's pool_address
+member) as needed.  Once carved off, a pool is in one of three states forever
+after:
+
+used == partially used, neither empty nor full
+    At least one block in the pool is currently allocated, and at least one
+    block in the pool is not currently allocated (note this implies a pool
+    has room for at least two blocks).
+    This is a pool's initial state, as a pool is created only when malloc
+    needs space.
+    The pool holds blocks of a fixed size, and is in the circular list headed
+    at usedpools[i] (see above).  It's linked to the other used pools of the
+    same size class via the pool_header's nextpool and prevpool members.
+    If all but one block is currently allocated, a malloc can cause a
+    transition to the full state.  If all but one block is not currently
+    allocated, a free can cause a transition to the empty state.
+
+full == all the pool's blocks are currently allocated
+    On transition to full, a pool is unlinked from its usedpools[] list.
+    It's not linked to from anything then anymore, and its nextpool and
+    prevpool members are meaningless until it transitions back to used.
+    A free of a block in a full pool puts the pool back in the used state.
+    Then it's linked in at the front of the appropriate usedpools[] list, so
+    that the next allocation for its size class will reuse the freed block.
+
+empty == all the pool's blocks are currently available for allocation
+    On transition to empty, a pool is unlinked from its usedpools[] list,
+    and linked to the front of its arena_object's singly-linked freepools list,
+    via its nextpool member.  The prevpool member has no meaning in this case.
+    Empty pools have no inherent size class:  the next time a malloc finds
+    an empty list in usedpools[], it takes the first pool off of freepools.
+    If the size class needed happens to be the same as the size class the pool
+    last had, some pool initialization can be skipped.
+
+
+Block Management
+
+Blocks within pools are again carved out as needed.  pool->freeblock points to
+the start of a singly-linked list of free blocks within the pool.  When a
+block is freed, it's inserted at the front of its pool's freeblock list.  Note
+that the available blocks in a pool are *not* linked all together when a pool
+is initialized.  Instead only "the first two" (lowest addresses) blocks are
+set up, returning the first such block, and setting pool->freeblock to a
+one-block list holding the second such block.  This is consistent with that
+pymalloc strives at all levels (arena, pool, and block) never to touch a piece
+of memory until it's actually needed.
+
+So long as a pool is in the used state, we're certain there *is* a block
+available for allocating, and pool->freeblock is not NULL.  If pool->freeblock
+points to the end of the free list before we've carved the entire pool into
+blocks, that means we simply haven't yet gotten to one of the higher-address
+blocks.  The offset from the pool_header to the start of "the next" virgin
+block is stored in the pool_header nextoffset member, and the largest value
+of nextoffset that makes sense is stored in the maxnextoffset member when a
+pool is initialized.  All the blocks in a pool have been passed out at least
+once when and only when nextoffset > maxnextoffset.
+
+
+Major obscurity:  While the usedpools vector is declared to have poolp
+entries, it doesn't really.  It really contains two pointers per (conceptual)
+poolp entry, the nextpool and prevpool members of a pool_header.  The
+excruciating initialization code below fools C so that
+
+    usedpool[i+i]
+
+"acts like" a genuine poolp, but only so long as you only reference its
+nextpool and prevpool members.  The "- 2*sizeof(block *)" gibberish is
+compensating for that a pool_header's nextpool and prevpool members
+immediately follow a pool_header's first two members:
+
+    union { block *_padding;
+            uint count; } ref;
+    block *freeblock;
+
+each of which consume sizeof(block *) bytes.  So what usedpools[i+i] really
+contains is a fudged-up pointer p such that *if* C believes it's a poolp
+pointer, then p->nextpool and p->prevpool are both p (meaning that the headed
+circular list is empty).
+
+It's unclear why the usedpools setup is so convoluted.  It could be to
+minimize the amount of cache required to hold this heavily-referenced table
+(which only *needs* the two interpool pointer members of a pool_header). OTOH,
+referencing code has to remember to "double the index" and doing so isn't
+free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying
+on that C doesn't insert any padding anywhere in a pool_header at or before
+the prevpool member.
+**************************************************************************** */
+
+#define PTA(x)  ((poolp )((uint8_t *)&(usedpools[2*(x)]) - 2*sizeof(block *)))
+#define PT(x)   PTA(x), PTA(x)
+
+static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8] = {
+    PT(0), PT(1), PT(2), PT(3), PT(4), PT(5), PT(6), PT(7)
+#if NB_SMALL_SIZE_CLASSES > 8
+    , PT(8), PT(9), PT(10), PT(11), PT(12), PT(13), PT(14), PT(15)
+#if NB_SMALL_SIZE_CLASSES > 16
+    , PT(16), PT(17), PT(18), PT(19), PT(20), PT(21), PT(22), PT(23)
+#if NB_SMALL_SIZE_CLASSES > 24
+    , PT(24), PT(25), PT(26), PT(27), PT(28), PT(29), PT(30), PT(31)
+#if NB_SMALL_SIZE_CLASSES > 32
+    , PT(32), PT(33), PT(34), PT(35), PT(36), PT(37), PT(38), PT(39)
+#if NB_SMALL_SIZE_CLASSES > 40
+    , PT(40), PT(41), PT(42), PT(43), PT(44), PT(45), PT(46), PT(47)
+#if NB_SMALL_SIZE_CLASSES > 48
+    , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55)
+#if NB_SMALL_SIZE_CLASSES > 56
+    , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63)
+#if NB_SMALL_SIZE_CLASSES > 64
+#error "NB_SMALL_SIZE_CLASSES should be less than 64"
+#endif /* NB_SMALL_SIZE_CLASSES > 64 */
+#endif /* NB_SMALL_SIZE_CLASSES > 56 */
+#endif /* NB_SMALL_SIZE_CLASSES > 48 */
+#endif /* NB_SMALL_SIZE_CLASSES > 40 */
+#endif /* NB_SMALL_SIZE_CLASSES > 32 */
+#endif /* NB_SMALL_SIZE_CLASSES > 24 */
+#endif /* NB_SMALL_SIZE_CLASSES > 16 */
+#endif /* NB_SMALL_SIZE_CLASSES >  8 */
+};
+
+/*==========================================================================
+Arena management.
+
+`arenas` is a vector of arena_objects.  It contains maxarenas entries, some of
+which may not be currently used (== they're arena_objects that aren't
+currently associated with an allocated arena).  Note that arenas proper are
+separately malloc'ed.
+
+Prior to Python 2.5, arenas were never free()'ed.  Starting with Python 2.5,
+we do try to free() arenas, and use some mild heuristic strategies to increase
+the likelihood that arenas eventually can be freed.
+
+unused_arena_objects
+
+    This is a singly-linked list of the arena_objects that are currently not
+    being used (no arena is associated with them).  Objects are taken off the
+    head of the list in new_arena(), and are pushed on the head of the list in
+    PyObject_Free() when the arena is empty.  Key invariant:  an arena_object
+    is on this list if and only if its .address member is 0.
+
+usable_arenas
+
+    This is a doubly-linked list of the arena_objects associated with arenas
+    that have pools available.  These pools are either waiting to be reused,
+    or have not been used before.  The list is sorted to have the most-
+    allocated arenas first (ascending order based on the nfreepools member).
+    This means that the next allocation will come from a heavily used arena,
+    which gives the nearly empty arenas a chance to be returned to the system.
+    In my unscientific tests this dramatically improved the number of arenas
+    that could be freed.
+
+Note that an arena_object associated with an arena all of whose pools are
+currently in use isn't on either list.
+*/
+
+/* Array of objects used to track chunks of memory (arenas). */
+static struct arena_object* arenas = NULL;
+/* Number of slots currently allocated in the `arenas` vector. */
+static uint maxarenas = 0;
+
+/* The head of the singly-linked, NULL-terminated list of available
+ * arena_objects.
+ */
+static struct arena_object* unused_arena_objects = NULL;
+
+/* The head of the doubly-linked, NULL-terminated at each end, list of
+ * arena_objects associated with arenas that have pools available.
+ */
+static struct arena_object* usable_arenas = NULL;
+
+/* How many arena_objects do we initially allocate?
+ * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
+ * `arenas` vector.
+ */
+#define INITIAL_ARENA_OBJECTS 16
+
+/* Number of arenas allocated that haven't been free()'d. */
+static size_t narenas_currently_allocated = 0;
+
+/* Total number of times malloc() called to allocate an arena. */
+static size_t ntimes_arena_allocated = 0;
+/* High water mark (max value ever seen) for narenas_currently_allocated. */
+static size_t narenas_highwater = 0;
+
+static Py_ssize_t _Py_AllocatedBlocks = 0;
+
 Py_ssize_t
 _Py_GetAllocatedBlocks(void)
 {
-    return _PyRuntime.mem.num_allocated_blocks;
+    return _Py_AllocatedBlocks;
 }
 
 
@@ -590,7 +1049,7 @@ new_arena(void)
     if (debug_stats)
         _PyObject_DebugMallocStats(stderr);
 
-    if (_PyRuntime.mem.unused_arena_objects == NULL) {
+    if (unused_arena_objects == NULL) {
         uint i;
         uint numarenas;
         size_t nbytes;
@@ -598,18 +1057,18 @@ new_arena(void)
         /* Double the number of arena objects on each allocation.
          * Note that it's possible for `numarenas` to overflow.
          */
-        numarenas = _PyRuntime.mem.maxarenas ? _PyRuntime.mem.maxarenas << 1 : INITIAL_ARENA_OBJECTS;
-        if (numarenas <= _PyRuntime.mem.maxarenas)
+        numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;
+        if (numarenas <= maxarenas)
             return NULL;                /* overflow */
 #if SIZEOF_SIZE_T <= SIZEOF_INT
-        if (numarenas > SIZE_MAX / sizeof(*_PyRuntime.mem.arenas))
+        if (numarenas > SIZE_MAX / sizeof(*arenas))
             return NULL;                /* overflow */
 #endif
-        nbytes = numarenas * sizeof(*_PyRuntime.mem.arenas);
-        arenaobj = (struct arena_object *)PyMem_RawRealloc(_PyRuntime.mem.arenas, nbytes);
+        nbytes = numarenas * sizeof(*arenas);
+        arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes);
         if (arenaobj == NULL)
             return NULL;
-        _PyRuntime.mem.arenas = arenaobj;
+        arenas = arenaobj;
 
         /* We might need to fix pointers that were copied.  However,
          * new_arena only gets called when all the pages in the
@@ -617,45 +1076,45 @@ new_arena(void)
          * into the old array. Thus, we don't have to worry about
          * invalid pointers.  Just to be sure, some asserts:
          */
-        assert(_PyRuntime.mem.usable_arenas == NULL);
-        assert(_PyRuntime.mem.unused_arena_objects == NULL);
+        assert(usable_arenas == NULL);
+        assert(unused_arena_objects == NULL);
 
         /* Put the new arenas on the unused_arena_objects list. */
-        for (i = _PyRuntime.mem.maxarenas; i < numarenas; ++i) {
-            _PyRuntime.mem.arenas[i].address = 0;              /* mark as unassociated */
-            _PyRuntime.mem.arenas[i].nextarena = i < numarenas - 1 ?
-                                   &_PyRuntime.mem.arenas[i+1] : NULL;
+        for (i = maxarenas; i < numarenas; ++i) {
+            arenas[i].address = 0;              /* mark as unassociated */
+            arenas[i].nextarena = i < numarenas - 1 ?
+                                   &arenas[i+1] : NULL;
         }
 
         /* Update globals. */
-        _PyRuntime.mem.unused_arena_objects = &_PyRuntime.mem.arenas[_PyRuntime.mem.maxarenas];
-        _PyRuntime.mem.maxarenas = numarenas;
+        unused_arena_objects = &arenas[maxarenas];
+        maxarenas = numarenas;
     }
 
     /* Take the next available arena object off the head of the list. */
-    assert(_PyRuntime.mem.unused_arena_objects != NULL);
-    arenaobj = _PyRuntime.mem.unused_arena_objects;
-    _PyRuntime.mem.unused_arena_objects = arenaobj->nextarena;
+    assert(unused_arena_objects != NULL);
+    arenaobj = unused_arena_objects;
+    unused_arena_objects = arenaobj->nextarena;
     assert(arenaobj->address == 0);
-    address = _PyRuntime.obj.allocator_arenas.alloc(_PyRuntime.obj.allocator_arenas.ctx, ARENA_SIZE);
+    address = _PyObject_Arena.alloc(_PyObject_Arena.ctx, ARENA_SIZE);
     if (address == NULL) {
         /* The allocation failed: return NULL after putting the
          * arenaobj back.
          */
-        arenaobj->nextarena = _PyRuntime.mem.unused_arena_objects;
-        _PyRuntime.mem.unused_arena_objects = arenaobj;
+        arenaobj->nextarena = unused_arena_objects;
+        unused_arena_objects = arenaobj;
         return NULL;
     }
     arenaobj->address = (uintptr_t)address;
 
-    ++_PyRuntime.mem.narenas_currently_allocated;
-    ++_PyRuntime.mem.ntimes_arena_allocated;
-    if (_PyRuntime.mem.narenas_currently_allocated > _PyRuntime.mem.narenas_highwater)
-        _PyRuntime.mem.narenas_highwater = _PyRuntime.mem.narenas_currently_allocated;
+    ++narenas_currently_allocated;
+    ++ntimes_arena_allocated;
+    if (narenas_currently_allocated > narenas_highwater)
+        narenas_highwater = narenas_currently_allocated;
     arenaobj->freepools = NULL;
     /* pool_address <- first pool-aligned address in the arena
        nfreepools <- number of whole pools that fit after alignment */
-    arenaobj->pool_address = (pyblock*)arenaobj->address;
+    arenaobj->pool_address = (block*)arenaobj->address;
     arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE;
     assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE);
     excess = (uint)(arenaobj->address & POOL_SIZE_MASK);
@@ -753,9 +1212,9 @@ address_in_range(void *p, poolp pool)
     // the GIL. The following dance forces the compiler to read pool->arenaindex
     // only once.
     uint arenaindex = *((volatile uint *)&pool->arenaindex);
-    return arenaindex < _PyRuntime.mem.maxarenas &&
-        (uintptr_t)p - _PyRuntime.mem.arenas[arenaindex].address < ARENA_SIZE &&
-        _PyRuntime.mem.arenas[arenaindex].address != 0;
+    return arenaindex < maxarenas &&
+        (uintptr_t)p - arenas[arenaindex].address < ARENA_SIZE &&
+        arenas[arenaindex].address != 0;
 }
 
 
@@ -777,7 +1236,7 @@ address_in_range(void *p, poolp pool)
 static int
 pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
 {
-    pyblock *bp;
+    block *bp;
     poolp pool;
     poolp next;
     uint size;
@@ -803,7 +1262,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
      * Most frequent paths first
      */
     size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
-    pool = _PyRuntime.mem.usedpools[size + size];
+    pool = usedpools[size + size];
     if (pool != pool->nextpool) {
         /*
          * There is a used pool for this size class.
@@ -812,7 +1271,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
         ++pool->ref.count;
         bp = pool->freeblock;
         assert(bp != NULL);
-        if ((pool->freeblock = *(pyblock **)bp) != NULL) {
+        if ((pool->freeblock = *(block **)bp) != NULL) {
             goto success;
         }
 
@@ -821,10 +1280,10 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
          */
         if (pool->nextoffset <= pool->maxnextoffset) {
             /* There is room for another block. */
-            pool->freeblock = (pyblock*)pool +
+            pool->freeblock = (block*)pool +
                               pool->nextoffset;
             pool->nextoffset += INDEX2SIZE(size);
-            *(pyblock **)(pool->freeblock) = NULL;
+            *(block **)(pool->freeblock) = NULL;
             goto success;
         }
 
@@ -839,27 +1298,27 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
     /* There isn't a pool of the right size class immediately
      * available:  use a free pool.
      */
-    if (_PyRuntime.mem.usable_arenas == NULL) {
+    if (usable_arenas == NULL) {
         /* No arena has a free pool:  allocate a new arena. */
 #ifdef WITH_MEMORY_LIMITS
-        if (_PyRuntime.mem.narenas_currently_allocated >= MAX_ARENAS) {
+        if (narenas_currently_allocated >= MAX_ARENAS) {
             goto failed;
         }
 #endif
-        _PyRuntime.mem.usable_arenas = new_arena();
-        if (_PyRuntime.mem.usable_arenas == NULL) {
+        usable_arenas = new_arena();
+        if (usable_arenas == NULL) {
             goto failed;
         }
-        _PyRuntime.mem.usable_arenas->nextarena =
-            _PyRuntime.mem.usable_arenas->prevarena = NULL;
+        usable_arenas->nextarena =
+            usable_arenas->prevarena = NULL;
     }
-    assert(_PyRuntime.mem.usable_arenas->address != 0);
+    assert(usable_arenas->address != 0);
 
     /* Try to get a cached free pool. */
-    pool = _PyRuntime.mem.usable_arenas->freepools;
+    pool = usable_arenas->freepools;
     if (pool != NULL) {
         /* Unlink from cached pools. */
-        _PyRuntime.mem.usable_arenas->freepools = pool->nextpool;
+        usable_arenas->freepools = pool->nextpool;
 
         /* This arena already had the smallest nfreepools
          * value, so decreasing nfreepools doesn't change
@@ -868,18 +1327,18 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
          * become wholly allocated, we need to remove its
          * arena_object from usable_arenas.
          */
-        --_PyRuntime.mem.usable_arenas->nfreepools;
-        if (_PyRuntime.mem.usable_arenas->nfreepools == 0) {
+        --usable_arenas->nfreepools;
+        if (usable_arenas->nfreepools == 0) {
             /* Wholly allocated:  remove. */
-            assert(_PyRuntime.mem.usable_arenas->freepools == NULL);
-            assert(_PyRuntime.mem.usable_arenas->nextarena == NULL ||
-                   _PyRuntime.mem.usable_arenas->nextarena->prevarena ==
-                   _PyRuntime.mem.usable_arenas);
-
-            _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena;
-            if (_PyRuntime.mem.usable_arenas != NULL) {
-                _PyRuntime.mem.usable_arenas->prevarena = NULL;
-                assert(_PyRuntime.mem.usable_arenas->address != 0);
+            assert(usable_arenas->freepools == NULL);
+            assert(usable_arenas->nextarena == NULL ||
+                   usable_arenas->nextarena->prevarena ==
+                   usable_arenas);
+
+            usable_arenas = usable_arenas->nextarena;
+            if (usable_arenas != NULL) {
+                usable_arenas->prevarena = NULL;
+                assert(usable_arenas->address != 0);
             }
         }
         else {
@@ -888,15 +1347,15 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
              * off all the arena's pools for the first
              * time.
              */
-            assert(_PyRuntime.mem.usable_arenas->freepools != NULL ||
-                   _PyRuntime.mem.usable_arenas->pool_address <=
-                   (pyblock*)_PyRuntime.mem.usable_arenas->address +
+            assert(usable_arenas->freepools != NULL ||
+                   usable_arenas->pool_address <=
+                   (block*)usable_arenas->address +
                        ARENA_SIZE - POOL_SIZE);
         }
 
     init_pool:
         /* Frontlink to used pools. */
-        next = _PyRuntime.mem.usedpools[size + size]; /* == prev */
+        next = usedpools[size + size]; /* == prev */
         pool->nextpool = next;
         pool->prevpool = next;
         next->nextpool = pool;
@@ -909,7 +1368,7 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
              */
             bp = pool->freeblock;
             assert(bp != NULL);
-            pool->freeblock = *(pyblock **)bp;
+            pool->freeblock = *(block **)bp;
             goto success;
         }
         /*
@@ -919,35 +1378,35 @@ pymalloc_alloc(void *ctx, void **ptr_p, size_t nbytes)
          */
         pool->szidx = size;
         size = INDEX2SIZE(size);
-        bp = (pyblock *)pool + POOL_OVERHEAD;
+        bp = (block *)pool + POOL_OVERHEAD;
         pool->nextoffset = POOL_OVERHEAD + (size << 1);
         pool->maxnextoffset = POOL_SIZE - size;
         pool->freeblock = bp + size;
-        *(pyblock **)(pool->freeblock) = NULL;
+        *(block **)(pool->freeblock) = NULL;
         goto success;
     }
 
     /* Carve off a new pool. */
-    assert(_PyRuntime.mem.usable_arenas->nfreepools > 0);
-    assert(_PyRuntime.mem.usable_arenas->freepools == NULL);
-    pool = (poolp)_PyRuntime.mem.usable_arenas->pool_address;
-    assert((pyblock*)pool <= (pyblock*)_PyRuntime.mem.usable_arenas->address +
+    assert(usable_arenas->nfreepools > 0);
+    assert(usable_arenas->freepools == NULL);
+    pool = (poolp)usable_arenas->pool_address;
+    assert((block*)pool <= (block*)usable_arenas->address +
                              ARENA_SIZE - POOL_SIZE);
-    pool->arenaindex = (uint)(_PyRuntime.mem.usable_arenas - _PyRuntime.mem.arenas);
-    assert(&_PyRuntime.mem.arenas[pool->arenaindex] == _PyRuntime.mem.usable_arenas);
+    pool->arenaindex = (uint)(usable_arenas - arenas);
+    assert(&arenas[pool->arenaindex] == usable_arenas);
     pool->szidx = DUMMY_SIZE_IDX;
-    _PyRuntime.mem.usable_arenas->pool_address += POOL_SIZE;
-    --_PyRuntime.mem.usable_arenas->nfreepools;
+    usable_arenas->pool_address += POOL_SIZE;
+    --usable_arenas->nfreepools;
 
-    if (_PyRuntime.mem.usable_arenas->nfreepools == 0) {
-        assert(_PyRuntime.mem.usable_arenas->nextarena == NULL ||
-               _PyRuntime.mem.usable_arenas->nextarena->prevarena ==
-               _PyRuntime.mem.usable_arenas);
+    if (usable_arenas->nfreepools == 0) {
+        assert(usable_arenas->nextarena == NULL ||
+               usable_arenas->nextarena->prevarena ==
+               usable_arenas);
         /* Unlink the arena:  it is completely allocated. */
-        _PyRuntime.mem.usable_arenas = _PyRuntime.mem.usable_arenas->nextarena;
-        if (_PyRuntime.mem.usable_arenas != NULL) {
-            _PyRuntime.mem.usable_arenas->prevarena = NULL;
-            assert(_PyRuntime.mem.usable_arenas->address != 0);
+        usable_arenas = usable_arenas->nextarena;
+        if (usable_arenas != NULL) {
+            usable_arenas->prevarena = NULL;
+            assert(usable_arenas->address != 0);
         }
     }
 
@@ -970,13 +1429,13 @@ _PyObject_Malloc(void *ctx, size_t nbytes)
 {
     void* ptr;
     if (pymalloc_alloc(ctx, &ptr, nbytes)) {
-        _PyRuntime.mem.num_allocated_blocks++;
+        _Py_AllocatedBlocks++;
         return ptr;
     }
 
     ptr = PyMem_RawMalloc(nbytes);
     if (ptr != NULL) {
-        _PyRuntime.mem.num_allocated_blocks++;
+        _Py_AllocatedBlocks++;
     }
     return ptr;
 }
@@ -992,13 +1451,13 @@ _PyObject_Calloc(void *ctx, size_t nelem, size_t elsize)
 
     if (pymalloc_alloc(ctx, &ptr, nbytes)) {
         memset(ptr, 0, nbytes);
-        _PyRuntime.mem.num_allocated_blocks++;
+        _Py_AllocatedBlocks++;
         return ptr;
     }
 
     ptr = PyMem_RawCalloc(nelem, elsize);
     if (ptr != NULL) {
-        _PyRuntime.mem.num_allocated_blocks++;
+        _Py_AllocatedBlocks++;
     }
     return ptr;
 }
@@ -1011,7 +1470,7 @@ static int
 pymalloc_free(void *ctx, void *p)
 {
     poolp pool;
-    pyblock *lastfree;
+    block *lastfree;
     poolp next, prev;
     uint size;
 
@@ -1038,8 +1497,8 @@ pymalloc_free(void *ctx, void *p)
      * list in any case).
      */
     assert(pool->ref.count > 0);            /* else it was empty */
-    *(pyblock **)p = lastfree = pool->freeblock;
-    pool->freeblock = (pyblock *)p;
+    *(block **)p = lastfree = pool->freeblock;
+    pool->freeblock = (block *)p;
     if (!lastfree) {
         /* Pool was full, so doesn't currently live in any list:
          * link it to the front of the appropriate usedpools[] list.
@@ -1050,7 +1509,7 @@ pymalloc_free(void *ctx, void *p)
         --pool->ref.count;
         assert(pool->ref.count > 0);            /* else the pool is empty */
         size = pool->szidx;
-        next = _PyRuntime.mem.usedpools[size + size];
+        next = usedpools[size + size];
         prev = next->prevpool;
 
         /* insert pool before next:   prev <-> pool <-> next */
@@ -1084,7 +1543,7 @@ pymalloc_free(void *ctx, void *p)
     /* Link the pool to freepools.  This is a singly-linked
      * list, and pool->prevpool isn't used there.
      */
-    ao = &_PyRuntime.mem.arenas[pool->arenaindex];
+    ao = &arenas[pool->arenaindex];
     pool->nextpool = ao->freepools;
     ao->freepools = pool;
     nf = ++ao->nfreepools;
@@ -1113,9 +1572,9 @@ pymalloc_free(void *ctx, void *p)
          * usable_arenas pointer.
          */
         if (ao->prevarena == NULL) {
-            _PyRuntime.mem.usable_arenas = ao->nextarena;
-            assert(_PyRuntime.mem.usable_arenas == NULL ||
-                   _PyRuntime.mem.usable_arenas->address != 0);
+            usable_arenas = ao->nextarena;
+            assert(usable_arenas == NULL ||
+                   usable_arenas->address != 0);
         }
         else {
             assert(ao->prevarena->nextarena == ao);
@@ -1131,14 +1590,14 @@ pymalloc_free(void *ctx, void *p)
         /* Record that this arena_object slot is
          * available to be reused.
          */
-        ao->nextarena = _PyRuntime.mem.unused_arena_objects;
-        _PyRuntime.mem.unused_arena_objects = ao;
+        ao->nextarena = unused_arena_objects;
+        unused_arena_objects = ao;
 
         /* Free the entire arena. */
-        _PyRuntime.obj.allocator_arenas.free(_PyRuntime.obj.allocator_arenas.ctx,
+        _PyObject_Arena.free(_PyObject_Arena.ctx,
                              (void *)ao->address, ARENA_SIZE);
         ao->address = 0;                        /* mark unassociated */
-        --_PyRuntime.mem.narenas_currently_allocated;
+        --narenas_currently_allocated;
 
         goto success;
     }
@@ -1149,12 +1608,12 @@ pymalloc_free(void *ctx, void *p)
          * ao->nfreepools was 0 before, ao isn't
          * currently on the usable_arenas list.
          */
-        ao->nextarena = _PyRuntime.mem.usable_arenas;
+        ao->nextarena = usable_arenas;
         ao->prevarena = NULL;
-        if (_PyRuntime.mem.usable_arenas)
-            _PyRuntime.mem.usable_arenas->prevarena = ao;
-        _PyRuntime.mem.usable_arenas = ao;
-        assert(_PyRuntime.mem.usable_arenas->address != 0);
+        if (usable_arenas)
+            usable_arenas->prevarena = ao;
+        usable_arenas = ao;
+        assert(usable_arenas->address != 0);
 
         goto success;
     }
@@ -1183,8 +1642,8 @@ pymalloc_free(void *ctx, void *p)
     }
     else {
         /* ao is at the head of the list */
-        assert(_PyRuntime.mem.usable_arenas == ao);
-        _PyRuntime.mem.usable_arenas = ao->nextarena;
+        assert(usable_arenas == ao);
+        usable_arenas = ao->nextarena;
     }
     ao->nextarena->prevarena = ao->prevarena;
 
@@ -1209,7 +1668,7 @@ pymalloc_free(void *ctx, void *p)
     assert(ao->nextarena == NULL || nf <= ao->nextarena->nfreepools);
     assert(ao->prevarena == NULL || nf > ao->prevarena->nfreepools);
     assert(ao->nextarena == NULL || ao->nextarena->prevarena == ao);
-    assert((_PyRuntime.mem.usable_arenas == ao && ao->prevarena == NULL)
+    assert((usable_arenas == ao && ao->prevarena == NULL)
            || ao->prevarena->nextarena == ao);
 
     goto success;
@@ -1228,7 +1687,7 @@ _PyObject_Free(void *ctx, void *p)
         return;
     }
 
-    _PyRuntime.mem.num_allocated_blocks--;
+    _Py_AllocatedBlocks--;
     if (!pymalloc_free(ctx, p)) {
         /* pymalloc didn't allocate this address */
         PyMem_RawFree(p);
@@ -1353,13 +1812,15 @@ _Py_GetAllocatedBlocks(void)
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
+static size_t serialno = 0;     /* incremented on each debug {m,re}alloc */
+
 /* serialno is always incremented via calling this routine.  The point is
  * to supply a single place to set a breakpoint.
  */
 static void
 bumpserialno(void)
 {
-    ++_PyRuntime.mem.serialno;
+    ++serialno;
 }
 
 #define SST SIZEOF_SIZE_T
@@ -1466,7 +1927,7 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
     /* at tail, write pad (SST bytes) and serialno (SST bytes) */
     tail = data + nbytes;
     memset(tail, FORBIDDENBYTE, SST);
-    write_size_t(tail + SST, _PyRuntime.mem.serialno);
+    write_size_t(tail + SST, serialno);
 
     return data;
 }
@@ -1526,7 +1987,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
     uint8_t *tail;        /* data + nbytes == pointer to tail pad bytes */
     size_t total;         /* 2 * SST + nbytes + 2 * SST */
     size_t original_nbytes;
-    size_t serialno;
+    size_t block_serialno;
 #define ERASED_SIZE 64
     uint8_t save[2*ERASED_SIZE];  /* A copy of erased bytes. */
 
@@ -1542,7 +2003,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
     total = nbytes + 4*SST;
 
     tail = data + original_nbytes;
-    serialno = read_size_t(tail + SST);
+    block_serialno = read_size_t(tail + SST);
     /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
        ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
      */
@@ -1565,7 +2026,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
     else {
         head = r;
         bumpserialno();
-        serialno = _PyRuntime.mem.serialno;
+        block_serialno = serialno;
     }
 
     write_size_t(head, nbytes);
@@ -1575,7 +2036,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
 
     tail = data + nbytes;
     memset(tail, FORBIDDENBYTE, SST);
-    write_size_t(tail + SST, serialno);
+    write_size_t(tail + SST, block_serialno);
 
     /* Restore saved bytes. */
     if (original_nbytes <= sizeof(save)) {
@@ -1923,16 +2384,16 @@ _PyObject_DebugMallocStats(FILE *out)
      * to march over all the arenas.  If we're lucky, most of the memory
      * will be living in full pools -- would be a shame to miss them.
      */
-    for (i = 0; i < _PyRuntime.mem.maxarenas; ++i) {
+    for (i = 0; i < maxarenas; ++i) {
         uint j;
-        uintptr_t base = _PyRuntime.mem.arenas[i].address;
+        uintptr_t base = arenas[i].address;
 
         /* Skip arenas which are not allocated. */
-        if (_PyRuntime.mem.arenas[i].address == (uintptr_t)NULL)
+        if (arenas[i].address == (uintptr_t)NULL)
             continue;
         narenas += 1;
 
-        numfreepools += _PyRuntime.mem.arenas[i].nfreepools;
+        numfreepools += arenas[i].nfreepools;
 
         /* round up to pool alignment */
         if (base & (uintptr_t)POOL_SIZE_MASK) {
@@ -1942,8 +2403,8 @@ _PyObject_DebugMallocStats(FILE *out)
         }
 
         /* visit every pool in the arena */
-        assert(base <= (uintptr_t) _PyRuntime.mem.arenas[i].pool_address);
-        for (j = 0; base < (uintptr_t) _PyRuntime.mem.arenas[i].pool_address;
+        assert(base <= (uintptr_t) arenas[i].pool_address);
+        for (j = 0; base < (uintptr_t) arenas[i].pool_address;
              ++j, base += POOL_SIZE) {
             poolp p = (poolp)base;
             const uint sz = p->szidx;
@@ -1952,7 +2413,7 @@ _PyObject_DebugMallocStats(FILE *out)
             if (p->ref.count == 0) {
                 /* currently unused */
 #ifdef Py_DEBUG
-                assert(pool_is_in_list(p, _PyRuntime.mem.arenas[i].freepools));
+                assert(pool_is_in_list(p, arenas[i].freepools));
 #endif
                 continue;
             }
@@ -1962,11 +2423,11 @@ _PyObject_DebugMallocStats(FILE *out)
             numfreeblocks[sz] += freeblocks;
 #ifdef Py_DEBUG
             if (freeblocks > 0)
-                assert(pool_is_in_list(p, _PyRuntime.mem.usedpools[sz + sz]));
+                assert(pool_is_in_list(p, usedpools[sz + sz]));
 #endif
         }
     }
-    assert(narenas == _PyRuntime.mem.narenas_currently_allocated);
+    assert(narenas == narenas_currently_allocated);
 
     fputc('\n', out);
     fputs("class   size   num pools   blocks in use  avail blocks\n"
@@ -1994,10 +2455,10 @@ _PyObject_DebugMallocStats(FILE *out)
     }
     fputc('\n', out);
     if (_PyMem_DebugEnabled())
-        (void)printone(out, "# times object malloc called", _PyRuntime.mem.serialno);
-    (void)printone(out, "# arenas allocated total", _PyRuntime.mem.ntimes_arena_allocated);
-    (void)printone(out, "# arenas reclaimed", _PyRuntime.mem.ntimes_arena_allocated - narenas);
-    (void)printone(out, "# arenas highwater mark", _PyRuntime.mem.narenas_highwater);
+        (void)printone(out, "# times object malloc called", serialno);
+    (void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
+    (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
+    (void)printone(out, "# arenas highwater mark", narenas_highwater);
     (void)printone(out, "# arenas allocated current", narenas);
 
     PyOS_snprintf(buf, sizeof(buf),