Move slabs out of chunks.

8 files changed, 147 insertions, 863 deletions

diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index b0c4b5f..d66548f 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -3,9 +3,9 @@
 
 #define	LARGE_MINCLASS		(ZU(1) << LG_LARGE_MINCLASS)
 
-/* Maximum number of regions in one run. */
-#define	LG_RUN_MAXREGS		(LG_PAGE - LG_TINY_MIN)
-#define	RUN_MAXREGS		(1U << LG_RUN_MAXREGS)
+/* Maximum number of regions in one slab. */
+#define	LG_SLAB_MAXREGS		(LG_PAGE - LG_TINY_MIN)
+#define	SLAB_MAXREGS		(1U << LG_SLAB_MAXREGS)
 
 /*
  * The minimum ratio of active:dirty pages per arena is computed as:
@@ -29,12 +29,7 @@ typedef enum {
 /* Number of event ticks between time checks. */
 #define	DECAY_NTICKS_PER_UPDATE	1000
 
-typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t;
-typedef struct arena_avail_links_s arena_avail_links_t;
-typedef struct arena_run_s arena_run_t;
-typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t;
-typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t;
-typedef struct arena_chunk_s arena_chunk_t;
+typedef struct arena_slab_data_s arena_slab_data_t;
 typedef struct arena_bin_info_s arena_bin_info_t;
 typedef struct arena_bin_s arena_bin_t;
 typedef struct arena_s arena_t;
@@ -45,152 +40,25 @@ typedef struct arena_tdata_s arena_tdata_t;
 #ifdef JEMALLOC_H_STRUCTS
 
 #ifdef JEMALLOC_ARENA_STRUCTS_A
-struct arena_run_s {
-	/* Index of bin this run is associated with. */
+struct arena_slab_data_s {
+	/* Index of bin this slab is associated with. */
 	szind_t		binind;
 
-	/* Number of free regions in run. */
+	/* Number of free regions in slab. */
 	unsigned	nfree;
 
 	/* Per region allocated/deallocated bitmap. */
 	bitmap_t	bitmap[BITMAP_GROUPS_MAX];
 };
-
-/* Each element of the chunk map corresponds to one page within the chunk. */
-struct arena_chunk_map_bits_s {
-	/*
-	 * Run address (or size) and various flags are stored together.  The bit
-	 * layout looks like (assuming 32-bit system):
-	 *
-	 *   ???????? ???????? ???nnnnn nnndumla
-	 *
-	 * ? : Unallocated: Run address for first/last pages, unset for internal
-	 *                  pages.
-	 *     Small: Run page offset.
-	 *     Large: Run page count for first page, unset for trailing pages.
-	 * n : binind for small size class, BININD_INVALID for large size class.
-	 * d : dirty?
-	 * u : unzeroed?
-	 * m : decommitted?
-	 * l : large?
-	 * a : allocated?
-	 *
-	 * Following are example bit patterns for the three types of runs.
-	 *
-	 * p : run page offset
-	 * s : run size
-	 * n : binind for size class; large objects set these to BININD_INVALID
-	 * x : don't care
-	 * - : 0
-	 * + : 1
-	 * [DUMLA] : bit set
-	 * [dumla] : bit unset
-	 *
-	 *   Unallocated (clean):
-	 *     ssssssss ssssssss sss+++++ +++dum-a
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxx-Uxxx
-	 *     ssssssss ssssssss sss+++++ +++dUm-a
-	 *
-	 *   Unallocated (dirty):
-	 *     ssssssss ssssssss sss+++++ +++D-m-a
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     ssssssss ssssssss sss+++++ +++D-m-a
-	 *
-	 *   Small:
-	 *     pppppppp pppppppp pppnnnnn nnnd---A
-	 *     pppppppp pppppppp pppnnnnn nnn----A
-	 *     pppppppp pppppppp pppnnnnn nnnd---A
-	 *
-	 *   Large:
-	 *     ssssssss ssssssss sss+++++ +++D--LA
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ---+++++ +++D--LA
-	 *
-	 *   Large (sampled, size <= LARGE_MINCLASS):
-	 *     ssssssss ssssssss sssnnnnn nnnD--LA
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ---+++++ +++D--LA
-	 *
-	 *   Large (not sampled, size == LARGE_MINCLASS):
-	 *     ssssssss ssssssss sss+++++ +++D--LA
-	 *     xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
-	 *     -------- -------- ---+++++ +++D--LA
-	 */
-	size_t				bits;
-#define	CHUNK_MAP_ALLOCATED	((size_t)0x01U)
-#define	CHUNK_MAP_LARGE		((size_t)0x02U)
-#define	CHUNK_MAP_STATE_MASK	((size_t)0x3U)
-
-#define	CHUNK_MAP_DECOMMITTED	((size_t)0x04U)
-#define	CHUNK_MAP_UNZEROED	((size_t)0x08U)
-#define	CHUNK_MAP_DIRTY		((size_t)0x10U)
-#define	CHUNK_MAP_FLAGS_MASK	((size_t)0x1cU)
-
-#define	CHUNK_MAP_BININD_SHIFT	5
-#define	BININD_INVALID		((size_t)0xffU)
-#define	CHUNK_MAP_BININD_MASK	(BININD_INVALID << CHUNK_MAP_BININD_SHIFT)
-#define	CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK
-
-#define	CHUNK_MAP_RUNIND_SHIFT	(CHUNK_MAP_BININD_SHIFT + 8)
-#define	CHUNK_MAP_SIZE_SHIFT	(CHUNK_MAP_RUNIND_SHIFT - LG_PAGE)
-#define	CHUNK_MAP_SIZE_MASK						\
-    (~(CHUNK_MAP_BININD_MASK | CHUNK_MAP_FLAGS_MASK | CHUNK_MAP_STATE_MASK))
-};
-
-struct arena_runs_dirty_link_s {
-	qr(arena_runs_dirty_link_t)	rd_link;
-};
-
-/*
- * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just
- * like arena_chunk_map_bits_t.  Two separate arrays are stored within each
- * chunk header in order to improve cache locality.
- */
-struct arena_chunk_map_misc_s {
-	/*
-	 * Linkage for run heaps.  There are two disjoint uses:
-	 *
-	 * 1) arena_t's runs_avail heaps.
-	 * 2) arena_run_t conceptually uses this linkage for in-use non-full
-	 *    runs, rather than directly embedding linkage.
-	 */
-	phn(arena_chunk_map_misc_t)		ph_link;
-
-	union {
-		/* Linkage for list of dirty runs. */
-		arena_runs_dirty_link_t		rd;
-
-		/* Profile counters, used for large object runs. */
-		union {
-			void			*prof_tctx_pun;
-			prof_tctx_t		*prof_tctx;
-		};
-
-		/* Small region run metadata. */
-		arena_run_t			run;
-	};
-};
-typedef ph(arena_chunk_map_misc_t) arena_run_heap_t;
 #endif /* JEMALLOC_ARENA_STRUCTS_A */
 
 #ifdef JEMALLOC_ARENA_STRUCTS_B
-/* Arena chunk header. */
-struct arena_chunk_s {
-	/*
-	 * Map of pages within chunk that keeps track of free/large/small.  The
-	 * first map_bias entries are omitted, since the chunk header does not
-	 * need to be tracked in the map.  This omission saves a header page
-	 * for common chunk sizes (e.g. 4 MiB).
-	 */
-	arena_chunk_map_bits_t	map_bits[1]; /* Dynamically sized. */
-};
-
 /*
  * Read-only information associated with each element of arena_t's bins array
  * is stored separately, partly to reduce memory usage (only one copy, rather
  * than one per arena), but mainly to avoid false cacheline sharing.
  *
- * Each run has the following layout:
+ * Each slab has the following layout:
  *
  *   /--------------------\
  *   | region 0           |
@@ -205,45 +73,42 @@ struct arena_chunk_s {
  *   \--------------------/
  */
 struct arena_bin_info_s {
-	/* Size of regions in a run for this bin's size class. */
+	/* Size of regions in a slab for this bin's size class. */
 	size_t			reg_size;
 
-	/* Total size of a run for this bin's size class. */
-	size_t			run_size;
+	/* Total size of a slab for this bin's size class. */
+	size_t			slab_size;
 
-	/* Total number of regions in a run for this bin's size class. */
+	/* Total number of regions in a slab for this bin's size class. */
 	uint32_t		nregs;
 
 	/*
-	 * Metadata used to manipulate bitmaps for runs associated with this
+	 * Metadata used to manipulate bitmaps for slabs associated with this
 	 * bin.
 	 */
 	bitmap_info_t		bitmap_info;
 };
 
 struct arena_bin_s {
-	/*
-	 * All operations on runcur, runs, and stats require that lock be
-	 * locked.  Run allocation/deallocation are protected by the arena lock,
-	 * which may be acquired while holding one or more bin locks, but not
-	 * vise versa.
-	 */
+	/* All operations on arena_bin_t fields require lock ownership. */
 	malloc_mutex_t		lock;
 
 	/*
-	 * Current run being used to service allocations of this bin's size
-	 * class.
+	 * Current slab being used to service allocations of this bin's size
+	 * class.  slabcur is independent of slabs_{nonfull,full}; whenever
+	 * slabcur is reassigned, the previous slab must be deallocated or
+	 * inserted into slabs_{nonfull,full}.
 	 */
-	arena_run_t		*runcur;
+	extent_t		*slabcur;
 
 	/*
-	 * Heap of non-full runs.  This heap is used when looking for an
-	 * existing run when runcur is no longer usable.  We choose the
-	 * non-full run that is lowest in memory; this policy tends to keep
-	 * objects packed well, and it can also help reduce the number of
-	 * almost-empty chunks.
+	 * Heap of non-full slabs.  This heap is used to assure that new
+	 * allocations come from the non-full slab that is lowest in memory.
 	 */
-	arena_run_heap_t	runs;
+	extent_heap_t		slabs_nonfull;
+
+	/* Ring sentinel used to track full slabs. */
+	extent_t		slabs_full;
 
 	/* Bin statistics. */
 	malloc_bin_stats_t	stats;
@@ -272,7 +137,7 @@ struct arena_s {
 	 * perspective:
 	 * 1) Thread assignment (modifies nthreads) is synchronized via atomics.
 	 * 2) Bin-related operations are protected by bin locks.
-	 * 3) Chunk- and run-related operations are protected by this mutex.
+	 * 3) Chunk-related operations are protected by this mutex.
 	 */
 	malloc_mutex_t		lock;
 
@@ -294,32 +159,17 @@ struct arena_s {
 
 	dss_prec_t		dss_prec;
 
-	/* Extant arena chunks. */
-	ql_head(extent_t)	achunks;
-
-	/*
-	 * In order to avoid rapid chunk allocation/deallocation when an arena
-	 * oscillates right on the cusp of needing a new chunk, cache the most
-	 * recently freed chunk.  The spare is left in the arena's chunk trees
-	 * until it is deleted.
-	 *
-	 * There is one spare chunk per arena, rather than one spare total, in
-	 * order to avoid interactions between multiple threads that could make
-	 * a single spare inadequate.
-	 */
-	extent_t		*spare;
-
 	/* Minimum ratio (log base 2) of nactive:ndirty. */
 	ssize_t			lg_dirty_mult;
 
 	/* True if a thread is currently executing arena_purge_to_limit(). */
 	bool			purging;
 
-	/* Number of pages in active runs and huge regions. */
+	/* Number of pages in active extents. */
 	size_t			nactive;
 
 	/*
-	 * Current count of pages within unused runs that are potentially
+	 * Current count of pages within unused extents that are potentially
 	 * dirty, and for which madvise(... MADV_DONTNEED) has not been called.
 	 * By tracking this, we can institute a limit on how much dirty unused
 	 * memory is mapped for each arena.
@@ -327,35 +177,10 @@ struct arena_s {
 	size_t			ndirty;
 
 	/*
-	 * Unused dirty memory this arena manages.  Dirty memory is conceptually
-	 * tracked as an arbitrarily interleaved LRU of dirty runs and cached
-	 * chunks, but the list linkage is actually semi-duplicated in order to
-	 * avoid extra arena_chunk_map_misc_t space overhead.
-	 *
-	 *   LRU-----------------------------------------------------------MRU
-	 *
-	 *        /-- arena ---\
-	 *        |            |
-	 *        |            |
-	 *        |------------|                             /-- chunk --\
-	 *   ...->|chunks_cache|<--------------------------->|  /------\ |<--...
-	 *        |------------|                             |  |extent| |
-	 *        |            |                             |  |      | |
-	 *        |            |    /- run -\    /- run -\   |  |      | |
-	 *        |            |    |       |    |       |   |  |      | |
-	 *        |            |    |       |    |       |   |  |      | |
-	 *        |------------|    |-------|    |-------|   |  |------| |
-	 *   ...->|runs_dirty  |<-->|rd     |<-->|rd     |<---->|rd    |<----...
-	 *        |------------|    |-------|    |-------|   |  |------| |
-	 *        |            |    |       |    |       |   |  |      | |
-	 *        |            |    |       |    |       |   |  \------/ |
-	 *        |            |    \-------/    \-------/   |           |
-	 *        |            |                             |           |
-	 *        |            |                             |           |
-	 *        \------------/                             \-----------/
+	 * Ring sentinel used to track unused dirty memory.  Dirty memory is
+	 * managed as an LRU of cached extents.
 	 */
-	arena_runs_dirty_link_t	runs_dirty;
-	extent_t		chunks_cache;
+	extent_t		extents_dirty;
 
 	/*
 	 * Approximate time in seconds from the creation of a set of unused
@@ -424,16 +249,8 @@ struct arena_s {
 	/* User-configurable chunk hook functions. */
 	chunk_hooks_t		chunk_hooks;
 
-	/* bins is used to store trees of free regions. */
+	/* bins is used to store heaps of free regions. */
 	arena_bin_t		bins[NBINS];
-
-	/*
-	 * Size-segregated address-ordered heaps of this arena's available runs,
-	 * used for first-best-fit run allocation.  Runs are quantized, i.e.
-	 * they reside in the last heap which corresponds to a size class less
-	 * than or equal to the run size.
-	 */
-	arena_run_heap_t	runs_avail[NPSIZES];
 };
 
 /* Used in conjunction with tsd for fast arena-related context lookup. */
@@ -461,15 +278,6 @@ extern ssize_t		opt_decay_time;
 
 extern const arena_bin_info_t	arena_bin_info[NBINS];
 
-extern size_t		map_bias; /* Number of arena chunk header pages. */
-extern size_t		map_misc_offset;
-extern size_t		arena_maxrun; /* Max run size for arenas. */
-
-#ifdef JEMALLOC_JET
-typedef size_t (run_quantize_t)(size_t);
-extern run_quantize_t *run_quantize_floor;
-extern run_quantize_t *run_quantize_ceil;
-#endif
 extent_t	*arena_chunk_cache_alloc(tsdn_t *tsdn, arena_t *arena,
     chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
     bool *zero);
@@ -514,10 +322,9 @@ void	arena_prof_promote(tsdn_t *tsdn, extent_t *extent, const void *ptr,
 void	arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr,
     tcache_t *tcache, bool slow_path);
 void	arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
-    arena_chunk_t *chunk, extent_t *extent, void *ptr,
-    arena_chunk_map_bits_t *bitselm);
-void	arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
-    extent_t *extent, void *ptr, size_t pageind);
+    extent_t *extent, void *ptr);
+void	arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
+    void *ptr);
 bool	arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
     size_t oldsize, size_t size, size_t extra, bool zero);
 void	*arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
@@ -552,70 +359,19 @@ void	arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
 #ifdef JEMALLOC_H_INLINES
 
 #ifndef JEMALLOC_ENABLE_INLINE
-arena_chunk_map_bits_t	*arena_bitselm_get_mutable(arena_chunk_t *chunk,
-    size_t pageind);
-const arena_chunk_map_bits_t	*arena_bitselm_get_const(
-    const arena_chunk_t *chunk, size_t pageind);
-arena_chunk_map_misc_t	*arena_miscelm_get_mutable(arena_chunk_t *chunk,
-    size_t pageind);
-const arena_chunk_map_misc_t	*arena_miscelm_get_const(
-    const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_miscelm_to_pageind(const extent_t *extent,
-    const arena_chunk_map_misc_t *miscelm);
-void	*arena_miscelm_to_rpages(const extent_t *extent,
-    const arena_chunk_map_misc_t *miscelm);
-arena_chunk_map_misc_t	*arena_rd_to_miscelm(const extent_t *extent,
-    arena_runs_dirty_link_t *rd);
-arena_chunk_map_misc_t	*arena_run_to_miscelm(const extent_t *extent,
-    arena_run_t *run);
-size_t	*arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind);
-const size_t	*arena_mapbitsp_get_const(const arena_chunk_t *chunk,
-    size_t pageind);
-size_t	arena_mapbitsp_read(const size_t *mapbitsp);
-size_t	arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_mapbits_size_decode(size_t mapbits);
-size_t	arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk,
-    size_t pageind);
-size_t	arena_mapbits_large_size_get(const arena_chunk_t *chunk,
-    size_t pageind);
-size_t	arena_mapbits_small_runind_get(const arena_chunk_t *chunk,
-    size_t pageind);
-szind_t	arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_mapbits_decommitted_get(const arena_chunk_t *chunk,
-    size_t pageind);
-size_t	arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind);
-size_t	arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind);
-void	arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits);
-size_t	arena_mapbits_size_encode(size_t size);
-void	arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind,
-    size_t size, size_t flags);
-void	arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
-    size_t size);
-void	arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind,
-    size_t flags);
-void	arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind,
-    size_t size, size_t flags);
-void	arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
-    szind_t binind);
-void	arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
-    size_t runind, szind_t binind, size_t flags);
 void	arena_metadata_allocated_add(arena_t *arena, size_t size);
 void	arena_metadata_allocated_sub(arena_t *arena, size_t size);
 size_t	arena_metadata_allocated_get(arena_t *arena);
 bool	arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes);
 bool	arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes);
 bool	arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes);
-szind_t	arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr,
-    size_t mapbits);
 szind_t	arena_bin_index(arena_t *arena, arena_bin_t *bin);
 prof_tctx_t	*arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
     const void *ptr);
 void	arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
     size_t usize, prof_tctx_t *tctx);
 void	arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, const void *old_ptr, prof_tctx_t *old_tctx);
+    prof_tctx_t *tctx);
 void	arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
 void	arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
 void	*arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
@@ -630,330 +386,6 @@ void	arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
 #  ifdef JEMALLOC_ARENA_INLINE_A
-JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t *
-arena_bitselm_get_mutable(arena_chunk_t *chunk, size_t pageind)
-{
-
-	assert(pageind >= map_bias);
-	assert(pageind < chunk_npages);
-
-	return (&chunk->map_bits[pageind-map_bias]);
-}
-
-JEMALLOC_ALWAYS_INLINE const arena_chunk_map_bits_t *
-arena_bitselm_get_const(const arena_chunk_t *chunk, size_t pageind)
-{
-
-	return (arena_bitselm_get_mutable((arena_chunk_t *)chunk, pageind));
-}
-
-JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
-arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind)
-{
-
-	assert(pageind >= map_bias);
-	assert(pageind < chunk_npages);
-
-	return ((arena_chunk_map_misc_t *)((uintptr_t)chunk +
-	    (uintptr_t)map_misc_offset) + pageind-map_bias);
-}
-
-JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t *
-arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind)
-{
-
-	return (arena_miscelm_get_mutable((arena_chunk_t *)chunk, pageind));
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_miscelm_to_pageind(const extent_t *extent,
-    const arena_chunk_map_misc_t *miscelm)
-{
-	arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
-	size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk +
-	    map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias;
-
-	assert(pageind >= map_bias);
-	assert(pageind < chunk_npages);
-
-	return (pageind);
-}
-
-JEMALLOC_ALWAYS_INLINE void *
-arena_miscelm_to_rpages(const extent_t *extent,
-    const arena_chunk_map_misc_t *miscelm)
-{
-	arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
-	size_t pageind = arena_miscelm_to_pageind(extent, miscelm);
-
-	return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE)));
-}
-
-JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
-arena_rd_to_miscelm(const extent_t *extent, arena_runs_dirty_link_t *rd)
-{
-	arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t
-	    *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd));
-
-	assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias);
-	assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages);
-
-	return (miscelm);
-}
-
-JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t *
-arena_run_to_miscelm(const extent_t *extent, arena_run_t *run)
-{
-	arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t
-	    *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run));
-
-	assert(arena_miscelm_to_pageind(extent, miscelm) >= map_bias);
-	assert(arena_miscelm_to_pageind(extent, miscelm) < chunk_npages);
-
-	return (miscelm);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t *
-arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind)
-{
-
-	return (&arena_bitselm_get_mutable(chunk, pageind)->bits);
-}
-
-JEMALLOC_ALWAYS_INLINE const size_t *
-arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind)
-{
-
-	return (arena_mapbitsp_get_mutable((arena_chunk_t *)chunk, pageind));
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbitsp_read(const size_t *mapbitsp)
-{
-
-	return (*mapbitsp);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_get(const arena_chunk_t *chunk, size_t pageind)
-{
-
-	return (arena_mapbitsp_read(arena_mapbitsp_get_const(chunk, pageind)));
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_size_decode(size_t mapbits)
-{
-	size_t size;
-
-#if CHUNK_MAP_SIZE_SHIFT > 0
-	size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT;
-#elif CHUNK_MAP_SIZE_SHIFT == 0
-	size = mapbits & CHUNK_MAP_SIZE_MASK;
-#else
-	size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT;
-#endif
-
-	return (size);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_unallocated_size_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
-	return (arena_mapbits_size_decode(mapbits));
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_large_size_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
-	    (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED));
-	return (arena_mapbits_size_decode(mapbits));
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_small_runind_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
-	    CHUNK_MAP_ALLOCATED);
-	return (mapbits >> CHUNK_MAP_RUNIND_SHIFT);
-}
-
-JEMALLOC_ALWAYS_INLINE szind_t
-arena_mapbits_binind_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-	szind_t binind;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
-	assert(binind < NBINS || binind == BININD_INVALID);
-	return (binind);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_dirty_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
-	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
-	return (mapbits & CHUNK_MAP_DIRTY);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_unzeroed_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
-	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
-	return (mapbits & CHUNK_MAP_UNZEROED);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_decommitted_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	assert((mapbits & CHUNK_MAP_DECOMMITTED) == 0 || (mapbits &
-	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
-	return (mapbits & CHUNK_MAP_DECOMMITTED);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_large_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	return (mapbits & CHUNK_MAP_LARGE);
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_allocated_get(const arena_chunk_t *chunk, size_t pageind)
-{
-	size_t mapbits;
-
-	mapbits = arena_mapbits_get(chunk, pageind);
-	return (mapbits & CHUNK_MAP_ALLOCATED);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits)
-{
-
-	*mapbitsp = mapbits;
-}
-
-JEMALLOC_ALWAYS_INLINE size_t
-arena_mapbits_size_encode(size_t size)
-{
-	size_t mapbits;
-
-#if CHUNK_MAP_SIZE_SHIFT > 0
-	mapbits = size << CHUNK_MAP_SIZE_SHIFT;
-#elif CHUNK_MAP_SIZE_SHIFT == 0
-	mapbits = size;
-#else
-	mapbits = size >> -CHUNK_MAP_SIZE_SHIFT;
-#endif
-
-	assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0);
-	return (mapbits);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
-    size_t flags)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-
-	assert((size & PAGE_MASK) == 0);
-	assert((flags & CHUNK_MAP_FLAGS_MASK) == flags);
-	assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags &
-	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
-	arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
-	    CHUNK_MAP_BININD_INVALID | flags);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
-    size_t size)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-	size_t mapbits = arena_mapbitsp_read(mapbitsp);
-
-	assert((size & PAGE_MASK) == 0);
-	assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
-	arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
-	    (mapbits & ~CHUNK_MAP_SIZE_MASK));
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, size_t flags)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-
-	assert((flags & CHUNK_MAP_UNZEROED) == flags);
-	arena_mapbitsp_write(mapbitsp, flags);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
-    size_t flags)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-
-	assert((size & PAGE_MASK) == 0);
-	assert((flags & CHUNK_MAP_FLAGS_MASK) == flags);
-	assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags &
-	    (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0);
-	arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) |
-	    CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE |
-	    CHUNK_MAP_ALLOCATED);
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
-    szind_t binind)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-	size_t mapbits = arena_mapbitsp_read(mapbitsp);
-
-	assert(binind <= BININD_INVALID);
-	assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS +
-	    large_pad);
-	arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) |
-	    (binind << CHUNK_MAP_BININD_SHIFT));
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
-    szind_t binind, size_t flags)
-{
-	size_t *mapbitsp = arena_mapbitsp_get_mutable(chunk, pageind);
-
-	assert(binind < BININD_INVALID);
-	assert(pageind - runind >= map_bias);
-	assert((flags & CHUNK_MAP_UNZEROED) == flags);
-	arena_mapbitsp_write(mapbitsp, (runind << CHUNK_MAP_RUNIND_SHIFT) |
-	    (binind << CHUNK_MAP_BININD_SHIFT) | flags | CHUNK_MAP_ALLOCATED);
-}
-
 JEMALLOC_INLINE void
 arena_metadata_allocated_add(arena_t *arena, size_t size)
 {
@@ -1022,54 +454,6 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes)
 #  endif /* JEMALLOC_ARENA_INLINE_A */
 
 #  ifdef JEMALLOC_ARENA_INLINE_B
-JEMALLOC_ALWAYS_INLINE szind_t
-arena_ptr_small_binind_get(tsdn_t *tsdn, const void *ptr, size_t mapbits)
-{
-	szind_t binind;
-
-	binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
-
-	if (config_debug) {
-		const extent_t *extent;
-		arena_chunk_t *chunk;
-		arena_t *arena;
-		size_t pageind;
-		size_t actual_mapbits;
-		size_t rpages_ind;
-		const arena_run_t *run;
-		arena_bin_t *bin;
-		szind_t run_binind, actual_binind;
-		const arena_bin_info_t *bin_info;
-		const arena_chunk_map_misc_t *miscelm;
-		const void *rpages;
-
-		assert(binind != BININD_INVALID);
-		assert(binind < NBINS);
-		extent = iealloc(tsdn, ptr);
-		chunk = (arena_chunk_t *)extent_base_get(extent);
-		arena = extent_arena_get(extent);
-		pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-		actual_mapbits = arena_mapbits_get(chunk, pageind);
-		assert(mapbits == actual_mapbits);
-		assert(arena_mapbits_large_get(chunk, pageind) == 0);
-		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
-		rpages_ind = pageind - arena_mapbits_small_runind_get(chunk,
-		    pageind);
-		miscelm = arena_miscelm_get_const(chunk, rpages_ind);
-		run = &miscelm->run;
-		run_binind = run->binind;
-		bin = &arena->bins[run_binind];
-		actual_binind = (szind_t)(bin - arena->bins);
-		assert(run_binind == actual_binind);
-		bin_info = &arena_bin_info[actual_binind];
-		rpages = arena_miscelm_to_rpages(extent, miscelm);
-		assert(((uintptr_t)ptr - (uintptr_t)rpages) % bin_info->reg_size
-		    == 0);
-	}
-
-	return (binind);
-}
-
 JEMALLOC_INLINE szind_t
 arena_bin_index(arena_t *arena, arena_bin_t *bin)
 {
@@ -1081,27 +465,13 @@ arena_bin_index(arena_t *arena, arena_bin_t *bin)
 JEMALLOC_INLINE prof_tctx_t *
 arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
 {
-	prof_tctx_t *ret;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	if (likely(extent_slab_get(extent))) {
-		arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
-		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-		size_t mapbits = arena_mapbits_get(chunk, pageind);
-		assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
-		if (likely((mapbits & CHUNK_MAP_LARGE) == 0))
-			ret = (prof_tctx_t *)(uintptr_t)1U;
-		else {
-			arena_chunk_map_misc_t *elm =
-			    arena_miscelm_get_mutable(chunk, pageind);
-			ret = atomic_read_p(&elm->prof_tctx_pun);
-		}
-	} else
-		ret = huge_prof_tctx_get(tsdn, extent);
-
-	return (ret);
+	if (unlikely(!extent_slab_get(extent)))
+		return (huge_prof_tctx_get(tsdn, extent));
+	return ((prof_tctx_t *)(uintptr_t)1U);
 }
 
 JEMALLOC_INLINE void
@@ -1112,61 +482,20 @@ arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	if (likely(extent_slab_get(extent))) {
-		arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
-		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-
-		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
-
-		if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx >
-		    (uintptr_t)1U)) {
-			arena_chunk_map_misc_t *elm;
-
-			assert(arena_mapbits_large_get(chunk, pageind) != 0);
-
-			elm = arena_miscelm_get_mutable(chunk, pageind);
-			atomic_write_p(&elm->prof_tctx_pun, tctx);
-		} else {
-			/*
-			 * tctx must always be initialized for large runs.
-			 * Assert that the surrounding conditional logic is
-			 * equivalent to checking whether ptr refers to a large
-			 * run.
-			 */
-			assert(arena_mapbits_large_get(chunk, pageind) == 0);
-		}
-	} else
+	if (unlikely(!extent_slab_get(extent)))
 		huge_prof_tctx_set(tsdn, extent, tctx);
 }
 
 JEMALLOC_INLINE void
 arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, const void *old_ptr, prof_tctx_t *old_tctx)
+    prof_tctx_t *tctx)
 {
 
 	cassert(config_prof);
 	assert(ptr != NULL);
+	assert(!extent_slab_get(extent));
 
-	if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr &&
-	    (uintptr_t)old_tctx > (uintptr_t)1U))) {
-		if (likely(extent_slab_get(extent))) {
-			arena_chunk_t *chunk =
-			    (arena_chunk_t *)extent_base_get(extent);
-			size_t pageind;
-			arena_chunk_map_misc_t *elm;
-
-			pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
-			    LG_PAGE;
-			assert(arena_mapbits_allocated_get(chunk, pageind) !=
-			    0);
-			assert(arena_mapbits_large_get(chunk, pageind) != 0);
-
-			elm = arena_miscelm_get_mutable(chunk, pageind);
-			atomic_write_p(&elm->prof_tctx_pun,
-			    (prof_tctx_t *)(uintptr_t)1U);
-		} else
-			huge_prof_tctx_reset(tsdn, extent);
-	}
+	huge_prof_tctx_reset(tsdn, extent);
 }
 
 JEMALLOC_ALWAYS_INLINE void
@@ -1231,20 +560,9 @@ arena_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
 
 	assert(ptr != NULL);
 
-	if (likely(extent_slab_get(extent))) {
-		const arena_chunk_t *chunk =
-		    (const arena_chunk_t *)extent_base_get(extent);
-		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-		szind_t binind;
-
-		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
-		binind = arena_mapbits_binind_get(chunk, pageind);
-		/* Small allocation. */
-		assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
-		    arena_ptr_small_binind_get(tsdn, ptr,
-		    arena_mapbits_get(chunk, pageind)) == binind);
-		ret = index2size(binind);
-	} else
+	if (likely(extent_slab_get(extent)))
+		ret = index2size(extent_slab_data_get_const(extent)->binind);
+	else
 		ret = huge_salloc(tsdn, extent);
 
 	return (ret);
@@ -1260,19 +578,13 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
 
 	if (likely(extent_slab_get(extent))) {
 		/* Small allocation. */
-		arena_chunk_t *chunk = (arena_chunk_t *)extent_base_get(extent);
-		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-		size_t mapbits = arena_mapbits_get(chunk, pageind);
-		assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
-		assert((mapbits & CHUNK_MAP_LARGE) == 0);
 		if (likely(tcache != NULL)) {
-			szind_t binind = arena_ptr_small_binind_get(tsdn, ptr,
-			    mapbits);
+			szind_t binind = extent_slab_data_get(extent)->binind;
 			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind,
 			    slow_path);
 		} else {
 			arena_dalloc_small(tsdn, extent_arena_get(extent),
-			    chunk, extent, ptr, pageind);
+			    extent, ptr);
 		}
 	} else {
 		size_t usize = extent_usize_get(extent);
@@ -1282,8 +594,8 @@ arena_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, tcache_t *tcache,
 				arena_dalloc_promoted(tsdn, extent, ptr,
 				    tcache, slow_path);
 			} else {
-				tcache_dalloc_huge(tsdn_tsd(tsdn), tcache, ptr,
-				    usize, slow_path);
+				tcache_dalloc_huge(tsdn_tsd(tsdn), tcache,
+				    ptr, usize, slow_path);
 			}
 		} else
 			huge_dalloc(tsdn, extent);
@@ -1302,15 +614,12 @@ arena_sdalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t size,
 		/* Small allocation. */
 		if (likely(tcache != NULL)) {
 			szind_t binind = size2index(size);
+			assert(binind == extent_slab_data_get(extent)->binind);
 			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, binind,
 			    slow_path);
 		} else {
-			arena_chunk_t *chunk =
-			    (arena_chunk_t *)extent_base_get(extent);
-			size_t pageind = ((uintptr_t)ptr -
-			    (uintptr_t)chunk) >> LG_PAGE;
 			arena_dalloc_small(tsdn, extent_arena_get(extent),
-			    chunk, extent, ptr, pageind);
+			    extent, ptr);
 		}
 	} else {
 		if (likely(tcache != NULL) && size <= tcache_maxclass) {
diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h
index 0d456e2..c2e3455 100644
--- a/include/jemalloc/internal/bitmap.h
+++ b/include/jemalloc/internal/bitmap.h
@@ -2,7 +2,7 @@
 #ifdef JEMALLOC_H_TYPES
 
 /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */
-#define	LG_BITMAP_MAXBITS	LG_RUN_MAXREGS
+#define	LG_BITMAP_MAXBITS	LG_SLAB_MAXREGS
 #define	BITMAP_MAXBITS		(ZU(1) << LG_BITMAP_MAXBITS)
 
 typedef struct bitmap_level_s bitmap_level_t;
diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h
index 4e1e97e..bfe6181 100644
--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -27,9 +27,6 @@ struct extent_s {
 	/* True if extent is active (in use). */
 	bool			e_active;
 
-	/* True if extent is dirty (touched). */
-	bool			e_dirty;
-
 	/*
 	 * The zeroed flag is used by chunk recycling code to track whether
 	 * memory is zero-filled.
@@ -50,21 +47,27 @@ struct extent_s {
 	 */
 	bool			e_slab;
 
-	/* Profile counters, used for huge objects. */
 	union {
-		void		*e_prof_tctx_pun;
-		prof_tctx_t	*e_prof_tctx;
+		/* Small region slab metadata. */
+		arena_slab_data_t	e_slab_data;
+
+		/* Profile counters, used for huge objects. */
+		union {
+			void		*e_prof_tctx_pun;
+			prof_tctx_t	*e_prof_tctx;
+		};
 	};
 
-	/* Linkage for arena's runs_dirty and chunks_cache rings. */
-	arena_runs_dirty_link_t	rd;
-	qr(extent_t)		cc_link;
+	/*
+	 * Linkage for arena's extents_dirty and arena_bin_t's slabs_full rings.
+	 */
+	qr(extent_t)		qr_link;
 
 	union {
 		/* Linkage for per size class address-ordered heaps. */
 		phn(extent_t)		ph_link;
 
-		/* Linkage for arena's achunks, huge, and node_cache lists. */
+		/* Linkage for arena's huge and extent_cache lists. */
 		ql_elm(extent_t)	ql_link;
 	};
 };
@@ -102,11 +105,12 @@ void	*extent_before_get(const extent_t *extent);
 void	*extent_last_get(const extent_t *extent);
 void	*extent_past_get(const extent_t *extent);
 bool	extent_active_get(const extent_t *extent);
-bool	extent_dirty_get(const extent_t *extent);
 bool	extent_retained_get(const extent_t *extent);
 bool	extent_zeroed_get(const extent_t *extent);
 bool	extent_committed_get(const extent_t *extent);
 bool	extent_slab_get(const extent_t *extent);
+arena_slab_data_t	*extent_slab_data_get(extent_t *extent);
+const arena_slab_data_t	*extent_slab_data_get_const(const extent_t *extent);
 prof_tctx_t	*extent_prof_tctx_get(const extent_t *extent);
 void	extent_arena_set(extent_t *extent, arena_t *arena);
 void	extent_addr_set(extent_t *extent, void *addr);
@@ -114,17 +118,15 @@ void	extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
 void	extent_size_set(extent_t *extent, size_t size);
 void	extent_usize_set(extent_t *extent, size_t usize);
 void	extent_active_set(extent_t *extent, bool active);
-void	extent_dirty_set(extent_t *extent, bool dirty);
 void	extent_zeroed_set(extent_t *extent, bool zeroed);
 void	extent_committed_set(extent_t *extent, bool committed);
 void	extent_slab_set(extent_t *extent, bool slab);
 void	extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
 void	extent_init(extent_t *extent, arena_t *arena, void *addr,
-    size_t size, size_t usize, bool active, bool dirty, bool zeroed,
-    bool committed, bool slab);
-void	extent_dirty_insert(extent_t *extent,
-    arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty);
-void	extent_dirty_remove(extent_t *extent);
+    size_t size, size_t usize, bool active, bool zeroed, bool committed,
+    bool slab);
+void	extent_ring_insert(extent_t *sentinel, extent_t *extent);
+void	extent_ring_remove(extent_t *extent);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
@@ -198,17 +200,10 @@ extent_active_get(const extent_t *extent)
 }
 
 JEMALLOC_INLINE bool
-extent_dirty_get(const extent_t *extent)
-{
-
-	return (extent->e_dirty);
-}
-
-JEMALLOC_INLINE bool
 extent_retained_get(const extent_t *extent)
 {
 
-	return (qr_next(&extent->rd, rd_link) == &extent->rd);
+	return (qr_next(extent, qr_link) == extent);
 }
 
 JEMALLOC_INLINE bool
@@ -232,6 +227,22 @@ extent_slab_get(const extent_t *extent)
 	return (extent->e_slab);
 }
 
+JEMALLOC_INLINE arena_slab_data_t *
+extent_slab_data_get(extent_t *extent)
+{
+
+	assert(extent->e_slab);
+	return (&extent->e_slab_data);
+}
+
+JEMALLOC_INLINE const arena_slab_data_t *
+extent_slab_data_get_const(const extent_t *extent)
+{
+
+	assert(extent->e_slab);
+	return (&extent->e_slab_data);
+}
+
 JEMALLOC_INLINE prof_tctx_t *
 extent_prof_tctx_get(const extent_t *extent)
 {
@@ -297,13 +308,6 @@ extent_active_set(extent_t *extent, bool active)
 }
 
 JEMALLOC_INLINE void
-extent_dirty_set(extent_t *extent, bool dirty)
-{
-
-	extent->e_dirty = dirty;
-}
-
-JEMALLOC_INLINE void
 extent_zeroed_set(extent_t *extent, bool zeroed)
 {
 
@@ -333,8 +337,7 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx)
 
 JEMALLOC_INLINE void
 extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
-    size_t usize, bool active, bool dirty, bool zeroed, bool committed,
-    bool slab)
+    size_t usize, bool active, bool zeroed, bool committed, bool slab)
 {
 
 	assert(addr == PAGE_ADDR2BASE(addr) || !slab);
@@ -344,31 +347,26 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
 	extent_size_set(extent, size);
 	extent_usize_set(extent, usize);
 	extent_active_set(extent, active);
-	extent_dirty_set(extent, dirty);
 	extent_zeroed_set(extent, zeroed);
 	extent_committed_set(extent, committed);
 	extent_slab_set(extent, slab);
 	if (config_prof)
 		extent_prof_tctx_set(extent, NULL);
-	qr_new(&extent->rd, rd_link);
-	qr_new(extent, cc_link);
+	qr_new(extent, qr_link);
 }
 
 JEMALLOC_INLINE void
-extent_dirty_insert(extent_t *extent,
-    arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty)
+extent_ring_insert(extent_t *sentinel, extent_t *extent)
 {
 
-	qr_meld(runs_dirty, &extent->rd, rd_link);
-	qr_meld(chunks_dirty, extent, cc_link);
+	qr_meld(sentinel, extent, qr_link);
 }
 
 JEMALLOC_INLINE void
-extent_dirty_remove(extent_t *extent)
+extent_ring_remove(extent_t *extent)
 {
 
-	qr_remove(&extent->rd, rd_link);
-	qr_remove(extent, cc_link);
+	qr_remove(extent, qr_link);
 }
 #endif
 
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 5f94d2c..676c243 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -5,8 +5,6 @@ arena_alloc_junk_small
 arena_basic_stats_merge
 arena_bin_index
 arena_bin_info
-arena_bitselm_get_const
-arena_bitselm_get_mutable
 arena_boot
 arena_choose
 arena_choose_hard
@@ -43,38 +41,11 @@ arena_lg_dirty_mult_get
 arena_lg_dirty_mult_set
 arena_malloc
 arena_malloc_hard
-arena_mapbits_allocated_get
-arena_mapbits_binind_get
-arena_mapbits_decommitted_get
-arena_mapbits_dirty_get
-arena_mapbits_get
-arena_mapbits_internal_set
-arena_mapbits_large_binind_set
-arena_mapbits_large_get
-arena_mapbits_large_set
-arena_mapbits_large_size_get
-arena_mapbits_size_decode
-arena_mapbits_size_encode
-arena_mapbits_small_runind_get
-arena_mapbits_small_set
-arena_mapbits_unallocated_set
-arena_mapbits_unallocated_size_get
-arena_mapbits_unallocated_size_set
-arena_mapbits_unzeroed_get
-arena_mapbitsp_get_const
-arena_mapbitsp_get_mutable
-arena_mapbitsp_read
-arena_mapbitsp_write
-arena_maxrun
 arena_maybe_purge
 arena_metadata_allocated_add
 arena_metadata_allocated_get
 arena_metadata_allocated_sub
 arena_migrate
-arena_miscelm_get_const
-arena_miscelm_get_mutable
-arena_miscelm_to_pageind
-arena_miscelm_to_rpages
 arena_new
 arena_nthreads_dec
 arena_nthreads_get
@@ -93,14 +64,11 @@ arena_prof_promote
 arena_prof_tctx_get
 arena_prof_tctx_reset
 arena_prof_tctx_set
-arena_ptr_small_binind_get
 arena_purge
 arena_ralloc
 arena_ralloc_junk_large
 arena_ralloc_no_move
-arena_rd_to_miscelm
 arena_reset
-arena_run_to_miscelm
 arena_salloc
 arena_sdalloc
 arena_stats_merge
@@ -213,22 +181,23 @@ extent_before_get
 extent_committed_get
 extent_committed_set
 extent_dalloc
-extent_dirty_get
-extent_dirty_insert
-extent_dirty_remove
-extent_dirty_set
 extent_init
 extent_last_get
 extent_past_get
 extent_prof_tctx_get
 extent_prof_tctx_set
 extent_retained_get
+extent_ring_insert
+extent_ring_remove
 extent_size_get
 extent_size_set
 extent_size_quantize_ceil
 extent_size_quantize_floor
+extent_slab_data_get
+extent_slab_data_get_const
 extent_slab_get
 extent_slab_set
+extent_slab_data_get
 extent_usize_get
 extent_zeroed_get
 extent_zeroed_set
@@ -309,8 +278,6 @@ malloc_tsd_no_cleanup
 malloc_vcprintf
 malloc_vsnprintf
 malloc_write
-map_bias
-map_misc_offset
 mb_write
 narenas_auto
 narenas_tdata_cleanup
@@ -451,8 +418,6 @@ rtree_subtree_read
 rtree_subtree_read_hard
 rtree_subtree_tryread
 rtree_write
-run_quantize_ceil
-run_quantize_floor
 s2u
 s2u_compute
 s2u_lookup
diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h
index 7da20ad..8fdc27f 100644
--- a/include/jemalloc/internal/prof.h
+++ b/include/jemalloc/internal/prof.h
@@ -335,8 +335,8 @@ prof_tctx_t	*prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
 void	prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
     size_t usize, prof_tctx_t *tctx);
 void	prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, const void *old_ptr, prof_tctx_t *tctx);
-bool	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit,
+    prof_tctx_t *tctx);
+bool	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
     prof_tdata_t **tdata_out);
 prof_tctx_t	*prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
     bool update);
@@ -344,7 +344,8 @@ void	prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
     size_t usize, prof_tctx_t *tctx);
 void	prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
     size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
-    const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx);
+    extent_t *old_extent, const void *old_ptr, size_t old_usize,
+    prof_tctx_t *old_tctx);
 void	prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
     size_t usize);
 #endif
@@ -421,14 +422,14 @@ prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
-    const void *old_ptr, prof_tctx_t *old_tctx)
+prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
+    prof_tctx_t *tctx)
 {
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	arena_prof_tctx_reset(tsdn, extent, ptr, usize, old_ptr, old_tctx);
+	arena_prof_tctx_reset(tsdn, extent, ptr, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -501,10 +502,10 @@ prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
 
 JEMALLOC_ALWAYS_INLINE void
 prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
-    prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
-    size_t old_usize, prof_tctx_t *old_tctx)
+    prof_tctx_t *tctx, bool prof_active, bool updated, extent_t *old_extent,
+    const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx)
 {
-	bool sampled, old_sampled;
+	bool sampled, old_sampled, moved;
 
 	cassert(config_prof);
 	assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U);
@@ -523,19 +524,30 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
 		}
 	}
 
+	/*
+	 * The following code must differentiate among eight possible cases,
+	 * based on three boolean conditions.
+	 */
 	sampled = ((uintptr_t)tctx > (uintptr_t)1U);
 	old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U);
+	moved = (ptr != old_ptr);
+
+	/*
+	 * The following block must only execute if this is a non-moving
+	 * reallocation, because for moving reallocation the old allocation will
+	 * be deallocated via a separate call.
+	 */
+	if (unlikely(old_sampled) && !moved)
+		prof_free_sampled_object(tsd, old_usize, old_tctx);
 
 	if (unlikely(sampled)) {
 		prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
 		    tctx);
-	} else {
-		prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, usize, old_ptr,
-		    old_tctx);
-	}
-
-	if (unlikely(old_sampled))
-		prof_free_sampled_object(tsd, old_usize, old_tctx);
+	} else if (moved) {
+		prof_tctx_set(tsd_tsdn(tsd), extent, ptr, usize,
+		    (prof_tctx_t *)(uintptr_t)1U);
+	} else if (unlikely(old_sampled))
+		prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE void
diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh
index 440953a..b73064d 100755
--- a/include/jemalloc/internal/size_classes.sh
+++ b/include/jemalloc/internal/size_classes.sh
@@ -50,7 +50,7 @@ reg_size_compute() {
   reg_size=$((${grp} + ${delta}*${ndelta}))
 }
 
-run_size() {
+slab_size() {
   lg_p=$1
   lg_grp=$2
   lg_delta=$3
@@ -59,22 +59,22 @@ run_size() {
   pow2 ${lg_p}; p=${pow2_result}
   reg_size_compute ${lg_grp} ${lg_delta} ${ndelta}
 
-  # Compute smallest run size that is an integer multiple of reg_size.
-  try_run_size=${p}
-  try_nregs=$((${try_run_size} / ${reg_size}))
+  # Compute smallest slab size that is an integer multiple of reg_size.
+  try_slab_size=${p}
+  try_nregs=$((${try_slab_size} / ${reg_size}))
   perfect=0
   while [ ${perfect} -eq 0 ] ; do
-    perfect_run_size=${try_run_size}
+    perfect_slab_size=${try_slab_size}
     perfect_nregs=${try_nregs}
 
-    try_run_size=$((${try_run_size} + ${p}))
-    try_nregs=$((${try_run_size} / ${reg_size}))
-    if [ ${perfect_run_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then
+    try_slab_size=$((${try_slab_size} + ${p}))
+    try_nregs=$((${try_slab_size} / ${reg_size}))
+    if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then
       perfect=1
     fi
   done
 
-  run_size_pgs=$((${perfect_run_size} / ${p}))
+  slab_size_pgs=$((${perfect_slab_size} / ${p}))
 }
 
 size_class() {
@@ -117,7 +117,7 @@ size_class() {
 
   if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then
     bin="yes"
-    run_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${run_size_pgs}
+    slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs}
   else
     bin="no"
     pgs=0
@@ -278,7 +278,7 @@ cat <<EOF
  *     ndelta: Delta multiplier.  size == 1<<lg_grp + ndelta<<lg_delta
  *     psz: 'yes' if a multiple of the page size, 'no' otherwise.
  *     bin: 'yes' if a small bin size class, 'no' otherwise.
- *     pgs: Run page count if a small bin size class, 0 otherwise.
+ *     pgs: Slab page count if a small bin size class, 0 otherwise.
  *     lg_delta_lookup: Same as lg_delta if a lookup table size class, 'no'
  *                      otherwise.
  *   NTBINS: Number of tiny bins.
diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h
index c9a716d..881faad 100644
--- a/include/jemalloc/internal/stats.h
+++ b/include/jemalloc/internal/stats.h
@@ -48,17 +48,17 @@ struct malloc_bin_stats_s {
 	/* Number of tcache flushes to this bin. */
 	uint64_t	nflushes;
 
-	/* Total number of runs created for this bin's size class. */
-	uint64_t	nruns;
+	/* Total number of slabs created for this bin's size class. */
+	uint64_t	nslabs;
 
 	/*
-	 * Total number of runs reused by extracting them from the runs tree for
-	 * this bin's size class.
+	 * Total number of slabs reused by extracting them from the slabs heap
+	 * for this bin's size class.
 	 */
-	uint64_t	reruns;
+	uint64_t	reslabs;
 
-	/* Current number of runs in this bin. */
-	size_t		curruns;
+	/* Current number of slabs in this bin. */
+	size_t		curslabs;
 };
 
 struct malloc_huge_stats_s {
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index 186adf2..f1182dc 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -24,7 +24,7 @@ typedef struct tcaches_s tcaches_t;
 /*
  * Absolute maximum number of cache slots for each small bin in the thread
  * cache.  This is an additional constraint beyond that imposed as: twice the
- * number of regions per run for this size class.
+ * number of regions per slab for this size class.
  *
  * This constant must be an even number.
  */