Normalize size classes.

Normalize size classes to use the same number of size classes per size
doubling (currently hard coded to 4), across the intire range of size
classes.  Small size classes already used this spacing, but in order to
support this change, additional small size classes now fill [4 KiB .. 16
KiB).  Large size classes range from [16 KiB .. 4 MiB).  Huge size
classes now support non-multiples of the chunk size in order to fill (4
MiB .. 16 MiB).

6 files changed, 231 insertions, 152 deletions

diff --git a/src/arena.c b/src/arena.c
index b7300a9..49a3057 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -7,42 +7,11 @@
 ssize_t		opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
 arena_bin_info_t	arena_bin_info[NBINS];
 
-JEMALLOC_ALIGNED(CACHELINE)
-const uint32_t	small_bin2size_tab[NBINS] = {
-#define	B2S_bin_yes(size) \
-	size,
-#define	B2S_bin_no(size)
-#define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
-	B2S_bin_##bin((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
-	SIZE_CLASSES
-#undef B2S_bin_yes
-#undef B2S_bin_no
-#undef SC
-};
-
-JEMALLOC_ALIGNED(CACHELINE)
-const uint8_t	small_size2bin_tab[] = {
-#define	S2B_3(i)	i,
-#define	S2B_4(i)	S2B_3(i) S2B_3(i)
-#define	S2B_5(i)	S2B_4(i) S2B_4(i)
-#define	S2B_6(i)	S2B_5(i) S2B_5(i)
-#define	S2B_7(i)	S2B_6(i) S2B_6(i)
-#define	S2B_8(i)	S2B_7(i) S2B_7(i)
-#define	S2B_9(i)	S2B_8(i) S2B_8(i)
-#define	S2B_no(i)
-#define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
-	S2B_##lg_delta_lookup(index)
-	SIZE_CLASSES
-#undef S2B_3
-#undef S2B_4
-#undef S2B_5
-#undef S2B_6
-#undef S2B_7
-#undef S2B_8
-#undef S2B_9
-#undef S2B_no
-#undef SC
-};
+size_t		map_bias;
+size_t		map_misc_offset;
+size_t		arena_maxrun; /* Max run size for arenas. */
+size_t		arena_maxclass; /* Max size class for arenas. */
+size_t		nlclasses; /* Number of large size classes. */
 
 /******************************************************************************/
 /*
@@ -198,7 +167,7 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr)
 	arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
 	size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 	size_t mapbits = arena_mapbits_get(chunk, pageind);
-	size_t binind = arena_ptr_small_binind_get(ptr, mapbits);
+	index_t binind = arena_ptr_small_binind_get(ptr, mapbits);
 	arena_bin_info_t *bin_info = &arena_bin_info[binind];
 	unsigned regind = arena_run_regind(run, bin_info, ptr);
 
@@ -375,7 +344,7 @@ arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
 
 static void
 arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
-    size_t binind)
+    index_t binind)
 {
 	arena_chunk_t *chunk;
 	arena_chunk_map_misc_t *miscelm;
@@ -429,9 +398,9 @@ arena_chunk_init_spare(arena_t *arena)
 	assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
 	assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
 	assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
-	    arena_maxclass);
+	    arena_maxrun);
 	assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
-	    arena_maxclass);
+	    arena_maxrun);
 	assert(arena_mapbits_dirty_get(chunk, map_bias) ==
 	    arena_mapbits_dirty_get(chunk, chunk_npages-1));
 
@@ -518,8 +487,7 @@ arena_chunk_init_hard(arena_t *arena)
 	 * the pages as zeroed iff chunk_alloc() returned a zeroed chunk.
 	 */
 	unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED;
-	arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass,
-	    unzeroed);
+	arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, unzeroed);
 	/*
 	 * There is no need to initialize the internal page map entries unless
 	 * the chunk is not zeroed.
@@ -544,7 +512,7 @@ arena_chunk_init_hard(arena_t *arena)
 			}
 		}
 	}
-	arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass,
+	arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun,
 	    unzeroed);
 
 	return (chunk);
@@ -607,9 +575,9 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
 	assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
 	assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0);
 	assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
-	    arena_maxclass);
+	    arena_maxrun);
 	assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) ==
-	    arena_maxclass);
+	    arena_maxrun);
 	assert(arena_mapbits_dirty_get(chunk, map_bias) ==
 	    arena_mapbits_dirty_get(chunk, chunk_npages-1));
 
@@ -682,7 +650,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
 }
 
 static arena_run_t *
-arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind)
+arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind)
 {
 	arena_run_t *run;
 	arena_chunk_map_misc_t *miscelm;
@@ -700,7 +668,7 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind)
 }
 
 static arena_run_t *
-arena_run_alloc_small(arena_t *arena, size_t size, size_t binind)
+arena_run_alloc_small(arena_t *arena, size_t size, index_t binind)
 {
 	arena_chunk_t *chunk;
 	arena_run_t *run;
@@ -1034,7 +1002,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned)
 		    arena_mapbits_large_size_get(chunk,
 		    run_ind+(size>>LG_PAGE)-1) == 0);
 	} else {
-		size_t binind = arena_bin_index(arena, run->bin);
+		index_t binind = arena_bin_index(arena, run->bin);
 		arena_bin_info_t *bin_info = &arena_bin_info[binind];
 		size = bin_info->run_size;
 	}
@@ -1079,9 +1047,9 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned)
 		arena_dirty_insert(arena, chunk, run_ind, run_pages);
 
 	/* Deallocate chunk if it is now completely unused. */
-	if (size == arena_maxclass) {
+	if (size == arena_maxrun) {
 		assert(run_ind == map_bias);
-		assert(run_pages == (arena_maxclass >> LG_PAGE));
+		assert(run_pages == (arena_maxrun >> LG_PAGE));
 		arena_chunk_dalloc(arena, chunk);
 	}
 
@@ -1212,7 +1180,7 @@ static arena_run_t *
 arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
 {
 	arena_run_t *run;
-	size_t binind;
+	index_t binind;
 	arena_bin_info_t *bin_info;
 
 	/* Look for a usable run. */
@@ -1264,7 +1232,7 @@ static void *
 arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
 {
 	void *ret;
-	size_t binind;
+	index_t binind;
 	arena_bin_info_t *bin_info;
 	arena_run_t *run;
 
@@ -1310,7 +1278,7 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
 }
 
 void
-arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind,
+arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind,
     uint64_t prof_accumbytes)
 {
 	unsigned i, nfill;
@@ -1450,14 +1418,14 @@ arena_dalloc_junk_small_t *arena_dalloc_junk_small =
 void
 arena_quarantine_junk_small(void *ptr, size_t usize)
 {
-	size_t binind;
+	index_t binind;
 	arena_bin_info_t *bin_info;
 	cassert(config_fill);
 	assert(opt_junk);
 	assert(opt_quarantine);
 	assert(usize <= SMALL_MAXCLASS);
 
-	binind = small_size2bin(usize);
+	binind = size2index(usize);
 	bin_info = &arena_bin_info[binind];
 	arena_redzones_validate(ptr, bin_info, true);
 }
@@ -1468,12 +1436,12 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 	void *ret;
 	arena_bin_t *bin;
 	arena_run_t *run;
-	size_t binind;
+	index_t binind;
 
-	binind = small_size2bin(size);
+	binind = size2index(size);
 	assert(binind < NBINS);
 	bin = &arena->bins[binind];
-	size = small_bin2size(binind);
+	size = index2size(binind);
 
 	malloc_mutex_lock(&bin->lock);
 	if ((run = bin->runcur) != NULL && run->nfree > 0)
@@ -1520,14 +1488,15 @@ void *
 arena_malloc_large(arena_t *arena, size_t size, bool zero)
 {
 	void *ret;
+	size_t usize;
 	arena_run_t *run;
 	arena_chunk_map_misc_t *miscelm;
 	UNUSED bool idump;
 
 	/* Large allocation. */
-	size = PAGE_CEILING(size);
+	usize = s2u(size);
 	malloc_mutex_lock(&arena->lock);
-	run = arena_run_alloc_large(arena, size, zero);
+	run = arena_run_alloc_large(arena, usize, zero);
 	if (run == NULL) {
 		malloc_mutex_unlock(&arena->lock);
 		return (NULL);
@@ -1535,15 +1504,17 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 	miscelm = arena_run_to_miscelm(run);
 	ret = arena_miscelm_to_rpages(miscelm);
 	if (config_stats) {
+		index_t index = size2index(usize) - NBINS;
+
 		arena->stats.nmalloc_large++;
 		arena->stats.nrequests_large++;
-		arena->stats.allocated_large += size;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
+		arena->stats.allocated_large += usize;
+		arena->stats.lstats[index].nmalloc++;
+		arena->stats.lstats[index].nrequests++;
+		arena->stats.lstats[index].curruns++;
 	}
 	if (config_prof)
-		idump = arena_prof_accum_locked(arena, size);
+		idump = arena_prof_accum_locked(arena, usize);
 	malloc_mutex_unlock(&arena->lock);
 	if (config_prof && idump)
 		prof_idump();
@@ -1551,9 +1522,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 	if (!zero) {
 		if (config_fill) {
 			if (unlikely(opt_junk))
-				memset(ret, 0xa5, size);
+				memset(ret, 0xa5, usize);
 			else if (unlikely(opt_zero))
-				memset(ret, 0, size);
+				memset(ret, 0, usize);
 		}
 	}
 
@@ -1610,12 +1581,14 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
 	ret = arena_miscelm_to_rpages(miscelm);
 
 	if (config_stats) {
+		index_t index = size2index(size) - NBINS;
+
 		arena->stats.nmalloc_large++;
 		arena->stats.nrequests_large++;
 		arena->stats.allocated_large += size;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
+		arena->stats.lstats[index].nmalloc++;
+		arena->stats.lstats[index].nrequests++;
+		arena->stats.lstats[index].curruns++;
 	}
 	malloc_mutex_unlock(&arena->lock);
 
@@ -1632,22 +1605,23 @@ void
 arena_prof_promoted(const void *ptr, size_t size)
 {
 	arena_chunk_t *chunk;
-	size_t pageind, binind;
+	size_t pageind;
+	index_t binind;
 
 	cassert(config_prof);
 	assert(ptr != NULL);
 	assert(CHUNK_ADDR2BASE(ptr) != ptr);
-	assert(isalloc(ptr, false) == PAGE);
-	assert(isalloc(ptr, true) == PAGE);
+	assert(isalloc(ptr, false) == LARGE_MINCLASS);
+	assert(isalloc(ptr, true) == LARGE_MINCLASS);
 	assert(size <= SMALL_MAXCLASS);
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
-	binind = small_size2bin(size);
+	binind = size2index(size);
 	assert(binind < NBINS);
 	arena_mapbits_large_binind_set(chunk, pageind, binind);
 
-	assert(isalloc(ptr, false) == PAGE);
+	assert(isalloc(ptr, false) == LARGE_MINCLASS);
 	assert(isalloc(ptr, true) == size);
 }
 
@@ -1660,7 +1634,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
 	if (run == bin->runcur)
 		bin->runcur = NULL;
 	else {
-		size_t binind = arena_bin_index(chunk->arena, bin);
+		index_t binind = arena_bin_index(chunk->arena, bin);
 		arena_bin_info_t *bin_info = &arena_bin_info[binind];
 
 		if (bin_info->nregs != 1) {
@@ -1678,7 +1652,7 @@ static void
 arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
     arena_bin_t *bin)
 {
-	size_t binind;
+	index_t binind;
 	arena_bin_info_t *bin_info;
 	size_t npages, run_ind, past;
 	arena_chunk_map_misc_t *miscelm;
@@ -1762,7 +1736,8 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 	arena_run_t *run;
 	arena_bin_t *bin;
 	arena_bin_info_t *bin_info;
-	size_t size, binind;
+	size_t size;
+	index_t binind;
 
 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 	rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
@@ -1851,10 +1826,12 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 
 		arena_dalloc_junk_large(ptr, usize);
 		if (config_stats) {
+			index_t index = size2index(usize) - NBINS;
+
 			arena->stats.ndalloc_large++;
 			arena->stats.allocated_large -= usize;
-			arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++;
-			arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--;
+			arena->stats.lstats[index].ndalloc++;
+			arena->stats.lstats[index].curruns--;
 		}
 	}
 
@@ -1887,17 +1864,20 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 	malloc_mutex_lock(&arena->lock);
 	arena_run_trim_tail(arena, chunk, run, oldsize, size, true);
 	if (config_stats) {
+		index_t oldindex = size2index(oldsize) - NBINS;
+		index_t index = size2index(size) - NBINS;
+
 		arena->stats.ndalloc_large++;
 		arena->stats.allocated_large -= oldsize;
-		arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++;
-		arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--;
+		arena->stats.lstats[oldindex].ndalloc++;
+		arena->stats.lstats[oldindex].curruns--;
 
 		arena->stats.nmalloc_large++;
 		arena->stats.nrequests_large++;
 		arena->stats.allocated_large += size;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++;
-		arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
+		arena->stats.lstats[index].nmalloc++;
+		arena->stats.lstats[index].nrequests++;
+		arena->stats.lstats[index].curruns++;
 	}
 	malloc_mutex_unlock(&arena->lock);
 }
@@ -1909,24 +1889,30 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 	size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
 	size_t npages = oldsize >> LG_PAGE;
 	size_t followsize;
+	size_t usize_min = s2u(size);
 
 	assert(oldsize == arena_mapbits_large_size_get(chunk, pageind));
 
 	/* Try to extend the run. */
-	assert(size + extra > oldsize);
+	assert(usize_min > oldsize);
 	malloc_mutex_lock(&arena->lock);
 	if (pageind + npages < chunk_npages &&
 	    arena_mapbits_allocated_get(chunk, pageind+npages) == 0 &&
 	    (followsize = arena_mapbits_unallocated_size_get(chunk,
-	    pageind+npages)) >= size - oldsize) {
+	    pageind+npages)) >= usize_min - oldsize) {
 		/*
 		 * The next run is available and sufficiently large.  Split the
 		 * following run, then merge the first part with the existing
 		 * allocation.
 		 */
-		size_t flag_dirty;
-		size_t splitsize = (oldsize + followsize <= size + extra)
-		    ? followsize : size + extra - oldsize;
+		size_t flag_dirty, splitsize, usize;
+
+		usize = s2u(size + extra);
+		while (oldsize + followsize < usize)
+			usize = index2size(size2index(usize)-1);
+		assert(usize >= usize_min);
+		splitsize = usize - oldsize;
+
 		arena_run_t *run = &arena_miscelm_get(chunk,
 		    pageind+npages)->run;
 		arena_run_split_large(arena, run, splitsize, zero);
@@ -1948,17 +1934,20 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 		arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty);
 
 		if (config_stats) {
+			index_t oldindex = size2index(oldsize) - NBINS;
+			index_t index = size2index(size) - NBINS;
+
 			arena->stats.ndalloc_large++;
 			arena->stats.allocated_large -= oldsize;
-			arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++;
-			arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--;
+			arena->stats.lstats[oldindex].ndalloc++;
+			arena->stats.lstats[oldindex].curruns--;
 
 			arena->stats.nmalloc_large++;
 			arena->stats.nrequests_large++;
 			arena->stats.allocated_large += size;
-			arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++;
-			arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++;
-			arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
+			arena->stats.lstats[index].nmalloc++;
+			arena->stats.lstats[index].nrequests++;
+			arena->stats.lstats[index].curruns++;
 		}
 		malloc_mutex_unlock(&arena->lock);
 		return (false);
@@ -1996,10 +1985,14 @@ static bool
 arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
     bool zero)
 {
-	size_t psize;
+	size_t usize;
 
-	psize = PAGE_CEILING(size + extra);
-	if (psize == oldsize) {
+	/* Make sure extra can't cause size_t overflow. */
+	if (extra >= arena_maxclass)
+		return (true);
+
+	usize = s2u(size + extra);
+	if (usize == oldsize) {
 		/* Same size class. */
 		return (false);
 	} else {
@@ -2009,16 +2002,15 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra,
 		chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 		arena = chunk->arena;
 
-		if (psize < oldsize) {
+		if (usize < oldsize) {
 			/* Fill before shrinking in order avoid a race. */
-			arena_ralloc_junk_large(ptr, oldsize, psize);
+			arena_ralloc_junk_large(ptr, oldsize, usize);
 			arena_ralloc_large_shrink(arena, chunk, ptr, oldsize,
-			    psize);
+			    usize);
 			return (false);
 		} else {
 			bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
-			    oldsize, PAGE_CEILING(size),
-			    psize - PAGE_CEILING(size), zero);
+			    oldsize, size, extra, zero);
 			if (config_fill && !ret && !zero) {
 				if (unlikely(opt_junk)) {
 					memset((void *)((uintptr_t)ptr +
@@ -2045,12 +2037,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 	 */
 	if (oldsize <= arena_maxclass) {
 		if (oldsize <= SMALL_MAXCLASS) {
-			assert(arena_bin_info[small_size2bin(oldsize)].reg_size
+			assert(arena_bin_info[size2index(oldsize)].reg_size
 			    == oldsize);
-			if ((size + extra <= SMALL_MAXCLASS &&
-			    small_size2bin(size + extra) ==
-			    small_size2bin(oldsize)) || (size <= oldsize &&
-			    size + extra >= oldsize))
+			if ((size + extra <= SMALL_MAXCLASS && size2index(size +
+			    extra) == size2index(oldsize)) || (size <= oldsize
+			    && size + extra >= oldsize))
 				return (false);
 		} else {
 			assert(size <= arena_maxclass);
@@ -2258,7 +2249,7 @@ arena_new(arena_t *arena, unsigned ind)
 /*
  * Calculate bin_info->run_size such that it meets the following constraints:
  *
- *   *) bin_info->run_size <= arena_maxclass
+ *   *) bin_info->run_size <= arena_maxrun
  *   *) bin_info->nregs <= RUN_MAXREGS
  *
  * bin_info->nregs and bin_info->reg0_offset are also calculated here, since
@@ -2330,7 +2321,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info)
 	/*
 	 * Make sure that the run will fit within an arena chunk.
 	 */
-	while (actual_run_size > arena_maxclass) {
+	while (actual_run_size > arena_maxrun) {
 		actual_run_size -= PAGE;
 		actual_nregs = (actual_run_size - pad_size) /
 		    bin_info->reg_interval;
@@ -2396,7 +2387,17 @@ arena_boot(void)
 	map_misc_offset = offsetof(arena_chunk_t, map_bits) +
 	    sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias);
 
-	arena_maxclass = chunksize - (map_bias << LG_PAGE);
+	arena_maxrun = chunksize - (map_bias << LG_PAGE);
+	arena_maxclass = index2size(size2index(chunksize)-1);
+	if (arena_maxclass > arena_maxrun) {
+		/*
+		 * For small chunk sizes it's possible for there to be fewer
+		 * non-header pages available than are necessary to serve the
+		 * size classes just below chunksize.
+		 */
+		arena_maxclass = arena_maxrun;
+	}
+	nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS);
 
 	bin_info_init();
 }
diff --git a/src/chunk.c b/src/chunk.c
index 32b8b3a..618aaca 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -27,9 +27,6 @@ rtree_t		*chunks_rtree;
 size_t		chunksize;
 size_t		chunksize_mask; /* (chunksize - 1). */
 size_t		chunk_npages;
-size_t		map_bias;
-size_t		map_misc_offset;
-size_t		arena_maxclass; /* Max size class for arenas. */
 
 /******************************************************************************/
 /*
diff --git a/src/ctl.c b/src/ctl.c
index 309f1f6..f1f3234 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -1628,7 +1628,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
 }
 
 CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t)
-CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t)
+CTL_RO_NL_GEN(arenas_lrun_i_size, index2size(NBINS+mib[2]), size_t)
 static const ctl_named_node_t *
 arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
 {
diff --git a/src/huge.c b/src/huge.c
index 6bdc076..ae41625 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -15,12 +15,19 @@ static extent_tree_t	huge;
 void *
 huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
 {
+	size_t usize;
 
-	return (huge_palloc(tsd, arena, size, chunksize, zero));
+	usize = s2u(size);
+	if (usize == 0) {
+		/* size_t overflow. */
+		return (NULL);
+	}
+
+	return (huge_palloc(tsd, arena, usize, chunksize, zero));
 }
 
 void *
-huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
+huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
     bool zero)
 {
 	void *ret;
@@ -30,11 +37,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
 
 	/* Allocate one or more contiguous chunks for this request. */
 
-	csize = CHUNK_CEILING(size);
-	if (csize == 0) {
-		/* size is large enough to cause size_t wrap-around. */
-		return (NULL);
-	}
+	csize = CHUNK_CEILING(usize);
+	assert(csize >= usize);
 
 	/* Allocate an extent node with which to track the chunk. */
 	node = base_node_alloc();
@@ -55,7 +59,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
 
 	/* Insert node into huge. */
 	node->addr = ret;
-	node->size = csize;
+	node->size = usize;
 	node->arena = arena;
 
 	malloc_mutex_lock(&huge_mtx);
@@ -64,9 +68,9 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment,
 
 	if (config_fill && !zero) {
 		if (unlikely(opt_junk))
-			memset(ret, 0xa5, csize);
+			memset(ret, 0xa5, usize);
 		else if (unlikely(opt_zero) && !is_zeroed)
-			memset(ret, 0, csize);
+			memset(ret, 0, usize);
 	}
 
 	return (ret);
@@ -97,7 +101,7 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
 
 static bool
 huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
-	size_t csize;
+	size_t usize;
 	void *expand_addr;
 	size_t expand_size;
 	extent_node_t *node, key;
@@ -105,14 +109,14 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
 	bool is_zeroed;
 	void *ret;
 
-	csize = CHUNK_CEILING(size);
-	if (csize == 0) {
-		/* size is large enough to cause size_t wrap-around. */
+	usize = s2u(size);
+	if (usize == 0) {
+		/* size_t overflow. */
 		return (true);
 	}
 
-	expand_addr = ptr + oldsize;
-	expand_size = csize - oldsize;
+	expand_addr = ptr + CHUNK_CEILING(oldsize);
+	expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
 
 	malloc_mutex_lock(&huge_mtx);
 
@@ -140,14 +144,14 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) {
 
 	malloc_mutex_lock(&huge_mtx);
 	/* Update the size of the huge allocation. */
-	node->size = csize;
+	node->size = usize;
 	malloc_mutex_unlock(&huge_mtx);
 
 	if (config_fill && !zero) {
 		if (unlikely(opt_junk))
-			memset(expand_addr, 0xa5, expand_size);
+			memset(ptr + oldsize, 0xa5, usize - oldsize);
 		else if (unlikely(opt_zero) && !is_zeroed)
-			memset(expand_addr, 0, expand_size);
+			memset(ptr + oldsize, 0, usize - oldsize);
 	}
 	return (false);
 }
@@ -156,27 +160,71 @@ bool
 huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
     bool zero)
 {
+	size_t usize;
 
 	/* Both allocations must be huge to avoid a move. */
-	if (oldsize <= arena_maxclass)
+	if (oldsize < chunksize)
 		return (true);
 
-	assert(CHUNK_CEILING(oldsize) == oldsize);
+	assert(s2u(oldsize) == oldsize);
+	usize = s2u(size);
+	if (usize == 0) {
+		/* size_t overflow. */
+		return (true);
+	}
 
 	/*
-	 * Avoid moving the allocation if the size class can be left the same.
+	 * Avoid moving the allocation if the existing chunk size accommodates
+	 * the new size.
 	 */
-	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
+	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)
 	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
+		size_t usize_next;
+
+		/* Increase usize to incorporate extra. */
+		while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) <
+		    oldsize)
+			usize = usize_next;
+
+		/* Update the size of the huge allocation if it changed. */
+		if (oldsize != usize) {
+			extent_node_t *node, key;
+
+			malloc_mutex_lock(&huge_mtx);
+
+			key.addr = ptr;
+			node = extent_tree_ad_search(&huge, &key);
+			assert(node != NULL);
+			assert(node->addr == ptr);
+
+			assert(node->size != usize);
+			node->size = usize;
+
+			malloc_mutex_unlock(&huge_mtx);
+
+			if (oldsize < usize) {
+				if (zero || (config_fill &&
+				    unlikely(opt_zero))) {
+					memset(ptr + oldsize, 0, usize -
+					    oldsize);
+				} else if (config_fill && unlikely(opt_junk)) {
+					memset(ptr + oldsize, 0xa5, usize -
+					    oldsize);
+				}
+			} else if (config_fill && unlikely(opt_junk) && oldsize
+			    > usize)
+				memset(ptr + usize, 0x5a, oldsize - usize);
+		}
 		return (false);
 	}
 
-	/* Overflow. */
-	if (CHUNK_CEILING(size) == 0)
-		return (true);
+	if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size)
+	    && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) {
+		return (false);
+	}
 
 	/* Shrink the allocation in-place. */
-	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(size)) {
+	if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize)) {
 		extent_node_t *node, key;
 		void *excess_addr;
 		size_t excess_size;
@@ -189,15 +237,15 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
 		assert(node->addr == ptr);
 
 		/* Update the size of the huge allocation. */
-		node->size = CHUNK_CEILING(size);
+		node->size = usize;
 
 		malloc_mutex_unlock(&huge_mtx);
 
-		excess_addr = node->addr + CHUNK_CEILING(size);
-		excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(size);
+		excess_addr = node->addr + CHUNK_CEILING(usize);
+		excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
 
 		/* Zap the excess chunks. */
-		huge_dalloc_junk(excess_addr, excess_size);
+		huge_dalloc_junk(ptr + usize, oldsize - usize);
 		arena_chunk_dalloc_huge(node->arena, excess_addr, excess_size);
 
 		return (false);
@@ -275,7 +323,8 @@ huge_dalloc(void *ptr)
 	malloc_mutex_unlock(&huge_mtx);
 
 	huge_dalloc_junk(node->addr, node->size);
-	arena_chunk_dalloc_huge(node->arena, node->addr, node->size);
+	arena_chunk_dalloc_huge(node->arena, node->addr,
+	    CHUNK_CEILING(node->size));
 	base_node_dalloc(node);
 }
 
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 3490ecd..f3750b4 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -42,6 +42,38 @@ unsigned		narenas_auto;
 /* Set to true once the allocator has been initialized. */
 static bool		malloc_initialized = false;
 
+JEMALLOC_ALIGNED(CACHELINE)
+const size_t	index2size_tab[NSIZES] = {
+#define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
+	((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
+	SIZE_CLASSES
+#undef SC
+};
+
+JEMALLOC_ALIGNED(CACHELINE)
+const uint8_t	size2index_tab[] = {
+#define	S2B_3(i)	i,
+#define	S2B_4(i)	S2B_3(i) S2B_3(i)
+#define	S2B_5(i)	S2B_4(i) S2B_4(i)
+#define	S2B_6(i)	S2B_5(i) S2B_5(i)
+#define	S2B_7(i)	S2B_6(i) S2B_6(i)
+#define	S2B_8(i)	S2B_7(i) S2B_7(i)
+#define	S2B_9(i)	S2B_8(i) S2B_8(i)
+#define	S2B_no(i)
+#define	SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
+	S2B_##lg_delta_lookup(index)
+	SIZE_CLASSES
+#undef S2B_3
+#undef S2B_4
+#undef S2B_5
+#undef S2B_6
+#undef S2B_7
+#undef S2B_8
+#undef S2B_9
+#undef S2B_no
+#undef SC
+};
+
 #ifdef JEMALLOC_THREADED_INIT
 /* Used to let the initializing thread recursively allocate. */
 #  define NO_INITIALIZER	((unsigned long)0)
@@ -1731,7 +1763,7 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra,
 		    alignment, zero))
 			return (old_usize);
 		usize = isalloc(ptr, config_prof);
-		if (max_usize < PAGE)
+		if (max_usize < LARGE_MINCLASS)
 			arena_prof_promoted(ptr, usize);
 	} else {
 		usize = ixallocx_helper(ptr, old_usize, size, extra, alignment,
diff --git a/src/tcache.c b/src/tcache.c
index 07167b6..2c968c6 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -24,7 +24,7 @@ size_t	tcache_salloc(const void *ptr)
 void
 tcache_event_hard(tcache_t *tcache)
 {
-	size_t binind = tcache->next_gc_bin;
+	index_t binind = tcache->next_gc_bin;
 	tcache_bin_t *tbin = &tcache->tbins[binind];
 	tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
 
@@ -62,7 +62,7 @@ tcache_event_hard(tcache_t *tcache)
 }
 
 void *
-tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
+tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, index_t binind)
 {
 	void *ret;
 
@@ -76,7 +76,7 @@ tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 }
 
 void
-tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
+tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem,
     tcache_t *tcache)
 {
 	void *ptr;
@@ -153,7 +153,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
 }
 
 void
-tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
+tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem,
     tcache_t *tcache)
 {
 	void *ptr;