Remove medium size classes.

Remove medium size classes, because concurrent dirty page purging is
no longer capable of purging inactive dirty pages inside active runs
(due to recent arena/bin locking changes).

Enhance tcache to support caching large objects, so that the same range
of size classes is still cached, despite the removal of medium size
class support.

5 files changed, 263 insertions, 325 deletions

diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c
index a3d6654..435cf69 100644
--- a/jemalloc/src/arena.c
+++ b/jemalloc/src/arena.c
@@ -6,7 +6,6 @@
 
 size_t	opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT;
 size_t	opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT;
-size_t	opt_lg_medium_max = LG_MEDIUM_MAX_DEFAULT;
 ssize_t		opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
 uint8_t const	*small_size2bin;
 
@@ -14,15 +13,12 @@ uint8_t const	*small_size2bin;
 unsigned	nqbins;
 unsigned	ncbins;
 unsigned	nsbins;
-unsigned	nmbins;
 unsigned	nbins;
-unsigned	mbin0;
 size_t		qspace_max;
 size_t		cspace_min;
 size_t		cspace_max;
 size_t		sspace_min;
 size_t		sspace_max;
-size_t		medium_max;
 
 size_t		lg_mspace;
 size_t		mspace_mask;
@@ -178,8 +174,6 @@ static void	arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
 static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
 static void	*arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
 static size_t	arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
-static void	*arena_malloc_large(arena_t *arena, size_t size, bool zero);
-static bool	arena_is_large(const void *ptr);
 static void	arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk,
     arena_run_t *run, arena_bin_t *bin);
 static void	arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
@@ -1077,7 +1071,7 @@ arena_prof_accum(arena_t *arena, uint64_t accumbytes)
 
 #ifdef JEMALLOC_TCACHE
 void
-arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
+arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind
 #  ifdef JEMALLOC_PROF
     , uint64_t prof_accumbytes
 #  endif
@@ -1239,7 +1233,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 	size_t binind;
 
 	binind = small_size2bin[size];
-	assert(binind < mbin0);
+	assert(binind < nbins);
 	bin = &arena->bins[binind];
 	size = bin->reg_size;
 
@@ -1282,58 +1276,6 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 }
 
 void *
-arena_malloc_medium(arena_t *arena, size_t size, bool zero)
-{
-	void *ret;
-	arena_bin_t *bin;
-	arena_run_t *run;
-	size_t binind;
-
-	size = MEDIUM_CEILING(size);
-	binind = mbin0 + ((size - medium_min) >> lg_mspace);
-	assert(binind < nbins);
-	bin = &arena->bins[binind];
-	assert(bin->reg_size == size);
-
-	malloc_mutex_lock(&bin->lock);
-	if ((run = bin->runcur) != NULL && run->nfree > 0)
-		ret = arena_run_reg_alloc(run, bin);
-	else
-		ret = arena_bin_malloc_hard(arena, bin);
-
-	if (ret == NULL) {
-		malloc_mutex_unlock(&bin->lock);
-		return (NULL);
-	}
-
-#ifdef JEMALLOC_STATS
-	bin->stats.allocated += size;
-	bin->stats.nmalloc++;
-	bin->stats.nrequests++;
-#endif
-	malloc_mutex_unlock(&bin->lock);
-#ifdef JEMALLOC_PROF
-	if (isthreaded == false) {
-		malloc_mutex_lock(&arena->lock);
-		arena_prof_accum(arena, size);
-		malloc_mutex_unlock(&arena->lock);
-	}
-#endif
-
-	if (zero == false) {
-#ifdef JEMALLOC_FILL
-		if (opt_junk)
-			memset(ret, 0xa5, size);
-		else if (opt_zero)
-			memset(ret, 0, size);
-#endif
-	} else
-		memset(ret, 0, size);
-
-	return (ret);
-}
-
-static void *
 arena_malloc_large(arena_t *arena, size_t size, bool zero)
 {
 	void *ret;
@@ -1348,7 +1290,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
 	}
 #ifdef JEMALLOC_STATS
 	arena->stats.nmalloc_large++;
+	arena->stats.nrequests_large++;
 	arena->stats.allocated_large += size;
+	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++;
 	if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns >
@@ -1381,21 +1325,31 @@ arena_malloc(size_t size, bool zero)
 	assert(size != 0);
 	assert(QUANTUM_CEILING(size) <= arena_maxclass);
 
-	if (size <= bin_maxclass) {
+	if (size <= small_maxclass) {
 #ifdef JEMALLOC_TCACHE
 		tcache_t *tcache;
 
 		if ((tcache = tcache_get()) != NULL)
-			return (tcache_alloc(tcache, size, zero));
+			return (tcache_alloc_small(tcache, size, zero));
+		else
+
 #endif
-		if (size <= small_maxclass)
 			return (arena_malloc_small(choose_arena(), size, zero));
-		else {
-			return (arena_malloc_medium(choose_arena(), size,
-			    zero));
-		}
-	} else
-		return (arena_malloc_large(choose_arena(), size, zero));
+	} else {
+#ifdef JEMALLOC_TCACHE
+		if (size <= tcache_maxclass) {
+			tcache_t *tcache;
+
+			if ((tcache = tcache_get()) != NULL)
+				return (tcache_alloc_large(tcache, size, zero));
+			else {
+				return (arena_malloc_large(choose_arena(),
+				    size, zero));
+			}
+		} else
+#endif
+			return (arena_malloc_large(choose_arena(), size, zero));
+	}
 }
 
 /* Only handles large allocations that require more than page alignment. */
@@ -1444,7 +1398,9 @@ arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
 
 #ifdef JEMALLOC_STATS
 	arena->stats.nmalloc_large++;
+	arena->stats.nrequests_large++;
 	arena->stats.allocated_large += size;
+	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++;
 	if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns >
@@ -1464,22 +1420,6 @@ arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
 	return (ret);
 }
 
-static bool
-arena_is_large(const void *ptr)
-{
-	arena_chunk_t *chunk;
-	size_t pageind, mapbits;
-
-	assert(ptr != NULL);
-	assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
-	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
-	pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT);
-	mapbits = chunk->map[pageind].bits;
-	assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
-	return ((mapbits & CHUNK_MAP_LARGE) != 0);
-}
-
 /* Return the size of the allocation pointed to by ptr. */
 size_t
 arena_salloc(const void *ptr)
@@ -1781,8 +1721,11 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
 	astats->allocated_large += arena->stats.allocated_large;
 	astats->nmalloc_large += arena->stats.nmalloc_large;
 	astats->ndalloc_large += arena->stats.ndalloc_large;
+	astats->nrequests_large += arena->stats.nrequests_large;
 
 	for (i = 0; i < nlclasses; i++) {
+		lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
+		lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
 		lstats[i].nrequests += arena->stats.lstats[i].nrequests;
 		lstats[i].highruns += arena->stats.lstats[i].highruns;
 		lstats[i].curruns += arena->stats.lstats[i].curruns;
@@ -1815,8 +1758,6 @@ arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 {
 
 	/* Large allocation. */
-	malloc_mutex_lock(&arena->lock);
-
 #ifdef JEMALLOC_FILL
 #  ifndef JEMALLOC_STATS
 	if (opt_junk)
@@ -1838,12 +1779,12 @@ arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 #ifdef JEMALLOC_STATS
 		arena->stats.ndalloc_large++;
 		arena->stats.allocated_large -= size;
+		arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++;
 		arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--;
 #endif
 	}
 
 	arena_run_dalloc(arena, (arena_run_t *)ptr, true);
-	malloc_mutex_unlock(&arena->lock);
 }
 
 static void
@@ -1863,10 +1804,13 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 #ifdef JEMALLOC_STATS
 	arena->stats.ndalloc_large++;
 	arena->stats.allocated_large -= oldsize;
+	arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++;
 	arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--;
 
 	arena->stats.nmalloc_large++;
+	arena->stats.nrequests_large++;
 	arena->stats.allocated_large += size;
+	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++;
 	if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns >
@@ -1910,10 +1854,13 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 #ifdef JEMALLOC_STATS
 	arena->stats.ndalloc_large++;
 	arena->stats.allocated_large -= oldsize;
+	arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++;
 	arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--;
 
 	arena->stats.nmalloc_large++;
+	arena->stats.nrequests_large++;
 	arena->stats.allocated_large += size;
+	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++;
 	arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++;
 	if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns >
@@ -1988,30 +1935,15 @@ arena_ralloc(void *ptr, size_t size, size_t oldsize)
 	void *ret;
 	size_t copysize;
 
-	/*
-	 * Try to avoid moving the allocation.
-	 *
-	 * posix_memalign() can cause allocation of "large" objects that are
-	 * smaller than bin_maxclass (in order to meet alignment requirements).
-	 * Therefore, do not assume that (oldsize <= bin_maxclass) indicates
-	 * ptr refers to a bin-allocated object.
-	 */
+	/* Try to avoid moving the allocation. */
 	if (oldsize <= arena_maxclass) {
-		if (arena_is_large(ptr) == false ) {
-			if (size <= small_maxclass) {
-				if (oldsize <= small_maxclass &&
-				    small_size2bin[size] ==
-				    small_size2bin[oldsize])
-					goto IN_PLACE;
-			} else if (size <= bin_maxclass) {
-				if (small_maxclass < oldsize && oldsize <=
-				    bin_maxclass && MEDIUM_CEILING(size) ==
-				    MEDIUM_CEILING(oldsize))
-					goto IN_PLACE;
-			}
+		if (oldsize <= small_maxclass) {
+			if (size <= small_maxclass && small_size2bin[size] ==
+			    small_size2bin[oldsize])
+				goto IN_PLACE;
 		} else {
 			assert(size <= arena_maxclass);
-			if (size > bin_maxclass) {
+			if (size > small_maxclass) {
 				if (arena_ralloc_large(ptr, size, oldsize) ==
 				    false)
 					return (ptr);
@@ -2019,23 +1951,6 @@ arena_ralloc(void *ptr, size_t size, size_t oldsize)
 		}
 	}
 
-	/* Try to avoid moving the allocation. */
-	if (size <= small_maxclass) {
-		if (oldsize <= small_maxclass && small_size2bin[size] ==
-		    small_size2bin[oldsize])
-			goto IN_PLACE;
-	} else if (size <= bin_maxclass) {
-		if (small_maxclass < oldsize && oldsize <= bin_maxclass &&
-		    MEDIUM_CEILING(size) == MEDIUM_CEILING(oldsize))
-			goto IN_PLACE;
-	} else {
-		if (bin_maxclass < oldsize && oldsize <= arena_maxclass) {
-			assert(size > bin_maxclass);
-			if (arena_ralloc_large(ptr, size, oldsize) == false)
-				return (ptr);
-		}
-	}
-
 	/*
 	 * If we get here, then size and oldsize are different enough that we
 	 * need to move the object.  In that case, fall back to allocating new
@@ -2074,13 +1989,12 @@ arena_new(arena_t *arena, unsigned ind)
 
 #ifdef JEMALLOC_STATS
 	memset(&arena->stats, 0, sizeof(arena_stats_t));
-	arena->stats.lstats = (malloc_large_stats_t *)base_alloc(
-	    sizeof(malloc_large_stats_t) * ((chunksize - PAGE_SIZE) >>
-	        PAGE_SHIFT));
+	arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses *
+	    sizeof(malloc_large_stats_t));
 	if (arena->stats.lstats == NULL)
 		return (true);
-	memset(arena->stats.lstats, 0, sizeof(malloc_large_stats_t) *
-	    ((chunksize - PAGE_SIZE) >> PAGE_SHIFT));
+	memset(arena->stats.lstats, 0, nlclasses *
+	    sizeof(malloc_large_stats_t));
 #  ifdef JEMALLOC_TCACHE
 	ql_new(&arena->tcache_ql);
 #  endif
@@ -2159,7 +2073,7 @@ arena_new(arena_t *arena, unsigned ind)
 	}
 
 	/* Subpage-spaced bins. */
-	for (; i < ntbins + nqbins + ncbins + nsbins; i++) {
+	for (; i < nbins; i++) {
 		bin = &arena->bins[i];
 		if (malloc_mutex_init(&bin->lock))
 			return (true);
@@ -2176,24 +2090,6 @@ arena_new(arena_t *arena, unsigned ind)
 #endif
 	}
 
-	/* Medium bins. */
-	for (; i < nbins; i++) {
-		bin = &arena->bins[i];
-		if (malloc_mutex_init(&bin->lock))
-			return (true);
-		bin->runcur = NULL;
-		arena_run_tree_new(&bin->runs);
-
-		bin->reg_size = medium_min + ((i - (ntbins + nqbins + ncbins +
-		    nsbins)) << lg_mspace);
-
-		prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
-
-#ifdef JEMALLOC_STATS
-		memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
-#endif
-	}
-
 #ifdef JEMALLOC_DEBUG
 	arena->magic = ARENA_MAGIC;
 #endif
@@ -2355,7 +2251,6 @@ arena_boot(void)
 		sspace_min += SUBPAGE;
 	assert(sspace_min < PAGE_SIZE);
 	sspace_max = PAGE_SIZE - SUBPAGE;
-	medium_max = (1U << opt_lg_medium_max);
 
 #ifdef JEMALLOC_TINY
 	assert(LG_QUANTUM >= LG_TINY_MIN);
@@ -2364,24 +2259,9 @@ arena_boot(void)
 	nqbins = qspace_max >> LG_QUANTUM;
 	ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1;
 	nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1;
+	nbins = ntbins + nqbins + ncbins + nsbins;
 
 	/*
-	 * Compute medium size class spacing and the number of medium size
-	 * classes.  Limit spacing to no more than pagesize, but if possible
-	 * use the smallest spacing that does not exceed NMBINS_MAX medium size
-	 * classes.
-	 */
-	lg_mspace = LG_SUBPAGE;
-	nmbins = ((medium_max - medium_min) >> lg_mspace) + 1;
-	while (lg_mspace < PAGE_SHIFT && nmbins > NMBINS_MAX) {
-		lg_mspace = lg_mspace + 1;
-		nmbins = ((medium_max - medium_min) >> lg_mspace) + 1;
-	}
-	mspace_mask = (1U << lg_mspace) - 1U;
-
-	mbin0 = ntbins + nqbins + ncbins + nsbins;
-	nbins = mbin0 + nmbins;
-	/*
 	 * The small_size2bin lookup table uses uint8_t to encode each bin
 	 * index, so we cannot support more than 256 small size classes.  This
 	 * limit is difficult to exceed (not even possible with 16B quantum and
@@ -2389,10 +2269,10 @@ arena_boot(void)
 	 * nonetheless we need to protect against this case in order to avoid
 	 * undefined behavior.
 	 */
-	if (mbin0 > 256) {
+	if (nbins > 256) {
 	    char line_buf[UMAX2S_BUFSIZE];
 	    malloc_write("<jemalloc>: Too many small size classes (");
-	    malloc_write(umax2s(mbin0, 10, line_buf));
+	    malloc_write(umax2s(nbins, 10, line_buf));
 	    malloc_write(" > max 256)\n");
 	    abort();
 	}
diff --git a/jemalloc/src/ctl.c b/jemalloc/src/ctl.c
index 1644f73..2249102 100644
--- a/jemalloc/src/ctl.c
+++ b/jemalloc/src/ctl.c
@@ -84,7 +84,6 @@ CTL_PROTO(opt_prof_leak)
 CTL_PROTO(opt_stats_print)
 CTL_PROTO(opt_lg_qspace_max)
 CTL_PROTO(opt_lg_cspace_max)
-CTL_PROTO(opt_lg_medium_max)
 CTL_PROTO(opt_lg_dirty_mult)
 CTL_PROTO(opt_lg_chunk)
 #ifdef JEMALLOC_SWAP
@@ -102,7 +101,6 @@ CTL_PROTO(arenas_quantum)
 CTL_PROTO(arenas_cacheline)
 CTL_PROTO(arenas_subpage)
 CTL_PROTO(arenas_pagesize)
-CTL_PROTO(arenas_medium)
 CTL_PROTO(arenas_chunksize)
 #ifdef JEMALLOC_TINY
 CTL_PROTO(arenas_tspace_min)
@@ -114,14 +112,17 @@ CTL_PROTO(arenas_cspace_min)
 CTL_PROTO(arenas_cspace_max)
 CTL_PROTO(arenas_sspace_min)
 CTL_PROTO(arenas_sspace_max)
-CTL_PROTO(arenas_medium_min)
-CTL_PROTO(arenas_medium_max)
+#ifdef JEMALLOC_TCACHE
+CTL_PROTO(arenas_tcache_max)
+#endif
 CTL_PROTO(arenas_ntbins)
 CTL_PROTO(arenas_nqbins)
 CTL_PROTO(arenas_ncbins)
 CTL_PROTO(arenas_nsbins)
-CTL_PROTO(arenas_nmbins)
 CTL_PROTO(arenas_nbins)
+#ifdef JEMALLOC_TCACHE
+CTL_PROTO(arenas_nhbins)
+#endif
 CTL_PROTO(arenas_nlruns)
 #ifdef JEMALLOC_PROF
 CTL_PROTO(prof_dump)
@@ -138,13 +139,10 @@ CTL_PROTO(stats_arenas_i_small_allocated)
 CTL_PROTO(stats_arenas_i_small_nmalloc)
 CTL_PROTO(stats_arenas_i_small_ndalloc)
 CTL_PROTO(stats_arenas_i_small_nrequests)
-CTL_PROTO(stats_arenas_i_medium_allocated)
-CTL_PROTO(stats_arenas_i_medium_nmalloc)
-CTL_PROTO(stats_arenas_i_medium_ndalloc)
-CTL_PROTO(stats_arenas_i_medium_nrequests)
 CTL_PROTO(stats_arenas_i_large_allocated)
 CTL_PROTO(stats_arenas_i_large_nmalloc)
 CTL_PROTO(stats_arenas_i_large_ndalloc)
+CTL_PROTO(stats_arenas_i_large_nrequests)
 CTL_PROTO(stats_arenas_i_bins_j_allocated)
 CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
 CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
@@ -158,6 +156,8 @@ CTL_PROTO(stats_arenas_i_bins_j_nreruns)
 CTL_PROTO(stats_arenas_i_bins_j_highruns)
 CTL_PROTO(stats_arenas_i_bins_j_curruns)
 INDEX_PROTO(stats_arenas_i_bins_j)
+CTL_PROTO(stats_arenas_i_lruns_j_nmalloc)
+CTL_PROTO(stats_arenas_i_lruns_j_ndalloc)
 CTL_PROTO(stats_arenas_i_lruns_j_nrequests)
 CTL_PROTO(stats_arenas_i_lruns_j_highruns)
 CTL_PROTO(stats_arenas_i_lruns_j_curruns)
@@ -255,7 +255,6 @@ static const ctl_node_t opt_node[] = {
 	{NAME("stats_print"),		CTL(opt_stats_print)},
 	{NAME("lg_qspace_max"),		CTL(opt_lg_qspace_max)},
 	{NAME("lg_cspace_max"),		CTL(opt_lg_cspace_max)},
-	{NAME("lg_medium_max"),		CTL(opt_lg_medium_max)},
 	{NAME("lg_dirty_mult"),		CTL(opt_lg_dirty_mult)},
 	{NAME("lg_chunk"),		CTL(opt_lg_chunk)}
 #ifdef JEMALLOC_SWAP
@@ -295,7 +294,6 @@ static const ctl_node_t arenas_node[] = {
 	{NAME("cacheline"),		CTL(arenas_cacheline)},
 	{NAME("subpage"),		CTL(arenas_subpage)},
 	{NAME("pagesize"),		CTL(arenas_pagesize)},
-	{NAME("medium"),		CTL(arenas_medium)},
 	{NAME("chunksize"),		CTL(arenas_chunksize)},
 #ifdef JEMALLOC_TINY
 	{NAME("tspace_min"),		CTL(arenas_tspace_min)},
@@ -307,14 +305,17 @@ static const ctl_node_t arenas_node[] = {
 	{NAME("cspace_max"),		CTL(arenas_cspace_max)},
 	{NAME("sspace_min"),		CTL(arenas_sspace_min)},
 	{NAME("sspace_max"),		CTL(arenas_sspace_max)},
-	{NAME("medium_min"),		CTL(arenas_medium_min)},
-	{NAME("medium_max"),		CTL(arenas_medium_max)},
+#ifdef JEMALLOC_TCACHE
+	{NAME("tcache_max"),		CTL(arenas_tcache_max)},
+#endif
 	{NAME("ntbins"),		CTL(arenas_ntbins)},
 	{NAME("nqbins"),		CTL(arenas_nqbins)},
 	{NAME("ncbins"),		CTL(arenas_ncbins)},
 	{NAME("nsbins"),		CTL(arenas_nsbins)},
-	{NAME("nmbins"),		CTL(arenas_nmbins)},
 	{NAME("nbins"),			CTL(arenas_nbins)},
+#ifdef JEMALLOC_TCACHE
+	{NAME("nhbins"),		CTL(arenas_nhbins)},
+#endif
 	{NAME("bin"),			CHILD(arenas_bin)},
 	{NAME("nlruns"),		CTL(arenas_nlruns)},
 	{NAME("lrun"),			CHILD(arenas_lrun)}
@@ -347,17 +348,11 @@ static const ctl_node_t stats_arenas_i_small_node[] = {
 	{NAME("nrequests"),		CTL(stats_arenas_i_small_nrequests)}
 };
 
-static const ctl_node_t stats_arenas_i_medium_node[] = {
-	{NAME("allocated"),		CTL(stats_arenas_i_medium_allocated)},
-	{NAME("nmalloc"),		CTL(stats_arenas_i_medium_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_medium_ndalloc)},
-	{NAME("nrequests"),		CTL(stats_arenas_i_medium_nrequests)}
-};
-
 static const ctl_node_t stats_arenas_i_large_node[] = {
 	{NAME("allocated"),		CTL(stats_arenas_i_large_allocated)},
 	{NAME("nmalloc"),		CTL(stats_arenas_i_large_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_large_ndalloc)}
+	{NAME("ndalloc"),		CTL(stats_arenas_i_large_ndalloc)},
+	{NAME("nrequests"),		CTL(stats_arenas_i_large_nrequests)}
 };
 
 static const ctl_node_t stats_arenas_i_bins_j_node[] = {
@@ -383,6 +378,8 @@ static const ctl_node_t stats_arenas_i_bins_node[] = {
 };
 
 static const ctl_node_t stats_arenas_i_lruns_j_node[] = {
+	{NAME("nmalloc"),		CTL(stats_arenas_i_lruns_j_nmalloc)},
+	{NAME("ndalloc"),		CTL(stats_arenas_i_lruns_j_ndalloc)},
 	{NAME("nrequests"),		CTL(stats_arenas_i_lruns_j_nrequests)},
 	{NAME("highruns"),		CTL(stats_arenas_i_lruns_j_highruns)},
 	{NAME("curruns"),		CTL(stats_arenas_i_lruns_j_curruns)}
@@ -406,7 +403,6 @@ static const ctl_node_t stats_arenas_i_node[] = {
 	{NAME("nmadvise"),		CTL(stats_arenas_i_nmadvise)},
 	{NAME("purged"),		CTL(stats_arenas_i_purged)},
 	{NAME("small"),			CHILD(stats_arenas_i_small)},
-	{NAME("medium"),		CHILD(stats_arenas_i_medium)},
 	{NAME("large"),			CHILD(stats_arenas_i_large)},
 	{NAME("bins"),			CHILD(stats_arenas_i_bins)},
 	{NAME("lruns"),		CHILD(stats_arenas_i_lruns)}
@@ -505,10 +501,6 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
 	astats->nmalloc_small = 0;
 	astats->ndalloc_small = 0;
 	astats->nrequests_small = 0;
-	astats->allocated_medium = 0;
-	astats->nmalloc_medium = 0;
-	astats->ndalloc_medium = 0;
-	astats->nrequests_medium = 0;
 	memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t));
 	memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t));
 #endif
@@ -523,19 +515,12 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
 	arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
 	    &cstats->astats, cstats->bstats, cstats->lstats);
 
-	for (i = 0; i < mbin0; i++) {
+	for (i = 0; i < nbins; i++) {
 		cstats->allocated_small += cstats->bstats[i].allocated;
 		cstats->nmalloc_small += cstats->bstats[i].nmalloc;
 		cstats->ndalloc_small += cstats->bstats[i].ndalloc;
 		cstats->nrequests_small += cstats->bstats[i].nrequests;
 	}
-
-	for (; i < nbins; i++) {
-		cstats->allocated_medium += cstats->bstats[i].allocated;
-		cstats->nmalloc_medium += cstats->bstats[i].nmalloc;
-		cstats->ndalloc_medium += cstats->bstats[i].ndalloc;
-		cstats->nrequests_medium += cstats->bstats[i].nrequests;
-	}
 }
 
 static void
@@ -556,16 +541,14 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
 	sstats->ndalloc_small += astats->ndalloc_small;
 	sstats->nrequests_small += astats->nrequests_small;
 
-	sstats->allocated_medium += astats->allocated_medium;
-	sstats->nmalloc_medium += astats->nmalloc_medium;
-	sstats->ndalloc_medium += astats->ndalloc_medium;
-	sstats->nrequests_medium += astats->nrequests_medium;
-
 	sstats->astats.allocated_large += astats->astats.allocated_large;
 	sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
 	sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
+	sstats->astats.nrequests_large += astats->astats.nrequests_large;
 
 	for (i = 0; i < nlclasses; i++) {
+		sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc;
+		sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc;
 		sstats->lstats[i].nrequests += astats->lstats[i].nrequests;
 		sstats->lstats[i].highruns += astats->lstats[i].highruns;
 		sstats->lstats[i].curruns += astats->lstats[i].curruns;
@@ -648,7 +631,6 @@ ctl_refresh(void)
 
 #ifdef JEMALLOC_STATS
 	ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
-	    + ctl_stats.arenas[narenas].allocated_medium
 	    + ctl_stats.arenas[narenas].astats.allocated_large
 	    + ctl_stats.huge.allocated;
 	ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT)
@@ -1178,7 +1160,6 @@ CTL_RO_GEN(opt_prof_leak, opt_prof_leak, bool)
 CTL_RO_GEN(opt_stats_print, opt_stats_print, bool)
 CTL_RO_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t)
 CTL_RO_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t)
-CTL_RO_GEN(opt_lg_medium_max, opt_lg_medium_max, size_t)
 CTL_RO_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
 CTL_RO_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
 #ifdef JEMALLOC_SWAP
@@ -1239,7 +1220,6 @@ CTL_RO_GEN(arenas_quantum, QUANTUM, size_t)
 CTL_RO_GEN(arenas_cacheline, CACHELINE, size_t)
 CTL_RO_GEN(arenas_subpage, SUBPAGE, size_t)
 CTL_RO_GEN(arenas_pagesize, PAGE_SIZE, size_t)
-CTL_RO_GEN(arenas_medium, (1U << lg_mspace), size_t)
 CTL_RO_GEN(arenas_chunksize, chunksize, size_t)
 #ifdef JEMALLOC_TINY
 CTL_RO_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t)
@@ -1251,14 +1231,17 @@ CTL_RO_GEN(arenas_cspace_min, cspace_min, size_t)
 CTL_RO_GEN(arenas_cspace_max, cspace_max, size_t)
 CTL_RO_GEN(arenas_sspace_min, sspace_min, size_t)
 CTL_RO_GEN(arenas_sspace_max, sspace_max, size_t)
-CTL_RO_GEN(arenas_medium_min, medium_min, size_t)
-CTL_RO_GEN(arenas_medium_max, medium_max, size_t)
+#ifdef JEMALLOC_TCACHE
+CTL_RO_GEN(arenas_tcache_max, tcache_maxclass, size_t)
+#endif
 CTL_RO_GEN(arenas_ntbins, ntbins, unsigned)
 CTL_RO_GEN(arenas_nqbins, nqbins, unsigned)
 CTL_RO_GEN(arenas_ncbins, ncbins, unsigned)
 CTL_RO_GEN(arenas_nsbins, nsbins, unsigned)
-CTL_RO_GEN(arenas_nmbins, nmbins, unsigned)
 CTL_RO_GEN(arenas_nbins, nbins, unsigned)
+#ifdef JEMALLOC_TCACHE
+CTL_RO_GEN(arenas_nhbins, nhbins, unsigned)
+#endif
 CTL_RO_GEN(arenas_nlruns, nlclasses, size_t)
 
 /******************************************************************************/
@@ -1304,20 +1287,14 @@ CTL_RO_GEN(stats_arenas_i_small_ndalloc,
     ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
 CTL_RO_GEN(stats_arenas_i_small_nrequests,
     ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_allocated,
-    ctl_stats.arenas[mib[2]].allocated_medium, size_t)
-CTL_RO_GEN(stats_arenas_i_medium_nmalloc,
-    ctl_stats.arenas[mib[2]].nmalloc_medium, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_ndalloc,
-    ctl_stats.arenas[mib[2]].ndalloc_medium, uint64_t)
-CTL_RO_GEN(stats_arenas_i_medium_nrequests,
-    ctl_stats.arenas[mib[2]].nrequests_medium, uint64_t)
 CTL_RO_GEN(stats_arenas_i_large_allocated,
     ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
 CTL_RO_GEN(stats_arenas_i_large_nmalloc,
     ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t)
 CTL_RO_GEN(stats_arenas_i_large_ndalloc,
     ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
+CTL_RO_GEN(stats_arenas_i_large_nrequests,
+    ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t)
 
 CTL_RO_GEN(stats_arenas_i_bins_j_allocated,
     ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t)
@@ -1351,6 +1328,10 @@ stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j)
 	return (super_stats_arenas_i_bins_j_node);
 }
 
+CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc,
+    ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t)
+CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc,
+    ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests,
     ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t)
 CTL_RO_GEN(stats_arenas_i_lruns_j_curruns,
diff --git a/jemalloc/src/jemalloc.c b/jemalloc/src/jemalloc.c
index 49c2b0b..d880769 100644
--- a/jemalloc/src/jemalloc.c
+++ b/jemalloc/src/jemalloc.c
@@ -52,17 +52,9 @@
  *   |          |                  |     3584 |
  *   |          |                  |     3840 |
  *   |========================================|
- *   | Medium                      |    4 KiB |
- *   |                             |    6 KiB |
+ *   | Large                       |    4 KiB |
  *   |                             |    8 KiB |
- *   |                             |      ... |
- *   |                             |   28 KiB |
- *   |                             |   30 KiB |
- *   |                             |   32 KiB |
- *   |========================================|
- *   | Large                       |   36 KiB |
- *   |                             |   40 KiB |
- *   |                             |   44 KiB |
+ *   |                             |   12 KiB |
  *   |                             |      ... |
  *   |                             | 1012 KiB |
  *   |                             | 1016 KiB |
@@ -76,9 +68,8 @@
  *
  * Different mechanisms are used accoding to category:
  *
- *   Small/medium : Each size class is segregated into its own set of runs.
- *                  Each run maintains a bitmap of which regions are
- *                  free/allocated.
+ *   Small: Each size class is segregated into its own set of runs.  Each run
+ *          maintains a bitmap of which regions are free/allocated.
  *
  *   Large : Each allocation is backed by a dedicated run.  Metadata are stored
  *           in the associated arena chunk header maps.
@@ -252,6 +243,11 @@ stats_print_atexit(void)
 		if (arena != NULL) {
 			tcache_t *tcache;
 
+			/*
+			 * tcache_stats_merge() locks bins, so if any code is
+			 * introduced that acquires both arena and bin locks in
+			 * the opposite order, deadlocks may result.
+			 */
 			malloc_mutex_lock(&arena->lock);
 			ql_foreach(tcache, &arena->tcache_ql, link) {
 				tcache_stats_merge(tcache, arena);
@@ -510,14 +506,10 @@ MALLOC_OUT:
 				case 'k':
 					/*
 					 * Chunks always require at least one
-					 * header page, plus enough room to
-					 * hold a run for the largest medium
-					 * size class (one page more than the
-					 * size).
+					 * header page, plus one data page.
 					 */
 					if ((1U << (opt_lg_chunk - 1)) >=
-					    (2U << PAGE_SHIFT) + (1U <<
-					    opt_lg_medium_max))
+					    (2U << PAGE_SHIFT))
 						opt_lg_chunk--;
 					break;
 				case 'K':
@@ -533,15 +525,17 @@ MALLOC_OUT:
 					opt_prof_leak = true;
 					break;
 #endif
+#ifdef JEMALLOC_TCACHE
 				case 'm':
-					if (opt_lg_medium_max > PAGE_SHIFT)
-						opt_lg_medium_max--;
+					if (opt_lg_tcache_maxclass >= 0)
+						opt_lg_tcache_maxclass--;
 					break;
 				case 'M':
-					if (opt_lg_medium_max + 1 <
-					    opt_lg_chunk)
-						opt_lg_medium_max++;
+					if (opt_lg_tcache_maxclass + 1 <
+					    (sizeof(size_t) << 3))
+						opt_lg_tcache_maxclass++;
 					break;
+#endif
 				case 'n':
 					opt_narenas_lshift--;
 					break;
@@ -725,33 +719,7 @@ MALLOC_OUT:
 		 * For SMP systems, create more than one arena per CPU by
 		 * default.
 		 */
-#ifdef JEMALLOC_TCACHE
-		if (opt_tcache
-#  ifdef JEMALLOC_PROF
-		    /*
-		     * Profile data storage concurrency is directly linked to
-		     * the number of arenas, so only drop the number of arenas
-		     * on behalf of enabled tcache if profiling is disabled.
-		     */
-		    && opt_prof == false
-#  endif
-		    ) {
-			/*
-			 * Only large object allocation/deallocation is
-			 * guaranteed to acquire an arena mutex, so we can get
-			 * away with fewer arenas than without thread caching.
-			 */
-			opt_narenas_lshift += 1;
-		} else {
-#endif
-			/*
-			 * All allocations must acquire an arena mutex, so use
-			 * plenty of arenas.
-			 */
-			opt_narenas_lshift += 2;
-#ifdef JEMALLOC_TCACHE
-		}
-#endif
+		opt_narenas_lshift += 2;
 	}
 
 	/* Determine how many arenas to use. */
diff --git a/jemalloc/src/stats.c b/jemalloc/src/stats.c
index c57db47..a5ec1f1 100644
--- a/jemalloc/src/stats.c
+++ b/jemalloc/src/stats.c
@@ -234,8 +234,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				    j,
 				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
 				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
-				    : "M",
+				    "S",
 				    reg_size, nregs, run_size / pagesize,
 				    allocated, nmalloc, ndalloc, nrequests,
 				    nfills, nflushes, nruns, reruns, highruns,
@@ -248,8 +247,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				    j,
 				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
 				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
-				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
-				    : "M",
+				    "S",
 				    reg_size, nregs, run_size / pagesize,
 				    allocated, nmalloc, ndalloc, nruns, reruns,
 				    highruns, curruns);
@@ -278,12 +276,17 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	CTL_GET("arenas.pagesize", &pagesize, size_t);
 
 	malloc_cprintf(write_cb, cbopaque,
-	    "large:   size pages    nrequests      maxruns      curruns\n");
+	    "large:   size pages      nmalloc      ndalloc    nrequests"
+	    "      maxruns      curruns\n");
 	CTL_GET("arenas.nlruns", &nlruns, size_t);
 	for (j = 0, gap_start = -1; j < nlruns; j++) {
-		uint64_t nrequests;
+		uint64_t nmalloc, ndalloc, nrequests;
 		size_t run_size, highruns, curruns;
 
+		CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc,
+		    uint64_t);
+		CTL_IJ_GET("stats.arenas.0.lruns.0.ndalloc", &ndalloc,
+		    uint64_t);
 		CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests,
 		    uint64_t);
 		if (nrequests == 0) {
@@ -301,9 +304,10 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				gap_start = -1;
 			}
 			malloc_cprintf(write_cb, cbopaque,
-			    "%13zu %5zu %12"PRIu64" %12zu %12zu\n",
-			    run_size, run_size / pagesize, nrequests, highruns,
-			    curruns);
+			    "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64
+			    " %12zu %12zu\n",
+			    run_size, run_size / pagesize, nmalloc, ndalloc,
+			    nrequests, highruns, curruns);
 		}
 	}
 	if (gap_start != -1)
@@ -318,10 +322,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	uint64_t npurge, nmadvise, purged;
 	size_t small_allocated;
 	uint64_t small_nmalloc, small_ndalloc, small_nrequests;
-	size_t medium_allocated;
-	uint64_t medium_nmalloc, medium_ndalloc, medium_nrequests;
 	size_t large_allocated;
-	uint64_t large_nmalloc, large_ndalloc;
+	uint64_t large_nmalloc, large_ndalloc, large_nrequests;
 
 	CTL_GET("arenas.pagesize", &pagesize, size_t);
 
@@ -345,26 +347,19 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	malloc_cprintf(write_cb, cbopaque,
 	    "small:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
 	    small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
-	CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t);
-	CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t);
-	CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t);
-	CTL_I_GET("stats.arenas.0.medium.nrequests", &medium_nrequests,
-	    uint64_t);
-	malloc_cprintf(write_cb, cbopaque,
-	    "medium:  %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	    medium_allocated, medium_nmalloc, medium_ndalloc, medium_nrequests);
 	CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t);
+	CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t);
 	malloc_cprintf(write_cb, cbopaque,
 	    "large:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	    large_allocated, large_nmalloc, large_ndalloc, large_nmalloc);
+	    large_allocated, large_nmalloc, large_ndalloc, large_nrequests);
 	malloc_cprintf(write_cb, cbopaque,
 	    "total:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
-	    small_allocated + medium_allocated + large_allocated,
-	    small_nmalloc + medium_nmalloc + large_nmalloc,
-	    small_ndalloc + medium_ndalloc + large_ndalloc,
-	    small_nrequests + medium_nrequests + large_nmalloc);
+	    small_allocated + large_allocated,
+	    small_nmalloc + large_nmalloc,
+	    small_ndalloc + large_ndalloc,
+	    small_nrequests + large_nrequests);
 	malloc_cprintf(write_cb, cbopaque, "active:  %12zu\n",
 	    pactive * pagesize );
 	CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);
@@ -511,11 +506,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		write_cb(cbopaque, umax2s(sv, 10, s));
 		write_cb(cbopaque, "\n");
 
-		CTL_GET("arenas.medium", &sv, size_t);
-		write_cb(cbopaque, "Medium spacing: ");
-		write_cb(cbopaque, umax2s(sv, 10, s));
-		write_cb(cbopaque, "\n");
-
 		if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz,
 		    NULL, 0)) == 0) {
 			write_cb(cbopaque, "Tiny 2^n-spaced sizes: [");
@@ -551,14 +541,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 		write_cb(cbopaque, umax2s(sv, 10, s));
 		write_cb(cbopaque, "]\n");
 
-		CTL_GET("arenas.medium_min", &sv, size_t);
-		write_cb(cbopaque, "Medium sizes: [");
-		write_cb(cbopaque, umax2s(sv, 10, s));
-		write_cb(cbopaque, "..");
-		CTL_GET("arenas.medium_max", &sv, size_t);
-		write_cb(cbopaque, umax2s(sv, 10, s));
-		write_cb(cbopaque, "]\n");
-
 		CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t);
 		if (ssv >= 0) {
 			write_cb(cbopaque,
@@ -569,6 +551,13 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			write_cb(cbopaque,
 			    "Min active:dirty page ratio per arena: N/A\n");
 		}
+		if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv,
+		    &ssz, NULL, 0)) == 0) {
+			write_cb(cbopaque,
+			    "Maximum thread-cached size class: ");
+			write_cb(cbopaque, umax2s(sv, 10, s));
+			write_cb(cbopaque, "\n");
+		}
 		if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv,
 		    &ssz, NULL, 0)) == 0) {
 			size_t tcache_gc_sweep = (1U << ssv);
@@ -705,7 +694,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
 
 				for (i = 0; i < narenas; i++) {
 					if (initialized[i]) {
-						malloc_cprintf(write_cb, cbopaque,
+						malloc_cprintf(write_cb,
+						    cbopaque,
 						    "\narenas[%u]:\n", i);
 						stats_arena_print(write_cb,
 						    cbopaque, i);
diff --git a/jemalloc/src/tcache.c b/jemalloc/src/tcache.c
index feba61b..6113dec 100644
--- a/jemalloc/src/tcache.c
+++ b/jemalloc/src/tcache.c
@@ -5,6 +5,7 @@
 /* Data. */
 
 bool	opt_tcache = true;
+ssize_t	opt_lg_tcache_maxclass = LG_TCACHE_MAXCLASS_DEFAULT;
 ssize_t	opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT;
 
 /* Map of thread-specific caches. */
@@ -16,6 +17,8 @@ __thread tcache_t	*tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
  */
 static pthread_key_t		tcache_tsd;
 
+size_t				nhbins;
+size_t				tcache_maxclass;
 unsigned			tcache_gc_incr;
 
 /******************************************************************************/
@@ -26,11 +29,11 @@ static void	tcache_thread_cleanup(void *arg);
 /******************************************************************************/
 
 void *
-tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
+tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 {
 	void *ret;
 
-	arena_tcache_fill(tcache->arena, tbin, binind
+	arena_tcache_fill_small(tcache->arena, tbin, binind
 #ifdef JEMALLOC_PROF
 	    , tcache->prof_accumbytes
 #endif
@@ -38,13 +41,13 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 #ifdef JEMALLOC_PROF
 	tcache->prof_accumbytes = 0;
 #endif
-	ret = tcache_bin_alloc(tbin);
+	ret = tcache_alloc_easy(tbin);
 
 	return (ret);
 }
 
 void
-tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
+tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
 #if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
     , tcache_t *tcache
 #endif
@@ -53,6 +56,7 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 	void *flush, *deferred, *ptr;
 	unsigned i, nflush, ndeferred;
 
+	assert(binind < nbins);
 	assert(rem <= tbin->ncached);
 
 	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
@@ -120,6 +124,79 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 		tbin->low_water = tbin->ncached;
 }
 
+void
+tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+    , tcache_t *tcache
+#endif
+    )
+{
+	void *flush, *deferred, *ptr;
+	unsigned i, nflush, ndeferred;
+
+	assert(binind < nhbins);
+	assert(rem <= tbin->ncached);
+
+	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
+	    flush = deferred, nflush = ndeferred) {
+		/* Lock the arena associated with the first object. */
+		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
+		arena_t *arena = chunk->arena;
+
+		malloc_mutex_lock(&arena->lock);
+#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
+		if (arena == tcache->arena) {
+#endif
+#ifdef JEMALLOC_PROF
+			arena_prof_accum(arena, tcache->prof_accumbytes);
+			tcache->prof_accumbytes = 0;
+#endif
+#ifdef JEMALLOC_STATS
+			arena->stats.nrequests_large += tbin->tstats.nrequests;
+			arena->stats.lstats[binind - nbins].nrequests +=
+			    tbin->tstats.nrequests;
+			tbin->tstats.nrequests = 0;
+#endif
+#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
+		}
+#endif
+		deferred = NULL;
+		ndeferred = 0;
+		for (i = 0; i < nflush; i++) {
+			ptr = flush;
+			assert(ptr != NULL);
+			flush = *(void **)ptr;
+			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+			if (chunk->arena == arena)
+				arena_dalloc_large(arena, chunk, ptr);
+			else {
+				/*
+				 * This object was allocated via a different
+				 * arena than the one that is currently locked.
+				 * Stash the object, so that it can be handled
+				 * in a future pass.
+				 */
+				*(void **)ptr = deferred;
+				deferred = ptr;
+				ndeferred++;
+			}
+		}
+		malloc_mutex_unlock(&arena->lock);
+
+		if (flush != NULL) {
+			/*
+			 * This was the first pass, and rem cached objects
+			 * remain.
+			 */
+			tbin->avail = flush;
+		}
+	}
+
+	tbin->ncached = rem;
+	if (tbin->ncached < tbin->low_water)
+		tbin->low_water = tbin->ncached;
+}
+
 tcache_t *
 tcache_create(arena_t *arena)
 {
@@ -127,7 +204,7 @@ tcache_create(arena_t *arena)
 	size_t size;
 	unsigned i;
 
-	size = sizeof(tcache_t) + (sizeof(tcache_bin_t) * (nbins - 1));
+	size = sizeof(tcache_t) + (sizeof(tcache_bin_t) * (nhbins - 1));
 	/*
 	 * Round up to the nearest multiple of the cacheline size, in order to
 	 * avoid the possibility of false cacheline sharing.
@@ -138,8 +215,6 @@ tcache_create(arena_t *arena)
 
 	if (size <= small_maxclass)
 		tcache = (tcache_t *)arena_malloc_small(arena, size, true);
-	else if (size <= bin_maxclass)
-		tcache = (tcache_t *)arena_malloc_medium(arena, size, true);
 	else
 		tcache = (tcache_t *)icalloc(size);
 
@@ -155,14 +230,16 @@ tcache_create(arena_t *arena)
 #endif
 
 	tcache->arena = arena;
-	assert((TCACHE_NSLOTS_MAX & 1U) == 0);
+	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
 	for (i = 0; i < nbins; i++) {
-		if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_MAX) {
+		if ((arena->bins[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) {
 			tcache->tbins[i].ncached_max = (arena->bins[i].nregs <<
 			    1);
 		} else
-			tcache->tbins[i].ncached_max = TCACHE_NSLOTS_MAX;
+			tcache->tbins[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX;
 	}
+	for (; i < nhbins; i++)
+		tcache->tbins[i].ncached_max = TCACHE_NSLOTS_LARGE;
 
 	tcache_tls = tcache;
 	pthread_setspecific(tcache_tsd, tcache);
@@ -185,7 +262,7 @@ tcache_destroy(tcache_t *tcache)
 
 	for (i = 0; i < nbins; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];
-		tcache_bin_flush(tbin, i, 0
+		tcache_bin_flush_small(tbin, i, 0
 #if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 		    , tcache
 #endif
@@ -202,6 +279,26 @@ tcache_destroy(tcache_t *tcache)
 #endif
 	}
 
+	for (; i < nhbins; i++) {
+		tcache_bin_t *tbin = &tcache->tbins[i];
+		tcache_bin_flush_large(tbin, i, 0
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
+		    , tcache
+#endif
+		    );
+
+#ifdef JEMALLOC_STATS
+		if (tbin->tstats.nrequests != 0) {
+			arena_t *arena = tcache->arena;
+			malloc_mutex_lock(&arena->lock);
+			arena->stats.nrequests_large += tbin->tstats.nrequests;
+			arena->stats.lstats[i - nbins].nrequests +=
+			    tbin->tstats.nrequests;
+			malloc_mutex_unlock(&arena->lock);
+		}
+#endif
+	}
+
 #ifdef JEMALLOC_PROF
 	if (tcache->prof_accumbytes > 0) {
 		malloc_mutex_lock(&tcache->arena->lock);
@@ -210,7 +307,7 @@ tcache_destroy(tcache_t *tcache)
 	}
 #endif
 
-	if (arena_salloc(tcache) <= bin_maxclass) {
+	if (arena_salloc(tcache) <= small_maxclass) {
 		arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
 		arena_t *arena = chunk->arena;
 		size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
@@ -256,6 +353,14 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 		malloc_mutex_unlock(&bin->lock);
 		tbin->tstats.nrequests = 0;
 	}
+
+	for (; i < nhbins; i++) {
+		malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins];
+		tcache_bin_t *tbin = &tcache->tbins[i];
+		arena->stats.nrequests_large += tbin->tstats.nrequests;
+		lstats->nrequests += tbin->tstats.nrequests;
+		tbin->tstats.nrequests = 0;
+	}
 }
 #endif
 
@@ -264,6 +369,20 @@ tcache_boot(void)
 {
 
 	if (opt_tcache) {
+		/*
+		 * If necessary, clamp opt_lg_tcache_maxclass, now that
+		 * small_maxclass and arena_maxclass are known.
+		 */
+		if (opt_lg_tcache_maxclass < 0 || (1U <<
+		    opt_lg_tcache_maxclass) < small_maxclass)
+			tcache_maxclass = small_maxclass;
+		else if ((1U << opt_lg_tcache_maxclass) > arena_maxclass)
+			tcache_maxclass = arena_maxclass;
+		else
+			tcache_maxclass = (1U << opt_lg_tcache_maxclass);
+
+		nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT);
+
 		/* Compute incremental GC event threshold. */
 		if (opt_lg_tcache_gc_sweep >= 0) {
 			tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) /