Push locks into arena bins.

For bin-related allocation, protect data structures with bin locks
rather than arena locks.  Arena locks remain for run
allocation/deallocation and other miscellaneous operations.

Restructure statistics counters to maintain per bin
allocated/nmalloc/ndalloc, but continue to provide arena-wide statistics
via aggregation in the ctl code.

9 files changed, 368 insertions, 191 deletions

diff --git a/jemalloc/doc/jemalloc.3.in b/jemalloc/doc/jemalloc.3.in
index 197e68c..71f058b 100644
--- a/jemalloc/doc/jemalloc.3.in
+++ b/jemalloc/doc/jemalloc.3.in
@@ -38,7 +38,7 @@
 .\"     @(#)malloc.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD: head/lib/libc/stdlib/malloc.3 182225 2008-08-27 02:00:53Z jasone $
 .\"
-.Dd March 4, 2010
+.Dd March 13, 2010
 .Dt JEMALLOC 3
 .Os
 .Sh NAME
@@ -1134,12 +1134,17 @@ has not been called.
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.small.nmalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of small allocation requests.
+@roff_stats@Cumulative number of allocation requests served by small bins.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.small.ndalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of small deallocation requests.
+@roff_stats@Cumulative number of small objects returned to bins.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.small.nrequests (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of small allocation requests.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.medium.allocated (size_t) r-"
@@ -1149,12 +1154,17 @@ has not been called.
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.medium.nmalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of medium allocation requests.
+@roff_stats@Cumulative number of allocation requests served by medium bins.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.medium.ndalloc (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
-@roff_stats@Cumulative number of medium deallocation requests.
+@roff_stats@Cumulative number of medium objects returned to bins.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.medium.nrequests (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of medium allocation requests.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.large.allocated (size_t) r-"
@@ -1172,6 +1182,21 @@ has not been called.
 @roff_stats@Cumulative number of large deallocation requests.
 @roff_stats@.Ed
 .\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.allocated (size_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Current number of bytes allocated by bin.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of allocations served by bin.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
+@roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) r-"
+@roff_stats@.Bd -ragged -offset indent -compact
+@roff_stats@Cumulative number of allocations returned to bin.
+@roff_stats@.Ed
+.\"-----------------------------------------------------------------------------
 @roff_stats@.It Sy "stats.arenas.<i>.bins.<j>.nrequests (uint64_t) r-"
 @roff_stats@.Bd -ragged -offset indent -compact
 @roff_stats@Cumulative number of allocation requests.
diff --git a/jemalloc/include/jemalloc/internal/arena.h b/jemalloc/include/jemalloc/internal/arena.h
index c6fe823..06b60ca 100644
--- a/jemalloc/include/jemalloc/internal/arena.h
+++ b/jemalloc/include/jemalloc/internal/arena.h
@@ -207,6 +207,14 @@ struct arena_run_s {
 
 struct arena_bin_s {
 	/*
+	 * All operations on runcur, runs, and stats require that lock be
+	 * locked.  Run allocation/deallocation are protected by the arena lock,
+	 * which may be acquired while holding one or more bin locks, but not
+	 * vise versa.
+	 */
+	malloc_mutex_t	lock;
+
+	/*
 	 * Current run being used to service allocations of this bin's size
 	 * class.
 	 */
@@ -256,7 +264,10 @@ struct arena_s {
 	/* This arena's index within the arenas array. */
 	unsigned		ind;
 
-	/* All operations on this arena require that lock be locked. */
+	/*
+	 * All non-bin-related operations on this arena require that lock be
+	 * locked.
+	 */
 	malloc_mutex_t		lock;
 
 #ifdef JEMALLOC_STATS
@@ -459,9 +470,18 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 			tcache_dalloc(tcache, ptr);
 		else {
 #endif
-			malloc_mutex_lock(&arena->lock);
+			arena_run_t *run;
+			arena_bin_t *bin;
+
+			run = (arena_run_t *)((uintptr_t)chunk +
+			    (uintptr_t)((pageind - ((mapelm->bits &
+			    CHUNK_MAP_PG_MASK) >> CHUNK_MAP_PG_SHIFT)) <<
+			    PAGE_SHIFT));
+			assert(run->magic == ARENA_RUN_MAGIC);
+			bin = run->bin;
+			malloc_mutex_lock(&bin->lock);
 			arena_dalloc_bin(arena, chunk, ptr, mapelm);
-			malloc_mutex_unlock(&arena->lock);
+			malloc_mutex_unlock(&bin->lock);
 #ifdef JEMALLOC_TCACHE
 		}
 #endif
diff --git a/jemalloc/include/jemalloc/internal/ctl.h b/jemalloc/include/jemalloc/internal/ctl.h
index d5b128b..9a39e14 100644
--- a/jemalloc/include/jemalloc/internal/ctl.h
+++ b/jemalloc/include/jemalloc/internal/ctl.h
@@ -33,6 +33,18 @@ struct ctl_arena_stats_s {
 	size_t			pdirty;
 #ifdef JEMALLOC_STATS
 	arena_stats_t		astats;
+
+	/* Aggregate stats for small/medium size classes, based on bin stats. */
+	size_t			allocated_small;
+	uint64_t		nmalloc_small;
+	uint64_t		ndalloc_small;
+	uint64_t		nrequests_small;
+
+	size_t			allocated_medium;
+	uint64_t		nmalloc_medium;
+	uint64_t		ndalloc_medium;
+	uint64_t		nrequests_medium;
+
 	malloc_bin_stats_t	*bstats;	/* nbins elements. */
 	malloc_large_stats_t	*lstats;	/* nlclasses elements. */
 #endif
diff --git a/jemalloc/include/jemalloc/internal/stats.h b/jemalloc/include/jemalloc/internal/stats.h
index 8e7f95a..47701e6 100644
--- a/jemalloc/include/jemalloc/internal/stats.h
+++ b/jemalloc/include/jemalloc/internal/stats.h
@@ -31,8 +31,23 @@ struct tcache_bin_stats_s {
 
 struct malloc_bin_stats_s {
 	/*
-	 * Number of allocation requests that corresponded to the size of this
-	 * bin.
+	 * Current number of bytes allocated, including objects currently
+	 * cached by tcache.
+	 */
+	size_t		allocated;
+	/*
+	 * Total number of allocation/deallocation requests served directly by
+	 * the bin.  Note that tcache may allocate an object, then recycle it
+	 * many times, resulting many increments to nrequests, but only one
+	 * each to nmalloc and ndalloc.
+	 */
+	uint64_t	nmalloc;
+	uint64_t	ndalloc;
+
+	/*
+	 * Number of allocation requests that correspond to the size of this
+	 * bin.  This includes requests served by tcache, though tcache only
+	 * periodically merges into this counter.
 	 */
 	uint64_t	nrequests;
 
@@ -87,14 +102,6 @@ struct arena_stats_s {
 	uint64_t	purged;
 
 	/* Per-size-category statistics. */
-	size_t		allocated_small;
-	uint64_t	nmalloc_small;
-	uint64_t	ndalloc_small;
-
-	size_t		allocated_medium;
-	uint64_t	nmalloc_medium;
-	uint64_t	ndalloc_medium;
-
 	size_t		allocated_large;
 	uint64_t	nmalloc_large;
 	uint64_t	ndalloc_large;
diff --git a/jemalloc/include/jemalloc/internal/tcache.h b/jemalloc/include/jemalloc/internal/tcache.h
index fa238bf..ad30b7c 100644
--- a/jemalloc/include/jemalloc/internal/tcache.h
+++ b/jemalloc/include/jemalloc/internal/tcache.h
@@ -64,7 +64,7 @@ extern __thread tcache_t	*tcache_tls
 extern unsigned			tcache_gc_incr;
 
 void	tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
     , tcache_t *tcache
 #endif
     );
@@ -130,7 +130,7 @@ tcache_event(tcache_t *tcache)
 			 */
 			tcache_bin_flush(tbin, binind, tbin->ncached -
 			    tbin->low_water + (tbin->low_water >> 2)
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 			    , tcache
 #endif
 			    );
@@ -234,7 +234,7 @@ tcache_dalloc(tcache_t *tcache, void *ptr)
 	tbin = &tcache->tbins[binind];
 	if (tbin->ncached == tbin->ncached_max) {
 		tcache_bin_flush(tbin, binind, (tbin->ncached_max >> 1)
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 		    , tcache
 #endif
 		    );
diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c
index e7ecb43..3981cbb 100644
--- a/jemalloc/src/arena.c
+++ b/jemalloc/src/arena.c
@@ -769,7 +769,9 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
 	/* No existing runs have any space available. */
 
 	/* Allocate a new run. */
+	malloc_mutex_lock(&arena->lock);
 	run = arena_run_alloc(arena, bin->run_size, false, false);
+	malloc_mutex_unlock(&arena->lock);
 	if (run == NULL)
 		return (NULL);
 
@@ -805,6 +807,21 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
 	return (arena_run_reg_alloc(bin->runcur, bin));
 }
 
+#ifdef JEMALLOC_PROF
+void
+arena_prof_accum(arena_t *arena, uint64_t accumbytes)
+{
+
+	if (prof_interval != 0) {
+		arena->prof_accumbytes += accumbytes;
+		if (arena->prof_accumbytes >= prof_interval) {
+			prof_idump();
+			arena->prof_accumbytes -= prof_interval;
+		}
+	}
+}
+#endif
+
 #ifdef JEMALLOC_TCACHE
 void
 arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
@@ -821,9 +838,11 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
 	assert(tbin->ncached == 0);
 
 	bin = &arena->bins[binind];
-	malloc_mutex_lock(&arena->lock);
+	malloc_mutex_lock(&bin->lock);
 #ifdef JEMALLOC_PROF
+	malloc_mutex_lock(&arena->lock);
 	arena_prof_accum(arena, prof_accumbytes);
+	malloc_mutex_unlock(&arena->lock);
 #endif
 	for (i = 0, nfill = (tbin->ncached_max >> 1); i < nfill; i++) {
 		if ((run = bin->runcur) != NULL && run->nfree > 0)
@@ -836,41 +855,19 @@ arena_tcache_fill(arena_t *arena, tcache_bin_t *tbin, size_t binind
 		tbin->avail = ptr;
 	}
 #ifdef JEMALLOC_STATS
-	bin->stats.nfills++;
+	bin->stats.allocated += (i - tbin->ncached) * bin->reg_size;
+	bin->stats.nmalloc += i;
 	bin->stats.nrequests += tbin->tstats.nrequests;
-	if (bin->reg_size <= small_maxclass) {
-		arena->stats.allocated_small += (i - tbin->ncached) *
-		    bin->reg_size;
-		arena->stats.nmalloc_small += tbin->tstats.nrequests;
-	} else {
-		arena->stats.allocated_medium += (i - tbin->ncached) *
-		    bin->reg_size;
-		arena->stats.nmalloc_medium += tbin->tstats.nrequests;
-	}
+	bin->stats.nfills++;
 	tbin->tstats.nrequests = 0;
 #endif
-	malloc_mutex_unlock(&arena->lock);
+	malloc_mutex_unlock(&bin->lock);
 	tbin->ncached = i;
 	if (tbin->ncached > tbin->high_water)
 		tbin->high_water = tbin->ncached;
 }
 #endif
 
-#ifdef JEMALLOC_PROF
-void
-arena_prof_accum(arena_t *arena, uint64_t accumbytes)
-{
-
-	if (prof_interval != 0) {
-		arena->prof_accumbytes += accumbytes;
-		if (arena->prof_accumbytes >= prof_interval) {
-			prof_idump();
-			arena->prof_accumbytes -= prof_interval;
-		}
-	}
-}
-#endif
-
 /*
  * Calculate bin->run_size such that it meets the following constraints:
  *
@@ -993,33 +990,30 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 	bin = &arena->bins[binind];
 	size = bin->reg_size;
 
-	malloc_mutex_lock(&arena->lock);
+	malloc_mutex_lock(&bin->lock);
 	if ((run = bin->runcur) != NULL && run->nfree > 0)
 		ret = arena_run_reg_alloc(run, bin);
 	else
 		ret = arena_bin_malloc_hard(arena, bin);
 
 	if (ret == NULL) {
-		malloc_mutex_unlock(&arena->lock);
+		malloc_mutex_unlock(&bin->lock);
 		return (NULL);
 	}
 
 #ifdef JEMALLOC_STATS
-#  ifdef JEMALLOC_TCACHE
-	if (isthreaded == false) {
-#  endif
-		bin->stats.nrequests++;
-		arena->stats.nmalloc_small++;
-#  ifdef JEMALLOC_TCACHE
-	}
-#  endif
-	arena->stats.allocated_small += size;
+	bin->stats.allocated += size;
+	bin->stats.nmalloc++;
+	bin->stats.nrequests++;
 #endif
+	malloc_mutex_unlock(&bin->lock);
 #ifdef JEMALLOC_PROF
-	if (isthreaded == false)
+	if (isthreaded == false) {
+		malloc_mutex_lock(&arena->lock);
 		arena_prof_accum(arena, size);
+		malloc_mutex_unlock(&arena->lock);
+	}
 #endif
-	malloc_mutex_unlock(&arena->lock);
 
 	if (zero == false) {
 #ifdef JEMALLOC_FILL
@@ -1048,33 +1042,30 @@ arena_malloc_medium(arena_t *arena, size_t size, bool zero)
 	bin = &arena->bins[binind];
 	assert(bin->reg_size == size);
 
-	malloc_mutex_lock(&arena->lock);
+	malloc_mutex_lock(&bin->lock);
 	if ((run = bin->runcur) != NULL && run->nfree > 0)
 		ret = arena_run_reg_alloc(run, bin);
 	else
 		ret = arena_bin_malloc_hard(arena, bin);
 
 	if (ret == NULL) {
-		malloc_mutex_unlock(&arena->lock);
+		malloc_mutex_unlock(&bin->lock);
 		return (NULL);
 	}
 
 #ifdef JEMALLOC_STATS
-#  ifdef JEMALLOC_TCACHE
-	if (isthreaded == false) {
-#  endif
-		bin->stats.nrequests++;
-		arena->stats.nmalloc_medium++;
-#  ifdef JEMALLOC_TCACHE
-	}
-#  endif
-	arena->stats.allocated_medium += size;
+	bin->stats.allocated += size;
+	bin->stats.nmalloc++;
+	bin->stats.nrequests++;
 #endif
+	malloc_mutex_unlock(&bin->lock);
 #ifdef JEMALLOC_PROF
-	if (isthreaded == false)
+	if (isthreaded == false) {
+		malloc_mutex_lock(&arena->lock);
 		arena_prof_accum(arena, size);
+		malloc_mutex_unlock(&arena->lock);
+	}
 #endif
-	malloc_mutex_unlock(&arena->lock);
 
 	if (zero == false) {
 #ifdef JEMALLOC_FILL
@@ -1417,6 +1408,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 	run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT);
 	past = (size_t)(((uintptr_t)run->next - (uintptr_t)1U -
 	    (uintptr_t)chunk) >> PAGE_SHIFT) + 1;
+
+	malloc_mutex_lock(&arena->lock);
 	chunk->ndirty += past - run_ind;
 	arena->ndirty += past - run_ind;
 	for (; run_ind < past; run_ind++) {
@@ -1428,9 +1421,6 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 	run->magic = 0;
 #endif
 	arena_run_dalloc(arena, run, false);
-#ifdef JEMALLOC_STATS
-	bin->stats.curruns--;
-#endif
 
 	if (chunk->dirtied == false) {
 		ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty);
@@ -1440,6 +1430,11 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
 	if (opt_lg_dirty_mult >= 0 && arena->ndirty > chunk_npages &&
 	    (arena->nactive >> opt_lg_dirty_mult) < arena->ndirty)
 		arena_purge(arena);
+
+	malloc_mutex_unlock(&arena->lock);
+#ifdef JEMALLOC_STATS
+	bin->stats.curruns--;
+#endif
 }
 
 void
@@ -1508,13 +1503,8 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 	}
 
 #ifdef JEMALLOC_STATS
-	if (size <= small_maxclass) {
-		arena->stats.allocated_small -= size;
-		arena->stats.ndalloc_small++;
-	} else {
-		arena->stats.allocated_medium -= size;
-		arena->stats.ndalloc_medium++;
-	}
+	bin->stats.allocated -= size;
+	bin->stats.ndalloc++;
 #endif
 }
 
@@ -1526,6 +1516,7 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
 {
 	unsigned i;
 
+	malloc_mutex_lock(&arena->lock);
 	*nactive += arena->nactive;
 	*ndirty += arena->ndirty;
 
@@ -1533,33 +1524,35 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
 	astats->npurge += arena->stats.npurge;
 	astats->nmadvise += arena->stats.nmadvise;
 	astats->purged += arena->stats.purged;
-	astats->allocated_small += arena->stats.allocated_small;
-	astats->nmalloc_small += arena->stats.nmalloc_small;
-	astats->ndalloc_small += arena->stats.ndalloc_small;
-	astats->allocated_medium += arena->stats.allocated_medium;
-	astats->nmalloc_medium += arena->stats.nmalloc_medium;
-	astats->ndalloc_medium += arena->stats.ndalloc_medium;
 	astats->allocated_large += arena->stats.allocated_large;
 	astats->nmalloc_large += arena->stats.nmalloc_large;
 	astats->ndalloc_large += arena->stats.ndalloc_large;
 
-	for (i = 0; i < nbins; i++) {
-		bstats[i].nrequests += arena->bins[i].stats.nrequests;
-#ifdef JEMALLOC_TCACHE
-		bstats[i].nfills += arena->bins[i].stats.nfills;
-		bstats[i].nflushes += arena->bins[i].stats.nflushes;
-#endif
-		bstats[i].nruns += arena->bins[i].stats.nruns;
-		bstats[i].reruns += arena->bins[i].stats.reruns;
-		bstats[i].highruns += arena->bins[i].stats.highruns;
-		bstats[i].curruns += arena->bins[i].stats.curruns;
-	}
-
 	for (i = 0; i < nlclasses; i++) {
 		lstats[i].nrequests += arena->stats.lstats[i].nrequests;
 		lstats[i].highruns += arena->stats.lstats[i].highruns;
 		lstats[i].curruns += arena->stats.lstats[i].curruns;
 	}
+	malloc_mutex_unlock(&arena->lock);
+
+	for (i = 0; i < nbins; i++) {
+		arena_bin_t *bin = &arena->bins[i];
+
+		malloc_mutex_lock(&bin->lock);
+		bstats[i].allocated += bin->stats.allocated;
+		bstats[i].nmalloc += bin->stats.nmalloc;
+		bstats[i].ndalloc += bin->stats.ndalloc;
+		bstats[i].nrequests += bin->stats.nrequests;
+#ifdef JEMALLOC_TCACHE
+		bstats[i].nfills += bin->stats.nfills;
+		bstats[i].nflushes += bin->stats.nflushes;
+#endif
+		bstats[i].nruns += bin->stats.nruns;
+		bstats[i].reruns += bin->stats.reruns;
+		bstats[i].highruns += bin->stats.highruns;
+		bstats[i].curruns += bin->stats.curruns;
+		malloc_mutex_unlock(&bin->lock);
+	}
 }
 #endif
 
@@ -1860,6 +1853,8 @@ arena_new(arena_t *arena, unsigned ind)
 	/* (2^n)-spaced tiny bins. */
 	for (; i < ntbins; i++) {
 		bin = &arena->bins[i];
+		if (malloc_mutex_init(&bin->lock))
+			return (true);
 		bin->runcur = NULL;
 		arena_run_tree_new(&bin->runs);
 
@@ -1876,6 +1871,8 @@ arena_new(arena_t *arena, unsigned ind)
 	/* Quantum-spaced bins. */
 	for (; i < ntbins + nqbins; i++) {
 		bin = &arena->bins[i];
+		if (malloc_mutex_init(&bin->lock))
+			return (true);
 		bin->runcur = NULL;
 		arena_run_tree_new(&bin->runs);
 
@@ -1891,6 +1888,8 @@ arena_new(arena_t *arena, unsigned ind)
 	/* Cacheline-spaced bins. */
 	for (; i < ntbins + nqbins + ncbins; i++) {
 		bin = &arena->bins[i];
+		if (malloc_mutex_init(&bin->lock))
+			return (true);
 		bin->runcur = NULL;
 		arena_run_tree_new(&bin->runs);
 
@@ -1907,6 +1906,8 @@ arena_new(arena_t *arena, unsigned ind)
 	/* Subpage-spaced bins. */
 	for (; i < ntbins + nqbins + ncbins + nsbins; i++) {
 		bin = &arena->bins[i];
+		if (malloc_mutex_init(&bin->lock))
+			return (true);
 		bin->runcur = NULL;
 		arena_run_tree_new(&bin->runs);
 
@@ -1923,6 +1924,8 @@ arena_new(arena_t *arena, unsigned ind)
 	/* Medium bins. */
 	for (; i < nbins; i++) {
 		bin = &arena->bins[i];
+		if (malloc_mutex_init(&bin->lock))
+			return (true);
 		bin->runcur = NULL;
 		arena_run_tree_new(&bin->runs);
 
diff --git a/jemalloc/src/ctl.c b/jemalloc/src/ctl.c
index 28e9368..1644f73 100644
--- a/jemalloc/src/ctl.c
+++ b/jemalloc/src/ctl.c
@@ -24,6 +24,12 @@ const ctl_node_t	*n##_index(const size_t *mib, size_t miblen,	\
 static bool	ctl_arena_init(ctl_arena_stats_t *astats);
 #endif
 static void	ctl_arena_clear(ctl_arena_stats_t *astats);
+#ifdef JEMALLOC_STATS
+static void	ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
+    arena_t *arena);
+static void	ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
+    ctl_arena_stats_t *astats);
+#endif
 static void	ctl_arena_refresh(arena_t *arena, unsigned i);
 static void	ctl_refresh(void);
 static bool	ctl_init(void);
@@ -131,12 +137,17 @@ CTL_PROTO(stats_huge_ndalloc)
 CTL_PROTO(stats_arenas_i_small_allocated)
 CTL_PROTO(stats_arenas_i_small_nmalloc)
 CTL_PROTO(stats_arenas_i_small_ndalloc)
+CTL_PROTO(stats_arenas_i_small_nrequests)
 CTL_PROTO(stats_arenas_i_medium_allocated)
 CTL_PROTO(stats_arenas_i_medium_nmalloc)
 CTL_PROTO(stats_arenas_i_medium_ndalloc)
+CTL_PROTO(stats_arenas_i_medium_nrequests)
 CTL_PROTO(stats_arenas_i_large_allocated)
 CTL_PROTO(stats_arenas_i_large_nmalloc)
 CTL_PROTO(stats_arenas_i_large_ndalloc)
+CTL_PROTO(stats_arenas_i_bins_j_allocated)
+CTL_PROTO(stats_arenas_i_bins_j_nmalloc)
+CTL_PROTO(stats_arenas_i_bins_j_ndalloc)
 CTL_PROTO(stats_arenas_i_bins_j_nrequests)
 #ifdef JEMALLOC_TCACHE
 CTL_PROTO(stats_arenas_i_bins_j_nfills)
@@ -332,13 +343,15 @@ static const ctl_node_t stats_huge_node[] = {
 static const ctl_node_t stats_arenas_i_small_node[] = {
 	{NAME("allocated"),		CTL(stats_arenas_i_small_allocated)},
 	{NAME("nmalloc"),		CTL(stats_arenas_i_small_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_small_ndalloc)}
+	{NAME("ndalloc"),		CTL(stats_arenas_i_small_ndalloc)},
+	{NAME("nrequests"),		CTL(stats_arenas_i_small_nrequests)}
 };
 
 static const ctl_node_t stats_arenas_i_medium_node[] = {
 	{NAME("allocated"),		CTL(stats_arenas_i_medium_allocated)},
 	{NAME("nmalloc"),		CTL(stats_arenas_i_medium_nmalloc)},
-	{NAME("ndalloc"),		CTL(stats_arenas_i_medium_ndalloc)}
+	{NAME("ndalloc"),		CTL(stats_arenas_i_medium_ndalloc)},
+	{NAME("nrequests"),		CTL(stats_arenas_i_medium_nrequests)}
 };
 
 static const ctl_node_t stats_arenas_i_large_node[] = {
@@ -348,6 +361,9 @@ static const ctl_node_t stats_arenas_i_large_node[] = {
 };
 
 static const ctl_node_t stats_arenas_i_bins_j_node[] = {
+	{NAME("allocated"),		CTL(stats_arenas_i_bins_j_allocated)},
+	{NAME("nmalloc"),		CTL(stats_arenas_i_bins_j_nmalloc)},
+	{NAME("ndalloc"),		CTL(stats_arenas_i_bins_j_ndalloc)},
 	{NAME("nrequests"),		CTL(stats_arenas_i_bins_j_nrequests)},
 #ifdef JEMALLOC_TCACHE
 	{NAME("nfills"),		CTL(stats_arenas_i_bins_j_nfills)},
@@ -485,11 +501,93 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
 	astats->pdirty = 0;
 #ifdef JEMALLOC_STATS
 	memset(&astats->astats, 0, sizeof(arena_stats_t));
+	astats->allocated_small = 0;
+	astats->nmalloc_small = 0;
+	astats->ndalloc_small = 0;
+	astats->nrequests_small = 0;
+	astats->allocated_medium = 0;
+	astats->nmalloc_medium = 0;
+	astats->ndalloc_medium = 0;
+	astats->nrequests_medium = 0;
 	memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t));
 	memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t));
 #endif
 }
 
+#ifdef JEMALLOC_STATS
+static void
+ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
+{
+	unsigned i;
+
+	arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
+	    &cstats->astats, cstats->bstats, cstats->lstats);
+
+	for (i = 0; i < mbin0; i++) {
+		cstats->allocated_small += cstats->bstats[i].allocated;
+		cstats->nmalloc_small += cstats->bstats[i].nmalloc;
+		cstats->ndalloc_small += cstats->bstats[i].ndalloc;
+		cstats->nrequests_small += cstats->bstats[i].nrequests;
+	}
+
+	for (; i < nbins; i++) {
+		cstats->allocated_medium += cstats->bstats[i].allocated;
+		cstats->nmalloc_medium += cstats->bstats[i].nmalloc;
+		cstats->ndalloc_medium += cstats->bstats[i].ndalloc;
+		cstats->nrequests_medium += cstats->bstats[i].nrequests;
+	}
+}
+
+static void
+ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
+{
+	unsigned i;
+
+	sstats->pactive += astats->pactive;
+	sstats->pdirty += astats->pdirty;
+
+	sstats->astats.mapped += astats->astats.mapped;
+	sstats->astats.npurge += astats->astats.npurge;
+	sstats->astats.nmadvise += astats->astats.nmadvise;
+	sstats->astats.purged += astats->astats.purged;
+
+	sstats->allocated_small += astats->allocated_small;
+	sstats->nmalloc_small += astats->nmalloc_small;
+	sstats->ndalloc_small += astats->ndalloc_small;
+	sstats->nrequests_small += astats->nrequests_small;
+
+	sstats->allocated_medium += astats->allocated_medium;
+	sstats->nmalloc_medium += astats->nmalloc_medium;
+	sstats->ndalloc_medium += astats->ndalloc_medium;
+	sstats->nrequests_medium += astats->nrequests_medium;
+
+	sstats->astats.allocated_large += astats->astats.allocated_large;
+	sstats->astats.nmalloc_large += astats->astats.nmalloc_large;
+	sstats->astats.ndalloc_large += astats->astats.ndalloc_large;
+
+	for (i = 0; i < nlclasses; i++) {
+		sstats->lstats[i].nrequests += astats->lstats[i].nrequests;
+		sstats->lstats[i].highruns += astats->lstats[i].highruns;
+		sstats->lstats[i].curruns += astats->lstats[i].curruns;
+	}
+
+	for (i = 0; i < nbins; i++) {
+		sstats->bstats[i].allocated += astats->bstats[i].allocated;
+		sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc;
+		sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc;
+		sstats->bstats[i].nrequests += astats->bstats[i].nrequests;
+#ifdef JEMALLOC_TCACHE
+		sstats->bstats[i].nfills += astats->bstats[i].nfills;
+		sstats->bstats[i].nflushes += astats->bstats[i].nflushes;
+#endif
+		sstats->bstats[i].nruns += astats->bstats[i].nruns;
+		sstats->bstats[i].reruns += astats->bstats[i].reruns;
+		sstats->bstats[i].highruns += astats->bstats[i].highruns;
+		sstats->bstats[i].curruns += astats->bstats[i].curruns;
+	}
+}
+#endif
+
 static void
 ctl_arena_refresh(arena_t *arena, unsigned i)
 {
@@ -498,13 +596,10 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 
 	ctl_arena_clear(astats);
 
-	malloc_mutex_lock(&arena->lock);
 #ifdef JEMALLOC_STATS
-	arena_stats_merge(arena, &astats->pactive, &astats->pdirty,
-	    &astats->astats, astats->bstats, astats->lstats);
+	ctl_arena_stats_amerge(astats, arena);
 	/* Merge into sum stats as well. */
-	arena_stats_merge(arena, &sstats->pactive, &sstats->pdirty,
-	    &sstats->astats, sstats->bstats, sstats->lstats);
+	ctl_arena_stats_smerge(sstats, astats);
 #else
 	astats->pactive += arena->nactive;
 	astats->pdirty += arena->ndirty;
@@ -512,7 +607,6 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 	sstats->pactive += arena->nactive;
 	sstats->pdirty += arena->ndirty;
 #endif
-	malloc_mutex_unlock(&arena->lock);
 }
 
 static void
@@ -553,8 +647,8 @@ ctl_refresh(void)
 	}
 
 #ifdef JEMALLOC_STATS
-	ctl_stats.allocated = ctl_stats.arenas[narenas].astats.allocated_small
-	    + ctl_stats.arenas[narenas].astats.allocated_medium
+	ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
+	    + ctl_stats.arenas[narenas].allocated_medium
 	    + ctl_stats.arenas[narenas].astats.allocated_large
 	    + ctl_stats.huge.allocated;
 	ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT)
@@ -1203,17 +1297,21 @@ CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t)
 CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t)
 CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_small_allocated,
-    ctl_stats.arenas[mib[2]].astats.allocated_small, size_t)
+    ctl_stats.arenas[mib[2]].allocated_small, size_t)
 CTL_RO_GEN(stats_arenas_i_small_nmalloc,
-    ctl_stats.arenas[mib[2]].astats.nmalloc_small, uint64_t)
+    ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t)
 CTL_RO_GEN(stats_arenas_i_small_ndalloc,
-    ctl_stats.arenas[mib[2]].astats.ndalloc_small, uint64_t)
+    ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t)
+CTL_RO_GEN(stats_arenas_i_small_nrequests,
+    ctl_stats.arenas[mib[2]].nrequests_small, uint64_t)
 CTL_RO_GEN(stats_arenas_i_medium_allocated,
-    ctl_stats.arenas[mib[2]].astats.allocated_medium, size_t)
+    ctl_stats.arenas[mib[2]].allocated_medium, size_t)
 CTL_RO_GEN(stats_arenas_i_medium_nmalloc,
-    ctl_stats.arenas[mib[2]].astats.nmalloc_medium, uint64_t)
+    ctl_stats.arenas[mib[2]].nmalloc_medium, uint64_t)
 CTL_RO_GEN(stats_arenas_i_medium_ndalloc,
-    ctl_stats.arenas[mib[2]].astats.ndalloc_medium, uint64_t)
+    ctl_stats.arenas[mib[2]].ndalloc_medium, uint64_t)
+CTL_RO_GEN(stats_arenas_i_medium_nrequests,
+    ctl_stats.arenas[mib[2]].nrequests_medium, uint64_t)
 CTL_RO_GEN(stats_arenas_i_large_allocated,
     ctl_stats.arenas[mib[2]].astats.allocated_large, size_t)
 CTL_RO_GEN(stats_arenas_i_large_nmalloc,
@@ -1221,6 +1319,12 @@ CTL_RO_GEN(stats_arenas_i_large_nmalloc,
 CTL_RO_GEN(stats_arenas_i_large_ndalloc,
     ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t)
 
+CTL_RO_GEN(stats_arenas_i_bins_j_allocated,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t)
+CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t)
+CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc,
+    ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t)
 CTL_RO_GEN(stats_arenas_i_bins_j_nrequests,
     ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t)
 #ifdef JEMALLOC_TCACHE
diff --git a/jemalloc/src/stats.c b/jemalloc/src/stats.c
index 236d7f8..dcd93f1 100644
--- a/jemalloc/src/stats.c
+++ b/jemalloc/src/stats.c
@@ -160,12 +160,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	CTL_GET("config.tcache", &config_tcache, bool);
 	if (config_tcache) {
 		malloc_cprintf(write_cb, cbopaque,
-		    "bins:     bin    size regs pgs  nrequests    "
-		    "nfills  nflushes   newruns    reruns maxruns curruns\n");
+		    "bins:     bin    size regs pgs allocated   nmalloc"
+		    "   ndalloc  nrequests    nfills  nflushes   newruns"
+		    "    reruns maxruns curruns\n");
 	} else {
 		malloc_cprintf(write_cb, cbopaque,
-		    "bins:     bin    size regs pgs  nrequests   "
-		    "newruns    reruns maxruns curruns\n");
+		    "bins:     bin    size regs pgs allocated   nmalloc"
+		    "   ndalloc   newruns    reruns maxruns curruns\n");
 	}
 	CTL_GET("arenas.nbins", &nbins, unsigned);
 	for (j = 0, gap_start = UINT_MAX; j < nbins; j++) {
@@ -177,9 +178,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 				gap_start = j;
 		} else {
 			unsigned ntbins_, nqbins, ncbins, nsbins;
-			size_t reg_size, run_size;
+			size_t reg_size, run_size, allocated;
 			uint32_t nregs;
-			uint64_t nrequests, nfills, nflushes, reruns;
+			uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes;
+			uint64_t reruns;
 			size_t highruns, curruns;
 
 			if (gap_start != UINT_MAX) {
@@ -202,9 +204,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			CTL_J_GET("arenas.bin.0.size", &reg_size, size_t);
 			CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t);
 			CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t);
-			CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
-			    &nrequests, uint64_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.allocated",
+			    &allocated, size_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc",
+			    &nmalloc, uint64_t);
+			CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc",
+			    &ndalloc, uint64_t);
 			if (config_tcache) {
+				CTL_IJ_GET("stats.arenas.0.bins.0.nrequests",
+				    &nrequests, uint64_t);
 				CTL_IJ_GET("stats.arenas.0.bins.0.nfills",
 				    &nfills, uint64_t);
 				CTL_IJ_GET("stats.arenas.0.bins.0.nflushes",
@@ -218,29 +226,32 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
 			    size_t);
 			if (config_tcache) {
 				malloc_cprintf(write_cb, cbopaque,
-				    "%13u %1s %5zu %4u %3zu %10"PRIu64
-				    " %9"PRIu64" %9"PRIu64" %9"PRIu64""
-				    " %9"PRIu64" %7zu %7zu\n",
+				    "%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
+				    " %9"PRIu64" %10"PRIu64" %9"PRIu64
+				    " %9"PRIu64" %9"PRIu64" %9"PRIu64
+				    " %7zu %7zu\n",
 				    j,
 				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
 				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
 				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
 				    : "M",
 				    reg_size, nregs, run_size / pagesize,
-				    nrequests, nfills, nflushes, nruns, reruns,
-				    highruns, curruns);
+				    allocated, nmalloc, ndalloc, nrequests,
+				    nfills, nflushes, nruns, reruns, highruns,
+				    curruns);
 			} else {
 				malloc_cprintf(write_cb, cbopaque,
-				    "%13u %1s %5zu %4u %3zu %10"PRIu64
-				    " %9"PRIu64" %9"PRIu64" %7zu %7zu\n",
+				    "%13u %1s %5zu %4u %3zu %9zu %9"PRIu64
+				    " %9"PRIu64" %9"PRIu64" %9"PRIu64
+				    " %7zu %7zu\n",
 				    j,
 				    j < ntbins_ ? "T" : j < ntbins_ + nqbins ?
 				    "Q" : j < ntbins_ + nqbins + ncbins ? "C" :
 				    j < ntbins_ + nqbins + ncbins + nsbins ? "S"
 				    : "M",
 				    reg_size, nregs, run_size / pagesize,
-				    nrequests, nruns, reruns, highruns,
-				    curruns);
+				    allocated, nmalloc, ndalloc, nruns, reruns,
+				    highruns, curruns);
 			}
 		}
 	}
@@ -305,9 +316,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	size_t pagesize, pactive, pdirty, mapped;
 	uint64_t npurge, nmadvise, purged;
 	size_t small_allocated;
-	uint64_t small_nmalloc, small_ndalloc;
+	uint64_t small_nmalloc, small_ndalloc, small_nrequests;
 	size_t medium_allocated;
-	uint64_t medium_nmalloc, medium_ndalloc;
+	uint64_t medium_nmalloc, medium_ndalloc, medium_nrequests;
 	size_t large_allocated;
 	uint64_t large_nmalloc, large_ndalloc;
 
@@ -325,30 +336,34 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
 	    nmadvise, nmadvise == 1 ? "" : "s", purged);
 
 	malloc_cprintf(write_cb, cbopaque,
-	    "            allocated      nmalloc      ndalloc\n");
+	    "            allocated      nmalloc      ndalloc    nrequests\n");
 	CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t);
+	CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t);
 	malloc_cprintf(write_cb, cbopaque,
-	    "small:   %12zu %12"PRIu64" %12"PRIu64"\n",
-	    small_allocated, small_nmalloc, small_ndalloc);
+	    "small:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
+	    small_allocated, small_nmalloc, small_ndalloc, small_nrequests);
 	CTL_I_GET("stats.arenas.0.medium.allocated", &medium_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.medium.nmalloc", &medium_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.medium.ndalloc", &medium_ndalloc, uint64_t);
+	CTL_I_GET("stats.arenas.0.medium.nrequests", &medium_nrequests,
+	    uint64_t);
 	malloc_cprintf(write_cb, cbopaque,
-	    "medium:  %12zu %12"PRIu64" %12"PRIu64"\n",
-	    medium_allocated, medium_nmalloc, medium_ndalloc);
+	    "medium:  %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
+	    medium_allocated, medium_nmalloc, medium_ndalloc, medium_nrequests);
 	CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t);
 	CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t);
 	CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t);
 	malloc_cprintf(write_cb, cbopaque,
-	    "large:   %12zu %12"PRIu64" %12"PRIu64"\n",
-	    large_allocated, large_nmalloc, large_ndalloc);
+	    "large:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
+	    large_allocated, large_nmalloc, large_ndalloc, large_nmalloc);
 	malloc_cprintf(write_cb, cbopaque,
-	    "total:   %12zu %12"PRIu64" %12"PRIu64"\n",
+	    "total:   %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n",
 	    small_allocated + medium_allocated + large_allocated,
 	    small_nmalloc + medium_nmalloc + large_nmalloc,
-	    small_ndalloc + medium_ndalloc + large_ndalloc);
+	    small_ndalloc + medium_ndalloc + large_ndalloc,
+	    small_nrequests + medium_nrequests + large_nmalloc);
 	malloc_cprintf(write_cb, cbopaque, "active:  %12zu\n",
 	    pactive * pagesize );
 	CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t);
diff --git a/jemalloc/src/tcache.c b/jemalloc/src/tcache.c
index dcb72c6..4192f98 100644
--- a/jemalloc/src/tcache.c
+++ b/jemalloc/src/tcache.c
@@ -45,7 +45,7 @@ tcache_alloc_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
 
 void
 tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
     , tcache_t *tcache
 #endif
     )
@@ -53,16 +53,29 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 	void *flush, *deferred, *ptr;
 	unsigned i, nflush, ndeferred;
 
+	assert(rem <= tbin->ncached);
+
 	for (flush = tbin->avail, nflush = tbin->ncached - rem; flush != NULL;
 	    flush = deferred, nflush = ndeferred) {
-		/* Lock the arena associated with the first object. */
+		/* Lock the arena bin associated with the first object. */
 		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(flush);
 		arena_t *arena = chunk->arena;
-		malloc_mutex_lock(&arena->lock);
-#ifdef JEMALLOC_PROF
+		arena_bin_t *bin = &arena->bins[binind];
+
+		malloc_mutex_lock(&bin->lock);
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 		if (arena == tcache->arena) {
+#  ifdef JEMALLOC_PROF
+			malloc_mutex_lock(&arena->lock);
 			arena_prof_accum(arena, tcache->prof_accumbytes);
+			malloc_mutex_unlock(&arena->lock);
 			tcache->prof_accumbytes = 0;
+#  endif
+#  ifdef JEMALLOC_STATS
+			bin->stats.nflushes++;
+			bin->stats.nrequests += tbin->tstats.nrequests;
+			tbin->tstats.nrequests = 0;
+#  endif
 		}
 #endif
 		deferred = NULL;
@@ -81,31 +94,16 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 			} else {
 				/*
 				 * This object was allocated via a different
-				 * arena than the one that is currently locked.
-				 * Stash the object, so that it can be handled
-				 * in a future pass.
+				 * arena bin than the one that is currently
+				 * locked.  Stash the object, so that it can be
+				 * handled in a future pass.
 				 */
 				*(void **)ptr = deferred;
 				deferred = ptr;
 				ndeferred++;
 			}
 		}
-#ifdef JEMALLOC_STATS
-		arena->bins[binind].stats.nflushes++;
-		{
-			arena_bin_t *bin = &arena->bins[binind];
-			bin->stats.nrequests += tbin->tstats.nrequests;
-			if (bin->reg_size <= small_maxclass) {
-				arena->stats.nmalloc_small +=
-				    tbin->tstats.nrequests;
-			} else {
-				arena->stats.nmalloc_medium +=
-				    tbin->tstats.nrequests;
-			}
-			tbin->tstats.nrequests = 0;
-		}
-#endif
-		malloc_mutex_unlock(&arena->lock);
+		malloc_mutex_unlock(&bin->lock);
 
 		if (flush != NULL) {
 			/*
@@ -117,6 +115,8 @@ tcache_bin_flush(tcache_bin_t *tbin, size_t binind, unsigned rem
 	}
 
 	tbin->ncached = rem;
+	if (tbin->ncached < tbin->low_water)
+		tbin->low_water = tbin->ncached;
 }
 
 tcache_t *
@@ -178,14 +178,14 @@ tcache_destroy(tcache_t *tcache)
 	/* Unlink from list of extant tcaches. */
 	malloc_mutex_lock(&tcache->arena->lock);
 	ql_remove(&tcache->arena->tcache_ql, tcache, link);
-	tcache_stats_merge(tcache, tcache->arena);
 	malloc_mutex_unlock(&tcache->arena->lock);
+	tcache_stats_merge(tcache, tcache->arena);
 #endif
 
 	for (i = 0; i < nbins; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];
 		tcache_bin_flush(tbin, i, 0
-#ifdef JEMALLOC_PROF
+#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
 		    , tcache
 #endif
 		    );
@@ -194,16 +194,9 @@ tcache_destroy(tcache_t *tcache)
 		if (tbin->tstats.nrequests != 0) {
 			arena_t *arena = tcache->arena;
 			arena_bin_t *bin = &arena->bins[i];
-			malloc_mutex_lock(&arena->lock);
+			malloc_mutex_lock(&bin->lock);
 			bin->stats.nrequests += tbin->tstats.nrequests;
-			if (bin->reg_size <= small_maxclass) {
-				arena->stats.nmalloc_small +=
-				    tbin->tstats.nrequests;
-			} else {
-				arena->stats.nmalloc_medium +=
-				    tbin->tstats.nrequests;
-			}
-			malloc_mutex_unlock(&arena->lock);
+			malloc_mutex_unlock(&bin->lock);
 		}
 #endif
 	}
@@ -222,10 +215,14 @@ tcache_destroy(tcache_t *tcache)
 		size_t pageind = (((uintptr_t)tcache - (uintptr_t)chunk) >>
 		    PAGE_SHIFT);
 		arena_chunk_map_t *mapelm = &chunk->map[pageind];
+		arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
+		    (uintptr_t)((pageind - ((mapelm->bits & CHUNK_MAP_PG_MASK)
+		    >> CHUNK_MAP_PG_SHIFT)) << PAGE_SHIFT));
+		arena_bin_t *bin = run->bin;
 
-		malloc_mutex_lock(&arena->lock);
+		malloc_mutex_lock(&bin->lock);
 		arena_dalloc_bin(arena, chunk, tcache, mapelm);
-		malloc_mutex_unlock(&arena->lock);
+		malloc_mutex_unlock(&bin->lock);
 	} else
 		idalloc(tcache);
 }
@@ -250,18 +247,12 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 	unsigned i;
 
 	/* Merge and reset tcache stats. */
-	for (i = 0; i < mbin0; i++) {
-		arena_bin_t *bin = &arena->bins[i];
-		tcache_bin_t *tbin = &tcache->tbins[i];
-		bin->stats.nrequests += tbin->tstats.nrequests;
-		arena->stats.nmalloc_small += tbin->tstats.nrequests;
-		tbin->tstats.nrequests = 0;
-	}
-	for (; i < nbins; i++) {
+	for (i = 0; i < nbins; i++) {
 		arena_bin_t *bin = &arena->bins[i];
 		tcache_bin_t *tbin = &tcache->tbins[i];
+		malloc_mutex_lock(&bin->lock);
 		bin->stats.nrequests += tbin->tstats.nrequests;
-		arena->stats.nmalloc_medium += tbin->tstats.nrequests;
+		malloc_mutex_unlock(&bin->lock);
 		tbin->tstats.nrequests = 0;
 	}
 }