Add arena-specific and selective dss allocation.

Add the "arenas.extend" mallctl, so that it is possible to create new
arenas that are outside the set that jemalloc automatically multiplexes
threads onto.

Add the ALLOCM_ARENA() flag for {,r,d}allocm(), so that it is possible
to explicitly allocate from a particular arena.

Add the "opt.dss" mallctl, which controls the default precedence of dss
allocation relative to mmap allocation.

Add the "arena.<i>.dss" mallctl, which makes it possible to set the
default dss precedence on a per arena or global basis.

Add the "arena.<i>.purge" mallctl, which obsoletes "arenas.purge".

Add the "stats.arenas.<i>.dss" mallctl.

8 files changed, 145 insertions, 35 deletions

diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index 0b0f640..49213e3 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -331,6 +331,8 @@ struct arena_s {
 
 	uint64_t		prof_accumbytes;
 
+	dss_prec_t		dss_prec;
+
 	/* List of dirty-page-containing chunks this arena manages. */
 	ql_head(arena_chunk_t)	chunks_dirty;
 
@@ -422,13 +424,16 @@ void	arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
 void	arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk,
     void *ptr);
 void	arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-void	arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
-    arena_stats_t *astats, malloc_bin_stats_t *bstats,
-    malloc_large_stats_t *lstats);
 void	*arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra, bool zero);
-void	*arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
-    size_t alignment, bool zero, bool try_tcache);
+void	*arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+    size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
+    bool try_tcache_dalloc);
+dss_prec_t	arena_dss_prec_get(arena_t *arena);
+void	arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+void	arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
+    size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
+    malloc_large_stats_t *lstats);
 bool	arena_new(arena_t *arena, unsigned ind);
 void	arena_boot(void);
 void	arena_prefork(arena_t *arena);
diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h
index c3c3e9c..87d8700 100644
--- a/include/jemalloc/internal/chunk.h
+++ b/include/jemalloc/internal/chunk.h
@@ -28,6 +28,7 @@
 #ifdef JEMALLOC_H_EXTERNS
 
 extern size_t		opt_lg_chunk;
+extern const char	*opt_dss;
 
 /* Protects stats_chunks; currently not used for any other purpose. */
 extern malloc_mutex_t	chunks_mtx;
@@ -42,7 +43,9 @@ extern size_t		chunk_npages;
 extern size_t		map_bias; /* Number of arena chunk header pages. */
 extern size_t		arena_maxclass; /* Max size class for arenas. */
 
-void	*chunk_alloc(size_t size, size_t alignment, bool base, bool *zero);
+void	*chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
+    dss_prec_t dss_prec);
+void	chunk_unmap(void *chunk, size_t size);
 void	chunk_dealloc(void *chunk, size_t size, bool unmap);
 bool	chunk_boot(void);
 void	chunk_prefork(void);
diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h
index 6e2643b..6585f07 100644
--- a/include/jemalloc/internal/chunk_dss.h
+++ b/include/jemalloc/internal/chunk_dss.h
@@ -1,14 +1,28 @@
 /******************************************************************************/
 #ifdef JEMALLOC_H_TYPES
 
+typedef enum {
+	dss_prec_disabled  = 0,
+	dss_prec_primary   = 1,
+	dss_prec_secondary = 2,
+
+	dss_prec_limit     = 3
+} dss_prec_t ;
+#define	DSS_PREC_DEFAULT	dss_prec_secondary
+#define	DSS_DEFAULT		"secondary"
+
 #endif /* JEMALLOC_H_TYPES */
 /******************************************************************************/
 #ifdef JEMALLOC_H_STRUCTS
 
+extern const char *dss_prec_names[];
+
 #endif /* JEMALLOC_H_STRUCTS */
 /******************************************************************************/
 #ifdef JEMALLOC_H_EXTERNS
 
+dss_prec_t	chunk_dss_prec_get(void);
+bool	chunk_dss_prec_set(dss_prec_t dss_prec);
 void	*chunk_alloc_dss(size_t size, size_t alignment, bool *zero);
 bool	chunk_in_dss(void *chunk);
 bool	chunk_dss_boot(void);
diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h
index 1d0c76a..0ffecc5 100644
--- a/include/jemalloc/internal/ctl.h
+++ b/include/jemalloc/internal/ctl.h
@@ -33,6 +33,7 @@ struct ctl_indexed_node_s {
 struct ctl_arena_stats_s {
 	bool			initialized;
 	unsigned		nthreads;
+	const char		*dss;
 	size_t			pactive;
 	size_t			pdirty;
 	arena_stats_t		astats;
@@ -61,6 +62,7 @@ struct ctl_stats_s {
 		uint64_t	nmalloc;	/* huge_nmalloc */
 		uint64_t	ndalloc;	/* huge_ndalloc */
 	} huge;
+	unsigned		narenas;
 	ctl_arena_stats_t	*arenas;	/* (narenas + 1) elements. */
 };
 
diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h
index e8513c9..d987d37 100644
--- a/include/jemalloc/internal/huge.h
+++ b/include/jemalloc/internal/huge.h
@@ -22,7 +22,7 @@ void	*huge_palloc(size_t size, size_t alignment, bool zero);
 void	*huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
     size_t extra);
 void	*huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
-    size_t alignment, bool zero);
+    size_t alignment, bool zero, bool try_tcache_dalloc);
 void	huge_dalloc(void *ptr, bool unmap);
 size_t	huge_salloc(const void *ptr);
 prof_ctx_t	*huge_prof_ctx_get(const void *ptr);
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index b5b09e2..475821a 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -514,13 +514,19 @@ extern size_t	opt_narenas;
 /* Number of CPUs. */
 extern unsigned		ncpus;
 
-extern malloc_mutex_t	arenas_lock; /* Protects arenas initialization. */
+/* Protects arenas initialization (arenas, arenas_total). */
+extern malloc_mutex_t	arenas_lock;
 /*
  * Arenas that are used to service external requests.  Not all elements of the
  * arenas array are necessarily used; arenas are created lazily as needed.
+ *
+ * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
+ * arenas.  arenas[narenas_auto..narenas_total) are only used if the application
+ * takes some action to create them and allocate from them.
  */
 extern arena_t		**arenas;
-extern unsigned		narenas;
+extern unsigned		narenas_total;
+extern unsigned		narenas_auto; /* Read-only after initialization. */
 
 arena_t	*arenas_extend(unsigned ind);
 void	arenas_cleanup(void *arg);
@@ -575,6 +581,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
 
 size_t	s2u(size_t size);
 size_t	sa2u(size_t size, size_t alignment);
+unsigned	narenas_total_get(void);
 arena_t	*choose_arena(arena_t *arena);
 #endif
 
@@ -679,6 +686,18 @@ sa2u(size_t size, size_t alignment)
 	}
 }
 
+JEMALLOC_INLINE unsigned
+narenas_total_get(void)
+{
+	unsigned narenas;
+
+	malloc_mutex_lock(&arenas_lock);
+	narenas = narenas_total;
+	malloc_mutex_unlock(&arenas_lock);
+
+	return (narenas);
+}
+
 /* Choose an arena based on a per-thread value. */
 JEMALLOC_INLINE arena_t *
 choose_arena(arena_t *arena)
@@ -714,15 +733,24 @@ choose_arena(arena_t *arena)
 #include "jemalloc/internal/quarantine.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
+void	*imallocx(size_t size, bool try_tcache, arena_t *arena);
 void	*imalloc(size_t size);
+void	*icallocx(size_t size, bool try_tcache, arena_t *arena);
 void	*icalloc(size_t size);
+void	*ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+    arena_t *arena);
 void	*ipalloc(size_t usize, size_t alignment, bool zero);
 size_t	isalloc(const void *ptr, bool demote);
 size_t	ivsalloc(const void *ptr, bool demote);
 size_t	u2rz(size_t usize);
 size_t	p2rz(const void *ptr);
+void	idallocx(void *ptr, bool try_tcache);
 void	idalloc(void *ptr);
+void	iqallocx(void *ptr, bool try_tcache);
 void	iqalloc(void *ptr);
+void	*irallocx(void *ptr, size_t size, size_t extra, size_t alignment,
+    bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc,
+    arena_t *arena);
 void	*iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
     bool zero, bool no_move);
 malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
@@ -730,29 +758,44 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
 JEMALLOC_INLINE void *
-imalloc(size_t size)
+imallocx(size_t size, bool try_tcache, arena_t *arena)
 {
 
 	assert(size != 0);
 
 	if (size <= arena_maxclass)
-		return (arena_malloc(NULL, size, false, true));
+		return (arena_malloc(arena, size, false, try_tcache));
 	else
 		return (huge_malloc(size, false));
 }
 
 JEMALLOC_INLINE void *
-icalloc(size_t size)
+imalloc(size_t size)
+{
+
+	return (imallocx(size, true, NULL));
+}
+
+JEMALLOC_INLINE void *
+icallocx(size_t size, bool try_tcache, arena_t *arena)
 {
 
 	if (size <= arena_maxclass)
-		return (arena_malloc(NULL, size, true, true));
+		return (arena_malloc(arena, size, true, try_tcache));
 	else
 		return (huge_malloc(size, true));
 }
 
 JEMALLOC_INLINE void *
-ipalloc(size_t usize, size_t alignment, bool zero)
+icalloc(size_t size)
+{
+
+	return (icallocx(size, true, NULL));
+}
+
+JEMALLOC_INLINE void *
+ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+    arena_t *arena)
 {
 	void *ret;
 
@@ -760,11 +803,11 @@ ipalloc(size_t usize, size_t alignment, bool zero)
 	assert(usize == sa2u(usize, alignment));
 
 	if (usize <= arena_maxclass && alignment <= PAGE)
-		ret = arena_malloc(NULL, usize, zero, true);
+		ret = arena_malloc(arena, usize, zero, try_tcache);
 	else {
 		if (usize <= arena_maxclass) {
-			ret = arena_palloc(choose_arena(NULL), usize, alignment,
-			    zero);
+			ret = arena_palloc(choose_arena(arena), usize,
+			    alignment, zero);
 		} else if (alignment <= chunksize)
 			ret = huge_malloc(usize, zero);
 		else
@@ -775,6 +818,13 @@ ipalloc(size_t usize, size_t alignment, bool zero)
 	return (ret);
 }
 
+JEMALLOC_INLINE void *
+ipalloc(size_t usize, size_t alignment, bool zero)
+{
+
+	return (ipallocx(usize, alignment, zero, true, NULL));
+}
+
 /*
  * Typical usage:
  *   void *ptr = [...]
@@ -833,7 +883,7 @@ p2rz(const void *ptr)
 }
 
 JEMALLOC_INLINE void
-idalloc(void *ptr)
+idallocx(void *ptr, bool try_tcache)
 {
 	arena_chunk_t *chunk;
 
@@ -841,24 +891,38 @@ idalloc(void *ptr)
 
 	chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 	if (chunk != ptr)
-		arena_dalloc(chunk->arena, chunk, ptr, true);
+		arena_dalloc(chunk->arena, chunk, ptr, try_tcache);
 	else
 		huge_dalloc(ptr, true);
 }
 
 JEMALLOC_INLINE void
-iqalloc(void *ptr)
+idalloc(void *ptr)
+{
+
+	idallocx(ptr, true);
+}
+
+JEMALLOC_INLINE void
+iqallocx(void *ptr, bool try_tcache)
 {
 
 	if (config_fill && opt_quarantine)
 		quarantine(ptr);
 	else
-		idalloc(ptr);
+		idallocx(ptr, try_tcache);
+}
+
+JEMALLOC_INLINE void
+iqalloc(void *ptr)
+{
+
+	iqallocx(ptr, true);
 }
 
 JEMALLOC_INLINE void *
-iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
-    bool no_move)
+irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+    bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
 {
 	void *ret;
 	size_t oldsize;
@@ -881,7 +945,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 		usize = sa2u(size + extra, alignment);
 		if (usize == 0)
 			return (NULL);
-		ret = ipalloc(usize, alignment, zero);
+		ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena);
 		if (ret == NULL) {
 			if (extra == 0)
 				return (NULL);
@@ -889,7 +953,8 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 			usize = sa2u(size, alignment);
 			if (usize == 0)
 				return (NULL);
-			ret = ipalloc(usize, alignment, zero);
+			ret = ipallocx(usize, alignment, zero, try_tcache_alloc,
+			    arena);
 			if (ret == NULL)
 				return (NULL);
 		}
@@ -900,7 +965,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 		 */
 		copysize = (size < oldsize) ? size : oldsize;
 		memcpy(ret, ptr, copysize);
-		iqalloc(ptr);
+		iqallocx(ptr, try_tcache_dalloc);
 		return (ret);
 	}
 
@@ -914,15 +979,25 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 		}
 	} else {
 		if (size + extra <= arena_maxclass) {
-			return (arena_ralloc(ptr, oldsize, size, extra,
-			    alignment, zero, true));
+			return (arena_ralloc(arena, ptr, oldsize, size, extra,
+			    alignment, zero, try_tcache_alloc,
+			    try_tcache_dalloc));
 		} else {
 			return (huge_ralloc(ptr, oldsize, size, extra,
-			    alignment, zero));
+			    alignment, zero, try_tcache_dalloc));
 		}
 	}
 }
 
+JEMALLOC_INLINE void *
+iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+    bool no_move)
+{
+
+	return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true,
+	    NULL));
+}
+
 malloc_tsd_externs(thread_allocated, thread_allocated_t)
 malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t,
     THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index 28686dc..06241cd 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -12,6 +12,8 @@
 #define	arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
 #define	arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked)
 #define	arena_dalloc_small JEMALLOC_N(arena_dalloc_small)
+#define	arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get)
+#define	arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set)
 #define	arena_malloc JEMALLOC_N(arena_malloc)
 #define	arena_malloc_large JEMALLOC_N(arena_malloc_large)
 #define	arena_malloc_small JEMALLOC_N(arena_malloc_small)
@@ -51,13 +53,11 @@
 #define	arena_stats_merge JEMALLOC_N(arena_stats_merge)
 #define	arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
 #define	arenas JEMALLOC_N(arenas)
-#define	arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
 #define	arenas_booted JEMALLOC_N(arenas_booted)
 #define	arenas_cleanup JEMALLOC_N(arenas_cleanup)
 #define	arenas_extend JEMALLOC_N(arenas_extend)
 #define	arenas_initialized JEMALLOC_N(arenas_initialized)
 #define	arenas_lock JEMALLOC_N(arenas_lock)
-#define	arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index)
 #define	arenas_tls JEMALLOC_N(arenas_tls)
 #define	arenas_tsd JEMALLOC_N(arenas_tsd)
 #define	arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot)
@@ -102,12 +102,15 @@
 #define	chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
 #define	chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child)
 #define	chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent)
+#define	chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get)
+#define	chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set)
 #define	chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork)
 #define	chunk_in_dss JEMALLOC_N(chunk_in_dss)
 #define	chunk_npages JEMALLOC_N(chunk_npages)
 #define	chunk_postfork_child JEMALLOC_N(chunk_postfork_child)
 #define	chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent)
 #define	chunk_prefork JEMALLOC_N(chunk_prefork)
+#define	chunk_unmap JEMALLOC_N(chunk_unmap)
 #define	chunks_mtx JEMALLOC_N(chunks_mtx)
 #define	chunks_rtree JEMALLOC_N(chunks_rtree)
 #define	chunksize JEMALLOC_N(chunksize)
@@ -136,6 +139,7 @@
 #define	ctl_postfork_child JEMALLOC_N(ctl_postfork_child)
 #define	ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent)
 #define	ctl_prefork JEMALLOC_N(ctl_prefork)
+#define	dss_prec_names JEMALLOC_N(dss_prec_names)
 #define	extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first)
 #define	extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert)
 #define	extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter)
@@ -188,11 +192,17 @@
 #define	huge_salloc JEMALLOC_N(huge_salloc)
 #define	iallocm JEMALLOC_N(iallocm)
 #define	icalloc JEMALLOC_N(icalloc)
+#define	icallocx JEMALLOC_N(icallocx)
 #define	idalloc JEMALLOC_N(idalloc)
+#define	idallocx JEMALLOC_N(idallocx)
 #define	imalloc JEMALLOC_N(imalloc)
+#define	imallocx JEMALLOC_N(imallocx)
 #define	ipalloc JEMALLOC_N(ipalloc)
+#define	ipallocx JEMALLOC_N(ipallocx)
 #define	iqalloc JEMALLOC_N(iqalloc)
+#define	iqallocx JEMALLOC_N(iqallocx)
 #define	iralloc JEMALLOC_N(iralloc)
+#define	irallocx JEMALLOC_N(irallocx)
 #define	isalloc JEMALLOC_N(isalloc)
 #define	isthreaded JEMALLOC_N(isthreaded)
 #define	ivsalloc JEMALLOC_N(ivsalloc)
@@ -220,7 +230,9 @@
 #define	map_bias JEMALLOC_N(map_bias)
 #define	mb_write JEMALLOC_N(mb_write)
 #define	mutex_boot JEMALLOC_N(mutex_boot)
-#define	narenas JEMALLOC_N(narenas)
+#define	narenas_auto JEMALLOC_N(narenas_auto)
+#define	narenas_total JEMALLOC_N(narenas_total)
+#define	narenas_total_get JEMALLOC_N(narenas_total_get)
 #define	ncpus JEMALLOC_N(ncpus)
 #define	nhbins JEMALLOC_N(nhbins)
 #define	opt_abort JEMALLOC_N(opt_abort)
@@ -297,9 +309,6 @@
 #define	s2u JEMALLOC_N(s2u)
 #define	sa2u JEMALLOC_N(sa2u)
 #define	set_errno JEMALLOC_N(set_errno)
-#define	stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index)
-#define	stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index)
-#define	stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index)
 #define	stats_cactive JEMALLOC_N(stats_cactive)
 #define	stats_cactive_add JEMALLOC_N(stats_cactive_add)
 #define	stats_cactive_get JEMALLOC_N(stats_cactive_get)
diff --git a/include/jemalloc/jemalloc.h.in b/include/jemalloc/jemalloc.h.in
index aeb5d2b..31b1304 100644
--- a/include/jemalloc/jemalloc.h.in
+++ b/include/jemalloc/jemalloc.h.in
@@ -25,6 +25,8 @@ extern "C" {
 #endif
 #define	ALLOCM_ZERO	((int)0x40)
 #define	ALLOCM_NO_MOVE	((int)0x80)
+/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
+#define	ALLOCM_ARENA(a)	((int)(((a)+1) << 8))
 
 #define	ALLOCM_SUCCESS		0
 #define	ALLOCM_ERR_OOM		1