Implement per-CPU arena.

The new feature, opt.percpu_arena, determines thread-arena association
dynamically based CPU id. Three modes are supported: "percpu", "phycpu"
and disabled.

"percpu" uses the current core id (with help from sched_getcpu())
directly as the arena index, while "phycpu" will assign threads on the
same physical CPU to the same arena. In other words, "percpu" means # of
arenas == # of CPUs, while "phycpu" has # of arenas == 1/2 * (# of
CPUs). Note that no runtime check on whether hyper threading is enabled
is added yet.

When enabled, threads will be migrated between arenas when a CPU change
is detected. In the current design, to reduce overhead from reading CPU
id, each arena tracks the thread accessed most recently. When a new
thread comes in, we will read CPU id and update arena if necessary.

2 files changed, 76 insertions, 61 deletions

diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c
index c931e37..1aedbe8 100644
--- a/test/unit/mallctl.c
+++ b/test/unit/mallctl.c
@@ -160,6 +160,7 @@ TEST_BEGIN(test_mallctl_opt) {
 	TEST_MALLCTL_OPT(bool, abort, always);
 	TEST_MALLCTL_OPT(const char *, dss, always);
 	TEST_MALLCTL_OPT(unsigned, narenas, always);
+	TEST_MALLCTL_OPT(const char *, percpu_arena, always);
 	TEST_MALLCTL_OPT(ssize_t, decay_time, always);
 	TEST_MALLCTL_OPT(bool, stats_print, always);
 	TEST_MALLCTL_OPT(const char *, junk, fill);
@@ -327,20 +328,38 @@ TEST_BEGIN(test_tcache) {
 TEST_END
 
 TEST_BEGIN(test_thread_arena) {
-	unsigned arena_old, arena_new, narenas;
-	size_t sz = sizeof(unsigned);
+	unsigned old_arena_ind, new_arena_ind, narenas;
+	const char *opt_percpu_arena;
+
+	size_t sz = sizeof(opt_percpu_arena);
+	assert_d_eq(mallctl("opt.percpu_arena", &opt_percpu_arena, &sz, NULL,
+	    0), 0, "Unexpected mallctl() failure");
 
+	sz = sizeof(unsigned);
 	assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0),
 	    0, "Unexpected mallctl() failure");
 	assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect");
-	arena_new = narenas - 1;
-	assert_d_eq(mallctl("thread.arena", (void *)&arena_old, &sz,
-	    (void *)&arena_new, sizeof(unsigned)), 0,
-	    "Unexpected mallctl() failure");
-	arena_new = 0;
-	assert_d_eq(mallctl("thread.arena", (void *)&arena_old, &sz,
-	    (void *)&arena_new, sizeof(unsigned)), 0,
-	    "Unexpected mallctl() failure");
+
+	if (strcmp(opt_percpu_arena, "disabled") == 0) {
+		new_arena_ind = narenas - 1;
+		assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
+		    (void *)&new_arena_ind, sizeof(unsigned)), 0,
+		    "Unexpected mallctl() failure");
+		new_arena_ind = 0;
+		assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
+		    (void *)&new_arena_ind, sizeof(unsigned)), 0,
+		    "Unexpected mallctl() failure");
+	} else {
+		assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
+		    NULL, 0), 0, "Unexpected mallctl() failure");
+		new_arena_ind = percpu_arena_ind_limit() - 1;
+		if (old_arena_ind != new_arena_ind) {
+			assert_d_eq(mallctl("thread.arena",
+			    (void *)&old_arena_ind, &sz, (void *)&new_arena_ind,
+			    sizeof(unsigned)), EPERM, "thread.arena ctl "
+			    "should not be allowed with percpu arena");
+		}
+	}
 }
 TEST_END
 
diff --git a/test/unit/stats.c b/test/unit/stats.c
index 948132c..c458d3f 100644
--- a/test/unit/stats.c
+++ b/test/unit/stats.c
@@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_large) {
 	size_t sz;
 	int expected = config_stats ? 0 : ENOENT;
 
-	p = mallocx(SMALL_MAXCLASS+1, 0);
+	p = mallocx(SMALL_MAXCLASS+1, MALLOCX_ARENA(0));
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
 	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
@@ -66,7 +66,6 @@ TEST_BEGIN(test_stats_large) {
 TEST_END
 
 TEST_BEGIN(test_stats_arenas_summary) {
-	unsigned arena;
 	void *little, *large;
 	uint64_t epoch;
 	size_t sz;
@@ -74,13 +73,9 @@ TEST_BEGIN(test_stats_arenas_summary) {
 	size_t mapped;
 	uint64_t npurge, nmadvise, purged;
 
-	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
-	    sizeof(arena)), 0, "Unexpected mallctl() failure");
-
-	little = mallocx(SMALL_MAXCLASS, 0);
+	little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0));
 	assert_ptr_not_null(little, "Unexpected mallocx() failure");
-	large = mallocx((1U << LG_LARGE_MINCLASS), 0);
+	large = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0));
 	assert_ptr_not_null(large, "Unexpected mallocx() failure");
 
 	dallocx(little, 0);
@@ -128,7 +123,6 @@ no_lazy_lock(void) {
 }
 
 TEST_BEGIN(test_stats_arenas_small) {
-	unsigned arena;
 	void *p;
 	size_t sz, allocated;
 	uint64_t epoch, nmalloc, ndalloc, nrequests;
@@ -136,11 +130,7 @@ TEST_BEGIN(test_stats_arenas_small) {
 
 	no_lazy_lock(); /* Lazy locking would dodge tcache testing. */
 
-	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
-	    sizeof(arena)), 0, "Unexpected mallctl() failure");
-
-	p = mallocx(SMALL_MAXCLASS, 0);
+	p = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0));
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
 	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
@@ -178,17 +168,12 @@ TEST_BEGIN(test_stats_arenas_small) {
 TEST_END
 
 TEST_BEGIN(test_stats_arenas_large) {
-	unsigned arena;
 	void *p;
 	size_t sz, allocated;
 	uint64_t epoch, nmalloc, ndalloc;
 	int expected = config_stats ? 0 : ENOENT;
 
-	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
-	    sizeof(arena)), 0, "Unexpected mallctl() failure");
-
-	p = mallocx((1U << LG_LARGE_MINCLASS), 0);
+	p = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0));
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
 	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
@@ -217,20 +202,29 @@ TEST_BEGIN(test_stats_arenas_large) {
 }
 TEST_END
 
+static void
+gen_mallctl_str(char *cmd, char *name, unsigned arena_ind) {
+	sprintf(cmd, "stats.arenas.%u.bins.0.%s", arena_ind, name);
+}
+
 TEST_BEGIN(test_stats_arenas_bins) {
-	unsigned arena;
 	void *p;
 	size_t sz, curslabs, curregs;
 	uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes;
 	uint64_t nslabs, nreslabs;
 	int expected = config_stats ? 0 : ENOENT;
 
-	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
-	    sizeof(arena)), 0, "Unexpected mallctl() failure");
+	unsigned arena_ind, old_arena_ind;
+	sz = sizeof(unsigned);
+	assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Arena creation failure");
+	sz = sizeof(arena_ind);
+	assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, &sz,
+	    (void *)&arena_ind, sizeof(arena_ind)), 0,
+	    "Unexpected mallctl() failure");
 
-	p = mallocx(arena_bin_info[0].reg_size, 0);
-	assert_ptr_not_null(p, "Unexpected mallocx() failure");
+	p = malloc(arena_bin_info[0].reg_size);
+	assert_ptr_not_null(p, "Unexpected malloc() failure");
 
 	assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0),
 	    config_tcache ? 0 : ENOENT, "Unexpected mallctl() result");
@@ -238,33 +232,40 @@ TEST_BEGIN(test_stats_arenas_bins) {
 	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),
 	    0, "Unexpected mallctl() failure");
 
+	char cmd[128];
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", (void *)&nmalloc,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.ndalloc", (void *)&ndalloc,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests",
-	    (void *)&nrequests, &sz, NULL, 0), expected,
+	gen_mallctl_str(cmd, "nmalloc", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nmalloc, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
+	gen_mallctl_str(cmd, "ndalloc", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&ndalloc, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
+	gen_mallctl_str(cmd, "nrequests", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nrequests, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.curregs", (void *)&curregs,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	gen_mallctl_str(cmd, "curregs", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&curregs, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 
 	sz = sizeof(uint64_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", (void *)&nfills,
-	    &sz, NULL, 0), config_tcache ? expected : ENOENT,
+	gen_mallctl_str(cmd, "nfills", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0),
+	    config_tcache ? expected : ENOENT, "Unexpected mallctl() result");
+	gen_mallctl_str(cmd, "nflushes", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0),
+	    config_tcache ? expected : ENOENT, "Unexpected mallctl() result");
+
+	gen_mallctl_str(cmd, "nslabs", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nslabs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nflushes", (void *)&nflushes,
-	    &sz, NULL, 0), config_tcache ? expected : ENOENT,
+	gen_mallctl_str(cmd, "nreslabs", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&nreslabs, &sz, NULL, 0), expected,
 	    "Unexpected mallctl() result");
-
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nslabs", (void *)&nslabs,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.nreslabs", (void *)&nreslabs,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
 	sz = sizeof(size_t);
-	assert_d_eq(mallctl("stats.arenas.0.bins.0.curslabs", (void *)&curslabs,
-	    &sz, NULL, 0), expected, "Unexpected mallctl() result");
+	gen_mallctl_str(cmd, "curslabs", arena_ind);
+	assert_d_eq(mallctl(cmd, (void *)&curslabs, &sz, NULL, 0), expected,
+	    "Unexpected mallctl() result");
 
 	if (config_stats) {
 		assert_u64_gt(nmalloc, 0,
@@ -292,21 +293,16 @@ TEST_BEGIN(test_stats_arenas_bins) {
 TEST_END
 
 TEST_BEGIN(test_stats_arenas_lextents) {
-	unsigned arena;
 	void *p;
 	uint64_t epoch, nmalloc, ndalloc;
 	size_t curlextents, sz, hsize;
 	int expected = config_stats ? 0 : ENOENT;
 
-	arena = 0;
-	assert_d_eq(mallctl("thread.arena", NULL, NULL, (void *)&arena,
-	    sizeof(arena)), 0, "Unexpected mallctl() failure");
-
 	sz = sizeof(size_t);
 	assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&hsize, &sz, NULL,
 	    0), 0, "Unexpected mallctl() failure");
 
-	p = mallocx(hsize, 0);
+	p = mallocx(hsize, MALLOCX_ARENA(0));
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");
 
 	assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)),