Add a per-arena oversize_threshold.

This can let manual arenas trade off memory and CPU the way auto arenas do.

10 files changed, 194 insertions, 12 deletions

diff --git a/Makefile.in b/Makefile.in
index ca9b17b..03dbbdf 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -229,6 +229,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/mq.c \
 	$(srcroot)test/unit/mtx.c \
 	$(srcroot)test/unit/nstime.c \
+	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h
index 5e97d0b..f1823e6 100644
--- a/include/jemalloc/internal/pa.h
+++ b/include/jemalloc/internal/pa.h
@@ -123,7 +123,8 @@ pa_shard_ehooks_get(pa_shard_t *shard) {
 /* Returns true on error. */
 bool pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
     unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
-    nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms);
+    nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
+    ssize_t muzzy_decay_ms);
 
 /*
  * This isn't exposed to users; we allow late enablement of the HPA shard so
diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h
index b998b69..6d4dfba 100644
--- a/include/jemalloc/internal/pac.h
+++ b/include/jemalloc/internal/pac.h
@@ -98,6 +98,9 @@ struct pac_s {
 	exp_grow_t exp_grow;
 	malloc_mutex_t grow_mtx;
 
+	/* How large extents should be before getting auto-purged. */
+	atomic_zu_t oversize_threshold;
+
 	/*
 	 * Decay-based purging state, responsible for scheduling extent state
 	 * transitions.
@@ -115,8 +118,9 @@ struct pac_s {
 };
 
 bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
-    edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
-    ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx);
+    edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
+    ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
+    malloc_mutex_t *stats_mtx);
 bool pac_retain_grow_limit_get_set(tsdn_t *tsdn, pac_t *pac, size_t *old_limit,
     size_t *new_limit);
 void pac_stats_merge(tsdn_t *tsdn, pac_t *pac, pac_stats_t *pac_stats_out,
diff --git a/src/arena.c b/src/arena.c
index 360827e..7099713 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -1500,7 +1500,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) {
 	nstime_init_update(&cur_time);
 	if (pa_shard_init(tsdn, &arena->pa_shard, &arena_emap_global, base, ind,
 	    &arena->stats.pa_shard_stats, LOCKEDINT_MTX(arena->stats.mtx),
-	    &cur_time, arena_dirty_decay_ms_default_get(),
+	    &cur_time, oversize_threshold, arena_dirty_decay_ms_default_get(),
 	    arena_muzzy_decay_ms_default_get())) {
 		goto label_error;
 	}
diff --git a/src/ctl.c b/src/ctl.c
index d5dd1d1..4bb422a 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -151,6 +151,7 @@ CTL_PROTO(arena_i_purge)
 CTL_PROTO(arena_i_reset)
 CTL_PROTO(arena_i_destroy)
 CTL_PROTO(arena_i_dss)
+CTL_PROTO(arena_i_oversize_threshold)
 CTL_PROTO(arena_i_dirty_decay_ms)
 CTL_PROTO(arena_i_muzzy_decay_ms)
 CTL_PROTO(arena_i_extent_hooks)
@@ -431,6 +432,11 @@ static const ctl_named_node_t arena_i_node[] = {
 	{NAME("reset"),		CTL(arena_i_reset)},
 	{NAME("destroy"),	CTL(arena_i_destroy)},
 	{NAME("dss"),		CTL(arena_i_dss)},
+	/*
+	 * Undocumented for now, since we anticipate an arena API in flux after
+	 * we cut the last 5-series release.
+	 */
+	{NAME("oversize_threshold"), CTL(arena_i_oversize_threshold)},
 	{NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)},
 	{NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)},
 	{NAME("extent_hooks"),	CTL(arena_i_extent_hooks)},
@@ -2531,6 +2537,38 @@ label_return:
 }
 
 static int
+arena_i_oversize_threshold_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+    void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+
+	unsigned arena_ind;
+	MIB_UNSIGNED(arena_ind, 1);
+
+	arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
+	if (arena == NULL) {
+		ret = EFAULT;
+		goto label_return;
+	}
+
+	if (oldp != NULL && oldlenp != NULL) {
+		size_t oldval = atomic_load_zu(
+		    &arena->pa_shard.pac.oversize_threshold, ATOMIC_RELAXED);
+		READ(oldval, size_t);
+	}
+	if (newp != NULL) {
+		if (newlen != sizeof(size_t)) {
+			ret = EINVAL;
+			goto label_return;
+		}
+		atomic_store_zu(&arena->pa_shard.pac.oversize_threshold,
+		    *(size_t *)newp, ATOMIC_RELAXED);
+	}
+	ret = 0;
+label_return:
+	return ret;
+}
+
+static int
 arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen,
     void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) {
 	int ret;
diff --git a/src/extent.c b/src/extent.c
index c7dcc2e..378bc73 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -983,8 +983,9 @@ extent_record(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
 			edata = extent_try_coalesce_large(tsdn, pac, ehooks,
 			    ecache, edata, &coalesced, growing_retained);
 		} while (coalesced);
-		if (edata_size_get(edata) >= oversize_threshold &&
-		    extent_may_force_decay(pac)) {
+		if (edata_size_get(edata) >=
+		    atomic_load_zu(&pac->oversize_threshold, ATOMIC_RELAXED)
+		    && extent_may_force_decay(pac)) {
 			/* Shortcut to purge the oversize extent eagerly. */
 			malloc_mutex_unlock(tsdn, &ecache->mtx);
 			extent_maximally_purge(tsdn, pac, ehooks, edata);
diff --git a/src/pa.c b/src/pa.c
index aee7bcd..e5fcbb7 100644
--- a/src/pa.c
+++ b/src/pa.c
@@ -17,7 +17,8 @@ pa_nactive_sub(pa_shard_t *shard, size_t sub_pages) {
 bool
 pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
     unsigned ind, pa_shard_stats_t *stats, malloc_mutex_t *stats_mtx,
-    nstime_t *cur_time, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
+    nstime_t *cur_time, size_t oversize_threshold, ssize_t dirty_decay_ms,
+    ssize_t muzzy_decay_ms) {
 	/* This will change eventually, but for now it should hold. */
 	assert(base_ind_get(base) == ind);
 	if (edata_cache_init(&shard->edata_cache, base)) {
@@ -25,8 +26,8 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, emap_t *emap, base_t *base,
 	}
 
 	if (pac_init(tsdn, &shard->pac, base, emap, &shard->edata_cache,
-	    cur_time, dirty_decay_ms, muzzy_decay_ms, &stats->pac_stats,
-	    stats_mtx)) {
+	    cur_time, oversize_threshold, dirty_decay_ms, muzzy_decay_ms,
+	    &stats->pac_stats, stats_mtx)) {
 		return true;
 	}
 
diff --git a/src/pac.c b/src/pac.c
index 07c9d23..8064615 100644
--- a/src/pac.c
+++ b/src/pac.c
@@ -37,8 +37,9 @@ pac_decay_data_get(pac_t *pac, extent_state_t state,
 
 bool
 pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
-    edata_cache_t *edata_cache, nstime_t *cur_time, ssize_t dirty_decay_ms,
-    ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx) {
+    edata_cache_t *edata_cache, nstime_t *cur_time, size_t oversize_threshold,
+    ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats,
+    malloc_mutex_t *stats_mtx) {
 	unsigned ind = base_ind_get(base);
 	/*
 	 * Delay coalescing for dirty extents despite the disruptive effect on
@@ -73,6 +74,8 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	    WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) {
 		return true;
 	}
+	atomic_store_zu(&pac->oversize_threshold, oversize_threshold,
+	    ATOMIC_RELAXED);
 	if (decay_init(&pac->decay_dirty, cur_time, dirty_decay_ms)) {
 		return true;
 	}
diff --git a/test/unit/oversize_threshold.c b/test/unit/oversize_threshold.c
new file mode 100644
index 0000000..e374b14
--- /dev/null
+++ b/test/unit/oversize_threshold.c
@@ -0,0 +1,131 @@
+#include "test/jemalloc_test.h"
+
+#include "jemalloc/internal/ctl.h"
+
+static void
+arena_mallctl(const char *mallctl_str, unsigned arena, void *oldp,
+    size_t *oldlen, void *newp, size_t newlen) {
+	int err;
+	char buf[100];
+	malloc_snprintf(buf, sizeof(buf), mallctl_str, arena);
+
+	err = mallctl(buf, oldp, oldlen, newp, newlen);
+	expect_d_eq(0, err, "Mallctl failed; %s", buf);
+}
+
+TEST_BEGIN(test_oversize_threshold_get_set) {
+	int err;
+	size_t old_threshold;
+	size_t new_threshold;
+	size_t threshold_sz = sizeof(old_threshold);
+
+	unsigned arena;
+	size_t arena_sz = sizeof(arena);
+	err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
+	expect_d_eq(0, err, "Arena creation failed");
+
+	/* Just a write. */
+	new_threshold = 1024 * 1024;
+	arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
+	    &new_threshold, threshold_sz);
+
+	/* Read and write */
+	new_threshold = 2 * 1024 * 1024;
+	arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
+	    &threshold_sz, &new_threshold, threshold_sz);
+	expect_zu_eq(1024 * 1024, old_threshold, "Should have read old value");
+
+	/* Just a read */
+	arena_mallctl("arena.%u.oversize_threshold", arena, &old_threshold,
+	    &threshold_sz, NULL, 0);
+	expect_zu_eq(2 * 1024 * 1024, old_threshold, "Should have read old value");
+}
+TEST_END
+
+static size_t max_purged = 0;
+static bool
+purge_forced_record_max(extent_hooks_t* hooks, void *addr, size_t sz,
+    size_t offset, size_t length, unsigned arena_ind) {
+	if (length > max_purged) {
+		max_purged = length;
+	}
+	return false;
+}
+
+static bool
+dalloc_record_max(extent_hooks_t *extent_hooks, void *addr, size_t sz,
+    bool comitted, unsigned arena_ind) {
+	if (sz > max_purged) {
+		max_purged = sz;
+	}
+	return false;
+}
+
+extent_hooks_t max_recording_extent_hooks;
+
+TEST_BEGIN(test_oversize_threshold) {
+	max_recording_extent_hooks = ehooks_default_extent_hooks;
+	max_recording_extent_hooks.purge_forced = &purge_forced_record_max;
+	max_recording_extent_hooks.dalloc = &dalloc_record_max;
+
+	extent_hooks_t *extent_hooks = &max_recording_extent_hooks;
+
+	int err;
+
+	unsigned arena;
+	size_t arena_sz = sizeof(arena);
+	err = mallctl("arenas.create", (void *)&arena, &arena_sz, NULL, 0);
+	expect_d_eq(0, err, "Arena creation failed");
+	arena_mallctl("arena.%u.extent_hooks", arena, NULL, NULL, &extent_hooks,
+	    sizeof(extent_hooks));
+
+	/*
+	 * This test will fundamentally race with purging, since we're going to
+	 * check the dirty stats to see if our oversized allocation got purged.
+	 * We don't want other purging to happen accidentally.  We can't just
+	 * disable purging entirely, though, since that will also disable
+	 * oversize purging.  Just set purging intervals to be very large.
+	 */
+	ssize_t decay_ms = 100 * 1000;
+	ssize_t decay_ms_sz = sizeof(decay_ms);
+	arena_mallctl("arena.%u.dirty_decay_ms", arena, NULL, NULL, &decay_ms,
+	    decay_ms_sz);
+	arena_mallctl("arena.%u.muzzy_decay_ms", arena, NULL, NULL, &decay_ms,
+	    decay_ms_sz);
+
+	/* Clean everything out. */
+	arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
+	max_purged = 0;
+
+	/* Set threshold to 1MB. */
+	size_t threshold = 1024 * 1024;
+	size_t threshold_sz = sizeof(threshold);
+	arena_mallctl("arena.%u.oversize_threshold", arena, NULL, NULL,
+	    &threshold, threshold_sz);
+
+	/* Allocating and freeing half a megabyte should leave them dirty. */
+	void *ptr = mallocx(512 * 1024, MALLOCX_ARENA(arena));
+	dallocx(ptr, MALLOCX_TCACHE_NONE);
+	expect_zu_lt(max_purged, 512 * 1024, "Expected no 512k purge");
+
+	/* Purge again to reset everything out. */
+	arena_mallctl("arena.%u.purge", arena, NULL, NULL, NULL, 0);
+	max_purged = 0;
+
+	/*
+	 * Allocating and freeing 2 megabytes should leave them dirty because of
+	 * the oversize threshold.
+	 */
+	ptr = mallocx(2 * 1024 * 1024, MALLOCX_ARENA(arena));
+	dallocx(ptr, MALLOCX_TCACHE_NONE);
+	expect_zu_ge(max_purged, 2 * 1024 * 1024, "Expected a 2MB purge");
+}
+TEST_END
+
+int
+main(void) {
+	return test_no_reentrancy(
+	    test_oversize_threshold_get_set,
+	    test_oversize_threshold);
+}
+
diff --git a/test/unit/pa.c b/test/unit/pa.c
index 3a91023..dacd8e7 100644
--- a/test/unit/pa.c
+++ b/test/unit/pa.c
@@ -63,9 +63,11 @@ test_data_t *init_test_data(ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms) {
 	nstime_t time;
 	nstime_init(&time, 0);
 
+	const size_t oversize_threshold = 8 * 1024 * 1024;
 	err = pa_shard_init(TSDN_NULL, &test_data->shard, &test_data->emap,
 	    test_data->base, /* ind */ 1, &test_data->stats,
-	    &test_data->stats_mtx, &time, dirty_decay_ms, muzzy_decay_ms);
+	    &test_data->stats_mtx, &time, oversize_threshold, dirty_decay_ms,
+	    muzzy_decay_ms);
 	assert_false(err, "");
 
 	return test_data;