Bypass extent tracking for auto arenas.

Tracking extents is required by arena_reset.  To support this, the extent
linkage was used for tracking 1) large allocations, and 2) full slabs.  However
modifying the extent linkage could be an expensive operation as it likely incurs
cache misses.  Since we forbid arena_reset on auto arenas, let's bypass the
linkage operations for auto arenas.

3 files changed, 10 insertions, 6 deletions

diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h
index 0d922f1..38fa3c7 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h
@@ -24,6 +24,7 @@ size_t sa2u(size_t size, size_t alignment);
 arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
 arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
 arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
+bool arena_is_auto(arena_t *arena);
 arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
     bool refresh_if_missing);
 arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
index f22708a..ab54a59 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
@@ -70,6 +70,12 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) {
 	return arena_choose_impl(tsd, arena, true);
 }
 
+JEMALLOC_INLINE bool
+arena_is_auto(arena_t *arena) {
+	assert(narenas_auto > 0);
+	return (arena_ind_get(arena) < narenas_auto);
+}
+
 JEMALLOC_ALWAYS_INLINE extent_t *
 iealloc(tsdn_t *tsdn, const void *ptr) {
 	rtree_ctx_t rtree_ctx_fallback;
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
index 8c79381..70ac666 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
@@ -54,8 +54,7 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache,
 
 	assert(size != 0);
 	assert(!is_internal || tcache == NULL);
-	assert(!is_internal || arena == NULL || arena_ind_get(arena) <
-	    narenas_auto);
+	assert(!is_internal || arena == NULL || arena_is_auto(arena));
 	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path);
@@ -79,8 +78,7 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
 	assert(usize != 0);
 	assert(usize == sa2u(usize, alignment));
 	assert(!is_internal || tcache == NULL);
-	assert(!is_internal || arena == NULL || arena_ind_get(arena) <
-	    narenas_auto);
+	assert(!is_internal || arena == NULL || arena_is_auto(arena));
 	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 
 	ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache);
@@ -113,8 +111,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, alloc_ctx_t *alloc_ctx,
     bool is_internal, bool slow_path) {
 	assert(ptr != NULL);
 	assert(!is_internal || tcache == NULL);
-	assert(!is_internal || arena_ind_get(iaalloc(tsdn, ptr)) <
-	    narenas_auto);
+	assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr)));
 	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
 	if (config_stats && is_internal) {
 		arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr));