Push down iealloc() calls.

Call iealloc() as deep into call chains as possible without causing
redundant calls.

5 files changed, 83 insertions, 100 deletions

diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h
index 9603d74..a35fe18 100644
--- a/include/jemalloc/internal/arena_externs.h
+++ b/include/jemalloc/internal/arena_externs.h
@@ -65,18 +65,16 @@ void	*arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
     szind_t ind, bool zero);
 void	*arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
     size_t alignment, bool zero, tcache_t *tcache);
-void	arena_prof_promote(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize);
-void	arena_dalloc_promoted(tsdn_t *tsdn, extent_t *extent, void *ptr,
-    tcache_t *tcache, bool slow_path);
+void	arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize);
+void	arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    bool slow_path);
 void	arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
     extent_t *extent, void *ptr);
-void	arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
-    void *ptr);
-bool	arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr,
-    size_t oldsize, size_t size, size_t extra, bool zero);
-void	*arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
-    size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache);
+void	arena_dalloc_small(tsdn_t *tsdn, void *ptr);
+bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+    size_t extra, bool zero);
+void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
+    size_t size, size_t alignment, bool zero, tcache_t *tcache);
 dss_prec_t	arena_dss_prec_get(arena_t *arena);
 bool	arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
 ssize_t arena_dirty_decay_time_default_get(void);
diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index cbd9e20..3c48ce4 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -3,12 +3,10 @@
 
 #ifndef JEMALLOC_ENABLE_INLINE
 szind_t	arena_bin_index(arena_t *arena, arena_bin_t *bin);
-prof_tctx_t	*arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
-    const void *ptr);
-void	arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, prof_tctx_t *tctx);
-void	arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
+prof_tctx_t	*arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr);
+void	arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
     prof_tctx_t *tctx);
+void	arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
 void	arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
 void	arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
 void	*arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
@@ -30,10 +28,11 @@ arena_bin_index(arena_t *arena, arena_bin_t *bin) {
 }
 
 JEMALLOC_INLINE prof_tctx_t *
-arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr) {
+arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
+	const extent_t *extent = iealloc(tsdn, ptr);
 	if (unlikely(!extent_slab_get(extent))) {
 		return large_prof_tctx_get(tsdn, extent);
 	}
@@ -41,21 +40,23 @@ arena_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr) {
 }
 
 JEMALLOC_INLINE void
-arena_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, prof_tctx_t *tctx) {
+arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
+    prof_tctx_t *tctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
+	extent_t *extent = iealloc(tsdn, ptr);
 	if (unlikely(!extent_slab_get(extent))) {
 		large_prof_tctx_set(tsdn, extent, tctx);
 	}
 }
 
 JEMALLOC_INLINE void
-arena_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    prof_tctx_t *tctx) {
+arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
+
+	extent_t *extent = iealloc(tsdn, ptr);
 	assert(!extent_slab_get(extent));
 
 	large_prof_tctx_reset(tsdn, extent);
@@ -187,16 +188,13 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
 			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
 			    slow_path);
 		} else {
-			extent_t *extent = iealloc(tsdn, ptr);
-			arena_dalloc_small(tsdn, extent_arena_get(extent),
-			    extent, ptr);
+			arena_dalloc_small(tsdn, ptr);
 		}
 	} else {
 		if (likely(tcache != NULL) && szind < nhbins) {
 			if (config_prof && unlikely(szind < NBINS)) {
-				extent_t *extent = iealloc(tsdn, ptr);
-				arena_dalloc_promoted(tsdn, extent, ptr,
-				    tcache, slow_path);
+				arena_dalloc_promoted(tsdn, ptr, tcache,
+				    slow_path);
 			} else {
 				tcache_dalloc_large(tsdn_tsd(tsdn), tcache,
 				    ptr, szind, slow_path);
@@ -250,16 +248,13 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
 			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
 			    slow_path);
 		} else {
-			extent_t *extent = iealloc(tsdn, ptr);
-			arena_dalloc_small(tsdn, extent_arena_get(extent),
-			    extent, ptr);
+			arena_dalloc_small(tsdn, ptr);
 		}
 	} else {
 		if (likely(tcache != NULL) && szind < nhbins) {
 			if (config_prof && unlikely(szind < NBINS)) {
-				extent_t *extent = iealloc(tsdn, ptr);
-				arena_dalloc_promoted(tsdn, extent, ptr,
-				    tcache, slow_path);
+				arena_dalloc_promoted(tsdn, ptr, tcache,
+				    slow_path);
 			} else {
 				tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
 				    szind, slow_path);
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index b351038..2fe2101 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -1007,32 +1007,32 @@ iealloc(tsdn_t *tsdn, const void *ptr) {
 #include "jemalloc/internal/hash_inlines.h"
 
 #ifndef JEMALLOC_ENABLE_INLINE
-arena_t	*iaalloc(tsdn_t *tsdn, const void *ptr);
-size_t	isalloc(tsdn_t *tsdn, const void *ptr);
-void	*iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
+arena_t *iaalloc(tsdn_t *tsdn, const void *ptr);
+size_t isalloc(tsdn_t *tsdn, const void *ptr);
+void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
     tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path);
-void	*ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
+void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
     bool slow_path);
-void	*ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
+void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, bool is_internal, arena_t *arena);
-void	*ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
+void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena);
-void	*ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
-size_t	ivsalloc(tsdn_t *tsdn, const void *ptr);
-void	idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
+void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
+size_t ivsalloc(tsdn_t *tsdn, const void *ptr);
+void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
     bool slow_path);
-void	idalloc(tsd_t *tsd, void *ptr);
-void	isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+void idalloc(tsd_t *tsd, void *ptr);
+void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     bool slow_path);
-void	*iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
     size_t extra, size_t alignment, bool zero, tcache_t *tcache,
     arena_t *arena);
-void	*iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
-void	*iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize,
-    size_t size, size_t alignment, bool zero);
-bool	ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
-    size_t size, size_t extra, size_t alignment, bool zero);
+void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
+    size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
+void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
+    size_t alignment, bool zero);
+bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
+    size_t alignment, bool zero);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -1176,8 +1176,8 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) {
+iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment,
+    bool zero, tcache_t *tcache, arena_t *arena) {
 	assert(ptr != NULL);
 	assert(size != 0);
 	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
@@ -1192,20 +1192,20 @@ iralloct(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
 		    zero, tcache, arena);
 	}
 
-	return arena_ralloc(tsdn, arena, extent, ptr, oldsize, size, alignment,
-	    zero, tcache);
+	return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
+	    tcache);
 }
 
 JEMALLOC_ALWAYS_INLINE void *
-iralloc(tsd_t *tsd, extent_t *extent, void *ptr, size_t oldsize, size_t size,
-    size_t alignment, bool zero) {
-	return iralloct(tsd_tsdn(tsd), extent, ptr, oldsize, size, alignment,
-	    zero, tcache_get(tsd, true), NULL);
+iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment,
+    bool zero) {
+	return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero,
+	    tcache_get(tsd, true), NULL);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
-ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
-    size_t extra, size_t alignment, bool zero) {
+ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
+    size_t alignment, bool zero) {
 	assert(ptr != NULL);
 	assert(size != 0);
 	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
@@ -1216,8 +1216,7 @@ ixalloc(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize, size_t size,
 		return true;
 	}
 
-	return arena_ralloc_no_move(tsdn, extent, ptr, oldsize, size, extra,
-	    zero);
+	return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero);
 }
 #endif
 
diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h
index f3b6f8d..985532f 100644
--- a/include/jemalloc/internal/prof_externs.h
+++ b/include/jemalloc/internal/prof_externs.h
@@ -39,8 +39,8 @@ extern uint64_t	prof_interval;
 extern size_t	lg_prof_sample;
 
 void	prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
-void	prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent,
-    const void *ptr, size_t usize, prof_tctx_t *tctx);
+void	prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
+    prof_tctx_t *tctx);
 void	prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
 void	bt_init(prof_bt_t *bt, void **vec);
 void	prof_backtrace(prof_bt_t *bt);
diff --git a/include/jemalloc/internal/prof_inlines_b.h b/include/jemalloc/internal/prof_inlines_b.h
index 29a2b52..5ee72c5 100644
--- a/include/jemalloc/internal/prof_inlines_b.h
+++ b/include/jemalloc/internal/prof_inlines_b.h
@@ -5,24 +5,20 @@
 bool	prof_active_get_unlocked(void);
 bool	prof_gdump_get_unlocked(void);
 prof_tdata_t	*prof_tdata_get(tsd_t *tsd, bool create);
-prof_tctx_t	*prof_tctx_get(tsdn_t *tsdn, const extent_t *extent,
-    const void *ptr);
-void	prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, prof_tctx_t *tctx);
-void	prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
+prof_tctx_t	*prof_tctx_get(tsdn_t *tsdn, const void *ptr);
+void	prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
     prof_tctx_t *tctx);
+void	prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
 bool	prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
     prof_tdata_t **tdata_out);
 prof_tctx_t	*prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
     bool update);
-void	prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    size_t usize, prof_tctx_t *tctx);
-void	prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr,
-    size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated,
-    extent_t *old_extent, const void *old_ptr, size_t old_usize,
-    prof_tctx_t *old_tctx);
-void	prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr,
-    size_t usize);
+void	prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize,
+    prof_tctx_t *tctx);
+void	prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
+    prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
+    size_t old_usize, prof_tctx_t *old_tctx);
+void	prof_free(tsd_t *tsd, const void *ptr, size_t usize);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
@@ -71,29 +67,27 @@ prof_tdata_get(tsd_t *tsd, bool create) {
 }
 
 JEMALLOC_ALWAYS_INLINE prof_tctx_t *
-prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr) {
+prof_tctx_get(tsdn_t *tsdn, const void *ptr) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	return arena_prof_tctx_get(tsdn, extent, ptr);
+	return arena_prof_tctx_get(tsdn, ptr);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
-    prof_tctx_t *tctx) {
+prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	arena_prof_tctx_set(tsdn, extent, ptr, usize, tctx);
+	arena_prof_tctx_set(tsdn, ptr, usize, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr,
-    prof_tctx_t *tctx) {
+prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 
-	arena_prof_tctx_reset(tsdn, extent, ptr, tctx);
+	arena_prof_tctx_reset(tsdn, ptr, tctx);
 }
 
 JEMALLOC_ALWAYS_INLINE bool
@@ -151,24 +145,22 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_malloc(tsdn_t *tsdn, extent_t *extent, const void *ptr, size_t usize,
-    prof_tctx_t *tctx) {
+prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) {
 	cassert(config_prof);
 	assert(ptr != NULL);
 	assert(usize == isalloc(tsdn, ptr));
 
 	if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) {
-		prof_malloc_sample_object(tsdn, extent, ptr, usize, tctx);
+		prof_malloc_sample_object(tsdn, ptr, usize, tctx);
 	} else {
-		prof_tctx_set(tsdn, extent, ptr, usize,
-		    (prof_tctx_t *)(uintptr_t)1U);
+		prof_tctx_set(tsdn, ptr, usize, (prof_tctx_t *)(uintptr_t)1U);
 	}
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
-    prof_tctx_t *tctx, bool prof_active, bool updated, extent_t *old_extent,
-    const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx) {
+prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx,
+    bool prof_active, bool updated, const void *old_ptr, size_t old_usize,
+    prof_tctx_t *old_tctx) {
 	bool sampled, old_sampled, moved;
 
 	cassert(config_prof);
@@ -194,10 +186,9 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
 	moved = (ptr != old_ptr);
 
 	if (unlikely(sampled)) {
-		prof_malloc_sample_object(tsd_tsdn(tsd), extent, ptr, usize,
-		    tctx);
+		prof_malloc_sample_object(tsd_tsdn(tsd), ptr, usize, tctx);
 	} else if (moved) {
-		prof_tctx_set(tsd_tsdn(tsd), extent, ptr, usize,
+		prof_tctx_set(tsd_tsdn(tsd), ptr, usize,
 		    (prof_tctx_t *)(uintptr_t)1U);
 	} else if (unlikely(old_sampled)) {
 		/*
@@ -206,9 +197,9 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
 		 * to do here in the presence of explicit knowledge re: moved
 		 * state.
 		 */
-		prof_tctx_reset(tsd_tsdn(tsd), extent, ptr, tctx);
+		prof_tctx_reset(tsd_tsdn(tsd), ptr, tctx);
 	} else {
-		assert((uintptr_t)prof_tctx_get(tsd_tsdn(tsd), extent, ptr) ==
+		assert((uintptr_t)prof_tctx_get(tsd_tsdn(tsd), ptr) ==
 		    (uintptr_t)1U);
 	}
 
@@ -225,8 +216,8 @@ prof_realloc(tsd_t *tsd, extent_t *extent, const void *ptr, size_t usize,
 }
 
 JEMALLOC_ALWAYS_INLINE void
-prof_free(tsd_t *tsd, const extent_t *extent, const void *ptr, size_t usize) {
-	prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
+prof_free(tsd_t *tsd, const void *ptr, size_t usize) {
+	prof_tctx_t *tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
 
 	cassert(config_prof);
 	assert(usize == isalloc(tsd_tsdn(tsd), ptr));