Pass dealloc_ctx down free() fast path.

This gets rid of the redundent rtree lookup down fast path.

4 files changed, 28 insertions, 11 deletions

diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index ea69a68..96889c1 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -15,7 +15,8 @@ arena_t *arena_aalloc(tsdn_t *tsdn, const void *ptr);
 size_t arena_salloc(tsdn_t *tsdn, const void *ptr);
 size_t arena_vsalloc(tsdn_t *tsdn, const void *ptr);
 void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr);
-void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    dalloc_ctx_t *dalloc_ctx, bool slow_path);
 void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size);
 void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     bool slow_path);
@@ -194,7 +195,8 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    dalloc_ctx_t *dalloc_ctx, bool slow_path) {
 	assert(!tsdn_null(tsdn) || tcache == NULL);
 	assert(ptr != NULL);
 
@@ -203,13 +205,21 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
 		return;
 	}
 
-	rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
 	szind_t szind;
 	bool slab;
-	rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr,
-	    true, &szind, &slab);
+	rtree_ctx_t *rtree_ctx;
+	if (dalloc_ctx != NULL) {
+		szind = dalloc_ctx->szind;
+		slab = dalloc_ctx->slab;
+		assert(szind != NSIZES);
+	} else {
+		rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
+		rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
+		    (uintptr_t)ptr, true, &szind, &slab);
+	}
 
 	if (config_debug) {
+		rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
 		extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
 		    rtree_ctx, (uintptr_t)ptr, true);
 		assert(szind == extent_szind_get(extent));
diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h
index 00e0d0c..b6ba3db 100644
--- a/include/jemalloc/internal/arena_structs_b.h
+++ b/include/jemalloc/internal/arena_structs_b.h
@@ -256,4 +256,10 @@ struct arena_tdata_s {
 	ticker_t		decay_ticker;
 };
 
+/* Used to pass rtree lookup context down the deallocation path. */
+struct dalloc_ctx_s {
+	szind_t szind;
+	bool slab;
+};
+
 #endif /* JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H */
diff --git a/include/jemalloc/internal/arena_types.h b/include/jemalloc/internal/arena_types.h
index ba53c40..435b930 100644
--- a/include/jemalloc/internal/arena_types.h
+++ b/include/jemalloc/internal/arena_types.h
@@ -19,6 +19,7 @@ typedef struct arena_decay_s arena_decay_t;
 typedef struct arena_bin_s arena_bin_t;
 typedef struct arena_s arena_t;
 typedef struct arena_tdata_s arena_tdata_t;
+typedef struct dalloc_ctx_s dalloc_ctx_t;
 
 typedef enum {
 	percpu_arena_disabled = 0,
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 62dae0c..17c77aa 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -1095,8 +1095,8 @@ void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
     tcache_t *tcache, arena_t *arena);
 void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
 size_t ivsalloc(tsdn_t *tsdn, const void *ptr);
-void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
-    bool slow_path);
+void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+    dalloc_ctx_t *dalloc_ctx, bool is_internal, bool slow_path);
 void idalloc(tsd_t *tsd, void *ptr);
 void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
     bool slow_path);
@@ -1188,8 +1188,8 @@ ivsalloc(tsdn_t *tsdn, const void *ptr) {
 }
 
 JEMALLOC_ALWAYS_INLINE void
-idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
-    bool slow_path) {
+idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, dalloc_ctx_t *dalloc_ctx,
+    bool is_internal, bool slow_path) {
 	assert(ptr != NULL);
 	assert(!is_internal || tcache == NULL);
 	assert(!is_internal || arena_ind_get(iaalloc(tsdn, ptr)) <
@@ -1201,12 +1201,12 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool is_internal,
 	if (!is_internal && *tsd_reentrancy_levelp_get(tsdn_tsd(tsdn)) != 0) {
 		tcache = NULL;
 	}
-	arena_dalloc(tsdn, ptr, tcache, slow_path);
+	arena_dalloc(tsdn, ptr, tcache, dalloc_ctx, slow_path);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 idalloc(tsd_t *tsd, void *ptr) {
-	idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), false, true);
+	idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), NULL, false, true);
 }
 
 JEMALLOC_ALWAYS_INLINE void