diff options
| -rw-r--r-- | include/jemalloc/internal/arena_inlines_b.h | 38 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_inlines_c.h | 10 | ||||
| -rw-r--r-- | src/arena.c | 2 | ||||
| -rw-r--r-- | src/jemalloc.c | 13 | ||||
| -rw-r--r-- | src/large.c | 2 |
5 files changed, 41 insertions, 24 deletions
diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 96889c1..382289e 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -19,7 +19,7 @@ void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, dalloc_ctx_t *dalloc_ctx, bool slow_path); void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size); void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - bool slow_path); + dalloc_ctx_t *dalloc_ctx, bool slow_path); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -293,7 +293,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { JEMALLOC_ALWAYS_INLINE void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - bool slow_path) { + dalloc_ctx_t *dalloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); assert(size <= LARGE_MAXCLASS); @@ -305,7 +305,22 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, szind_t szind; bool slab; - if (!config_prof || !opt_prof) { + UNUSED dalloc_ctx_t local_ctx; + if (config_prof && opt_prof) { + if (dalloc_ctx == NULL) { + /* Uncommon case and should be a static check. */ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, + &rtree_ctx_fallback); + rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true, &local_ctx.szind, + &local_ctx.slab); + assert(local_ctx.szind == size2index(size)); + dalloc_ctx = &local_ctx; + } + slab = dalloc_ctx->slab; + szind = dalloc_ctx->szind; + } else { /* * There is no risk of being confused by a promoted sampled * object, so base szind and slab on the given size. @@ -314,21 +329,14 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, slab = (szind < NBINS); } - if ((config_prof && opt_prof) || config_debug) { + if (config_debug) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); - rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &szind, &slab); - - assert(szind == size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < NBINS)); - - if (config_debug) { - extent_t *extent = rtree_extent_read(tsdn, - &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind == extent_szind_get(extent)); - assert(slab == extent_slab_get(extent)); - } + extent_t *extent = rtree_extent_read(tsdn, + &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); + assert(szind == extent_szind_get(extent)); + assert(slab == extent_slab_get(extent)); } if (likely(slab)) { diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 05debd2..4fb3424 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -18,7 +18,7 @@ void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, dalloc_ctx_t *dalloc_ctx, bool is_internal, bool slow_path); void idalloc(tsd_t *tsd, void *ptr); void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - bool slow_path); + dalloc_ctx_t *dalloc_ctx, bool slow_path); void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); @@ -129,10 +129,10 @@ idalloc(tsd_t *tsd, void *ptr) { } JEMALLOC_ALWAYS_INLINE void -isdalloct(tsdn_t *tsdn, void *ptr, size_t size, - tcache_t *tcache, bool slow_path) { +isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, + dalloc_ctx_t *dalloc_ctx, bool slow_path) { witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - arena_sdalloc(tsdn, ptr, size, tcache, slow_path); + arena_sdalloc(tsdn, ptr, size, tcache, dalloc_ctx, slow_path); } JEMALLOC_ALWAYS_INLINE void * @@ -168,7 +168,7 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - isdalloct(tsdn, ptr, oldsize, tcache, true); + isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } diff --git a/src/arena.c b/src/arena.c index 5d313e3..16728b3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1752,7 +1752,7 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, size_t copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); - isdalloct(tsdn, ptr, oldsize, tcache, true); + isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return ret; } diff --git a/src/jemalloc.c b/src/jemalloc.c index 77ee857..e71949a 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2083,17 +2083,26 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); + dalloc_ctx_t dalloc_ctx, *ctx; if (config_prof && opt_prof) { + rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); + rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true, &dalloc_ctx.szind, &dalloc_ctx.slab); + assert(dalloc_ctx.szind == size2index(usize)); prof_free(tsd, ptr, usize); + ctx = &dalloc_ctx; + } else { + ctx = NULL; } + if (config_stats) { *tsd_thread_deallocatedp_get(tsd) += usize; } if (likely(!slow_path)) { - isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, false); + isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, false); } else { - isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, true); + isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, ctx, true); } } diff --git a/src/large.c b/src/large.c index 18987c1..3b53eb3 100644 --- a/src/large.c +++ b/src/large.c @@ -304,7 +304,7 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, extent_addr_get(extent), copysize); - isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, true); + isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); return ret; } |