summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorJason Evans <jasone@canonware.com>2016-03-24 03:29:33 (GMT)
committerJason Evans <jasone@canonware.com>2016-06-03 19:27:41 (GMT)
commitdb72272bef91fa1b4709e89168aede0f01206d55 (patch)
tree75d99bde1468ee4fb3bcbfd57752d16afb953ec1 /src
parent2d2b4e98c947f9fcaf4a9fd2215b685057e89212 (diff)
downloadjemalloc-db72272bef91fa1b4709e89168aede0f01206d55.zip
jemalloc-db72272bef91fa1b4709e89168aede0f01206d55.tar.gz
jemalloc-db72272bef91fa1b4709e89168aede0f01206d55.tar.bz2
Use rtree-based chunk lookups rather than pointer bit twiddling.
Look up chunk metadata via the radix tree, rather than using CHUNK_ADDR2BASE(). Propagate pointer's containing extent. Minimize extent lookups by doing a single lookup (e.g. in free()) and propagating the pointer's extent into nearly all the functions that may need it.
Diffstat (limited to 'src')
-rw-r--r--src/arena.c245
-rw-r--r--src/chunk.c9
-rw-r--r--src/ckh.c10
-rw-r--r--src/huge.c149
-rw-r--r--src/jemalloc.c210
-rw-r--r--src/prof.c32
-rw-r--r--src/tcache.c31
7 files changed, 352 insertions, 334 deletions
diff --git a/src/arena.c b/src/arena.c
index b59f7f1..3abbc62 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -45,10 +45,10 @@ unsigned nhclasses; /* Number of huge size classes. */
static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena,
size_t ndirty_limit);
-static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run,
- bool dirty, bool cleaned, bool decommitted);
+static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
+ arena_run_t *run, bool dirty, bool cleaned, bool decommitted);
static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena,
- arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin);
+ arena_chunk_t *chunk, extent_t *extent, arena_run_t *run, arena_bin_t *bin);
static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, arena_bin_t *bin);
@@ -264,9 +264,9 @@ arena_run_reg_alloc(arena_run_t *run, const arena_bin_info_t *bin_info)
}
JEMALLOC_INLINE_C void
-arena_run_reg_dalloc(arena_run_t *run, void *ptr)
+arena_run_reg_dalloc(arena_run_t *run, extent_t *extent, void *ptr)
{
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
szind_t binind = arena_ptr_small_binind_get(ptr, mapbits);
@@ -375,15 +375,15 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
}
static bool
-arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
- bool remove, bool zero)
+arena_run_split_large_helper(arena_t *arena, extent_t *extent, arena_run_t *run,
+ size_t size, bool remove, bool zero)
{
arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm;
size_t flag_dirty, flag_decommitted, run_ind, need_pages;
size_t flag_unzeroed_mask;
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm);
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
@@ -439,22 +439,26 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
}
static bool
-arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
+arena_run_split_large(arena_t *arena, extent_t *extent, arena_run_t *run,
+ size_t size, bool zero)
{
- return (arena_run_split_large_helper(arena, run, size, true, zero));
+ return (arena_run_split_large_helper(arena, extent, run, size, true,
+ zero));
}
static bool
-arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
+arena_run_init_large(arena_t *arena, extent_t *extent, arena_run_t *run,
+ size_t size, bool zero)
{
- return (arena_run_split_large_helper(arena, run, size, false, zero));
+ return (arena_run_split_large_helper(arena, extent, run, size, false,
+ zero));
}
static bool
-arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
- szind_t binind)
+arena_run_split_small(arena_t *arena, extent_t *extent, arena_run_t *run,
+ size_t size, szind_t binind)
{
arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm;
@@ -462,7 +466,7 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
assert(binind != BININD_INVALID);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm);
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
@@ -1037,7 +1041,7 @@ arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
{
arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
if (run != NULL) {
- if (arena_run_split_large(arena, run, size, zero))
+ if (arena_run_split_large(arena, iealloc(run), run, size, zero))
run = NULL;
}
return (run);
@@ -1063,7 +1067,7 @@ arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero)
chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
- if (arena_run_split_large(arena, run, size, zero))
+ if (arena_run_split_large(arena, iealloc(run), run, size, zero))
run = NULL;
return (run);
}
@@ -1081,7 +1085,8 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
{
arena_run_t *run = arena_run_first_best_fit(arena, size);
if (run != NULL) {
- if (arena_run_split_small(arena, run, size, binind))
+ if (arena_run_split_small(arena, iealloc(run), run, size,
+ binind))
run = NULL;
}
return (run);
@@ -1108,7 +1113,8 @@ arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind)
chunk = arena_chunk_alloc(tsdn, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
- if (arena_run_split_small(arena, run, size, binind))
+ if (arena_run_split_small(arena, iealloc(run), run, size,
+ binind))
run = NULL;
return (run);
}
@@ -1435,8 +1441,9 @@ arena_dirty_count(arena_t *arena)
npages = extent_size_get(chunkselm) >> LG_PAGE;
chunkselm = qr_next(chunkselm, cc_link);
} else {
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
- rdelm);
+ extent_t *extent = iealloc(rdelm);
+ arena_chunk_t *chunk =
+ (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -1497,8 +1504,9 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
LG_PAGE));
chunkselm = chunkselm_next;
} else {
+ extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk =
- (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
+ (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -1523,7 +1531,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_chunk_alloc(tsdn, arena);
/* Temporarily allocate the free dirty run. */
- arena_run_split_large(arena, run, run_size, false);
+ arena_run_split_large(arena, extent, run, run_size,
+ false);
/* Stash. */
if (false)
qr_new(rdelm, rd_link); /* Redundant. */
@@ -1577,8 +1586,9 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
} else {
size_t pageind, run_size, flag_unzeroed, flags, i;
bool decommitted;
+ extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk =
- (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
+ (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
pageind = arena_miscelm_to_pageind(miscelm);
@@ -1661,8 +1671,9 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr,
size, zeroed, committed);
} else {
+ extent_t *extent = iealloc(rdelm);
arena_chunk_t *chunk =
- (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
+ (arena_chunk_t *)extent_addr_get(extent);
arena_chunk_map_misc_t *miscelm =
arena_rd_to_miscelm(rdelm);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -1670,7 +1681,7 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
pageind) != 0);
arena_run_t *run = &miscelm->run;
qr_remove(rdelm, rd_link);
- arena_run_dalloc(tsdn, arena, run, false, true,
+ arena_run_dalloc(tsdn, arena, extent, run, false, true,
decommitted);
}
}
@@ -1755,10 +1766,10 @@ arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
if (arena_mapbits_large_get(chunk, pageind) != 0) {
void *ptr = (void *)((uintptr_t)chunk + (pageind
<< LG_PAGE));
- size_t usize = isalloc(tsd_tsdn(tsd), ptr,
- config_prof);
+ size_t usize = isalloc(tsd_tsdn(tsd),
+ &chunk->extent, ptr, config_prof);
- prof_free(tsd, ptr, usize);
+ prof_free(tsd, &chunk->extent, ptr, usize);
npages = arena_mapbits_large_size_get(chunk,
pageind) >> LG_PAGE;
} else {
@@ -1820,12 +1831,14 @@ arena_reset(tsd_t *tsd, arena_t *arena)
size_t usize;
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx);
- if (config_stats || (config_prof && opt_prof))
- usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
+ if (config_stats || (config_prof && opt_prof)) {
+ usize = isalloc(tsd_tsdn(tsd), extent, ptr,
+ config_prof);
+ }
/* Remove huge allocation from prof sample set. */
if (config_prof && opt_prof)
- prof_free(tsd, ptr, usize);
- huge_dalloc(tsd_tsdn(tsd), ptr);
+ prof_free(tsd, extent, ptr, usize);
+ huge_dalloc(tsd_tsdn(tsd), extent, ptr);
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
/* Cancel out unwanted effects on stats. */
if (config_stats)
@@ -1997,14 +2010,14 @@ arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
}
static void
-arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty,
- bool cleaned, bool decommitted)
+arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
+ arena_run_t *run, bool dirty, bool cleaned, bool decommitted)
{
arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm;
size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run);
run_ind = arena_miscelm_to_pageind(miscelm);
assert(run_ind >= map_bias);
@@ -2074,7 +2087,7 @@ arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty,
static void
arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- arena_run_t *run, size_t oldsize, size_t newsize)
+ extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize)
{
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -2109,13 +2122,14 @@ arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
pageind+head_npages)));
- arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted !=
- 0));
+ arena_run_dalloc(tsdn, arena, extent, run, false, false,
+ (flag_decommitted != 0));
}
static void
arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- arena_run_t *run, size_t oldsize, size_t newsize, bool dirty)
+ extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize,
+ bool dirty)
{
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -2154,8 +2168,8 @@ arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages);
tail_run = &tail_miscelm->run;
- arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted
- != 0));
+ arena_run_dalloc(tsdn, arena, extent, tail_run, dirty, false,
+ (flag_decommitted != 0));
}
static void
@@ -2251,6 +2265,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
assert(bin->runcur->nfree > 0);
ret = arena_run_reg_alloc(bin->runcur, bin_info);
if (run != NULL) {
+ extent_t *extent;
arena_chunk_t *chunk;
/*
@@ -2261,10 +2276,11 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
* arena_bin_lower_run() must be called, as if a region
* were just deallocated from the run.
*/
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ extent = iealloc(run);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
if (run->nfree == bin_info->nregs) {
- arena_dalloc_bin_run(tsdn, arena, chunk, run,
- bin);
+ arena_dalloc_bin_run(tsdn, arena, chunk, extent,
+ run, bin);
} else
arena_bin_lower_run(arena, chunk, run, bin);
}
@@ -2499,6 +2515,7 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
void *ret;
size_t alloc_size, leadsize, trailsize;
arena_run_t *run;
+ extent_t *extent;
arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm;
void *rpages;
@@ -2520,7 +2537,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
malloc_mutex_unlock(tsdn, &arena->lock);
return (NULL);
}
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
+ extent = iealloc(run);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
miscelm = arena_run_to_miscelm(run);
rpages = arena_miscelm_to_rpages(miscelm);
@@ -2531,20 +2549,22 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
if (leadsize != 0) {
arena_chunk_map_misc_t *head_miscelm = miscelm;
arena_run_t *head_run = run;
+ extent_t *head_extent = extent;
miscelm = arena_miscelm_get_mutable(chunk,
arena_miscelm_to_pageind(head_miscelm) + (leadsize >>
LG_PAGE));
run = &miscelm->run;
+ extent = iealloc(run);
- arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size,
- alloc_size - leadsize);
+ arena_run_trim_head(tsdn, arena, chunk, head_extent, head_run,
+ alloc_size, alloc_size - leadsize);
}
if (trailsize != 0) {
- arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad +
- trailsize, usize + large_pad, false);
+ arena_run_trim_tail(tsdn, arena, chunk, extent, run, usize +
+ large_pad + trailsize, usize + large_pad, false);
}
- if (arena_run_init_large(arena, run, usize + large_pad, zero)) {
+ if (arena_run_init_large(arena, extent, run, usize + large_pad, zero)) {
size_t run_ind =
arena_miscelm_to_pageind(arena_run_to_miscelm(run));
bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);
@@ -2552,7 +2572,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
run_ind) != 0);
assert(decommitted); /* Cause of OOM. */
- arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted);
+ arena_run_dalloc(tsdn, arena, extent, run, dirty, false,
+ decommitted);
malloc_mutex_unlock(tsdn, &arena->lock);
return (NULL);
}
@@ -2616,7 +2637,8 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
}
void
-arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size)
+arena_prof_promoted(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
+ size_t size)
{
arena_chunk_t *chunk;
size_t pageind;
@@ -2624,32 +2646,30 @@ arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size)
cassert(config_prof);
assert(ptr != NULL);
- assert(CHUNK_ADDR2BASE(ptr) != ptr);
- assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS);
- assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS);
+ assert(extent_addr_get(extent) != ptr);
+ assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
+ assert(isalloc(tsdn, extent, ptr, true) == LARGE_MINCLASS);
assert(size <= SMALL_MAXCLASS);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
binind = size2index(size);
assert(binind < NBINS);
arena_mapbits_large_binind_set(chunk, pageind, binind);
- assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS);
- assert(isalloc(tsdn, ptr, true) == size);
+ assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
+ assert(isalloc(tsdn, extent, ptr, true) == size);
}
static void
-arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
- arena_bin_t *bin)
+arena_dissociate_bin_run(extent_t *extent, arena_run_t *run, arena_bin_t *bin)
{
/* Dissociate run from bin. */
if (run == bin->runcur)
bin->runcur = NULL;
else {
- szind_t binind = arena_bin_index(extent_arena_get(
- &chunk->extent), bin);
+ szind_t binind = arena_bin_index(extent_arena_get(extent), bin);
const arena_bin_info_t *bin_info = &arena_bin_info[binind];
/*
@@ -2668,7 +2688,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
static void
arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- arena_run_t *run, arena_bin_t *bin)
+ extent_t *extent, arena_run_t *run, arena_bin_t *bin)
{
assert(run != bin->runcur);
@@ -2676,7 +2696,7 @@ arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
malloc_mutex_unlock(tsdn, &bin->lock);
/******************************/
malloc_mutex_lock(tsdn, &arena->lock);
- arena_run_dalloc(tsdn, arena, run, true, false, false);
+ arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
malloc_mutex_unlock(tsdn, &arena->lock);
/****************************/
malloc_mutex_lock(tsdn, &bin->lock);
@@ -2707,7 +2727,7 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
static void
arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
+ extent_t *extent, void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
{
size_t pageind, rpages_ind;
arena_run_t *run;
@@ -2725,10 +2745,10 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
if (!junked && config_fill && unlikely(opt_junk_free))
arena_dalloc_junk_small(ptr, bin_info);
- arena_run_reg_dalloc(run, ptr);
+ arena_run_reg_dalloc(run, extent, ptr);
if (run->nfree == bin_info->nregs) {
- arena_dissociate_bin_run(chunk, run, bin);
- arena_dalloc_bin_run(tsdn, arena, chunk, run, bin);
+ arena_dissociate_bin_run(extent, run, bin);
+ arena_dalloc_bin_run(tsdn, arena, chunk, extent, run, bin);
} else if (run->nfree == 1 && run != bin->runcur)
arena_bin_lower_run(arena, chunk, run, bin);
@@ -2740,15 +2760,17 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
void
arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
- arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm)
+ arena_chunk_t *chunk, extent_t *extent, void *ptr,
+ arena_chunk_map_bits_t *bitselm)
{
- arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true);
+ arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
+ true);
}
-void
-arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr,
- size_t pageind, arena_chunk_map_bits_t *bitselm)
+static void
+arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
+ extent_t *extent, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm)
{
arena_run_t *run;
arena_bin_t *bin;
@@ -2758,13 +2780,14 @@ arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr,
run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run;
bin = &arena->bins[run->binind];
malloc_mutex_lock(tsdn, &bin->lock);
- arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false);
+ arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
+ false);
malloc_mutex_unlock(tsdn, &bin->lock);
}
void
arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- void *ptr, size_t pageind)
+ extent_t *extent, void *ptr, size_t pageind)
{
arena_chunk_map_bits_t *bitselm;
@@ -2774,7 +2797,7 @@ arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
pageind)) != BININD_INVALID);
}
bitselm = arena_bitselm_get_mutable(chunk, pageind);
- arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm);
+ arena_dalloc_bin(tsdn, arena, chunk, extent, ptr, pageind, bitselm);
arena_decay_tick(tsdn, arena);
}
@@ -2798,7 +2821,7 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
static void
arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
- arena_chunk_t *chunk, void *ptr, bool junked)
+ arena_chunk_t *chunk, extent_t *extent, void *ptr, bool junked)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
@@ -2821,31 +2844,31 @@ arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
}
}
- arena_run_dalloc(tsdn, arena, run, true, false, false);
+ arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
}
void
arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena,
- arena_chunk_t *chunk, void *ptr)
+ arena_chunk_t *chunk, extent_t *extent, void *ptr)
{
- arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true);
+ arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, true);
}
void
arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- void *ptr)
+ extent_t *extent, void *ptr)
{
malloc_mutex_lock(tsdn, &arena->lock);
- arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false);
+ arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, false);
malloc_mutex_unlock(tsdn, &arena->lock);
arena_decay_tick(tsdn, arena);
}
static void
arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- void *ptr, size_t oldsize, size_t size)
+ extent_t *extent, void *ptr, size_t oldsize, size_t size)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
@@ -2859,8 +2882,8 @@ arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
* allocations.
*/
malloc_mutex_lock(tsdn, &arena->lock);
- arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size +
- large_pad, true);
+ arena_run_trim_tail(tsdn, arena, chunk, extent, run, oldsize +
+ large_pad, size + large_pad, true);
if (config_stats) {
szind_t oldindex = size2index(oldsize) - NBINS;
szind_t index = size2index(size) - NBINS;
@@ -2916,7 +2939,8 @@ arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
goto label_fail;
run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run;
- if (arena_run_split_large(arena, run, splitsize, zero))
+ if (arena_run_split_large(arena, iealloc(run), run, splitsize,
+ zero))
goto label_fail;
if (config_cache_oblivious && zero) {
@@ -3005,8 +3029,8 @@ arena_ralloc_junk_large_t *arena_ralloc_junk_large =
* always fail if growing an object, and the following run is already in use.
*/
static bool
-arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
- size_t usize_max, bool zero)
+arena_ralloc_large(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
+ size_t usize_min, size_t usize_max, bool zero)
{
arena_chunk_t *chunk;
arena_t *arena;
@@ -3016,8 +3040,8 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
return (false);
}
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- arena = extent_arena_get(&chunk->extent);
+ chunk = (arena_chunk_t *)extent_addr_get(extent);
+ arena = extent_arena_get(extent);
if (oldsize < usize_max) {
bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr,
@@ -3026,10 +3050,12 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
if (unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)ptr + oldsize),
JEMALLOC_ALLOC_JUNK,
- isalloc(tsdn, ptr, config_prof) - oldsize);
+ isalloc(tsdn, extent, ptr, config_prof) -
+ oldsize);
} else if (unlikely(opt_zero)) {
memset((void *)((uintptr_t)ptr + oldsize), 0,
- isalloc(tsdn, ptr, config_prof) - oldsize);
+ isalloc(tsdn, extent, ptr, config_prof) -
+ oldsize);
}
}
return (ret);
@@ -3038,13 +3064,14 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
assert(oldsize > usize_max);
/* Fill before shrinking in order avoid a race. */
arena_ralloc_junk_large(ptr, oldsize, usize_max);
- arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max);
+ arena_ralloc_large_shrink(tsdn, arena, chunk, extent, ptr, oldsize,
+ usize_max);
return (false);
}
bool
-arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
- size_t extra, bool zero)
+arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
+ size_t size, size_t extra, bool zero)
{
size_t usize_min, usize_max;
@@ -3057,8 +3084,6 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
usize_min = s2u(size);
usize_max = s2u(size + extra);
if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
- arena_chunk_t *chunk;
-
/*
* Avoid moving the allocation if the size class can be left the
* same.
@@ -3073,17 +3098,16 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
} else {
if (usize_max <= SMALL_MAXCLASS)
return (true);
- if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min,
- usize_max, zero))
+ if (arena_ralloc_large(tsdn, extent, ptr, oldsize,
+ usize_min, usize_max, zero))
return (true);
}
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- arena_decay_tick(tsdn, extent_arena_get(&chunk->extent));
+ arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
} else {
- return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min,
- usize_max, zero));
+ return (huge_ralloc_no_move(tsdn, extent, ptr, oldsize,
+ usize_min, usize_max, zero));
}
}
@@ -3102,8 +3126,8 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
}
void *
-arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
- size_t size, size_t alignment, bool zero, tcache_t *tcache)
+arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
+ size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache)
{
void *ret;
size_t usize;
@@ -3116,7 +3140,8 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t copysize;
/* Try to avoid moving the allocation. */
- if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero))
+ if (!arena_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, 0,
+ zero))
return (ptr);
/*
@@ -3136,10 +3161,10 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
- isdalloct(tsdn, ptr, oldsize, tcache, true);
+ isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
} else {
- ret = huge_ralloc(tsdn, arena, ptr, oldsize, usize, alignment,
- zero, tcache);
+ ret = huge_ralloc(tsdn, arena, extent, ptr, oldsize, usize,
+ alignment, zero, tcache);
}
return (ret);
}
diff --git a/src/chunk.c b/src/chunk.c
index 31b8645..e35bb30 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -180,6 +180,15 @@ chunk_deregister(const void *chunk, const extent_t *extent)
}
}
+void
+chunk_reregister(tsdn_t *tsdn, const void *chunk, const extent_t *extent)
+{
+ bool err;
+
+ err = chunk_register(tsdn, chunk, extent);
+ assert(!err);
+}
+
/*
* Do first-best-fit chunk selection, i.e. select the lowest chunk that best
* fits.
diff --git a/src/ckh.c b/src/ckh.c
index 747c1c8..3135ee7 100644
--- a/src/ckh.c
+++ b/src/ckh.c
@@ -283,12 +283,12 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) {
- idalloctm(tsdn, tab, NULL, true, true);
+ idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
break;
}
/* Rebuilding failed, so back out partially rebuilt table. */
- idalloctm(tsdn, ckh->tab, NULL, true, true);
+ idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
}
@@ -330,7 +330,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) {
- idalloctm(tsdn, tab, NULL, true, true);
+ idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
#ifdef CKH_COUNT
ckh->nshrinks++;
#endif
@@ -338,7 +338,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
}
/* Rebuilding failed, so back out partially rebuilt table. */
- idalloctm(tsdn, ckh->tab, NULL, true, true);
+ idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
#ifdef CKH_COUNT
@@ -421,7 +421,7 @@ ckh_delete(tsdn_t *tsdn, ckh_t *ckh)
(unsigned long long)ckh->nrelocs);
#endif
- idalloctm(tsdn, ckh->tab, NULL, true, true);
+ idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
if (config_debug)
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
}
diff --git a/src/huge.c b/src/huge.c
index c30e78d..e42ea9c 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -3,42 +3,6 @@
/******************************************************************************/
-static extent_t *
-huge_extent_get(const void *ptr)
-{
- extent_t *extent;
-
- extent = chunk_lookup(ptr, true);
- assert(!extent_achunk_get(extent));
-
- return (extent);
-}
-
-static bool
-huge_extent_set(tsdn_t *tsdn, const void *ptr, extent_t *extent)
-{
-
- assert(extent_addr_get(extent) == ptr);
- assert(!extent_achunk_get(extent));
- return (chunk_register(tsdn, ptr, extent));
-}
-
-static void
-huge_extent_reset(tsdn_t *tsdn, const void *ptr, extent_t *extent)
-{
- bool err;
-
- err = huge_extent_set(tsdn, ptr, extent);
- assert(!err);
-}
-
-static void
-huge_extent_unset(const void *ptr, const extent_t *extent)
-{
-
- chunk_deregister(ptr, extent);
-}
-
void *
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
{
@@ -81,15 +45,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
arena = arena_choose(tsdn_tsd(tsdn), arena);
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
arena, usize, alignment, &is_zeroed)) == NULL) {
- idalloctm(tsdn, extent, NULL, true, true);
+ idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
return (NULL);
}
extent_init(extent, arena, ret, usize, is_zeroed, true);
- if (huge_extent_set(tsdn, ret, extent)) {
+ if (chunk_register(tsdn, ret, extent)) {
arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
- idalloctm(tsdn, extent, NULL, true, true);
+ idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
return (NULL);
}
@@ -133,11 +97,10 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
#endif
static void
-huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
- size_t usize_min, size_t usize_max, bool zero)
+huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
+ size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
{
size_t usize, usize_next;
- extent_t *extent;
arena_t *arena;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
bool pre_zeroed, post_zeroed;
@@ -150,7 +113,6 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
if (oldsize == usize)
return;
- extent = huge_extent_get(ptr);
arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent);
@@ -169,15 +131,15 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
} else
post_zeroed = pre_zeroed;
- malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */
assert(extent_size_get(extent) != usize);
- huge_extent_unset(ptr, extent);
+ chunk_deregister(tsdn, ptr, extent);
+ malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize);
- huge_extent_reset(tsdn, ptr, extent);
+ malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ chunk_reregister(tsdn, ptr, extent);
/* Update zeroed. */
extent_zeroed_set(extent, post_zeroed);
- malloc_mutex_unlock(tsdn, &arena->huge_mtx);
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
@@ -196,16 +158,14 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
}
static bool
-huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
- size_t usize)
+huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, void *ptr,
+ size_t oldsize, size_t usize)
{
- extent_t *extent;
arena_t *arena;
chunk_hooks_t chunk_hooks;
size_t cdiff;
bool pre_zeroed, post_zeroed;
- extent = huge_extent_get(ptr);
arena = extent_arena_get(extent);
pre_zeroed = extent_zeroed_get(extent);
chunk_hooks = chunk_hooks_get(tsdn, arena);
@@ -233,14 +193,14 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
} else
post_zeroed = pre_zeroed;
- malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */
- huge_extent_unset(ptr, extent);
+ chunk_deregister(ptr, extent);
+ malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize);
- huge_extent_reset(tsdn, ptr, extent);
/* Update zeroed. */
extent_zeroed_set(extent, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ chunk_reregister(tsdn, ptr, extent);
/* Zap the excess chunks. */
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
@@ -249,14 +209,12 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
}
static bool
-huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
- size_t usize, bool zero)
+huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, void *ptr,
+ size_t oldsize, size_t usize, bool zero)
{
- extent_t *extent;
arena_t *arena;
bool is_zeroed_subchunk, is_zeroed_chunk;
- extent = huge_extent_get(ptr);
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
is_zeroed_subchunk = extent_zeroed_get(extent);
@@ -272,12 +230,12 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
&is_zeroed_chunk))
return (true);
- malloc_mutex_lock(tsdn, &arena->huge_mtx);
/* Update the size of the huge allocation. */
- huge_extent_unset(ptr, extent);
+ chunk_deregister(ptr, extent);
+ malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize);
- huge_extent_reset(tsdn, ptr, extent);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ chunk_reregister(tsdn, ptr, extent);
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed_subchunk) {
@@ -298,8 +256,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
}
bool
-huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
- size_t usize_max, bool zero)
+huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
+ size_t usize_min, size_t usize_max, bool zero)
{
assert(s2u(oldsize) == oldsize);
@@ -312,16 +270,16 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
/* Attempt to expand the allocation in-place. */
- if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
- zero)) {
- arena_decay_tick(tsdn, huge_aalloc(ptr));
+ if (!huge_ralloc_no_move_expand(tsdn, extent, ptr, oldsize,
+ usize_max, zero)) {
+ arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
/* Try again, this time with usize_min. */
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
- ptr, oldsize, usize_min, zero)) {
- arena_decay_tick(tsdn, huge_aalloc(ptr));
+ extent, ptr, oldsize, usize_min, zero)) {
+ arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
}
@@ -332,17 +290,17 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
*/
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
- huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
- usize_max, zero);
- arena_decay_tick(tsdn, huge_aalloc(ptr));
+ huge_ralloc_no_move_similar(tsdn, extent, ptr, oldsize,
+ usize_min, usize_max, zero);
+ arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
/* Attempt to shrink the allocation in-place. */
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
- if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
+ if (!huge_ralloc_no_move_shrink(tsdn, extent, ptr, oldsize,
usize_max)) {
- arena_decay_tick(tsdn, huge_aalloc(ptr));
+ arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
}
@@ -360,8 +318,8 @@ huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
}
void *
-huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
- size_t usize, size_t alignment, bool zero, tcache_t *tcache)
+huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
+ size_t oldsize, size_t usize, size_t alignment, bool zero, tcache_t *tcache)
{
void *ret;
size_t copysize;
@@ -370,7 +328,8 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
assert(usize > 0 && usize <= HUGE_MAXCLASS);
/* Try to avoid moving the allocation. */
- if (!huge_ralloc_no_move(tsdn, ptr, oldsize, usize, usize, zero))
+ if (!huge_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, usize,
+ zero))
return (ptr);
/*
@@ -384,19 +343,17 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
- isdalloct(tsdn, ptr, oldsize, tcache, true);
+ isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
return (ret);
}
void
-huge_dalloc(tsdn_t *tsdn, void *ptr)
+huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr)
{
- extent_t *extent;
arena_t *arena;
- extent = huge_extent_get(ptr);
arena = extent_arena_get(extent);
- huge_extent_unset(ptr, extent);
+ chunk_deregister(ptr, extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
ql_remove(&arena->huge, extent, ql_link);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
@@ -405,26 +362,17 @@ huge_dalloc(tsdn_t *tsdn, void *ptr)
extent_size_get(extent));
arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent),
extent_addr_get(extent), extent_size_get(extent));
- idalloctm(tsdn, extent, NULL, true, true);
+ idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
arena_decay_tick(tsdn, arena);
}
-arena_t *
-huge_aalloc(const void *ptr)
-{
-
- return (extent_arena_get(huge_extent_get(ptr)));
-}
-
size_t
-huge_salloc(tsdn_t *tsdn, const void *ptr)
+huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{
size_t size;
- extent_t *extent;
arena_t *arena;
- extent = huge_extent_get(ptr);
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
size = extent_size_get(extent);
@@ -434,13 +382,13 @@ huge_salloc(tsdn_t *tsdn, const void *ptr)
}
prof_tctx_t *
-huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
+huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
{
prof_tctx_t *tctx;
- extent_t *extent;
arena_t *arena;
- extent = huge_extent_get(ptr);
+ assert(extent == iealloc(ptr));
+
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
tctx = extent_prof_tctx_get(extent);
@@ -450,12 +398,13 @@ huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
}
void
-huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
+huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
+ prof_tctx_t *tctx)
{
- extent_t *extent;
arena_t *arena;
- extent = huge_extent_get(ptr);
+ assert(extent == iealloc(ptr));
+
arena = extent_arena_get(extent);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_prof_tctx_set(extent, tctx);
@@ -463,8 +412,8 @@ huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
}
void
-huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
+huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr)
{
- huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
+ huge_prof_tctx_set(tsdn, extent, ptr, (prof_tctx_t *)(uintptr_t)1U);
}
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 929f3b8..67a3b56 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -308,10 +308,10 @@ a0ialloc(size_t size, bool zero, bool is_metadata)
}
static void
-a0idalloc(void *ptr, bool is_metadata)
+a0idalloc(extent_t *extent, void *ptr, bool is_metadata)
{
- idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
+ idalloctm(TSDN_NULL, extent, ptr, false, is_metadata, true);
}
void *
@@ -325,7 +325,7 @@ void
a0dalloc(void *ptr)
{
- a0idalloc(ptr, true);
+ a0idalloc(iealloc(ptr), ptr, true);
}
/*
@@ -365,7 +365,7 @@ bootstrap_free(void *ptr)
if (unlikely(ptr == NULL))
return;
- a0idalloc(ptr, false);
+ a0idalloc(iealloc(ptr), ptr, false);
}
static void
@@ -1401,7 +1401,7 @@ ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
if (p == NULL)
return (NULL);
- arena_prof_promoted(tsd_tsdn(tsd), p, usize);
+ arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else
p = ialloc(tsd, usize, ind, zero, slow_path);
@@ -1423,7 +1423,7 @@ ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
+ prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
return (p);
}
@@ -1482,7 +1482,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
- assert(usize == isalloc(tsdn, ret, config_prof));
+ assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
}
witness_assert_lockless(tsdn);
@@ -1525,7 +1525,7 @@ imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
if (p == NULL)
return (NULL);
- arena_prof_promoted(tsd_tsdn(tsd), p, usize);
+ arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else
p = ipalloc(tsd, usize, alignment, false);
@@ -1547,7 +1547,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
+ prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
return (p);
}
@@ -1604,7 +1604,8 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
ret = 0;
label_return:
if (config_stats && likely(result != NULL)) {
- assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
+ assert(usize == isalloc(tsd_tsdn(tsd), iealloc(result), result,
+ config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, result);
@@ -1683,44 +1684,49 @@ je_calloc(size_t num, size_t size)
}
static void *
-irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
- prof_tctx_t *tctx)
+irealloc_prof_sample(tsd_t *tsd, extent_t *extent, void *old_ptr,
+ size_t old_usize, size_t usize, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
- p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
+ p = iralloc(tsd, extent, old_ptr, old_usize, LARGE_MINCLASS, 0,
+ false);
if (p == NULL)
return (NULL);
- arena_prof_promoted(tsd_tsdn(tsd), p, usize);
+ arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
} else
- p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
+ p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
-irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
+irealloc_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
+ size_t usize)
{
void *p;
+ extent_t *e;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
+ old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
tctx = prof_alloc_prep(tsd, usize, prof_active, true);
- if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
- p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
- else
- p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
+ if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
+ p = irealloc_prof_sample(tsd, extent, old_ptr, old_usize, usize,
+ tctx);
+ } else
+ p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
- old_tctx);
+ e = (p == old_ptr) ? extent : iealloc(p);
+ prof_realloc(tsd, e, p, usize, tctx, prof_active, true,
+ old_ptr, old_usize, old_tctx);
return (p);
}
@@ -1728,6 +1734,7 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
JEMALLOC_INLINE_C void
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
{
+ extent_t *extent;
size_t usize;
witness_assert_lockless(tsd_tsdn(tsd));
@@ -1735,22 +1742,24 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
+ extent = iealloc(ptr);
if (config_prof && opt_prof) {
- usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
- prof_free(tsd, ptr, usize);
+ usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
+ prof_free(tsd, extent, ptr, usize);
} else if (config_stats)
- usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
+ usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path))
- idalloctm(tsd_tsdn(tsd), ptr, tcache, false, false);
+ idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, false);
else
- idalloctm(tsd_tsdn(tsd), ptr, tcache, false, true);
+ idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, true);
}
JEMALLOC_INLINE_C void
-isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
+isfree(tsd_t *tsd, extent_t *extent, void *ptr, size_t usize, tcache_t *tcache,
+ bool slow_path)
{
witness_assert_lockless(tsd_tsdn(tsd));
@@ -1759,14 +1768,14 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
assert(malloc_initialized() || IS_INITIALIZER);
if (config_prof && opt_prof)
- prof_free(tsd, ptr, usize);
+ prof_free(tsd, extent, ptr, usize);
if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize;
if (likely(!slow_path))
- isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, false);
+ isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, false);
else
- isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, true);
+ isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, true);
}
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
@@ -1794,22 +1803,26 @@ je_realloc(void *ptr, size_t size)
if (likely(ptr != NULL)) {
tsd_t *tsd;
+ extent_t *extent;
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
- old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
+ extent = iealloc(ptr);
+ old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_prof && opt_prof) {
usize = s2u(size);
ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
- NULL : irealloc_prof(tsd, ptr, old_usize, usize);
+ NULL : irealloc_prof(tsd, extent, ptr, old_usize,
+ usize);
} else {
if (config_stats)
usize = s2u(size);
- ret = iralloc(tsd, ptr, old_usize, size, 0, false);
+ ret = iralloc(tsd, extent, ptr, old_usize, size, 0,
+ false);
}
tsdn = tsd_tsdn(tsd);
} else {
@@ -1832,7 +1845,7 @@ je_realloc(void *ptr, size_t size)
if (config_stats && likely(ret != NULL)) {
tsd_t *tsd;
- assert(usize == isalloc(tsdn, ret, config_prof));
+ assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
tsd = tsdn_tsd(tsdn);
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
@@ -1986,11 +1999,10 @@ imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
tcache, arena, slow_path);
if (p == NULL)
return (NULL);
- arena_prof_promoted(tsdn, p, usize);
- } else {
+ arena_prof_promoted(tsdn, iealloc(p), p, usize);
+ } else
p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
slow_path);
- }
return (p);
}
@@ -2021,7 +2033,7 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
+ prof_malloc(tsd_tsdn(tsd), iealloc(p), p, *usize, tctx);
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
return (p);
@@ -2109,46 +2121,47 @@ je_mallocx(size_t size, int flags)
}
static void *
-irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
- size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
- prof_tctx_t *tctx)
+irallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *old_ptr,
+ size_t old_usize, size_t usize, size_t alignment, bool zero,
+ tcache_t *tcache, arena_t *arena, prof_tctx_t *tctx)
{
void *p;
if (tctx == NULL)
return (NULL);
if (usize <= SMALL_MAXCLASS) {
- p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS,
+ p = iralloct(tsdn, extent, old_ptr, old_usize, LARGE_MINCLASS,
alignment, zero, tcache, arena);
if (p == NULL)
return (NULL);
- arena_prof_promoted(tsdn, p, usize);
+ arena_prof_promoted(tsdn, iealloc(p), p, usize);
} else {
- p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
- tcache, arena);
+ p = iralloct(tsdn, extent, old_ptr, old_usize, usize, alignment,
+ zero, tcache, arena);
}
return (p);
}
JEMALLOC_ALWAYS_INLINE_C void *
-irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
- size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
+irallocx_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
+ size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
arena_t *arena)
{
void *p;
+ extent_t *e;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
+ old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
tctx = prof_alloc_prep(tsd, *usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
- p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
- *usize, alignment, zero, tcache, arena, tctx);
+ p = irallocx_prof_sample(tsd_tsdn(tsd), extent, old_ptr,
+ old_usize, *usize, alignment, zero, tcache, arena, tctx);
} else {
- p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
- zero, tcache, arena);
+ p = iralloct(tsd_tsdn(tsd), extent, old_ptr, old_usize, size,
+ alignment, zero, tcache, arena);
}
if (unlikely(p == NULL)) {
prof_alloc_rollback(tsd, tctx, true);
@@ -2164,9 +2177,11 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
* be the same as the current usize because of in-place large
* reallocation. Therefore, query the actual value of usize.
*/
- *usize = isalloc(tsd_tsdn(tsd), p, config_prof);
- }
- prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr,
+ e = extent;
+ *usize = isalloc(tsd_tsdn(tsd), e, p, config_prof);
+ } else
+ e = iealloc(p);
+ prof_realloc(tsd, e, p, *usize, tctx, prof_active, true, old_ptr,
old_usize, old_tctx);
return (p);
@@ -2179,6 +2194,7 @@ je_rallocx(void *ptr, size_t size, int flags)
{
void *p;
tsd_t *tsd;
+ extent_t *extent;
size_t usize;
size_t old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags);
@@ -2191,6 +2207,7 @@ je_rallocx(void *ptr, size_t size, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
+ extent = iealloc(ptr);
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
@@ -2208,23 +2225,25 @@ je_rallocx(void *ptr, size_t size, int flags)
} else
tcache = tcache_get(tsd, true);
- old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
+ old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
if (config_prof && opt_prof) {
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
goto label_oom;
- p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
- zero, tcache, arena);
+ p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
+ &usize, zero, tcache, arena);
if (unlikely(p == NULL))
goto label_oom;
} else {
- p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
- zero, tcache, arena);
+ p = iralloct(tsd_tsdn(tsd), extent, ptr, old_usize, size,
+ alignment, zero, tcache, arena);
if (unlikely(p == NULL))
goto label_oom;
- if (config_stats)
- usize = isalloc(tsd_tsdn(tsd), p, config_prof);
+ if (config_stats) {
+ usize = isalloc(tsd_tsdn(tsd), iealloc(p), p,
+ config_prof);
+ }
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
@@ -2246,42 +2265,43 @@ label_oom:
}
JEMALLOC_ALWAYS_INLINE_C size_t
-ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
- size_t extra, size_t alignment, bool zero)
+ixallocx_helper(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t old_usize,
+ size_t size, size_t extra, size_t alignment, bool zero)
{
size_t usize;
- if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero))
+ if (ixalloc(tsdn, extent, ptr, old_usize, size, extra, alignment, zero))
return (old_usize);
- usize = isalloc(tsdn, ptr, config_prof);
+ usize = isalloc(tsdn, extent, ptr, config_prof);
return (usize);
}
static size_t
-ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
- size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
+ixallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *ptr,
+ size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero,
+ prof_tctx_t *tctx)
{
size_t usize;
if (tctx == NULL)
return (old_usize);
- usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
- zero);
+ usize = ixallocx_helper(tsdn, extent, ptr, old_usize, size, extra,
+ alignment, zero);
return (usize);
}
JEMALLOC_ALWAYS_INLINE_C size_t
-ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
- size_t extra, size_t alignment, bool zero)
+ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize,
+ size_t size, size_t extra, size_t alignment, bool zero)
{
size_t usize_max, usize;
bool prof_active;
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
+ old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
/*
* usize isn't knowable before ixalloc() returns when extra is non-zero.
* Therefore, compute its maximum possible value and use that in
@@ -2306,18 +2326,18 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
- usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
- size, extra, alignment, zero, tctx);
+ usize = ixallocx_prof_sample(tsd_tsdn(tsd), extent, ptr,
+ old_usize, size, extra, alignment, zero, tctx);
} else {
- usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
- extra, alignment, zero);
+ usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
+ size, extra, alignment, zero);
}
if (usize == old_usize) {
prof_alloc_rollback(tsd, tctx, false);
return (usize);
}
- prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
- old_tctx);
+ prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, ptr,
+ old_usize, old_tctx);
return (usize);
}
@@ -2326,6 +2346,7 @@ JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_xallocx(void *ptr, size_t size, size_t extra, int flags)
{
tsd_t *tsd;
+ extent_t *extent;
size_t usize, old_usize;
size_t alignment = MALLOCX_ALIGN_GET(flags);
bool zero = flags & MALLOCX_ZERO;
@@ -2336,8 +2357,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
witness_assert_lockless(tsd_tsdn(tsd));
+ extent = iealloc(ptr);
- old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
+ old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
/*
* The API explicitly absolves itself of protecting against (size +
@@ -2356,11 +2378,11 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
extra = HUGE_MAXCLASS - size;
if (config_prof && opt_prof) {
- usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
+ usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
alignment, zero);
} else {
- usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
- extra, alignment, zero);
+ usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
+ size, extra, alignment, zero);
}
if (unlikely(usize == old_usize))
goto label_not_resized;
@@ -2390,7 +2412,7 @@ je_sallocx(const void *ptr, int flags)
if (config_ivsalloc)
usize = ivsalloc(tsdn, ptr, config_prof);
else
- usize = isalloc(tsdn, ptr, config_prof);
+ usize = isalloc(tsdn, iealloc(ptr), ptr, config_prof);
witness_assert_lockless(tsdn);
return (usize);
@@ -2442,14 +2464,16 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_sdallocx(void *ptr, size_t size, int flags)
{
tsd_t *tsd;
- tcache_t *tcache;
+ extent_t *extent;
size_t usize;
+ tcache_t *tcache;
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
+ extent = iealloc(ptr);
usize = inallocx(tsd_tsdn(tsd), size, flags);
- assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
+ assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, config_prof));
witness_assert_lockless(tsd_tsdn(tsd));
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
@@ -2462,9 +2486,9 @@ je_sdallocx(void *ptr, size_t size, int flags)
UTRACE(ptr, 0, 0);
if (likely(!malloc_slow))
- isfree(tsd, ptr, usize, tcache, false);
+ isfree(tsd, extent, ptr, usize, tcache, false);
else
- isfree(tsd, ptr, usize, tcache, true);
+ isfree(tsd, extent, ptr, usize, tcache, true);
witness_assert_lockless(tsd_tsdn(tsd));
}
@@ -2566,8 +2590,10 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
if (config_ivsalloc)
ret = ivsalloc(tsdn, ptr, config_prof);
- else
- ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
+ else {
+ ret = (ptr == NULL) ? 0 : isalloc(tsdn, iealloc(ptr), ptr,
+ config_prof);
+ }
witness_assert_lockless(tsdn);
return (ret);
diff --git a/src/prof.c b/src/prof.c
index c1f58d4..121dcd9 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -223,11 +223,11 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
}
void
-prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
- prof_tctx_t *tctx)
+prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent, const void *ptr,
+ size_t usize, prof_tctx_t *tctx)
{
- prof_tctx_set(tsdn, ptr, usize, tctx);
+ prof_tctx_set(tsdn, extent, ptr, usize, tctx);
malloc_mutex_lock(tsdn, tctx->tdata->lock);
tctx->cnts.curobjs++;
@@ -596,7 +596,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
prof_leave(tsd, tdata_self);
/* Destroy gctx. */
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
- idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(gctx), gctx, NULL, true, true);
} else {
/*
* Compensate for increment in prof_tctx_destroy() or
@@ -707,7 +707,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
prof_tdata_destroy(tsd_tsdn(tsd), tdata, false);
if (destroy_tctx)
- idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(tctx), tctx, NULL, true, true);
}
static bool
@@ -736,7 +736,8 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) {
/* OOM. */
prof_leave(tsd, tdata);
- idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(gctx.v), gctx.v, NULL,
+ true, true);
return (true);
}
new_gctx = true;
@@ -816,7 +817,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
if (error) {
if (new_gctx)
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
- idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(ret.v), ret.v, NULL,
+ true, true);
return (NULL);
}
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
@@ -1238,7 +1240,8 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
to_destroy);
tctx_tree_remove(&gctx->tctxs,
to_destroy);
- idalloctm(tsd_tsdn(tsd), to_destroy,
+ idalloctm(tsd_tsdn(tsd),
+ iealloc(to_destroy), to_destroy,
NULL, true, true);
} else
next = NULL;
@@ -1815,7 +1818,7 @@ prof_tdata_init_impl(tsdn_t *tsdn, uint64_t thr_uid, uint64_t thr_discrim,
if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS,
prof_bt_hash, prof_bt_keycomp)) {
- idalloctm(tsdn, tdata, NULL, true, true);
+ idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
return (NULL);
}
@@ -1878,10 +1881,12 @@ prof_tdata_destroy_locked(tsdn_t *tsdn, prof_tdata_t *tdata,
assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
- if (tdata->thread_name != NULL)
- idalloctm(tsdn, tdata->thread_name, NULL, true, true);
+ if (tdata->thread_name != NULL) {
+ idalloctm(tsdn, iealloc(tdata->thread_name), tdata->thread_name,
+ NULL, true, true);
+ }
ckh_delete(tsdn, &tdata->bt2tctx);
- idalloctm(tsdn, tdata, NULL, true, true);
+ idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
}
static void
@@ -2075,7 +2080,8 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name)
return (EAGAIN);
if (tdata->thread_name != NULL) {
- idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(tdata->thread_name),
+ tdata->thread_name, NULL, true, true);
tdata->thread_name = NULL;
}
if (strlen(s) > 0)
diff --git a/src/tcache.c b/src/tcache.c
index c4a9900..c02f0f0 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -27,7 +27,7 @@ size_t
tcache_salloc(tsdn_t *tsdn, const void *ptr)
{
- return (arena_salloc(tsdn, ptr, false));
+ return (arena_salloc(tsdn, iealloc(ptr), ptr, false));
}
void
@@ -101,9 +101,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena bin associated with the first object. */
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
- *(tbin->avail - 1));
- arena_t *bin_arena = extent_arena_get(&chunk->extent);
+ extent_t *extent = iealloc(*(tbin->avail - 1));
+ arena_t *bin_arena = extent_arena_get(extent);
arena_bin_t *bin = &bin_arena->bins[binind];
if (config_prof && bin_arena == arena) {
@@ -125,14 +124,17 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
for (i = 0; i < nflush; i++) {
ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- if (extent_arena_get(&chunk->extent) == bin_arena) {
+
+ extent = iealloc(ptr);
+ if (extent_arena_get(extent) == bin_arena) {
+ arena_chunk_t *chunk =
+ (arena_chunk_t *)extent_addr_get(extent);
size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_bits_t *bitselm =
arena_bitselm_get_mutable(chunk, pageind);
arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
- bin_arena, chunk, ptr, bitselm);
+ bin_arena, chunk, extent, ptr, bitselm);
} else {
/*
* This object was allocated via a different
@@ -183,9 +185,8 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
assert(arena != NULL);
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
/* Lock the arena associated with the first object. */
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
- *(tbin->avail - 1));
- arena_t *locked_arena = extent_arena_get(&chunk->extent);
+ extent_t *extent = iealloc(*(tbin->avail - 1));
+ arena_t *locked_arena = extent_arena_get(extent);
UNUSED bool idump;
if (config_prof)
@@ -210,10 +211,12 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
for (i = 0; i < nflush; i++) {
ptr = *(tbin->avail - 1 - i);
assert(ptr != NULL);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- if (extent_arena_get(&chunk->extent) == locked_arena) {
+ extent = iealloc(ptr);
+ if (extent_arena_get(extent) == locked_arena) {
+ arena_chunk_t *chunk =
+ (arena_chunk_t *)extent_addr_get(extent);
arena_dalloc_large_junked_locked(tsd_tsdn(tsd),
- locked_arena, chunk, ptr);
+ locked_arena, chunk, extent, ptr);
} else {
/*
* This object was allocated via a different
@@ -391,7 +394,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes))
prof_idump(tsd_tsdn(tsd));
- idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true);
+ idalloctm(tsd_tsdn(tsd), iealloc(tcache), tcache, NULL, true, true);
}
void