summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorJason Evans <jasone@canonware.com>2016-04-14 06:36:15 (GMT)
committerJason Evans <jasone@canonware.com>2016-04-14 09:09:28 (GMT)
commitb2c0d6322d2307458ae2b28545f8a5c9903d7ef5 (patch)
tree0ab71b856358e4705440ace0661ac4ed9c3e5766 /src
parent8413463f3a334f14c55589e57d3e82dd594ef479 (diff)
downloadjemalloc-b2c0d6322d2307458ae2b28545f8a5c9903d7ef5.zip
jemalloc-b2c0d6322d2307458ae2b28545f8a5c9903d7ef5.tar.gz
jemalloc-b2c0d6322d2307458ae2b28545f8a5c9903d7ef5.tar.bz2
Add witness, a simple online locking validator.
This resolves #358.
Diffstat (limited to 'src')
-rw-r--r--src/arena.c568
-rw-r--r--src/base.c26
-rw-r--r--src/chunk.c186
-rw-r--r--src/chunk_dss.c46
-rw-r--r--src/ctl.c387
-rw-r--r--src/huge.c106
-rw-r--r--src/jemalloc.c377
-rw-r--r--src/mutex.c21
-rw-r--r--src/prof.c489
-rw-r--r--src/quarantine.c4
-rw-r--r--src/tcache.c91
-rw-r--r--src/tsd.c20
-rw-r--r--src/witness.c206
-rw-r--r--src/zone.c8
14 files changed, 1452 insertions, 1083 deletions
diff --git a/src/arena.c b/src/arena.c
index a9566af..cc648e3 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -37,11 +37,12 @@ static szind_t runs_avail_nclasses; /* Number of runs_avail trees. */
* definition.
*/
-static void arena_purge_to_limit(arena_t *arena, size_t ndirty_limit);
-static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty,
- bool cleaned, bool decommitted);
-static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk,
- arena_run_t *run, arena_bin_t *bin);
+static void arena_purge_to_limit(tsd_t *tsd, arena_t *arena,
+ size_t ndirty_limit);
+static void arena_run_dalloc(tsd_t *tsd, arena_t *arena, arena_run_t *run,
+ bool dirty, bool cleaned, bool decommitted);
+static void arena_dalloc_bin_run(tsd_t *tsd, arena_t *arena,
+ arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin);
static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
arena_run_t *run, arena_bin_t *bin);
@@ -591,7 +592,8 @@ arena_chunk_init_spare(arena_t *arena)
}
static bool
-arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero)
+arena_chunk_register(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ bool zero)
{
/*
@@ -602,62 +604,62 @@ arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero)
*/
extent_node_init(&chunk->node, arena, chunk, chunksize, zero, true);
extent_node_achunk_set(&chunk->node, true);
- return (chunk_register(chunk, &chunk->node));
+ return (chunk_register(tsd, chunk, &chunk->node));
}
static arena_chunk_t *
-arena_chunk_alloc_internal_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
- bool *zero, bool *commit)
+arena_chunk_alloc_internal_hard(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks, bool *zero, bool *commit)
{
arena_chunk_t *chunk;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
- chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL,
- chunksize, chunksize, zero, commit);
+ chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsd, arena, chunk_hooks,
+ NULL, chunksize, chunksize, zero, commit);
if (chunk != NULL && !*commit) {
/* Commit header. */
if (chunk_hooks->commit(chunk, chunksize, 0, map_bias <<
LG_PAGE, arena->ind)) {
- chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk,
- chunksize, *zero, *commit);
+ chunk_dalloc_wrapper(tsd, arena, chunk_hooks,
+ (void *)chunk, chunksize, *zero, *commit);
chunk = NULL;
}
}
- if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) {
+ if (chunk != NULL && arena_chunk_register(tsd, arena, chunk, *zero)) {
if (!*commit) {
/* Undo commit of header. */
chunk_hooks->decommit(chunk, chunksize, 0, map_bias <<
LG_PAGE, arena->ind);
}
- chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk,
+ chunk_dalloc_wrapper(tsd, arena, chunk_hooks, (void *)chunk,
chunksize, *zero, *commit);
chunk = NULL;
}
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
return (chunk);
}
static arena_chunk_t *
-arena_chunk_alloc_internal(arena_t *arena, bool *zero, bool *commit)
+arena_chunk_alloc_internal(tsd_t *tsd, arena_t *arena, bool *zero, bool *commit)
{
arena_chunk_t *chunk;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
- chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize,
+ chunk = chunk_alloc_cache(tsd, arena, &chunk_hooks, NULL, chunksize,
chunksize, zero, true);
if (chunk != NULL) {
- if (arena_chunk_register(arena, chunk, *zero)) {
- chunk_dalloc_cache(arena, &chunk_hooks, chunk,
+ if (arena_chunk_register(tsd, arena, chunk, *zero)) {
+ chunk_dalloc_cache(tsd, arena, &chunk_hooks, chunk,
chunksize, true);
return (NULL);
}
*commit = true;
}
if (chunk == NULL) {
- chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks,
- zero, commit);
+ chunk = arena_chunk_alloc_internal_hard(tsd, arena,
+ &chunk_hooks, zero, commit);
}
if (config_stats && chunk != NULL) {
@@ -669,7 +671,7 @@ arena_chunk_alloc_internal(arena_t *arena, bool *zero, bool *commit)
}
static arena_chunk_t *
-arena_chunk_init_hard(arena_t *arena)
+arena_chunk_init_hard(tsd_t *tsd, arena_t *arena)
{
arena_chunk_t *chunk;
bool zero, commit;
@@ -679,7 +681,7 @@ arena_chunk_init_hard(arena_t *arena)
zero = false;
commit = false;
- chunk = arena_chunk_alloc_internal(arena, &zero, &commit);
+ chunk = arena_chunk_alloc_internal(tsd, arena, &zero, &commit);
if (chunk == NULL)
return (NULL);
@@ -724,14 +726,14 @@ arena_chunk_init_hard(arena_t *arena)
}
static arena_chunk_t *
-arena_chunk_alloc(arena_t *arena)
+arena_chunk_alloc(tsd_t *tsd, arena_t *arena)
{
arena_chunk_t *chunk;
if (arena->spare != NULL)
chunk = arena_chunk_init_spare(arena);
else {
- chunk = arena_chunk_init_hard(arena);
+ chunk = arena_chunk_init_hard(tsd, arena);
if (chunk == NULL)
return (NULL);
}
@@ -742,7 +744,7 @@ arena_chunk_alloc(arena_t *arena)
}
static void
-arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
+arena_chunk_dalloc(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
{
assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
@@ -782,12 +784,12 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk)
* potential for causing later access of decommitted
* memory.
*/
- chunk_hooks = chunk_hooks_get(arena);
+ chunk_hooks = chunk_hooks_get(tsd, arena);
chunk_hooks.decommit(spare, chunksize, 0, map_bias <<
LG_PAGE, arena->ind);
}
- chunk_dalloc_cache(arena, &chunk_hooks, (void *)spare,
+ chunk_dalloc_cache(tsd, arena, &chunk_hooks, (void *)spare,
chunksize, committed);
if (config_stats) {
@@ -868,63 +870,64 @@ arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize,
}
extent_node_t *
-arena_node_alloc(arena_t *arena)
+arena_node_alloc(tsd_t *tsd, arena_t *arena)
{
extent_node_t *node;
- malloc_mutex_lock(&arena->node_cache_mtx);
+ malloc_mutex_lock(tsd, &arena->node_cache_mtx);
node = ql_last(&arena->node_cache, ql_link);
if (node == NULL) {
- malloc_mutex_unlock(&arena->node_cache_mtx);
- return (base_alloc(sizeof(extent_node_t)));
+ malloc_mutex_unlock(tsd, &arena->node_cache_mtx);
+ return (base_alloc(tsd, sizeof(extent_node_t)));
}
ql_tail_remove(&arena->node_cache, extent_node_t, ql_link);
- malloc_mutex_unlock(&arena->node_cache_mtx);
+ malloc_mutex_unlock(tsd, &arena->node_cache_mtx);
return (node);
}
void
-arena_node_dalloc(arena_t *arena, extent_node_t *node)
+arena_node_dalloc(tsd_t *tsd, arena_t *arena, extent_node_t *node)
{
- malloc_mutex_lock(&arena->node_cache_mtx);
+ malloc_mutex_lock(tsd, &arena->node_cache_mtx);
ql_elm_new(node, ql_link);
ql_tail_insert(&arena->node_cache, node, ql_link);
- malloc_mutex_unlock(&arena->node_cache_mtx);
+ malloc_mutex_unlock(tsd, &arena->node_cache_mtx);
}
static void *
-arena_chunk_alloc_huge_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
- size_t usize, size_t alignment, bool *zero, size_t csize)
+arena_chunk_alloc_huge_hard(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero,
+ size_t csize)
{
void *ret;
bool commit = true;
- ret = chunk_alloc_wrapper(arena, chunk_hooks, NULL, csize, alignment,
- zero, &commit);
+ ret = chunk_alloc_wrapper(tsd, arena, chunk_hooks, NULL, csize,
+ alignment, zero, &commit);
if (ret == NULL) {
/* Revert optimistic stats updates. */
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats) {
arena_huge_malloc_stats_update_undo(arena, usize);
arena->stats.mapped -= usize;
}
arena_nactive_sub(arena, usize >> LG_PAGE);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
return (ret);
}
void *
-arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
- bool *zero)
+arena_chunk_alloc_huge(tsd_t *tsd, arena_t *arena, size_t usize,
+ size_t alignment, bool *zero)
{
void *ret;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
size_t csize = CHUNK_CEILING(usize);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
/* Optimistically update stats. */
if (config_stats) {
@@ -933,61 +936,61 @@ arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment,
}
arena_nactive_add(arena, usize >> LG_PAGE);
- ret = chunk_alloc_cache(arena, &chunk_hooks, NULL, csize, alignment,
- zero, true);
- malloc_mutex_unlock(&arena->lock);
+ ret = chunk_alloc_cache(tsd, arena, &chunk_hooks, NULL, csize,
+ alignment, zero, true);
+ malloc_mutex_unlock(tsd, &arena->lock);
if (ret == NULL) {
- ret = arena_chunk_alloc_huge_hard(arena, &chunk_hooks, usize,
- alignment, zero, csize);
+ ret = arena_chunk_alloc_huge_hard(tsd, arena, &chunk_hooks,
+ usize, alignment, zero, csize);
}
return (ret);
}
void
-arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize)
+arena_chunk_dalloc_huge(tsd_t *tsd, arena_t *arena, void *chunk, size_t usize)
{
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
size_t csize;
csize = CHUNK_CEILING(usize);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats) {
arena_huge_dalloc_stats_update(arena, usize);
arena->stats.mapped -= usize;
}
arena_nactive_sub(arena, usize >> LG_PAGE);
- chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize, true);
- malloc_mutex_unlock(&arena->lock);
+ chunk_dalloc_cache(tsd, arena, &chunk_hooks, chunk, csize, true);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
void
-arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize,
- size_t usize)
+arena_chunk_ralloc_huge_similar(tsd_t *tsd, arena_t *arena, void *chunk,
+ size_t oldsize, size_t usize)
{
assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize));
assert(oldsize != usize);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats)
arena_huge_ralloc_stats_update(arena, oldsize, usize);
if (oldsize < usize)
arena_nactive_add(arena, (usize - oldsize) >> LG_PAGE);
else
arena_nactive_sub(arena, (oldsize - usize) >> LG_PAGE);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
void
-arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize,
- size_t usize)
+arena_chunk_ralloc_huge_shrink(tsd_t *tsd, arena_t *arena, void *chunk,
+ size_t oldsize, size_t usize)
{
size_t udiff = oldsize - usize;
size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats) {
arena_huge_ralloc_stats_update(arena, oldsize, usize);
if (cdiff != 0)
@@ -1000,51 +1003,52 @@ arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize,
void *nchunk = (void *)((uintptr_t)chunk +
CHUNK_CEILING(usize));
- chunk_dalloc_cache(arena, &chunk_hooks, nchunk, cdiff, true);
+ chunk_dalloc_cache(tsd, arena, &chunk_hooks, nchunk, cdiff,
+ true);
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
static bool
-arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_hooks_t *chunk_hooks,
- void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk,
- size_t udiff, size_t cdiff)
+arena_chunk_ralloc_huge_expand_hard(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks, void *chunk, size_t oldsize, size_t usize,
+ bool *zero, void *nchunk, size_t udiff, size_t cdiff)
{
bool err;
bool commit = true;
- err = (chunk_alloc_wrapper(arena, chunk_hooks, nchunk, cdiff, chunksize,
- zero, &commit) == NULL);
+ err = (chunk_alloc_wrapper(tsd, arena, chunk_hooks, nchunk, cdiff,
+ chunksize, zero, &commit) == NULL);
if (err) {
/* Revert optimistic stats updates. */
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats) {
arena_huge_ralloc_stats_update_undo(arena, oldsize,
usize);
arena->stats.mapped -= cdiff;
}
arena_nactive_sub(arena, udiff >> LG_PAGE);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
} else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk,
cdiff, true, arena->ind)) {
- chunk_dalloc_wrapper(arena, chunk_hooks, nchunk, cdiff, *zero,
- true);
+ chunk_dalloc_wrapper(tsd, arena, chunk_hooks, nchunk, cdiff,
+ *zero, true);
err = true;
}
return (err);
}
bool
-arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize,
- size_t usize, bool *zero)
+arena_chunk_ralloc_huge_expand(tsd_t *tsd, arena_t *arena, void *chunk,
+ size_t oldsize, size_t usize, bool *zero)
{
bool err;
- chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
+ chunk_hooks_t chunk_hooks = chunk_hooks_get(tsd, arena);
void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize));
size_t udiff = usize - oldsize;
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
/* Optimistically update stats. */
if (config_stats) {
@@ -1053,17 +1057,17 @@ arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize,
}
arena_nactive_add(arena, udiff >> LG_PAGE);
- err = (chunk_alloc_cache(arena, &chunk_hooks, nchunk, cdiff, chunksize,
- zero, true) == NULL);
- malloc_mutex_unlock(&arena->lock);
+ err = (chunk_alloc_cache(tsd, arena, &chunk_hooks, nchunk, cdiff,
+ chunksize, zero, true) == NULL);
+ malloc_mutex_unlock(tsd, &arena->lock);
if (err) {
- err = arena_chunk_ralloc_huge_expand_hard(arena, &chunk_hooks,
- chunk, oldsize, usize, zero, nchunk, udiff,
+ err = arena_chunk_ralloc_huge_expand_hard(tsd, arena,
+ &chunk_hooks, chunk, oldsize, usize, zero, nchunk, udiff,
cdiff);
} else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk,
cdiff, true, arena->ind)) {
- chunk_dalloc_wrapper(arena, &chunk_hooks, nchunk, cdiff, *zero,
- true);
+ chunk_dalloc_wrapper(tsd, arena, &chunk_hooks, nchunk, cdiff,
+ *zero, true);
err = true;
}
@@ -1103,7 +1107,7 @@ arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
}
static arena_run_t *
-arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
+arena_run_alloc_large(tsd_t *tsd, arena_t *arena, size_t size, bool zero)
{
arena_chunk_t *chunk;
arena_run_t *run;
@@ -1119,7 +1123,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero)
/*
* No usable runs. Create a new chunk from which to allocate the run.
*/
- chunk = arena_chunk_alloc(arena);
+ chunk = arena_chunk_alloc(tsd, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
if (arena_run_split_large(arena, run, size, zero))
@@ -1147,7 +1151,7 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
}
static arena_run_t *
-arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind)
+arena_run_alloc_small(tsd_t *tsd, arena_t *arena, size_t size, szind_t binind)
{
arena_chunk_t *chunk;
arena_run_t *run;
@@ -1164,7 +1168,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind)
/*
* No usable runs. Create a new chunk from which to allocate the run.
*/
- chunk = arena_chunk_alloc(arena);
+ chunk = arena_chunk_alloc(tsd, arena);
if (chunk != NULL) {
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
if (arena_run_split_small(arena, run, size, binind))
@@ -1189,28 +1193,28 @@ arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult)
}
ssize_t
-arena_lg_dirty_mult_get(arena_t *arena)
+arena_lg_dirty_mult_get(tsd_t *tsd, arena_t *arena)
{
ssize_t lg_dirty_mult;
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
lg_dirty_mult = arena->lg_dirty_mult;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (lg_dirty_mult);
}
bool
-arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult)
+arena_lg_dirty_mult_set(tsd_t *tsd, arena_t *arena, ssize_t lg_dirty_mult)
{
if (!arena_lg_dirty_mult_valid(lg_dirty_mult))
return (true);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena->lg_dirty_mult = lg_dirty_mult;
- arena_maybe_purge(arena);
- malloc_mutex_unlock(&arena->lock);
+ arena_maybe_purge(tsd, arena);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (false);
}
@@ -1367,25 +1371,25 @@ arena_decay_time_valid(ssize_t decay_time)
}
ssize_t
-arena_decay_time_get(arena_t *arena)
+arena_decay_time_get(tsd_t *tsd, arena_t *arena)
{
ssize_t decay_time;
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
decay_time = arena->decay_time;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (decay_time);
}
bool
-arena_decay_time_set(arena_t *arena, ssize_t decay_time)
+arena_decay_time_set(tsd_t *tsd, arena_t *arena, ssize_t decay_time)
{
if (!arena_decay_time_valid(decay_time))
return (true);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
/*
* Restart decay backlog from scratch, which may cause many dirty pages
* to be immediately purged. It would conceptually be possible to map
@@ -1395,14 +1399,14 @@ arena_decay_time_set(arena_t *arena, ssize_t decay_time)
* arbitrary change during initial arena configuration.
*/
arena_decay_init(arena, decay_time);
- arena_maybe_purge(arena);
- malloc_mutex_unlock(&arena->lock);
+ arena_maybe_purge(tsd, arena);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (false);
}
static void
-arena_maybe_purge_ratio(arena_t *arena)
+arena_maybe_purge_ratio(tsd_t *tsd, arena_t *arena)
{
assert(opt_purge == purge_mode_ratio);
@@ -1425,12 +1429,12 @@ arena_maybe_purge_ratio(arena_t *arena)
*/
if (arena->ndirty <= threshold)
return;
- arena_purge_to_limit(arena, threshold);
+ arena_purge_to_limit(tsd, arena, threshold);
}
}
static void
-arena_maybe_purge_decay(arena_t *arena)
+arena_maybe_purge_decay(tsd_t *tsd, arena_t *arena)
{
nstime_t time;
size_t ndirty_limit;
@@ -1440,7 +1444,7 @@ arena_maybe_purge_decay(arena_t *arena)
/* Purge all or nothing if the option is disabled. */
if (arena->decay_time <= 0) {
if (arena->decay_time == 0)
- arena_purge_to_limit(arena, 0);
+ arena_purge_to_limit(tsd, arena, 0);
return;
}
@@ -1461,11 +1465,11 @@ arena_maybe_purge_decay(arena_t *arena)
*/
if (arena->ndirty <= ndirty_limit)
return;
- arena_purge_to_limit(arena, ndirty_limit);
+ arena_purge_to_limit(tsd, arena, ndirty_limit);
}
void
-arena_maybe_purge(arena_t *arena)
+arena_maybe_purge(tsd_t *tsd, arena_t *arena)
{
/* Don't recursively purge. */
@@ -1473,9 +1477,9 @@ arena_maybe_purge(arena_t *arena)
return;
if (opt_purge == purge_mode_ratio)
- arena_maybe_purge_ratio(arena);
+ arena_maybe_purge_ratio(tsd, arena);
else
- arena_maybe_purge_decay(arena);
+ arena_maybe_purge_decay(tsd, arena);
}
static size_t
@@ -1513,7 +1517,7 @@ arena_dirty_count(arena_t *arena)
}
static size_t
-arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
+arena_stash_dirty(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
size_t ndirty_limit, arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel)
{
@@ -1544,7 +1548,7 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
* dalloc_node=false argument to chunk_alloc_cache().
*/
zero = false;
- chunk = chunk_alloc_cache(arena, chunk_hooks,
+ chunk = chunk_alloc_cache(tsd, arena, chunk_hooks,
extent_node_addr_get(chunkselm),
extent_node_size_get(chunkselm), chunksize, &zero,
false);
@@ -1579,7 +1583,7 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
* prior to allocation.
*/
if (chunk == arena->spare)
- arena_chunk_alloc(arena);
+ arena_chunk_alloc(tsd, arena);
/* Temporarily allocate the free dirty run. */
arena_run_split_large(arena, run, run_size, false);
@@ -1603,7 +1607,7 @@ arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks,
}
static size_t
-arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
+arena_purge_stashed(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel)
{
@@ -1615,7 +1619,7 @@ arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
nmadvise = 0;
npurged = 0;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
for (rdelm = qr_next(purge_runs_sentinel, rd_link),
chunkselm = qr_next(purge_chunks_sentinel, cc_link);
rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) {
@@ -1654,7 +1658,7 @@ arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
flag_unzeroed = 0;
flags = CHUNK_MAP_DECOMMITTED;
} else {
- flag_unzeroed = chunk_purge_wrapper(arena,
+ flag_unzeroed = chunk_purge_wrapper(tsd, arena,
chunk_hooks, chunk, chunksize, pageind <<
LG_PAGE, run_size) ? CHUNK_MAP_UNZEROED : 0;
flags = flag_unzeroed;
@@ -1685,7 +1689,7 @@ arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
if (config_stats)
nmadvise++;
}
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_stats) {
arena->stats.nmadvise += nmadvise;
@@ -1696,7 +1700,7 @@ arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks,
}
static void
-arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
+arena_unstash_purged(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_runs_dirty_link_t *purge_runs_sentinel,
extent_node_t *purge_chunks_sentinel)
{
@@ -1716,10 +1720,10 @@ arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
bool zeroed = extent_node_zeroed_get(chunkselm);
bool committed = extent_node_committed_get(chunkselm);
extent_node_dirty_remove(chunkselm);
- arena_node_dalloc(arena, chunkselm);
+ arena_node_dalloc(tsd, arena, chunkselm);
chunkselm = chunkselm_next;
- chunk_dalloc_wrapper(arena, chunk_hooks, addr, size,
- zeroed, committed);
+ chunk_dalloc_wrapper(tsd, arena, chunk_hooks, addr,
+ size, zeroed, committed);
} else {
arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
@@ -1730,7 +1734,8 @@ arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
pageind) != 0);
arena_run_t *run = &miscelm->run;
qr_remove(rdelm, rd_link);
- arena_run_dalloc(arena, run, false, true, decommitted);
+ arena_run_dalloc(tsd, arena, run, false, true,
+ decommitted);
}
}
}
@@ -1745,9 +1750,9 @@ arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks,
* (arena->ndirty >= ndirty_limit)
*/
static void
-arena_purge_to_limit(arena_t *arena, size_t ndirty_limit)
+arena_purge_to_limit(tsd_t *tsd, arena_t *arena, size_t ndirty_limit)
{
- chunk_hooks_t chunk_hooks = chunk_hooks_get(arena);
+ chunk_hooks_t chunk_hooks = chunk_hooks_get(tsd, arena);
size_t npurge, npurged;
arena_runs_dirty_link_t purge_runs_sentinel;
extent_node_t purge_chunks_sentinel;
@@ -1768,14 +1773,14 @@ arena_purge_to_limit(arena_t *arena, size_t ndirty_limit)
qr_new(&purge_runs_sentinel, rd_link);
extent_node_dirty_linkage_init(&purge_chunks_sentinel);
- npurge = arena_stash_dirty(arena, &chunk_hooks, ndirty_limit,
+ npurge = arena_stash_dirty(tsd, arena, &chunk_hooks, ndirty_limit,
&purge_runs_sentinel, &purge_chunks_sentinel);
if (npurge == 0)
goto label_return;
- npurged = arena_purge_stashed(arena, &chunk_hooks, &purge_runs_sentinel,
- &purge_chunks_sentinel);
+ npurged = arena_purge_stashed(tsd, arena, &chunk_hooks,
+ &purge_runs_sentinel, &purge_chunks_sentinel);
assert(npurged == npurge);
- arena_unstash_purged(arena, &chunk_hooks, &purge_runs_sentinel,
+ arena_unstash_purged(tsd, arena, &chunk_hooks, &purge_runs_sentinel,
&purge_chunks_sentinel);
if (config_stats)
@@ -1786,15 +1791,15 @@ label_return:
}
void
-arena_purge(arena_t *arena, bool all)
+arena_purge(tsd_t *tsd, arena_t *arena, bool all)
{
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (all)
- arena_purge_to_limit(arena, 0);
+ arena_purge_to_limit(tsd, arena, 0);
else
- arena_maybe_purge(arena);
- malloc_mutex_unlock(&arena->lock);
+ arena_maybe_purge(tsd, arena);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
static void
@@ -1911,8 +1916,8 @@ arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
}
static void
-arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,
- bool decommitted)
+arena_run_dalloc(tsd_t *tsd, arena_t *arena, arena_run_t *run, bool dirty,
+ bool cleaned, bool decommitted)
{
arena_chunk_t *chunk;
arena_chunk_map_misc_t *miscelm;
@@ -1972,7 +1977,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,
if (size == arena_maxrun) {
assert(run_ind == map_bias);
assert(run_pages == (arena_maxrun >> LG_PAGE));
- arena_chunk_dalloc(arena, chunk);
+ arena_chunk_dalloc(tsd, arena, chunk);
}
/*
@@ -1983,12 +1988,12 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned,
* chances of spuriously crossing the dirty page purging threshold.
*/
if (dirty)
- arena_maybe_purge(arena);
+ arena_maybe_purge(tsd, arena);
}
static void
-arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
- size_t oldsize, size_t newsize)
+arena_run_trim_head(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ arena_run_t *run, size_t oldsize, size_t newsize)
{
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -2023,12 +2028,13 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
pageind+head_npages)));
- arena_run_dalloc(arena, run, false, false, (flag_decommitted != 0));
+ arena_run_dalloc(tsd, arena, run, false, false, (flag_decommitted !=
+ 0));
}
static void
-arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
- size_t oldsize, size_t newsize, bool dirty)
+arena_run_trim_tail(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ arena_run_t *run, size_t oldsize, size_t newsize, bool dirty)
{
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
size_t pageind = arena_miscelm_to_pageind(miscelm);
@@ -2067,8 +2073,8 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages);
tail_run = &tail_miscelm->run;
- arena_run_dalloc(arena, tail_run, dirty, false, (flag_decommitted !=
- 0));
+ arena_run_dalloc(tsd, arena, tail_run, dirty, false, (flag_decommitted
+ != 0));
}
static void
@@ -2094,7 +2100,7 @@ arena_bin_nonfull_run_tryget(arena_bin_t *bin)
}
static arena_run_t *
-arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
+arena_bin_nonfull_run_get(tsd_t *tsd, arena_t *arena, arena_bin_t *bin)
{
arena_run_t *run;
szind_t binind;
@@ -2110,19 +2116,19 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
bin_info = &arena_bin_info[binind];
/* Allocate a new run. */
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
/******************************/
- malloc_mutex_lock(&arena->lock);
- run = arena_run_alloc_small(arena, bin_info->run_size, binind);
+ malloc_mutex_lock(tsd, &arena->lock);
+ run = arena_run_alloc_small(tsd, arena, bin_info->run_size, binind);
if (run != NULL) {
/* Initialize run internals. */
run->binind = binind;
run->nfree = bin_info->nregs;
bitmap_init(run->bitmap, &bin_info->bitmap_info);
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
/********************************/
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
if (run != NULL) {
if (config_stats) {
bin->stats.nruns++;
@@ -2145,7 +2151,7 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */
static void *
-arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
+arena_bin_malloc_hard(tsd_t *tsd, arena_t *arena, arena_bin_t *bin)
{
szind_t binind;
arena_bin_info_t *bin_info;
@@ -2154,7 +2160,7 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
binind = arena_bin_index(arena, bin);
bin_info = &arena_bin_info[binind];
bin->runcur = NULL;
- run = arena_bin_nonfull_run_get(arena, bin);
+ run = arena_bin_nonfull_run_get(tsd, arena, bin);
if (bin->runcur != NULL && bin->runcur->nfree > 0) {
/*
* Another thread updated runcur while this one ran without the
@@ -2175,9 +2181,10 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
* were just deallocated from the run.
*/
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
- if (run->nfree == bin_info->nregs)
- arena_dalloc_bin_run(arena, chunk, run, bin);
- else
+ if (run->nfree == bin_info->nregs) {
+ arena_dalloc_bin_run(tsd, arena, chunk, run,
+ bin);
+ } else
arena_bin_lower_run(arena, chunk, run, bin);
}
return (ret);
@@ -2202,10 +2209,10 @@ arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin,
assert(tbin->ncached == 0);
- if (config_prof && arena_prof_accum(arena, prof_accumbytes))
- prof_idump();
+ if (config_prof && arena_prof_accum(tsd, arena, prof_accumbytes))
+ prof_idump(tsd);
bin = &arena->bins[binind];
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >>
tbin->lg_fill_div); i < nfill; i++) {
arena_run_t *run;
@@ -2213,7 +2220,7 @@ arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin,
if ((run = bin->runcur) != NULL && run->nfree > 0)
ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]);
else
- ptr = arena_bin_malloc_hard(arena, bin);
+ ptr = arena_bin_malloc_hard(tsd, arena, bin);
if (ptr == NULL) {
/*
* OOM. tbin->avail isn't yet filled down to its first
@@ -2240,7 +2247,7 @@ arena_tcache_fill_small(tsd_t *tsd, arena_t *arena, tcache_bin_t *tbin,
bin->stats.nfills++;
tbin->tstats.nrequests = 0;
}
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
tbin->ncached = i;
arena_decay_tick(tsd, arena);
}
@@ -2365,14 +2372,14 @@ arena_malloc_small(tsd_t *tsd, arena_t *arena, szind_t binind, bool zero)
bin = &arena->bins[binind];
usize = index2size(binind);
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
if ((run = bin->runcur) != NULL && run->nfree > 0)
ret = arena_run_reg_alloc(run, &arena_bin_info[binind]);
else
- ret = arena_bin_malloc_hard(arena, bin);
+ ret = arena_bin_malloc_hard(tsd, arena, bin);
if (ret == NULL) {
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
return (NULL);
}
@@ -2381,9 +2388,9 @@ arena_malloc_small(tsd_t *tsd, arena_t *arena, szind_t binind, bool zero)
bin->stats.nrequests++;
bin->stats.curregs++;
}
- malloc_mutex_unlock(&bin->lock);
- if (config_prof && !isthreaded && arena_prof_accum(arena, usize))
- prof_idump();
+ malloc_mutex_unlock(tsd, &bin->lock);
+ if (config_prof && !isthreaded && arena_prof_accum(tsd, arena, usize))
+ prof_idump(tsd);
if (!zero) {
if (config_fill) {
@@ -2419,7 +2426,7 @@ arena_malloc_large(tsd_t *tsd, arena_t *arena, szind_t binind, bool zero)
/* Large allocation. */
usize = index2size(binind);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_cache_oblivious) {
uint64_t r;
@@ -2432,9 +2439,9 @@ arena_malloc_large(tsd_t *tsd, arena_t *arena, szind_t binind, bool zero)
random_offset = ((uintptr_t)r) << LG_CACHELINE;
} else
random_offset = 0;
- run = arena_run_alloc_large(arena, usize + large_pad, zero);
+ run = arena_run_alloc_large(tsd, arena, usize + large_pad, zero);
if (run == NULL) {
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (NULL);
}
miscelm = arena_run_to_miscelm(run);
@@ -2452,9 +2459,9 @@ arena_malloc_large(tsd_t *tsd, arena_t *arena, szind_t binind, bool zero)
}
if (config_prof)
idump = arena_prof_accum_locked(arena, usize);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
if (config_prof && idump)
- prof_idump();
+ prof_idump(tsd);
if (!zero) {
if (config_fill) {
@@ -2506,10 +2513,10 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
alignment = PAGE_CEILING(alignment);
alloc_size = usize + large_pad + alignment;
- malloc_mutex_lock(&arena->lock);
- run = arena_run_alloc_large(arena, alloc_size, false);
+ malloc_mutex_lock(tsd, &arena->lock);
+ run = arena_run_alloc_large(tsd, arena, alloc_size, false);
if (run == NULL) {
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (NULL);
}
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
@@ -2529,11 +2536,11 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
LG_PAGE));
run = &miscelm->run;
- arena_run_trim_head(arena, chunk, head_run, alloc_size,
+ arena_run_trim_head(tsd, arena, chunk, head_run, alloc_size,
alloc_size - leadsize);
}
if (trailsize != 0) {
- arena_run_trim_tail(arena, chunk, run, usize + large_pad +
+ arena_run_trim_tail(tsd, arena, chunk, run, usize + large_pad +
trailsize, usize + large_pad, false);
}
if (arena_run_init_large(arena, run, usize + large_pad, zero)) {
@@ -2544,8 +2551,8 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
run_ind) != 0);
assert(decommitted); /* Cause of OOM. */
- arena_run_dalloc(arena, run, dirty, false, decommitted);
- malloc_mutex_unlock(&arena->lock);
+ arena_run_dalloc(tsd, arena, run, dirty, false, decommitted);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (NULL);
}
ret = arena_miscelm_to_rpages(miscelm);
@@ -2560,7 +2567,7 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
arena->stats.lstats[index].nrequests++;
arena->stats.lstats[index].curruns++;
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
if (config_fill && !zero) {
if (unlikely(opt_junk_alloc))
@@ -2609,7 +2616,7 @@ arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
}
void
-arena_prof_promoted(const void *ptr, size_t size)
+arena_prof_promoted(tsd_t *tsd, const void *ptr, size_t size)
{
arena_chunk_t *chunk;
size_t pageind;
@@ -2618,8 +2625,8 @@ arena_prof_promoted(const void *ptr, size_t size)
cassert(config_prof);
assert(ptr != NULL);
assert(CHUNK_ADDR2BASE(ptr) != ptr);
- assert(isalloc(ptr, false) == LARGE_MINCLASS);
- assert(isalloc(ptr, true) == LARGE_MINCLASS);
+ assert(isalloc(tsd, ptr, false) == LARGE_MINCLASS);
+ assert(isalloc(tsd, ptr, true) == LARGE_MINCLASS);
assert(size <= SMALL_MAXCLASS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
@@ -2628,8 +2635,8 @@ arena_prof_promoted(const void *ptr, size_t size)
assert(binind < NBINS);
arena_mapbits_large_binind_set(chunk, pageind, binind);
- assert(isalloc(ptr, false) == LARGE_MINCLASS);
- assert(isalloc(ptr, true) == size);
+ assert(isalloc(tsd, ptr, false) == LARGE_MINCLASS);
+ assert(isalloc(tsd, ptr, true) == size);
}
static void
@@ -2660,19 +2667,19 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
}
static void
-arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
- arena_bin_t *bin)
+arena_dalloc_bin_run(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ arena_run_t *run, arena_bin_t *bin)
{
assert(run != bin->runcur);
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
/******************************/
- malloc_mutex_lock(&arena->lock);
- arena_run_dalloc(arena, run, true, false, false);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
+ arena_run_dalloc(tsd, arena, run, true, false, false);
+ malloc_mutex_unlock(tsd, &arena->lock);
/****************************/
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
if (config_stats)
bin->stats.curruns--;
}
@@ -2699,8 +2706,8 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
}
static void
-arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
- arena_chunk_map_bits_t *bitselm, bool junked)
+arena_dalloc_bin_locked_impl(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
{
size_t pageind, rpages_ind;
arena_run_t *run;
@@ -2721,7 +2728,7 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_run_reg_dalloc(run, ptr);
if (run->nfree == bin_info->nregs) {
arena_dissociate_bin_run(chunk, run, bin);
- arena_dalloc_bin_run(arena, chunk, run, bin);
+ arena_dalloc_bin_run(tsd, arena, chunk, run, bin);
} else if (run->nfree == 1 && run != bin->runcur)
arena_bin_lower_run(arena, chunk, run, bin);
@@ -2732,15 +2739,15 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr,
}
void
-arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
- arena_chunk_map_bits_t *bitselm)
+arena_dalloc_bin_junked_locked(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ void *ptr, arena_chunk_map_bits_t *bitselm)
{
- arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true);
+ arena_dalloc_bin_locked_impl(tsd, arena, chunk, ptr, bitselm, true);
}
void
-arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
+arena_dalloc_bin(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind, arena_chunk_map_bits_t *bitselm)
{
arena_run_t *run;
@@ -2750,9 +2757,9 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind);
run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run;
bin = &arena->bins[run->binind];
- malloc_mutex_lock(&bin->lock);
- arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false);
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
+ arena_dalloc_bin_locked_impl(tsd, arena, chunk, ptr, bitselm, false);
+ malloc_mutex_unlock(tsd, &bin->lock);
}
void
@@ -2767,7 +2774,7 @@ arena_dalloc_small(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, void *ptr,
pageind)) != BININD_INVALID);
}
bitselm = arena_bitselm_get_mutable(chunk, pageind);
- arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm);
+ arena_dalloc_bin(tsd, arena, chunk, ptr, pageind, bitselm);
arena_decay_tick(tsd, arena);
}
@@ -2790,7 +2797,7 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
#endif
static void
-arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
+arena_dalloc_large_locked_impl(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
void *ptr, bool junked)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
@@ -2814,30 +2821,30 @@ arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk,
}
}
- arena_run_dalloc(arena, run, true, false, false);
+ arena_run_dalloc(tsd, arena, run, true, false, false);
}
void
-arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk,
- void *ptr)
+arena_dalloc_large_junked_locked(tsd_t *tsd, arena_t *arena,
+ arena_chunk_t *chunk, void *ptr)
{
- arena_dalloc_large_locked_impl(arena, chunk, ptr, true);
+ arena_dalloc_large_locked_impl(tsd, arena, chunk, ptr, true);
}
void
arena_dalloc_large(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk, void *ptr)
{
- malloc_mutex_lock(&arena->lock);
- arena_dalloc_large_locked_impl(arena, chunk, ptr, false);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
+ arena_dalloc_large_locked_impl(tsd, arena, chunk, ptr, false);
+ malloc_mutex_unlock(tsd, &arena->lock);
arena_decay_tick(tsd, arena);
}
static void
-arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
- size_t oldsize, size_t size)
+arena_ralloc_large_shrink(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ void *ptr, size_t oldsize, size_t size)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
@@ -2850,8 +2857,8 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
* Shrink the run, and make trailing pages available for other
* allocations.
*/
- malloc_mutex_lock(&arena->lock);
- arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size +
+ malloc_mutex_lock(tsd, &arena->lock);
+ arena_run_trim_tail(tsd, arena, chunk, run, oldsize + large_pad, size +
large_pad, true);
if (config_stats) {
szind_t oldindex = size2index(oldsize) - NBINS;
@@ -2869,12 +2876,12 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena->stats.lstats[index].nrequests++;
arena->stats.lstats[index].curruns++;
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
static bool
-arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
- size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
+arena_ralloc_large_grow(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk,
+ void *ptr, size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
{
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t npages = (oldsize + large_pad) >> LG_PAGE;
@@ -2884,7 +2891,7 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
large_pad);
/* Try to extend the run. */
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk,
pageind+npages) != 0)
goto label_fail;
@@ -2964,11 +2971,11 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena->stats.lstats[index].nrequests++;
arena->stats.lstats[index].curruns++;
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (false);
}
label_fail:
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (true);
}
@@ -2997,7 +3004,7 @@ arena_ralloc_junk_large_t *arena_ralloc_junk_large =
* always fail if growing an object, and the following run is already in use.
*/
static bool
-arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
+arena_ralloc_large(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min,
size_t usize_max, bool zero)
{
arena_chunk_t *chunk;
@@ -3012,16 +3019,16 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
arena = extent_node_arena_get(&chunk->node);
if (oldsize < usize_max) {
- bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize,
- usize_min, usize_max, zero);
+ bool ret = arena_ralloc_large_grow(tsd, arena, chunk, ptr,
+ oldsize, usize_min, usize_max, zero);
if (config_fill && !ret && !zero) {
if (unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)ptr + oldsize),
JEMALLOC_ALLOC_JUNK,
- isalloc(ptr, config_prof) - oldsize);
+ isalloc(tsd, ptr, config_prof) - oldsize);
} else if (unlikely(opt_zero)) {
memset((void *)((uintptr_t)ptr + oldsize), 0,
- isalloc(ptr, config_prof) - oldsize);
+ isalloc(tsd, ptr, config_prof) - oldsize);
}
}
return (ret);
@@ -3030,7 +3037,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min,
assert(oldsize > usize_max);
/* Fill before shrinking in order avoid a race. */
arena_ralloc_junk_large(ptr, oldsize, usize_max);
- arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max);
+ arena_ralloc_large_shrink(tsd, arena, chunk, ptr, oldsize, usize_max);
return (false);
}
@@ -3065,7 +3072,7 @@ arena_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
} else {
if (usize_max <= SMALL_MAXCLASS)
return (true);
- if (arena_ralloc_large(ptr, oldsize, usize_min,
+ if (arena_ralloc_large(tsd, ptr, oldsize, usize_min,
usize_max, zero))
return (true);
}
@@ -3138,25 +3145,25 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size,
}
dss_prec_t
-arena_dss_prec_get(arena_t *arena)
+arena_dss_prec_get(tsd_t *tsd, arena_t *arena)
{
dss_prec_t ret;
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
ret = arena->dss_prec;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (ret);
}
bool
-arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec)
+arena_dss_prec_set(tsd_t *tsd, arena_t *arena, dss_prec_t dss_prec)
{
if (!have_dss)
return (dss_prec != dss_prec_disabled);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena->dss_prec = dss_prec;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
return (false);
}
@@ -3213,28 +3220,29 @@ arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads,
}
void
-arena_basic_stats_merge(arena_t *arena, unsigned *nthreads, const char **dss,
- ssize_t *lg_dirty_mult, ssize_t *decay_time, size_t *nactive,
- size_t *ndirty)
+arena_basic_stats_merge(tsd_t *tsd, arena_t *arena, unsigned *nthreads,
+ const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
+ size_t *nactive, size_t *ndirty)
{
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult,
decay_time, nactive, ndirty);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
void
-arena_stats_merge(arena_t *arena, unsigned *nthreads, const char **dss,
- ssize_t *lg_dirty_mult, ssize_t *decay_time, size_t *nactive,
- size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
- malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats)
+arena_stats_merge(tsd_t *tsd, arena_t *arena, unsigned *nthreads,
+ const char **dss, ssize_t *lg_dirty_mult, ssize_t *decay_time,
+ size_t *nactive, size_t *ndirty, arena_stats_t *astats,
+ malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats,
+ malloc_huge_stats_t *hstats)
{
unsigned i;
cassert(config_stats);
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena_basic_stats_merge_locked(arena, nthreads, dss, lg_dirty_mult,
decay_time, nactive, ndirty);
@@ -3264,12 +3272,12 @@ arena_stats_merge(arena_t *arena, unsigned *nthreads, const char **dss,
hstats[i].ndalloc += arena->stats.hstats[i].ndalloc;
hstats[i].curhchunks += arena->stats.hstats[i].curhchunks;
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
for (i = 0; i < NBINS; i++) {
arena_bin_t *bin = &arena->bins[i];
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
bstats[i].nmalloc += bin->stats.nmalloc;
bstats[i].ndalloc += bin->stats.ndalloc;
bstats[i].nrequests += bin->stats.nrequests;
@@ -3281,7 +3289,7 @@ arena_stats_merge(arena_t *arena, unsigned *nthreads, const char **dss,
bstats[i].nruns += bin->stats.nruns;
bstats[i].reruns += bin->stats.reruns;
bstats[i].curruns += bin->stats.curruns;
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
}
}
@@ -3307,7 +3315,7 @@ arena_nthreads_dec(arena_t *arena)
}
arena_t *
-arena_new(unsigned ind)
+arena_new(tsd_t *tsd, unsigned ind)
{
arena_t *arena;
size_t arena_size;
@@ -3322,17 +3330,17 @@ arena_new(unsigned ind)
* because there is no way to clean up if base_alloc() OOMs.
*/
if (config_stats) {
- arena = (arena_t *)base_alloc(CACHELINE_CEILING(arena_size) +
- QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) +
+ arena = (arena_t *)base_alloc(tsd, CACHELINE_CEILING(arena_size)
+ + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) +
nhclasses) * sizeof(malloc_huge_stats_t));
} else
- arena = (arena_t *)base_alloc(arena_size);
+ arena = (arena_t *)base_alloc(tsd, arena_size);
if (arena == NULL)
return (NULL);
arena->ind = ind;
arena->nthreads = 0;
- if (malloc_mutex_init(&arena->lock))
+ if (malloc_mutex_init(&arena->lock, "arena", WITNESS_RANK_ARENA))
return (NULL);
if (config_stats) {
@@ -3365,7 +3373,7 @@ arena_new(unsigned ind)
(uint64_t)(uintptr_t)arena;
}
- arena->dss_prec = chunk_dss_prec_get();
+ arena->dss_prec = chunk_dss_prec_get(tsd);
arena->spare = NULL;
@@ -3383,17 +3391,20 @@ arena_new(unsigned ind)
arena_decay_init(arena, arena_decay_time_default_get());
ql_new(&arena->huge);
- if (malloc_mutex_init(&arena->huge_mtx))
+ if (malloc_mutex_init(&arena->huge_mtx, "arena_huge",
+ WITNESS_RANK_ARENA_HUGE))
return (NULL);
extent_tree_szad_new(&arena->chunks_szad_cached);
extent_tree_ad_new(&arena->chunks_ad_cached);
extent_tree_szad_new(&arena->chunks_szad_retained);
extent_tree_ad_new(&arena->chunks_ad_retained);
- if (malloc_mutex_init(&arena->chunks_mtx))
+ if (malloc_mutex_init(&arena->chunks_mtx, "arena_chunks",
+ WITNESS_RANK_ARENA_CHUNKS))
return (NULL);
ql_new(&arena->node_cache);
- if (malloc_mutex_init(&arena->node_cache_mtx))
+ if (malloc_mutex_init(&arena->node_cache_mtx, "arena_node_cache",
+ WITNESS_RANK_ARENA_NODE_CACHE))
return (NULL);
arena->chunk_hooks = chunk_hooks_default;
@@ -3401,7 +3412,8 @@ arena_new(unsigned ind)
/* Initialize bins. */
for (i = 0; i < NBINS; i++) {
bin = &arena->bins[i];
- if (malloc_mutex_init(&bin->lock))
+ if (malloc_mutex_init(&bin->lock, "arena_bin",
+ WITNESS_RANK_ARENA_BIN))
return (NULL);
bin->runcur = NULL;
arena_run_heap_new(&bin->runs);
@@ -3533,7 +3545,7 @@ small_run_size_init(void)
assert(small_maxrun != 0);
- small_run_tab = (bool *)base_alloc(sizeof(bool) * (small_maxrun >>
+ small_run_tab = (bool *)base_alloc(NULL, sizeof(bool) * (small_maxrun >>
LG_PAGE));
if (small_run_tab == NULL)
return (true);
@@ -3560,12 +3572,12 @@ run_quantize_init(void)
run_quantize_max = chunksize + large_pad;
- run_quantize_floor_tab = (size_t *)base_alloc(sizeof(size_t) *
+ run_quantize_floor_tab = (size_t *)base_alloc(NULL, sizeof(size_t) *
(run_quantize_max >> LG_PAGE));
if (run_quantize_floor_tab == NULL)
return (true);
- run_quantize_ceil_tab = (size_t *)base_alloc(sizeof(size_t) *
+ run_quantize_ceil_tab = (size_t *)base_alloc(NULL, sizeof(size_t) *
(run_quantize_max >> LG_PAGE));
if (run_quantize_ceil_tab == NULL)
return (true);
@@ -3642,40 +3654,40 @@ arena_boot(void)
}
void
-arena_prefork(arena_t *arena)
+arena_prefork(tsd_t *tsd, arena_t *arena)
{
unsigned i;
- malloc_mutex_prefork(&arena->lock);
- malloc_mutex_prefork(&arena->huge_mtx);
- malloc_mutex_prefork(&arena->chunks_mtx);
- malloc_mutex_prefork(&arena->node_cache_mtx);
+ malloc_mutex_prefork(tsd, &arena->lock);
+ malloc_mutex_prefork(tsd, &arena->huge_mtx);
+ malloc_mutex_prefork(tsd, &arena->chunks_mtx);
+ malloc_mutex_prefork(tsd, &arena->node_cache_mtx);
for (i = 0; i < NBINS; i++)
- malloc_mutex_prefork(&arena->bins[i].lock);
+ malloc_mutex_prefork(tsd, &arena->bins[i].lock);
}
void
-arena_postfork_parent(arena_t *arena)
+arena_postfork_parent(tsd_t *tsd, arena_t *arena)
{
unsigned i;
for (i = 0; i < NBINS; i++)
- malloc_mutex_postfork_parent(&arena->bins[i].lock);
- malloc_mutex_postfork_parent(&arena->node_cache_mtx);
- malloc_mutex_postfork_parent(&arena->chunks_mtx);
- malloc_mutex_postfork_parent(&arena->huge_mtx);
- malloc_mutex_postfork_parent(&arena->lock);
+ malloc_mutex_postfork_parent(tsd, &arena->bins[i].lock);
+ malloc_mutex_postfork_parent(tsd, &arena->node_cache_mtx);
+ malloc_mutex_postfork_parent(tsd, &arena->chunks_mtx);
+ malloc_mutex_postfork_parent(tsd, &arena->huge_mtx);
+ malloc_mutex_postfork_parent(tsd, &arena->lock);
}
void
-arena_postfork_child(arena_t *arena)
+arena_postfork_child(tsd_t *tsd, arena_t *arena)
{
unsigned i;
for (i = 0; i < NBINS; i++)
- malloc_mutex_postfork_child(&arena->bins[i].lock);
- malloc_mutex_postfork_child(&arena->node_cache_mtx);
- malloc_mutex_postfork_child(&arena->chunks_mtx);
- malloc_mutex_postfork_child(&arena->huge_mtx);
- malloc_mutex_postfork_child(&arena->lock);
+ malloc_mutex_postfork_child(tsd, &arena->bins[i].lock);
+ malloc_mutex_postfork_child(tsd, &arena->node_cache_mtx);
+ malloc_mutex_postfork_child(tsd, &arena->chunks_mtx);
+ malloc_mutex_postfork_child(tsd, &arena->huge_mtx);
+ malloc_mutex_postfork_child(tsd, &arena->lock);
}
diff --git a/src/base.c b/src/base.c
index 7cdcfed..87b376b 100644
--- a/src/base.c
+++ b/src/base.c
@@ -76,7 +76,7 @@ base_chunk_alloc(size_t minsize)
* physical memory usage.
*/
void *
-base_alloc(size_t size)
+base_alloc(tsd_t *tsd, size_t size)
{
void *ret;
size_t csize, usize;
@@ -91,7 +91,7 @@ base_alloc(size_t size)
usize = s2u(csize);
extent_node_init(&key, NULL, NULL, usize, false, false);
- malloc_mutex_lock(&base_mtx);
+ malloc_mutex_lock(tsd, &base_mtx);
node = extent_tree_szad_nsearch(&base_avail_szad, &key);
if (node != NULL) {
/* Use existing space. */
@@ -123,28 +123,28 @@ base_alloc(size_t size)
}
JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, csize);
label_return:
- malloc_mutex_unlock(&base_mtx);
+ malloc_mutex_unlock(tsd, &base_mtx);
return (ret);
}
void
-base_stats_get(size_t *allocated, size_t *resident, size_t *mapped)
+base_stats_get(tsd_t *tsd, size_t *allocated, size_t *resident, size_t *mapped)
{
- malloc_mutex_lock(&base_mtx);
+ malloc_mutex_lock(tsd, &base_mtx);
assert(base_allocated <= base_resident);
assert(base_resident <= base_mapped);
*allocated = base_allocated;
*resident = base_resident;
*mapped = base_mapped;
- malloc_mutex_unlock(&base_mtx);
+ malloc_mutex_unlock(tsd, &base_mtx);
}
bool
base_boot(void)
{
- if (malloc_mutex_init(&base_mtx))
+ if (malloc_mutex_init(&base_mtx, "base", WITNESS_RANK_BASE))
return (true);
extent_tree_szad_new(&base_avail_szad);
base_nodes = NULL;
@@ -153,22 +153,22 @@ base_boot(void)
}
void
-base_prefork(void)
+base_prefork(tsd_t *tsd)
{
- malloc_mutex_prefork(&base_mtx);
+ malloc_mutex_prefork(tsd, &base_mtx);
}
void
-base_postfork_parent(void)
+base_postfork_parent(tsd_t *tsd)
{
- malloc_mutex_postfork_parent(&base_mtx);
+ malloc_mutex_postfork_parent(tsd, &base_mtx);
}
void
-base_postfork_child(void)
+base_postfork_child(tsd_t *tsd)
{
- malloc_mutex_postfork_child(&base_mtx);
+ malloc_mutex_postfork_child(tsd, &base_mtx);
}
diff --git a/src/chunk.c b/src/chunk.c
index 304d4e5..0ee2a1a 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -49,9 +49,10 @@ const chunk_hooks_t chunk_hooks_default = {
* definition.
*/
-static void chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
- extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache,
- void *chunk, size_t size, bool zeroed, bool committed);
+static void chunk_record(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks, extent_tree_t *chunks_szad,
+ extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed,
+ bool committed);
/******************************************************************************/
@@ -63,23 +64,23 @@ chunk_hooks_get_locked(arena_t *arena)
}
chunk_hooks_t
-chunk_hooks_get(arena_t *arena)
+chunk_hooks_get(tsd_t *tsd, arena_t *arena)
{
chunk_hooks_t chunk_hooks;
- malloc_mutex_lock(&arena->chunks_mtx);
+ malloc_mutex_lock(tsd, &arena->chunks_mtx);
chunk_hooks = chunk_hooks_get_locked(arena);
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
return (chunk_hooks);
}
chunk_hooks_t
-chunk_hooks_set(arena_t *arena, const chunk_hooks_t *chunk_hooks)
+chunk_hooks_set(tsd_t *tsd, arena_t *arena, const chunk_hooks_t *chunk_hooks)
{
chunk_hooks_t old_chunk_hooks;
- malloc_mutex_lock(&arena->chunks_mtx);
+ malloc_mutex_lock(tsd, &arena->chunks_mtx);
old_chunk_hooks = arena->chunk_hooks;
/*
* Copy each field atomically so that it is impossible for readers to
@@ -104,14 +105,14 @@ chunk_hooks_set(arena_t *arena, const chunk_hooks_t *chunk_hooks)
ATOMIC_COPY_HOOK(split);
ATOMIC_COPY_HOOK(merge);
#undef ATOMIC_COPY_HOOK
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
return (old_chunk_hooks);
}
static void
-chunk_hooks_assure_initialized_impl(arena_t *arena, chunk_hooks_t *chunk_hooks,
- bool locked)
+chunk_hooks_assure_initialized_impl(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks, bool locked)
{
static const chunk_hooks_t uninitialized_hooks =
CHUNK_HOOKS_INITIALIZER;
@@ -119,27 +120,28 @@ chunk_hooks_assure_initialized_impl(arena_t *arena, chunk_hooks_t *chunk_hooks,
if (memcmp(chunk_hooks, &uninitialized_hooks, sizeof(chunk_hooks_t)) ==
0) {
*chunk_hooks = locked ? chunk_hooks_get_locked(arena) :
- chunk_hooks_get(arena);
+ chunk_hooks_get(tsd, arena);
}
}
static void
-chunk_hooks_assure_initialized_locked(arena_t *arena,
+chunk_hooks_assure_initialized_locked(tsd_t *tsd, arena_t *arena,
chunk_hooks_t *chunk_hooks)
{
- chunk_hooks_assure_initialized_impl(arena, chunk_hooks, true);
+ chunk_hooks_assure_initialized_impl(tsd, arena, chunk_hooks, true);
}
static void
-chunk_hooks_assure_initialized(arena_t *arena, chunk_hooks_t *chunk_hooks)
+chunk_hooks_assure_initialized(tsd_t *tsd, arena_t *arena,
+ chunk_hooks_t *chunk_hooks)
{
- chunk_hooks_assure_initialized_impl(arena, chunk_hooks, false);
+ chunk_hooks_assure_initialized_impl(tsd, arena, chunk_hooks, false);
}
bool
-chunk_register(const void *chunk, const extent_node_t *node)
+chunk_register(tsd_t *tsd, const void *chunk, const extent_node_t *node)
{
assert(extent_node_addr_get(node) == chunk);
@@ -159,7 +161,7 @@ chunk_register(const void *chunk, const extent_node_t *node)
high = atomic_read_z(&highchunks);
}
if (cur > high && prof_gdump_get_unlocked())
- prof_gdump();
+ prof_gdump(tsd);
}
return (false);
@@ -197,7 +199,7 @@ chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad,
}
static void *
-chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
+chunk_recycle(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache,
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
bool dalloc_node)
@@ -219,8 +221,8 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Beware size_t wrap-around. */
if (alloc_size < size)
return (NULL);
- malloc_mutex_lock(&arena->chunks_mtx);
- chunk_hooks_assure_initialized_locked(arena, chunk_hooks);
+ malloc_mutex_lock(tsd, &arena->chunks_mtx);
+ chunk_hooks_assure_initialized_locked(tsd, arena, chunk_hooks);
if (new_addr != NULL) {
extent_node_t key;
extent_node_init(&key, arena, new_addr, alloc_size, false,
@@ -232,7 +234,7 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
}
if (node == NULL || (new_addr != NULL && extent_node_size_get(node) <
size)) {
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
return (NULL);
}
leadsize = ALIGNMENT_CEILING((uintptr_t)extent_node_addr_get(node),
@@ -251,7 +253,7 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
if (leadsize != 0 &&
chunk_hooks->split(extent_node_addr_get(node),
extent_node_size_get(node), leadsize, size, false, arena->ind)) {
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
return (NULL);
}
/* Remove node from the tree. */
@@ -271,20 +273,21 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
if (chunk_hooks->split(ret, size + trailsize, size,
trailsize, false, arena->ind)) {
if (dalloc_node && node != NULL)
- arena_node_dalloc(arena, node);
- malloc_mutex_unlock(&arena->chunks_mtx);
- chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad,
- cache, ret, size + trailsize, zeroed, committed);
+ arena_node_dalloc(tsd, arena, node);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
+ chunk_record(tsd, arena, chunk_hooks, chunks_szad,
+ chunks_ad, cache, ret, size + trailsize, zeroed,
+ committed);
return (NULL);
}
/* Insert the trailing space as a smaller chunk. */
if (node == NULL) {
- node = arena_node_alloc(arena);
+ node = arena_node_alloc(tsd, arena);
if (node == NULL) {
- malloc_mutex_unlock(&arena->chunks_mtx);
- chunk_record(arena, chunk_hooks, chunks_szad,
- chunks_ad, cache, ret, size + trailsize,
- zeroed, committed);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
+ chunk_record(tsd, arena, chunk_hooks,
+ chunks_szad, chunks_ad, cache, ret, size +
+ trailsize, zeroed, committed);
return (NULL);
}
}
@@ -296,16 +299,16 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
node = NULL;
}
if (!committed && chunk_hooks->commit(ret, size, 0, size, arena->ind)) {
- malloc_mutex_unlock(&arena->chunks_mtx);
- chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad, cache,
- ret, size, zeroed, committed);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
+ chunk_record(tsd, arena, chunk_hooks, chunks_szad, chunks_ad,
+ cache, ret, size, zeroed, committed);
return (NULL);
}
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
assert(dalloc_node || node != NULL);
if (dalloc_node && node != NULL)
- arena_node_dalloc(arena, node);
+ arena_node_dalloc(tsd, arena, node);
if (*zero) {
if (!zeroed)
memset(ret, 0, size);
@@ -328,8 +331,8 @@ chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks,
* them if they are returned.
*/
static void *
-chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment,
- bool *zero, bool *commit, dss_prec_t dss_prec)
+chunk_alloc_core(tsd_t *tsd, arena_t *arena, void *new_addr, size_t size,
+ size_t alignment, bool *zero, bool *commit, dss_prec_t dss_prec)
{
void *ret;
@@ -340,8 +343,8 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment,
/* "primary" dss. */
if (have_dss && dss_prec == dss_prec_primary && (ret =
- chunk_alloc_dss(arena, new_addr, size, alignment, zero, commit)) !=
- NULL)
+ chunk_alloc_dss(tsd, arena, new_addr, size, alignment, zero,
+ commit)) != NULL)
return (ret);
/* mmap. */
if ((ret = chunk_alloc_mmap(new_addr, size, alignment, zero, commit)) !=
@@ -349,8 +352,8 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment,
return (ret);
/* "secondary" dss. */
if (have_dss && dss_prec == dss_prec_secondary && (ret =
- chunk_alloc_dss(arena, new_addr, size, alignment, zero, commit)) !=
- NULL)
+ chunk_alloc_dss(tsd, arena, new_addr, size, alignment, zero,
+ commit)) != NULL)
return (ret);
/* All strategies for allocation failed. */
@@ -380,8 +383,8 @@ chunk_alloc_base(size_t size)
}
void *
-chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
- size_t size, size_t alignment, bool *zero, bool dalloc_node)
+chunk_alloc_cache(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *new_addr, size_t size, size_t alignment, bool *zero, bool dalloc_node)
{
void *ret;
bool commit;
@@ -392,7 +395,7 @@ chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
assert((alignment & chunksize_mask) == 0);
commit = true;
- ret = chunk_recycle(arena, chunk_hooks, &arena->chunks_szad_cached,
+ ret = chunk_recycle(tsd, arena, chunk_hooks, &arena->chunks_szad_cached,
&arena->chunks_ad_cached, true, new_addr, size, alignment, zero,
&commit, dalloc_node);
if (ret == NULL)
@@ -404,11 +407,11 @@ chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
}
static arena_t *
-chunk_arena_get(unsigned arena_ind)
+chunk_arena_get(tsd_t *tsd, unsigned arena_ind)
{
arena_t *arena;
- arena = arena_get(arena_ind, false);
+ arena = arena_get(tsd, arena_ind, false);
/*
* The arena we're allocating on behalf of must have been initialized
* already.
@@ -422,11 +425,13 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit, unsigned arena_ind)
{
void *ret;
+ tsd_t *tsd;
arena_t *arena;
- arena = chunk_arena_get(arena_ind);
- ret = chunk_alloc_core(arena, new_addr, size, alignment, zero, commit,
- arena->dss_prec);
+ tsd = tsd_fetch();
+ arena = chunk_arena_get(tsd, arena_ind);
+ ret = chunk_alloc_core(tsd, arena, new_addr, size, alignment, zero,
+ commit, arena->dss_prec);
if (ret == NULL)
return (NULL);
if (config_valgrind)
@@ -436,8 +441,8 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero,
}
static void *
-chunk_alloc_retained(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
- size_t size, size_t alignment, bool *zero, bool *commit)
+chunk_alloc_retained(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
assert(size != 0);
@@ -445,20 +450,20 @@ chunk_alloc_retained(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
- return (chunk_recycle(arena, chunk_hooks, &arena->chunks_szad_retained,
- &arena->chunks_ad_retained, false, new_addr, size, alignment, zero,
- commit, true));
+ return (chunk_recycle(tsd, arena, chunk_hooks,
+ &arena->chunks_szad_retained, &arena->chunks_ad_retained, false,
+ new_addr, size, alignment, zero, commit, true));
}
void *
-chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
- size_t size, size_t alignment, bool *zero, bool *commit)
+chunk_alloc_wrapper(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit)
{
void *ret;
- chunk_hooks_assure_initialized(arena, chunk_hooks);
+ chunk_hooks_assure_initialized(tsd, arena, chunk_hooks);
- ret = chunk_alloc_retained(arena, chunk_hooks, new_addr, size,
+ ret = chunk_alloc_retained(tsd, arena, chunk_hooks, new_addr, size,
alignment, zero, commit);
if (ret == NULL) {
ret = chunk_hooks->alloc(new_addr, size, alignment, zero,
@@ -473,7 +478,7 @@ chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr,
}
static void
-chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
+chunk_record(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache,
void *chunk, size_t size, bool zeroed, bool committed)
{
@@ -485,8 +490,8 @@ chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
unzeroed = cache || !zeroed;
JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size);
- malloc_mutex_lock(&arena->chunks_mtx);
- chunk_hooks_assure_initialized_locked(arena, chunk_hooks);
+ malloc_mutex_lock(tsd, &arena->chunks_mtx);
+ chunk_hooks_assure_initialized_locked(tsd, arena, chunk_hooks);
extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0,
false, false);
node = extent_tree_ad_nsearch(chunks_ad, &key);
@@ -511,7 +516,7 @@ chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
arena_chunk_cache_maybe_insert(arena, node, cache);
} else {
/* Coalescing forward failed, so insert a new node. */
- node = arena_node_alloc(arena);
+ node = arena_node_alloc(tsd, arena);
if (node == NULL) {
/*
* Node allocation failed, which is an exceedingly
@@ -520,8 +525,8 @@ chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
* a virtual memory leak.
*/
if (cache) {
- chunk_purge_wrapper(arena, chunk_hooks, chunk,
- size, 0, size);
+ chunk_purge_wrapper(tsd, arena, chunk_hooks,
+ chunk, size, 0, size);
}
goto label_return;
}
@@ -557,16 +562,16 @@ chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks,
extent_tree_szad_insert(chunks_szad, node);
arena_chunk_cache_maybe_insert(arena, node, cache);
- arena_node_dalloc(arena, prev);
+ arena_node_dalloc(tsd, arena, prev);
}
label_return:
- malloc_mutex_unlock(&arena->chunks_mtx);
+ malloc_mutex_unlock(tsd, &arena->chunks_mtx);
}
void
-chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
- size_t size, bool committed)
+chunk_dalloc_cache(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *chunk, size_t size, bool committed)
{
assert(chunk != NULL);
@@ -574,9 +579,9 @@ chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
assert(size != 0);
assert((size & chunksize_mask) == 0);
- chunk_record(arena, chunk_hooks, &arena->chunks_szad_cached,
+ chunk_record(tsd, arena, chunk_hooks, &arena->chunks_szad_cached,
&arena->chunks_ad_cached, true, chunk, size, false, committed);
- arena_maybe_purge(arena);
+ arena_maybe_purge(tsd, arena);
}
static bool
@@ -584,14 +589,14 @@ chunk_dalloc_default(void *chunk, size_t size, bool committed,
unsigned arena_ind)
{
- if (!have_dss || !chunk_in_dss(chunk))
+ if (!have_dss || !chunk_in_dss(tsd_fetch(), chunk))
return (chunk_dalloc_mmap(chunk, size));
return (true);
}
void
-chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
- size_t size, bool zeroed, bool committed)
+chunk_dalloc_wrapper(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *chunk, size_t size, bool zeroed, bool committed)
{
assert(chunk != NULL);
@@ -599,7 +604,7 @@ chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
assert(size != 0);
assert((size & chunksize_mask) == 0);
- chunk_hooks_assure_initialized(arena, chunk_hooks);
+ chunk_hooks_assure_initialized(tsd, arena, chunk_hooks);
/* Try to deallocate. */
if (!chunk_hooks->dalloc(chunk, size, committed, arena->ind))
return;
@@ -610,7 +615,7 @@ chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
}
zeroed = !committed || !chunk_hooks->purge(chunk, size, 0, size,
arena->ind);
- chunk_record(arena, chunk_hooks, &arena->chunks_szad_retained,
+ chunk_record(tsd, arena, chunk_hooks, &arena->chunks_szad_retained,
&arena->chunks_ad_retained, false, chunk, size, zeroed, committed);
}
@@ -648,11 +653,11 @@ chunk_purge_default(void *chunk, size_t size, size_t offset, size_t length,
}
bool
-chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk,
- size_t size, size_t offset, size_t length)
+chunk_purge_wrapper(tsd_t *tsd, arena_t *arena, chunk_hooks_t *chunk_hooks,
+ void *chunk, size_t size, size_t offset, size_t length)
{
- chunk_hooks_assure_initialized(arena, chunk_hooks);
+ chunk_hooks_assure_initialized(tsd, arena, chunk_hooks);
return (chunk_hooks->purge(chunk, size, offset, length, arena->ind));
}
@@ -673,8 +678,11 @@ chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
if (!maps_coalesce)
return (true);
- if (have_dss && chunk_in_dss(chunk_a) != chunk_in_dss(chunk_b))
- return (true);
+ if (have_dss) {
+ tsd_t *tsd = tsd_fetch();
+ if (chunk_in_dss(tsd, chunk_a) != chunk_in_dss(tsd, chunk_b))
+ return (true);
+ }
return (false);
}
@@ -683,7 +691,7 @@ static rtree_node_elm_t *
chunks_rtree_node_alloc(size_t nelms)
{
- return ((rtree_node_elm_t *)base_alloc(nelms *
+ return ((rtree_node_elm_t *)base_alloc(tsd_fetch(), nelms *
sizeof(rtree_node_elm_t)));
}
@@ -730,22 +738,22 @@ chunk_boot(void)
}
void
-chunk_prefork(void)
+chunk_prefork(tsd_t *tsd)
{
- chunk_dss_prefork();
+ chunk_dss_prefork(tsd);
}
void
-chunk_postfork_parent(void)
+chunk_postfork_parent(tsd_t *tsd)
{
- chunk_dss_postfork_parent();
+ chunk_dss_postfork_parent(tsd);
}
void
-chunk_postfork_child(void)
+chunk_postfork_child(tsd_t *tsd)
{
- chunk_dss_postfork_child();
+ chunk_dss_postfork_child(tsd);
}
diff --git a/src/chunk_dss.c b/src/chunk_dss.c
index 943d0e9..3b3f243 100644
--- a/src/chunk_dss.c
+++ b/src/chunk_dss.c
@@ -41,33 +41,33 @@ chunk_dss_sbrk(intptr_t increment)
}
dss_prec_t
-chunk_dss_prec_get(void)
+chunk_dss_prec_get(tsd_t *tsd)
{
dss_prec_t ret;
if (!have_dss)
return (dss_prec_disabled);
- malloc_mutex_lock(&dss_mtx);
+ malloc_mutex_lock(tsd, &dss_mtx);
ret = dss_prec_default;
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
return (ret);
}
bool
-chunk_dss_prec_set(dss_prec_t dss_prec)
+chunk_dss_prec_set(tsd_t *tsd, dss_prec_t dss_prec)
{
if (!have_dss)
return (dss_prec != dss_prec_disabled);
- malloc_mutex_lock(&dss_mtx);
+ malloc_mutex_lock(tsd, &dss_mtx);
dss_prec_default = dss_prec;
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
return (false);
}
void *
-chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
- bool *zero, bool *commit)
+chunk_alloc_dss(tsd_t *tsd, arena_t *arena, void *new_addr, size_t size,
+ size_t alignment, bool *zero, bool *commit)
{
cassert(have_dss);
assert(size > 0 && (size & chunksize_mask) == 0);
@@ -80,7 +80,7 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
if ((intptr_t)size < 0)
return (NULL);
- malloc_mutex_lock(&dss_mtx);
+ malloc_mutex_lock(tsd, &dss_mtx);
if (dss_prev != (void *)-1) {
/*
@@ -122,7 +122,7 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
if ((uintptr_t)ret < (uintptr_t)dss_max ||
(uintptr_t)dss_next < (uintptr_t)dss_max) {
/* Wrap-around. */
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
return (NULL);
}
incr = gap_size + cpad_size + size;
@@ -130,11 +130,11 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
if (dss_prev == dss_max) {
/* Success. */
dss_max = dss_next;
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
if (cpad_size != 0) {
chunk_hooks_t chunk_hooks =
CHUNK_HOOKS_INITIALIZER;
- chunk_dalloc_wrapper(arena,
+ chunk_dalloc_wrapper(tsd, arena,
&chunk_hooks, cpad, cpad_size,
false, true);
}
@@ -149,25 +149,25 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment,
}
} while (dss_prev != (void *)-1);
}
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
return (NULL);
}
bool
-chunk_in_dss(void *chunk)
+chunk_in_dss(tsd_t *tsd, void *chunk)
{
bool ret;
cassert(have_dss);
- malloc_mutex_lock(&dss_mtx);
+ malloc_mutex_lock(tsd, &dss_mtx);
if ((uintptr_t)chunk >= (uintptr_t)dss_base
&& (uintptr_t)chunk < (uintptr_t)dss_max)
ret = true;
else
ret = false;
- malloc_mutex_unlock(&dss_mtx);
+ malloc_mutex_unlock(tsd, &dss_mtx);
return (ret);
}
@@ -178,7 +178,7 @@ chunk_dss_boot(void)
cassert(have_dss);
- if (malloc_mutex_init(&dss_mtx))
+ if (malloc_mutex_init(&dss_mtx, "dss", WITNESS_RANK_DSS))
return (true);
dss_base = chunk_dss_sbrk(0);
dss_prev = dss_base;
@@ -188,27 +188,27 @@ chunk_dss_boot(void)
}
void
-chunk_dss_prefork(void)
+chunk_dss_prefork(tsd_t *tsd)
{
if (have_dss)
- malloc_mutex_prefork(&dss_mtx);
+ malloc_mutex_prefork(tsd, &dss_mtx);
}
void
-chunk_dss_postfork_parent(void)
+chunk_dss_postfork_parent(tsd_t *tsd)
{
if (have_dss)
- malloc_mutex_postfork_parent(&dss_mtx);
+ malloc_mutex_postfork_parent(tsd, &dss_mtx);
}
void
-chunk_dss_postfork_child(void)
+chunk_dss_postfork_child(tsd_t *tsd)
{
if (have_dss)
- malloc_mutex_postfork_child(&dss_mtx);
+ malloc_mutex_postfork_child(tsd, &dss_mtx);
}
/******************************************************************************/
diff --git a/src/ctl.c b/src/ctl.c
index 17bd071..50faee7 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -42,25 +42,25 @@ ctl_indexed_node(const ctl_node_t *node)
/* Function prototypes for non-inline static functions. */
#define CTL_PROTO(n) \
-static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \
- size_t *oldlenp, void *newp, size_t newlen);
+static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen);
#define INDEX_PROTO(n) \
-static const ctl_named_node_t *n##_index(const size_t *mib, \
- size_t miblen, size_t i);
+static const ctl_named_node_t *n##_index(tsd_t *tsd, \
+ const size_t *mib, size_t miblen, size_t i);
static bool ctl_arena_init(ctl_arena_stats_t *astats);
static void ctl_arena_clear(ctl_arena_stats_t *astats);
-static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
+static void ctl_arena_stats_amerge(tsd_t *tsd, ctl_arena_stats_t *cstats,
arena_t *arena);
static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
ctl_arena_stats_t *astats);
-static void ctl_arena_refresh(arena_t *arena, unsigned i);
-static bool ctl_grow(void);
-static void ctl_refresh(void);
-static bool ctl_init(void);
-static int ctl_lookup(const char *name, ctl_node_t const **nodesp,
- size_t *mibp, size_t *depthp);
+static void ctl_arena_refresh(tsd_t *tsd, arena_t *arena, unsigned i);
+static bool ctl_grow(tsd_t *tsd);
+static void ctl_refresh(tsd_t *tsd);
+static bool ctl_init(tsd_t *tsd);
+static int ctl_lookup(tsd_t *tsd, const char *name,
+ ctl_node_t const **nodesp, size_t *mibp, size_t *depthp);
CTL_PROTO(version)
CTL_PROTO(epoch)
@@ -117,7 +117,7 @@ CTL_PROTO(opt_prof_accum)
CTL_PROTO(tcache_create)
CTL_PROTO(tcache_flush)
CTL_PROTO(tcache_destroy)
-static void arena_i_purge(unsigned arena_ind, bool all);
+static void arena_i_purge(tsd_t *tsd, unsigned arena_ind, bool all);
CTL_PROTO(arena_i_purge)
CTL_PROTO(arena_i_decay)
CTL_PROTO(arena_i_dss)
@@ -554,12 +554,12 @@ ctl_arena_clear(ctl_arena_stats_t *astats)
}
static void
-ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
+ctl_arena_stats_amerge(tsd_t *tsd, ctl_arena_stats_t *cstats, arena_t *arena)
{
unsigned i;
if (config_stats) {
- arena_stats_merge(arena, &cstats->nthreads, &cstats->dss,
+ arena_stats_merge(tsd, arena, &cstats->nthreads, &cstats->dss,
&cstats->lg_dirty_mult, &cstats->decay_time,
&cstats->pactive, &cstats->pdirty, &cstats->astats,
cstats->bstats, cstats->lstats, cstats->hstats);
@@ -572,8 +572,8 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
cstats->nrequests_small += cstats->bstats[i].nrequests;
}
} else {
- arena_basic_stats_merge(arena, &cstats->nthreads, &cstats->dss,
- &cstats->lg_dirty_mult, &cstats->decay_time,
+ arena_basic_stats_merge(tsd, arena, &cstats->nthreads,
+ &cstats->dss, &cstats->lg_dirty_mult, &cstats->decay_time,
&cstats->pactive, &cstats->pdirty);
}
}
@@ -649,24 +649,24 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats)
}
static void
-ctl_arena_refresh(arena_t *arena, unsigned i)
+ctl_arena_refresh(tsd_t *tsd, arena_t *arena, unsigned i)
{
ctl_arena_stats_t *astats = &ctl_stats.arenas[i];
ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas];
ctl_arena_clear(astats);
- ctl_arena_stats_amerge(astats, arena);
+ ctl_arena_stats_amerge(tsd, astats, arena);
/* Merge into sum stats as well. */
ctl_arena_stats_smerge(sstats, astats);
}
static bool
-ctl_grow(void)
+ctl_grow(tsd_t *tsd)
{
ctl_arena_stats_t *astats;
/* Initialize new arena. */
- if (arena_init(ctl_stats.narenas) == NULL)
+ if (arena_init(tsd, ctl_stats.narenas) == NULL)
return (true);
/* Allocate extended arena stats. */
@@ -701,7 +701,7 @@ ctl_grow(void)
}
static void
-ctl_refresh(void)
+ctl_refresh(tsd_t *tsd)
{
unsigned i;
VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
@@ -713,19 +713,20 @@ ctl_refresh(void)
ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]);
for (i = 0; i < ctl_stats.narenas; i++)
- tarenas[i] = arena_get(i, false);
+ tarenas[i] = arena_get(tsd, i, false);
for (i = 0; i < ctl_stats.narenas; i++) {
bool initialized = (tarenas[i] != NULL);
ctl_stats.arenas[i].initialized = initialized;
if (initialized)
- ctl_arena_refresh(tarenas[i], i);
+ ctl_arena_refresh(tsd, tarenas[i], i);
}
if (config_stats) {
size_t base_allocated, base_resident, base_mapped;
- base_stats_get(&base_allocated, &base_resident, &base_mapped);
+ base_stats_get(tsd, &base_allocated, &base_resident,
+ &base_mapped);
ctl_stats.allocated =
ctl_stats.arenas[ctl_stats.narenas].allocated_small +
ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large +
@@ -748,11 +749,11 @@ ctl_refresh(void)
}
static bool
-ctl_init(void)
+ctl_init(tsd_t *tsd)
{
bool ret;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
if (!ctl_initialized) {
/*
* Allocate space for one extra arena stats element, which
@@ -794,19 +795,19 @@ ctl_init(void)
ctl_stats.arenas[ctl_stats.narenas].initialized = true;
ctl_epoch = 0;
- ctl_refresh();
+ ctl_refresh(tsd);
ctl_initialized = true;
}
ret = false;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static int
-ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
- size_t *depthp)
+ctl_lookup(tsd_t *tsd, const char *name, ctl_node_t const **nodesp,
+ size_t *mibp, size_t *depthp)
{
int ret;
const char *elm, *tdot, *dot;
@@ -858,7 +859,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
}
inode = ctl_indexed_node(node->children);
- node = inode->index(mibp, *depthp, (size_t)index);
+ node = inode->index(tsd, mibp, *depthp, (size_t)index);
if (node == NULL) {
ret = ENOENT;
goto label_return;
@@ -902,8 +903,8 @@ label_return:
}
int
-ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
- size_t newlen)
+ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
+ void *newp, size_t newlen)
{
int ret;
size_t depth;
@@ -911,19 +912,19 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t mib[CTL_MAX_DEPTH];
const ctl_named_node_t *node;
- if (!ctl_initialized && ctl_init()) {
+ if (!ctl_initialized && ctl_init(tsd)) {
ret = EAGAIN;
goto label_return;
}
depth = CTL_MAX_DEPTH;
- ret = ctl_lookup(name, nodes, mib, &depth);
+ ret = ctl_lookup(tsd, name, nodes, mib, &depth);
if (ret != 0)
goto label_return;
node = ctl_named_node(nodes[depth-1]);
if (node != NULL && node->ctl)
- ret = node->ctl(mib, depth, oldp, oldlenp, newp, newlen);
+ ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
else {
/* The name refers to a partial path through the ctl tree. */
ret = ENOENT;
@@ -934,29 +935,29 @@ label_return:
}
int
-ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp)
+ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp)
{
int ret;
- if (!ctl_initialized && ctl_init()) {
+ if (!ctl_initialized && ctl_init(tsd)) {
ret = EAGAIN;
goto label_return;
}
- ret = ctl_lookup(name, NULL, mibp, miblenp);
+ ret = ctl_lookup(tsd, name, NULL, mibp, miblenp);
label_return:
return(ret);
}
int
-ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
const ctl_named_node_t *node;
size_t i;
- if (!ctl_initialized && ctl_init()) {
+ if (!ctl_initialized && ctl_init(tsd)) {
ret = EAGAIN;
goto label_return;
}
@@ -978,7 +979,7 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
/* Indexed element. */
inode = ctl_indexed_node(node->children);
- node = inode->index(mib, miblen, mib[i]);
+ node = inode->index(tsd, mib, miblen, mib[i]);
if (node == NULL) {
ret = ENOENT;
goto label_return;
@@ -988,7 +989,7 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
/* Call the ctl function. */
if (node && node->ctl)
- ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen);
+ ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
else {
/* Partial MIB. */
ret = ENOENT;
@@ -1002,7 +1003,7 @@ bool
ctl_boot(void)
{
- if (malloc_mutex_init(&ctl_mtx))
+ if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL))
return (true);
ctl_initialized = false;
@@ -1011,24 +1012,24 @@ ctl_boot(void)
}
void
-ctl_prefork(void)
+ctl_prefork(tsd_t *tsd)
{
- malloc_mutex_prefork(&ctl_mtx);
+ malloc_mutex_prefork(tsd, &ctl_mtx);
}
void
-ctl_postfork_parent(void)
+ctl_postfork_parent(tsd_t *tsd)
{
- malloc_mutex_postfork_parent(&ctl_mtx);
+ malloc_mutex_postfork_parent(tsd, &ctl_mtx);
}
void
-ctl_postfork_child(void)
+ctl_postfork_child(tsd_t *tsd)
{
- malloc_mutex_postfork_child(&ctl_mtx);
+ malloc_mutex_postfork_child(tsd, &ctl_mtx);
}
/******************************************************************************/
@@ -1085,8 +1086,8 @@ ctl_postfork_child(void)
*/
#define CTL_RO_CLGEN(c, l, n, v, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
@@ -1094,7 +1095,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
if (!(c)) \
return (ENOENT); \
if (l) \
- malloc_mutex_lock(&ctl_mtx); \
+ malloc_mutex_lock(tsd, &ctl_mtx); \
READONLY(); \
oldval = (v); \
READ(oldval, t); \
@@ -1102,47 +1103,47 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
ret = 0; \
label_return: \
if (l) \
- malloc_mutex_unlock(&ctl_mtx); \
+ malloc_mutex_unlock(tsd, &ctl_mtx); \
return (ret); \
}
#define CTL_RO_CGEN(c, n, v, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
\
if (!(c)) \
return (ENOENT); \
- malloc_mutex_lock(&ctl_mtx); \
+ malloc_mutex_lock(tsd, &ctl_mtx); \
READONLY(); \
oldval = (v); \
READ(oldval, t); \
\
ret = 0; \
label_return: \
- malloc_mutex_unlock(&ctl_mtx); \
+ malloc_mutex_unlock(tsd, &ctl_mtx); \
return (ret); \
}
#define CTL_RO_GEN(n, v, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
\
- malloc_mutex_lock(&ctl_mtx); \
+ malloc_mutex_lock(tsd, &ctl_mtx); \
READONLY(); \
oldval = (v); \
READ(oldval, t); \
\
ret = 0; \
label_return: \
- malloc_mutex_unlock(&ctl_mtx); \
+ malloc_mutex_unlock(tsd, &ctl_mtx); \
return (ret); \
}
@@ -1152,8 +1153,8 @@ label_return: \
*/
#define CTL_RO_NL_CGEN(c, n, v, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
@@ -1171,8 +1172,8 @@ label_return: \
#define CTL_RO_NL_GEN(n, v, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
@@ -1188,17 +1189,15 @@ label_return: \
#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
- tsd_t *tsd; \
\
if (!(c)) \
return (ENOENT); \
READONLY(); \
- tsd = tsd_fetch(); \
oldval = (m(tsd)); \
READ(oldval, t); \
\
@@ -1209,8 +1208,8 @@ label_return: \
#define CTL_RO_CONFIG_GEN(n, t) \
static int \
-n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
- void *newp, size_t newlen) \
+n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \
+ size_t *oldlenp, void *newp, size_t newlen) \
{ \
int ret; \
t oldval; \
@@ -1229,21 +1228,21 @@ label_return: \
CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *)
static int
-epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
UNUSED uint64_t newval;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
WRITE(newval, uint64_t);
if (newp != NULL)
- ctl_refresh();
+ ctl_refresh(tsd);
READ(ctl_epoch, uint64_t);
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
@@ -1298,20 +1297,18 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool)
/******************************************************************************/
static int
-thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
- tsd_t *tsd;
arena_t *oldarena;
unsigned newind, oldind;
- tsd = tsd_fetch();
oldarena = arena_choose(tsd, NULL);
if (oldarena == NULL)
return (EAGAIN);
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
newind = oldind = oldarena->ind;
WRITE(newind, unsigned);
READ(oldind, unsigned);
@@ -1325,7 +1322,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
}
/* Initialize arena if necessary. */
- newarena = arena_get(newind, true);
+ newarena = arena_get(tsd, newind, true);
if (newarena == NULL) {
ret = EAGAIN;
goto label_return;
@@ -1335,7 +1332,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
if (config_tcache) {
tcache_t *tcache = tsd_tcache_get(tsd);
if (tcache != NULL) {
- tcache_arena_reassociate(tcache, oldarena,
+ tcache_arena_reassociate(tsd, tcache, oldarena,
newarena);
}
}
@@ -1343,7 +1340,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
@@ -1357,8 +1354,8 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp,
tsd_thread_deallocatedp_get, uint64_t *)
static int
-thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
@@ -1382,8 +1379,8 @@ label_return:
}
static int
-thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
@@ -1401,7 +1398,7 @@ label_return:
}
static int
-thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp,
+thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
@@ -1412,20 +1409,16 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp,
READ_XOR_WRITE();
if (newp != NULL) {
- tsd_t *tsd;
-
if (newlen != sizeof(const char *)) {
ret = EINVAL;
goto label_return;
}
- tsd = tsd_fetch();
-
if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) !=
0)
goto label_return;
} else {
- const char *oldname = prof_thread_name_get();
+ const char *oldname = prof_thread_name_get(tsd);
READ(oldname, const char *);
}
@@ -1435,7 +1428,7 @@ label_return:
}
static int
-thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp,
+thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
@@ -1444,13 +1437,13 @@ thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp,
if (!config_prof)
return (ENOENT);
- oldval = prof_thread_active_get();
+ oldval = prof_thread_active_get(tsd);
if (newp != NULL) {
if (newlen != sizeof(bool)) {
ret = EINVAL;
goto label_return;
}
- if (prof_thread_active_set(*(bool *)newp)) {
+ if (prof_thread_active_set(tsd, *(bool *)newp)) {
ret = EAGAIN;
goto label_return;
}
@@ -1465,19 +1458,16 @@ label_return:
/******************************************************************************/
static int
-tcache_create_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
- tsd_t *tsd;
unsigned tcache_ind;
if (!config_tcache)
return (ENOENT);
- tsd = tsd_fetch();
-
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
READONLY();
if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT;
@@ -1487,23 +1477,20 @@ tcache_create_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static int
-tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
- tsd_t *tsd;
unsigned tcache_ind;
if (!config_tcache)
return (ENOENT);
- tsd = tsd_fetch();
-
WRITEONLY();
tcache_ind = UINT_MAX;
WRITE(tcache_ind, unsigned);
@@ -1519,18 +1506,15 @@ label_return:
}
static int
-tcache_destroy_ctl(const size_t *mib, size_t miblen, void *oldp,
+tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
- tsd_t *tsd;
unsigned tcache_ind;
if (!config_tcache)
return (ENOENT);
- tsd = tsd_fetch();
-
WRITEONLY();
tcache_ind = UINT_MAX;
WRITE(tcache_ind, unsigned);
@@ -1548,10 +1532,10 @@ label_return:
/******************************************************************************/
static void
-arena_i_purge(unsigned arena_ind, bool all)
+arena_i_purge(tsd_t *tsd, unsigned arena_ind, bool all)
{
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
{
unsigned narenas = ctl_stats.narenas;
@@ -1560,43 +1544,43 @@ arena_i_purge(unsigned arena_ind, bool all)
VARIABLE_ARRAY(arena_t *, tarenas, narenas);
for (i = 0; i < narenas; i++)
- tarenas[i] = arena_get(i, false);
+ tarenas[i] = arena_get(tsd, i, false);
/*
* No further need to hold ctl_mtx, since narenas and
* tarenas contain everything needed below.
*/
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
for (i = 0; i < narenas; i++) {
if (tarenas[i] != NULL)
- arena_purge(tarenas[i], all);
+ arena_purge(tsd, tarenas[i], all);
}
} else {
arena_t *tarena;
assert(arena_ind < narenas);
- tarena = arena_get(arena_ind, false);
+ tarena = arena_get(tsd, arena_ind, false);
/* No further need to hold ctl_mtx. */
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
if (tarena != NULL)
- arena_purge(tarena, all);
+ arena_purge(tsd, tarena, all);
}
}
}
static int
-arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+arena_i_purge_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
READONLY();
WRITEONLY();
- arena_i_purge((unsigned)mib[1], true);
+ arena_i_purge(tsd, (unsigned)mib[1], true);
ret = 0;
label_return:
@@ -1604,14 +1588,14 @@ label_return:
}
static int
-arena_i_decay_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+arena_i_decay_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
READONLY();
WRITEONLY();
- arena_i_purge((unsigned)mib[1], false);
+ arena_i_purge(tsd, (unsigned)mib[1], false);
ret = 0;
label_return:
@@ -1619,8 +1603,8 @@ label_return:
}
static int
-arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
const char *dss = NULL;
@@ -1628,7 +1612,7 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
dss_prec_t dss_prec_old = dss_prec_limit;
dss_prec_t dss_prec = dss_prec_limit;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
WRITE(dss, const char *);
if (dss != NULL) {
int i;
@@ -1649,20 +1633,20 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
}
if (arena_ind < ctl_stats.narenas) {
- arena_t *arena = arena_get(arena_ind, false);
+ arena_t *arena = arena_get(tsd, arena_ind, false);
if (arena == NULL || (dss_prec != dss_prec_limit &&
- arena_dss_prec_set(arena, dss_prec))) {
+ arena_dss_prec_set(tsd, arena, dss_prec))) {
ret = EFAULT;
goto label_return;
}
- dss_prec_old = arena_dss_prec_get(arena);
+ dss_prec_old = arena_dss_prec_get(tsd, arena);
} else {
if (dss_prec != dss_prec_limit &&
- chunk_dss_prec_set(dss_prec)) {
+ chunk_dss_prec_set(tsd, dss_prec)) {
ret = EFAULT;
goto label_return;
}
- dss_prec_old = chunk_dss_prec_get();
+ dss_prec_old = chunk_dss_prec_get(tsd);
}
dss = dss_prec_names[dss_prec_old];
@@ -1670,26 +1654,26 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static int
-arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+arena_i_lg_dirty_mult_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned arena_ind = (unsigned)mib[1];
arena_t *arena;
- arena = arena_get(arena_ind, false);
+ arena = arena_get(tsd, arena_ind, false);
if (arena == NULL) {
ret = EFAULT;
goto label_return;
}
if (oldp != NULL && oldlenp != NULL) {
- size_t oldval = arena_lg_dirty_mult_get(arena);
+ size_t oldval = arena_lg_dirty_mult_get(tsd, arena);
READ(oldval, ssize_t);
}
if (newp != NULL) {
@@ -1697,7 +1681,7 @@ arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL;
goto label_return;
}
- if (arena_lg_dirty_mult_set(arena, *(ssize_t *)newp)) {
+ if (arena_lg_dirty_mult_set(tsd, arena, *(ssize_t *)newp)) {
ret = EFAULT;
goto label_return;
}
@@ -1709,21 +1693,21 @@ label_return:
}
static int
-arena_i_decay_time_ctl(const size_t *mib, size_t miblen, void *oldp,
+arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned arena_ind = (unsigned)mib[1];
arena_t *arena;
- arena = arena_get(arena_ind, false);
+ arena = arena_get(tsd, arena_ind, false);
if (arena == NULL) {
ret = EFAULT;
goto label_return;
}
if (oldp != NULL && oldlenp != NULL) {
- size_t oldval = arena_decay_time_get(arena);
+ size_t oldval = arena_decay_time_get(tsd, arena);
READ(oldval, ssize_t);
}
if (newp != NULL) {
@@ -1731,7 +1715,7 @@ arena_i_decay_time_ctl(const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL;
goto label_return;
}
- if (arena_decay_time_set(arena, *(ssize_t *)newp)) {
+ if (arena_decay_time_set(tsd, arena, *(ssize_t *)newp)) {
ret = EFAULT;
goto label_return;
}
@@ -1743,24 +1727,25 @@ label_return:
}
static int
-arena_i_chunk_hooks_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+arena_i_chunk_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned arena_ind = (unsigned)mib[1];
arena_t *arena;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
if (arena_ind < narenas_total_get() && (arena =
- arena_get(arena_ind, false)) != NULL) {
+ arena_get(tsd, arena_ind, false)) != NULL) {
if (newp != NULL) {
chunk_hooks_t old_chunk_hooks, new_chunk_hooks;
WRITE(new_chunk_hooks, chunk_hooks_t);
- old_chunk_hooks = chunk_hooks_set(arena,
+ old_chunk_hooks = chunk_hooks_set(tsd, arena,
&new_chunk_hooks);
READ(old_chunk_hooks, chunk_hooks_t);
} else {
- chunk_hooks_t old_chunk_hooks = chunk_hooks_get(arena);
+ chunk_hooks_t old_chunk_hooks = chunk_hooks_get(tsd,
+ arena);
READ(old_chunk_hooks, chunk_hooks_t);
}
} else {
@@ -1769,16 +1754,16 @@ arena_i_chunk_hooks_ctl(const size_t *mib, size_t miblen, void *oldp,
}
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static const ctl_named_node_t *
-arena_i_index(const size_t *mib, size_t miblen, size_t i)
+arena_i_index(tsd_t *tsd, const size_t *mib, size_t miblen, size_t i)
{
- const ctl_named_node_t * ret;
+ const ctl_named_node_t *ret;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
if (i > ctl_stats.narenas) {
ret = NULL;
goto label_return;
@@ -1786,20 +1771,20 @@ arena_i_index(const size_t *mib, size_t miblen, size_t i)
ret = super_arena_i_node;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
/******************************************************************************/
static int
-arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp,
+arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned narenas;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
READONLY();
if (*oldlenp != sizeof(unsigned)) {
ret = EINVAL;
@@ -1810,18 +1795,18 @@ arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp,
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static int
-arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp,
+arenas_initialized_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned nread, i;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
READONLY();
if (*oldlenp != ctl_stats.narenas * sizeof(bool)) {
ret = EINVAL;
@@ -1836,13 +1821,13 @@ arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp,
((bool *)oldp)[i] = ctl_stats.arenas[i].initialized;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
static int
-arenas_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+arenas_lg_dirty_mult_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
@@ -1867,7 +1852,7 @@ label_return:
}
static int
-arenas_decay_time_ctl(const size_t *mib, size_t miblen, void *oldp,
+arenas_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
@@ -1901,7 +1886,7 @@ CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t)
static const ctl_named_node_t *
-arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
+arenas_bin_i_index(tsd_t *tsd, const size_t *mib, size_t miblen, size_t i)
{
if (i > NBINS)
@@ -1912,7 +1897,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
CTL_RO_NL_GEN(arenas_nlruns, nlclasses, unsigned)
CTL_RO_NL_GEN(arenas_lrun_i_size, index2size(NBINS+(szind_t)mib[2]), size_t)
static const ctl_named_node_t *
-arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
+arenas_lrun_i_index(tsd_t *tsd, const size_t *mib, size_t miblen, size_t i)
{
if (i > nlclasses)
@@ -1924,7 +1909,7 @@ CTL_RO_NL_GEN(arenas_nhchunks, nhclasses, unsigned)
CTL_RO_NL_GEN(arenas_hchunk_i_size, index2size(NBINS+nlclasses+(szind_t)mib[2]),
size_t)
static const ctl_named_node_t *
-arenas_hchunk_i_index(const size_t *mib, size_t miblen, size_t i)
+arenas_hchunk_i_index(tsd_t *tsd, const size_t *mib, size_t miblen, size_t i)
{
if (i > nhclasses)
@@ -1933,15 +1918,15 @@ arenas_hchunk_i_index(const size_t *mib, size_t miblen, size_t i)
}
static int
-arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+arenas_extend_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
unsigned narenas;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
READONLY();
- if (ctl_grow()) {
+ if (ctl_grow(tsd)) {
ret = EAGAIN;
goto label_return;
}
@@ -1950,15 +1935,15 @@ arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = 0;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
/******************************************************************************/
static int
-prof_thread_active_init_ctl(const size_t *mib, size_t miblen, void *oldp,
- size_t *oldlenp, void *newp, size_t newlen)
+prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
+ void *oldp, size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
@@ -1971,9 +1956,9 @@ prof_thread_active_init_ctl(const size_t *mib, size_t miblen, void *oldp,
ret = EINVAL;
goto label_return;
}
- oldval = prof_thread_active_init_set(*(bool *)newp);
+ oldval = prof_thread_active_init_set(tsd, *(bool *)newp);
} else
- oldval = prof_thread_active_init_get();
+ oldval = prof_thread_active_init_get(tsd);
READ(oldval, bool);
ret = 0;
@@ -1982,8 +1967,8 @@ label_return:
}
static int
-prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
@@ -1996,9 +1981,9 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = EINVAL;
goto label_return;
}
- oldval = prof_active_set(*(bool *)newp);
+ oldval = prof_active_set(tsd, *(bool *)newp);
} else
- oldval = prof_active_get();
+ oldval = prof_active_get(tsd);
READ(oldval, bool);
ret = 0;
@@ -2007,8 +1992,8 @@ label_return:
}
static int
-prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
const char *filename = NULL;
@@ -2019,7 +2004,7 @@ prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
WRITEONLY();
WRITE(filename, const char *);
- if (prof_mdump(filename)) {
+ if (prof_mdump(tsd, filename)) {
ret = EFAULT;
goto label_return;
}
@@ -2030,8 +2015,8 @@ label_return:
}
static int
-prof_gdump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
bool oldval;
@@ -2044,9 +2029,9 @@ prof_gdump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = EINVAL;
goto label_return;
}
- oldval = prof_gdump_set(*(bool *)newp);
+ oldval = prof_gdump_set(tsd, *(bool *)newp);
} else
- oldval = prof_gdump_get();
+ oldval = prof_gdump_get(tsd);
READ(oldval, bool);
ret = 0;
@@ -2055,12 +2040,11 @@ label_return:
}
static int
-prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
- void *newp, size_t newlen)
+prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
{
int ret;
size_t lg_sample = lg_prof_sample;
- tsd_t *tsd;
if (!config_prof)
return (ENOENT);
@@ -2070,8 +2054,6 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
if (lg_sample >= (sizeof(uint64_t) << 3))
lg_sample = (sizeof(uint64_t) << 3) - 1;
- tsd = tsd_fetch();
-
prof_reset(tsd, lg_sample);
ret = 0;
@@ -2157,7 +2139,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns,
ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t)
static const ctl_named_node_t *
-stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j)
+stats_arenas_i_bins_j_index(tsd_t *tsd, const size_t *mib, size_t miblen,
+ size_t j)
{
if (j > NBINS)
@@ -2175,7 +2158,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns,
ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t)
static const ctl_named_node_t *
-stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j)
+stats_arenas_i_lruns_j_index(tsd_t *tsd, const size_t *mib, size_t miblen,
+ size_t j)
{
if (j > nlclasses)
@@ -2194,7 +2178,8 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_curhchunks,
ctl_stats.arenas[mib[2]].hstats[mib[4]].curhchunks, size_t)
static const ctl_named_node_t *
-stats_arenas_i_hchunks_j_index(const size_t *mib, size_t miblen, size_t j)
+stats_arenas_i_hchunks_j_index(tsd_t *tsd, const size_t *mib, size_t miblen,
+ size_t j)
{
if (j > nhclasses)
@@ -2203,11 +2188,11 @@ stats_arenas_i_hchunks_j_index(const size_t *mib, size_t miblen, size_t j)
}
static const ctl_named_node_t *
-stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i)
+stats_arenas_i_index(tsd_t *tsd, const size_t *mib, size_t miblen, size_t i)
{
const ctl_named_node_t * ret;
- malloc_mutex_lock(&ctl_mtx);
+ malloc_mutex_lock(tsd, &ctl_mtx);
if (i > ctl_stats.narenas || !ctl_stats.arenas[i].initialized) {
ret = NULL;
goto label_return;
@@ -2215,6 +2200,6 @@ stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i)
ret = super_stats_arenas_i_node;
label_return:
- malloc_mutex_unlock(&ctl_mtx);
+ malloc_mutex_unlock(tsd, &ctl_mtx);
return (ret);
}
diff --git a/src/huge.c b/src/huge.c
index a63c825..3a802de 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -15,12 +15,12 @@ huge_node_get(const void *ptr)
}
static bool
-huge_node_set(const void *ptr, extent_node_t *node)
+huge_node_set(tsd_t *tsd, const void *ptr, extent_node_t *node)
{
assert(extent_node_addr_get(node) == ptr);
assert(!extent_node_achunk_get(node));
- return (chunk_register(ptr, node));
+ return (chunk_register(tsd, ptr, node));
}
static void
@@ -68,7 +68,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
*/
is_zeroed = zero;
arena = arena_choose(tsd, arena);
- if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena,
+ if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsd, arena,
usize, alignment, &is_zeroed)) == NULL) {
idalloctm(tsd, node, tcache, true, true);
return (NULL);
@@ -76,17 +76,17 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
extent_node_init(node, arena, ret, usize, is_zeroed, true);
- if (huge_node_set(ret, node)) {
- arena_chunk_dalloc_huge(arena, ret, usize);
+ if (huge_node_set(tsd, ret, node)) {
+ arena_chunk_dalloc_huge(tsd, arena, ret, usize);
idalloctm(tsd, node, tcache, true, true);
return (NULL);
}
/* Insert node into huge. */
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
ql_elm_new(node, ql_link);
ql_tail_insert(&arena->huge, node, ql_link);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed)
@@ -103,7 +103,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment,
#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
#endif
static void
-huge_dalloc_junk(void *ptr, size_t usize)
+huge_dalloc_junk(tsd_t *tsd, void *ptr, size_t usize)
{
if (config_fill && have_dss && unlikely(opt_junk_free)) {
@@ -111,7 +111,7 @@ huge_dalloc_junk(void *ptr, size_t usize)
* Only bother junk filling if the chunk isn't about to be
* unmapped.
*/
- if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
+ if (!config_munmap || (have_dss && chunk_in_dss(tsd, ptr)))
memset(ptr, JEMALLOC_FREE_JUNK, usize);
}
}
@@ -122,8 +122,8 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
#endif
static void
-huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
- size_t usize_max, bool zero)
+huge_ralloc_no_move_similar(tsd_t *tsd, void *ptr, size_t oldsize,
+ size_t usize_min, size_t usize_max, bool zero)
{
size_t usize, usize_next;
extent_node_t *node;
@@ -151,21 +151,22 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
JEMALLOC_FREE_JUNK, sdiff);
post_zeroed = false;
} else {
- post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
- ptr, CHUNK_CEILING(oldsize), usize, sdiff);
+ post_zeroed = !chunk_purge_wrapper(tsd, arena,
+ &chunk_hooks, ptr, CHUNK_CEILING(oldsize), usize,
+ sdiff);
}
} else
post_zeroed = pre_zeroed;
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
/* Update the size of the huge allocation. */
assert(extent_node_size_get(node) != usize);
extent_node_size_set(node, usize);
/* Update zeroed. */
extent_node_zeroed_set(node, post_zeroed);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
- arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize);
+ arena_chunk_ralloc_huge_similar(tsd, arena, ptr, oldsize, usize);
/* Fill if necessary (growing). */
if (oldsize < usize) {
@@ -182,7 +183,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min,
}
static bool
-huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
+huge_ralloc_no_move_shrink(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize)
{
extent_node_t *node;
arena_t *arena;
@@ -193,7 +194,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
pre_zeroed = extent_node_zeroed_get(node);
- chunk_hooks = chunk_hooks_get(arena);
+ chunk_hooks = chunk_hooks_get(tsd, arena);
assert(oldsize > usize);
@@ -206,42 +207,43 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize)
if (oldsize > usize) {
size_t sdiff = oldsize - usize;
if (config_fill && unlikely(opt_junk_free)) {
- huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
+ huge_dalloc_junk(tsd, (void *)((uintptr_t)ptr + usize),
sdiff);
post_zeroed = false;
} else {
- post_zeroed = !chunk_purge_wrapper(arena, &chunk_hooks,
- CHUNK_ADDR2BASE((uintptr_t)ptr + usize),
- CHUNK_CEILING(oldsize),
+ post_zeroed = !chunk_purge_wrapper(tsd, arena,
+ &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr +
+ usize), CHUNK_CEILING(oldsize),
CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff);
}
} else
post_zeroed = pre_zeroed;
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
/* Update the size of the huge allocation. */
extent_node_size_set(node, usize);
/* Update zeroed. */
extent_node_zeroed_set(node, post_zeroed);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
/* Zap the excess chunks. */
- arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize);
+ arena_chunk_ralloc_huge_shrink(tsd, arena, ptr, oldsize, usize);
return (false);
}
static bool
-huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
+huge_ralloc_no_move_expand(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize,
+ bool zero) {
extent_node_t *node;
arena_t *arena;
bool is_zeroed_subchunk, is_zeroed_chunk;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
is_zeroed_subchunk = extent_node_zeroed_get(node);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
/*
* Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so
@@ -249,14 +251,14 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) {
*/
is_zeroed_chunk = zero;
- if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize,
+ if (arena_chunk_ralloc_huge_expand(tsd, arena, ptr, oldsize, usize,
&is_zeroed_chunk))
return (true);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
/* Update the size of the huge allocation. */
extent_node_size_set(node, usize);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed_subchunk) {
@@ -291,15 +293,15 @@ huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min,
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
/* Attempt to expand the allocation in-place. */
- if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max,
+ if (!huge_ralloc_no_move_expand(tsd, ptr, oldsize, usize_max,
zero)) {
arena_decay_tick(tsd, huge_aalloc(ptr));
return (false);
}
/* Try again, this time with usize_min. */
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
- CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr,
- oldsize, usize_min, zero)) {
+ CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsd,
+ ptr, oldsize, usize_min, zero)) {
arena_decay_tick(tsd, huge_aalloc(ptr));
return (false);
}
@@ -311,15 +313,15 @@ huge_ralloc_no_move(tsd_t *tsd, void *ptr, size_t oldsize, size_t usize_min,
*/
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
- huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max,
- zero);
+ huge_ralloc_no_move_similar(tsd, ptr, oldsize, usize_min,
+ usize_max, zero);
arena_decay_tick(tsd, huge_aalloc(ptr));
return (false);
}
/* Attempt to shrink the allocation in-place. */
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
- if (!huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)) {
+ if (!huge_ralloc_no_move_shrink(tsd, ptr, oldsize, usize_max)) {
arena_decay_tick(tsd, huge_aalloc(ptr));
return (false);
}
@@ -376,13 +378,13 @@ huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache)
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
huge_node_unset(ptr, node);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
ql_remove(&arena->huge, node, ql_link);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
- huge_dalloc_junk(extent_node_addr_get(node),
+ huge_dalloc_junk(tsd, extent_node_addr_get(node),
extent_node_size_get(node));
- arena_chunk_dalloc_huge(extent_node_arena_get(node),
+ arena_chunk_dalloc_huge(tsd, extent_node_arena_get(node),
extent_node_addr_get(node), extent_node_size_get(node));
idalloctm(tsd, node, tcache, true, true);
@@ -397,7 +399,7 @@ huge_aalloc(const void *ptr)
}
size_t
-huge_salloc(const void *ptr)
+huge_salloc(tsd_t *tsd, const void *ptr)
{
size_t size;
extent_node_t *node;
@@ -405,15 +407,15 @@ huge_salloc(const void *ptr)
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
size = extent_node_size_get(node);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
return (size);
}
prof_tctx_t *
-huge_prof_tctx_get(const void *ptr)
+huge_prof_tctx_get(tsd_t *tsd, const void *ptr)
{
prof_tctx_t *tctx;
extent_node_t *node;
@@ -421,29 +423,29 @@ huge_prof_tctx_get(const void *ptr)
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
tctx = extent_node_prof_tctx_get(node);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
return (tctx);
}
void
-huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx)
+huge_prof_tctx_set(tsd_t *tsd, const void *ptr, prof_tctx_t *tctx)
{
extent_node_t *node;
arena_t *arena;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
- malloc_mutex_lock(&arena->huge_mtx);
+ malloc_mutex_lock(tsd, &arena->huge_mtx);
extent_node_prof_tctx_set(node, tctx);
- malloc_mutex_unlock(&arena->huge_mtx);
+ malloc_mutex_unlock(tsd, &arena->huge_mtx);
}
void
-huge_prof_tctx_reset(const void *ptr)
+huge_prof_tctx_reset(tsd_t *tsd, const void *ptr)
{
- huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U);
+ huge_prof_tctx_set(tsd, ptr, (prof_tctx_t *)(uintptr_t)1U);
}
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 0735376..7543dff 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -212,7 +212,7 @@ _init_init_lock(void)
* really only matters early in the process creation, before any
* separate thread normally starts doing anything. */
if (!init_lock_initialized)
- malloc_mutex_init(&init_lock);
+ malloc_mutex_init(&init_lock, "init", WITNESS_RANK_INIT);
init_lock_initialized = true;
}
@@ -254,7 +254,7 @@ typedef struct {
* definition.
*/
-static bool malloc_init_hard_a0(void);
+static bool malloc_init_hard_a0(tsd_t *tsd);
static bool malloc_init_hard(void);
/******************************************************************************/
@@ -291,7 +291,7 @@ malloc_init_a0(void)
{
if (unlikely(malloc_init_state == malloc_init_uninitialized))
- return (malloc_init_hard_a0());
+ return (malloc_init_hard_a0(NULL));
return (false);
}
@@ -319,7 +319,7 @@ a0ialloc(size_t size, bool zero, bool is_metadata)
return (NULL);
return (iallocztm(NULL, size, size2index(size), zero, false,
- is_metadata, arena_get(0, false), true));
+ is_metadata, arena_get(NULL, 0, false), true));
}
static void
@@ -413,7 +413,7 @@ narenas_total_get(void)
/* Create a new arena and insert it into the arenas array at index ind. */
static arena_t *
-arena_init_locked(unsigned ind)
+arena_init_locked(tsd_t *tsd, unsigned ind)
{
arena_t *arena;
@@ -427,26 +427,26 @@ arena_init_locked(unsigned ind)
* Another thread may have already initialized arenas[ind] if it's an
* auto arena.
*/
- arena = arena_get(ind, false);
+ arena = arena_get(tsd, ind, false);
if (arena != NULL) {
assert(ind < narenas_auto);
return (arena);
}
/* Actually initialize the arena. */
- arena = arena_new(ind);
+ arena = arena_new(tsd, ind);
arena_set(ind, arena);
return (arena);
}
arena_t *
-arena_init(unsigned ind)
+arena_init(tsd_t *tsd, unsigned ind)
{
arena_t *arena;
- malloc_mutex_lock(&arenas_lock);
- arena = arena_init_locked(ind);
- malloc_mutex_unlock(&arenas_lock);
+ malloc_mutex_lock(tsd, &arenas_lock);
+ arena = arena_init_locked(tsd, ind);
+ malloc_mutex_unlock(tsd, &arenas_lock);
return (arena);
}
@@ -455,7 +455,7 @@ arena_bind(tsd_t *tsd, unsigned ind)
{
arena_t *arena;
- arena = arena_get(ind, false);
+ arena = arena_get(tsd, ind, false);
arena_nthreads_inc(arena);
if (tsd_nominal(tsd))
@@ -467,8 +467,8 @@ arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind)
{
arena_t *oldarena, *newarena;
- oldarena = arena_get(oldind, false);
- newarena = arena_get(newind, false);
+ oldarena = arena_get(tsd, oldind, false);
+ newarena = arena_get(tsd, newind, false);
arena_nthreads_dec(oldarena);
arena_nthreads_inc(newarena);
tsd_arena_set(tsd, newarena);
@@ -479,7 +479,7 @@ arena_unbind(tsd_t *tsd, unsigned ind)
{
arena_t *arena;
- arena = arena_get(ind, false);
+ arena = arena_get(tsd, ind, false);
arena_nthreads_dec(arena);
tsd_arena_set(tsd, NULL);
}
@@ -571,16 +571,16 @@ arena_choose_hard(tsd_t *tsd)
choose = 0;
first_null = narenas_auto;
- malloc_mutex_lock(&arenas_lock);
- assert(arena_get(0, false) != NULL);
+ malloc_mutex_lock(tsd, &arenas_lock);
+ assert(arena_get(tsd, 0, false) != NULL);
for (i = 1; i < narenas_auto; i++) {
- if (arena_get(i, false) != NULL) {
+ if (arena_get(tsd, i, false) != NULL) {
/*
* Choose the first arena that has the lowest
* number of threads assigned to it.
*/
- if (arena_nthreads_get(arena_get(i, false)) <
- arena_nthreads_get(arena_get(choose,
+ if (arena_nthreads_get(arena_get(tsd, i, false))
+ < arena_nthreads_get(arena_get(tsd, choose,
false)))
choose = i;
} else if (first_null == narenas_auto) {
@@ -597,26 +597,26 @@ arena_choose_hard(tsd_t *tsd)
}
}
- if (arena_nthreads_get(arena_get(choose, false)) == 0
+ if (arena_nthreads_get(arena_get(tsd, choose, false)) == 0
|| first_null == narenas_auto) {
/*
* Use an unloaded arena, or the least loaded arena if
* all arenas are already initialized.
*/
- ret = arena_get(choose, false);
+ ret = arena_get(tsd, choose, false);
} else {
/* Initialize a new arena. */
choose = first_null;
- ret = arena_init_locked(choose);
+ ret = arena_init_locked(tsd, choose);
if (ret == NULL) {
- malloc_mutex_unlock(&arenas_lock);
+ malloc_mutex_unlock(tsd, &arenas_lock);
return (NULL);
}
}
arena_bind(tsd, choose);
- malloc_mutex_unlock(&arenas_lock);
+ malloc_mutex_unlock(tsd, &arenas_lock);
} else {
- ret = arena_get(0, false);
+ ret = arena_get(tsd, 0, false);
arena_bind(tsd, 0);
}
@@ -681,8 +681,11 @@ stats_print_atexit(void)
{
if (config_tcache && config_stats) {
+ tsd_t *tsd;
unsigned narenas, i;
+ tsd = tsd_fetch();
+
/*
* Merge stats from extant threads. This is racy, since
* individual threads do not lock when recording tcache stats
@@ -691,7 +694,7 @@ stats_print_atexit(void)
* continue to allocate.
*/
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
- arena_t *arena = arena_get(i, false);
+ arena_t *arena = arena_get(tsd, i, false);
if (arena != NULL) {
tcache_t *tcache;
@@ -701,11 +704,11 @@ stats_print_atexit(void)
* and bin locks in the opposite order,
* deadlocks may result.
*/
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
ql_foreach(tcache, &arena->tcache_ql, link) {
- tcache_stats_merge(tcache, arena);
+ tcache_stats_merge(tsd, tcache, arena);
}
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
}
}
@@ -1056,7 +1059,8 @@ malloc_conf_init(void)
for (i = 0; i < dss_prec_limit; i++) {
if (strncmp(dss_prec_names[i], v, vlen)
== 0) {
- if (chunk_dss_prec_set(i)) {
+ if (chunk_dss_prec_set(NULL,
+ i)) {
malloc_conf_error(
"Error setting dss",
k, klen, v, vlen);
@@ -1186,7 +1190,6 @@ malloc_conf_init(void)
}
}
-/* init_lock must be held. */
static bool
malloc_init_hard_needed(void)
{
@@ -1204,9 +1207,9 @@ malloc_init_hard_needed(void)
if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
/* Busy-wait until the initializing thread completes. */
do {
- malloc_mutex_unlock(&init_lock);
+ malloc_mutex_unlock(NULL, &init_lock);
CPU_SPINWAIT;
- malloc_mutex_lock(&init_lock);
+ malloc_mutex_lock(NULL, &init_lock);
} while (!malloc_initialized());
return (false);
}
@@ -1214,9 +1217,8 @@ malloc_init_hard_needed(void)
return (true);
}
-/* init_lock must be held. */
static bool
-malloc_init_hard_a0_locked(void)
+malloc_init_hard_a0_locked(tsd_t *tsd)
{
malloc_initializer = INITIALIZER;
@@ -1242,9 +1244,9 @@ malloc_init_hard_a0_locked(void)
prof_boot1();
if (arena_boot())
return (true);
- if (config_tcache && tcache_boot())
+ if (config_tcache && tcache_boot(tsd))
return (true);
- if (malloc_mutex_init(&arenas_lock))
+ if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))
return (true);
/*
* Create enough scaffolding to allow recursive allocation in
@@ -1258,38 +1260,35 @@ malloc_init_hard_a0_locked(void)
* Initialize one arena here. The rest are lazily created in
* arena_choose_hard().
*/
- if (arena_init(0) == NULL)
+ if (arena_init(tsd, 0) == NULL)
return (true);
malloc_init_state = malloc_init_a0_initialized;
return (false);
}
static bool
-malloc_init_hard_a0(void)
+malloc_init_hard_a0(tsd_t *tsd)
{
bool ret;
- malloc_mutex_lock(&init_lock);
- ret = malloc_init_hard_a0_locked();
- malloc_mutex_unlock(&init_lock);
+ malloc_mutex_lock(tsd, &init_lock);
+ ret = malloc_init_hard_a0_locked(tsd);
+ malloc_mutex_unlock(tsd, &init_lock);
return (ret);
}
-/*
- * Initialize data structures which may trigger recursive allocation.
- *
- * init_lock must be held.
- */
+/* Initialize data structures which may trigger recursive allocation. */
static bool
-malloc_init_hard_recursible(void)
+malloc_init_hard_recursible(tsd_t **tsd)
{
- bool ret = false;
+ bool ret;
malloc_init_state = malloc_init_recursible;
- malloc_mutex_unlock(&init_lock);
+ malloc_mutex_unlock(*tsd, &init_lock);
/* LinuxThreads' pthread_setspecific() allocates. */
- if (malloc_tsd_boot0()) {
+ *tsd = malloc_tsd_boot0();
+ if (*tsd == NULL) {
ret = true;
goto label_return;
}
@@ -1308,17 +1307,17 @@ malloc_init_hard_recursible(void)
}
#endif
+ ret = false;
label_return:
- malloc_mutex_lock(&init_lock);
+ malloc_mutex_lock(*tsd, &init_lock);
return (ret);
}
-/* init_lock must be held. */
static bool
-malloc_init_hard_finish(void)
+malloc_init_hard_finish(tsd_t *tsd)
{
- if (mutex_boot())
+ if (malloc_mutex_boot())
return (true);
if (opt_narenas == 0) {
@@ -1343,7 +1342,7 @@ malloc_init_hard_finish(void)
narenas_total_set(narenas_auto);
/* Allocate and initialize arenas. */
- arenas = (arena_t **)base_alloc(sizeof(arena_t *) *
+ arenas = (arena_t **)base_alloc(tsd, sizeof(arena_t *) *
(MALLOCX_ARENA_MAX+1));
if (arenas == NULL)
return (true);
@@ -1359,38 +1358,39 @@ malloc_init_hard_finish(void)
static bool
malloc_init_hard(void)
{
+ tsd_t *tsd = NULL;
#if defined(_WIN32) && _WIN32_WINNT < 0x0600
_init_init_lock();
#endif
- malloc_mutex_lock(&init_lock);
+ malloc_mutex_lock(tsd, &init_lock);
if (!malloc_init_hard_needed()) {
- malloc_mutex_unlock(&init_lock);
+ malloc_mutex_unlock(tsd, &init_lock);
return (false);
}
if (malloc_init_state != malloc_init_a0_initialized &&
- malloc_init_hard_a0_locked()) {
- malloc_mutex_unlock(&init_lock);
+ malloc_init_hard_a0_locked(tsd)) {
+ malloc_mutex_unlock(tsd, &init_lock);
return (true);
}
- if (malloc_init_hard_recursible()) {
- malloc_mutex_unlock(&init_lock);
+ if (malloc_init_hard_recursible(&tsd)) {
+ malloc_mutex_unlock(tsd, &init_lock);
return (true);
}
- if (config_prof && prof_boot2()) {
- malloc_mutex_unlock(&init_lock);
+ if (config_prof && prof_boot2(tsd)) {
+ malloc_mutex_unlock(tsd, &init_lock);
return (true);
}
- if (malloc_init_hard_finish()) {
- malloc_mutex_unlock(&init_lock);
+ if (malloc_init_hard_finish(tsd)) {
+ malloc_mutex_unlock(tsd, &init_lock);
return (true);
}
- malloc_mutex_unlock(&init_lock);
+ malloc_mutex_unlock(tsd, &init_lock);
malloc_tsd_boot1();
return (false);
}
@@ -1416,7 +1416,7 @@ imalloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind,
p = imalloc(tsd, LARGE_MINCLASS, ind_large, slow_path);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else
p = imalloc(tsd, usize, ind, slow_path);
@@ -1438,7 +1438,7 @@ imalloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool slow_path)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(p, usize, tctx);
+ prof_malloc(tsd, p, usize, tctx);
return (p);
}
@@ -1450,7 +1450,11 @@ imalloc_body(size_t size, tsd_t **tsd, size_t *usize, bool slow_path)
if (slow_path && unlikely(malloc_init()))
return (NULL);
+
*tsd = tsd_fetch();
+
+ witness_assert_lockless(*tsd);
+
ind = size2index(size);
if (unlikely(ind >= NSIZES))
return (NULL);
@@ -1479,7 +1483,7 @@ imalloc_post_check(void *ret, tsd_t *tsd, size_t usize, bool slow_path)
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
- assert(usize == isalloc(ret, config_prof));
+ assert(usize == isalloc(tsd, ret, config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
}
}
@@ -1507,9 +1511,10 @@ je_malloc(size_t size)
ret = imalloc_body(size, &tsd, &usize, true);
imalloc_post_check(ret, tsd, usize, true);
UTRACE(0, size, ret);
- JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsd, ret, usize, false);
}
+ witness_assert_lockless(tsd);
return (ret);
}
@@ -1526,7 +1531,7 @@ imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else
p = ipalloc(tsd, usize, alignment, false);
@@ -1548,7 +1553,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(p, usize, tctx);
+ prof_malloc(tsd, p, usize, tctx);
return (p);
}
@@ -1565,10 +1570,12 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
assert(min_alignment != 0);
if (unlikely(malloc_init())) {
+ tsd = NULL;
result = NULL;
goto label_oom;
}
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
if (size == 0)
size = 1;
@@ -1603,10 +1610,12 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
ret = 0;
label_return:
if (config_stats && likely(result != NULL)) {
- assert(usize == isalloc(result, config_prof));
+ assert(usize == isalloc(tsd, result, config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, result);
+ JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd, result, usize, false);
+ witness_assert_lockless(tsd);
return (ret);
label_oom:
assert(result == NULL);
@@ -1616,6 +1625,7 @@ label_oom:
abort();
}
ret = ENOMEM;
+ witness_assert_lockless(tsd);
goto label_return;
}
@@ -1623,9 +1633,10 @@ JEMALLOC_EXPORT int JEMALLOC_NOTHROW
JEMALLOC_ATTR(nonnull(1))
je_posix_memalign(void **memptr, size_t alignment, size_t size)
{
- int ret = imemalign(memptr, alignment, size, sizeof(void *));
- JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr,
- config_prof), false);
+ int ret;
+
+ ret = imemalign(memptr, alignment, size, sizeof(void *));
+
return (ret);
}
@@ -1641,8 +1652,7 @@ je_aligned_alloc(size_t alignment, size_t size)
ret = NULL;
set_errno(err);
}
- JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
- false);
+
return (ret);
}
@@ -1658,7 +1668,7 @@ icalloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, prof_tctx_t *tctx)
p = icalloc(tsd, LARGE_MINCLASS, ind_large);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else
p = icalloc(tsd, usize, ind);
@@ -1680,7 +1690,7 @@ icalloc_prof(tsd_t *tsd, size_t usize, szind_t ind)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(p, usize, tctx);
+ prof_malloc(tsd, p, usize, tctx);
return (p);
}
@@ -1697,11 +1707,13 @@ je_calloc(size_t num, size_t size)
size_t usize JEMALLOC_CC_SILENCE_INIT(0);
if (unlikely(malloc_init())) {
+ tsd = NULL;
num_size = 0;
ret = NULL;
goto label_return;
}
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
num_size = num * size;
if (unlikely(num_size == 0)) {
@@ -1747,11 +1759,12 @@ label_return:
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
- assert(usize == isalloc(ret, config_prof));
+ assert(usize == isalloc(tsd, ret, config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, num_size, ret);
- JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true);
+ JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsd, ret, usize, true);
+ witness_assert_lockless(tsd);
return (ret);
}
@@ -1767,7 +1780,7 @@ irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
@@ -1782,7 +1795,7 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(old_ptr);
+ old_tctx = prof_tctx_get(tsd, old_ptr);
tctx = prof_alloc_prep(tsd, usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
@@ -1804,14 +1817,16 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
size_t usize;
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
+ witness_assert_lockless(tsd);
+
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
if (config_prof && opt_prof) {
- usize = isalloc(ptr, config_prof);
+ usize = isalloc(tsd, ptr, config_prof);
prof_free(tsd, ptr, usize);
} else if (config_stats || config_valgrind)
- usize = isalloc(ptr, config_prof);
+ usize = isalloc(tsd, ptr, config_prof);
if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize;
@@ -1819,7 +1834,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
iqalloc(tsd, ptr, tcache, false);
else {
if (config_valgrind && unlikely(in_valgrind))
- rzsize = p2rz(ptr);
+ rzsize = p2rz(tsd, ptr);
iqalloc(tsd, ptr, tcache, true);
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
}
@@ -1830,6 +1845,8 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache)
{
UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
+ witness_assert_lockless(tsd);
+
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
@@ -1838,7 +1855,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache)
if (config_stats)
*tsd_thread_deallocatedp_get(tsd) += usize;
if (config_valgrind && unlikely(in_valgrind))
- rzsize = p2rz(ptr);
+ rzsize = p2rz(tsd, ptr);
isqalloc(tsd, ptr, usize, tcache);
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
}
@@ -1869,10 +1886,13 @@ je_realloc(void *ptr, size_t size)
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
- old_usize = isalloc(ptr, config_prof);
- if (config_valgrind && unlikely(in_valgrind))
- old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize);
+ old_usize = isalloc(tsd, ptr, config_prof);
+ if (config_valgrind && unlikely(in_valgrind)) {
+ old_rzsize = config_prof ? p2rz(tsd, ptr) :
+ u2rz(old_usize);
+ }
if (config_prof && opt_prof) {
usize = s2u(size);
@@ -1901,13 +1921,14 @@ je_realloc(void *ptr, size_t size)
set_errno(ENOMEM);
}
if (config_stats && likely(ret != NULL)) {
- assert(usize == isalloc(ret, config_prof));
+ assert(usize == isalloc(tsd, ret, config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, ret);
- JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize,
+ JEMALLOC_VALGRIND_REALLOC(true, tsd, ret, usize, true, ptr, old_usize,
old_rzsize, true, false);
+ witness_assert_lockless(tsd);
return (ret);
}
@@ -1922,6 +1943,7 @@ je_free(void *ptr)
ifree(tsd, ptr, tcache_get(tsd, false), false);
else
ifree(tsd, ptr, tcache_get(tsd, false), true);
+ witness_assert_lockless(tsd);
}
}
@@ -1942,7 +1964,6 @@ je_memalign(size_t alignment, size_t size)
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
if (unlikely(imemalign(&ret, alignment, size, 1) != 0))
ret = NULL;
- JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
return (ret);
}
#endif
@@ -1956,7 +1977,6 @@ je_valloc(size_t size)
void *ret JEMALLOC_CC_SILENCE_INIT(NULL);
if (unlikely(imemalign(&ret, PAGE, size, 1) != 0))
ret = NULL;
- JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false);
return (ret);
}
#endif
@@ -2020,7 +2040,7 @@ imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize,
*tcache = tcache_get(tsd, true);
if ((flags & MALLOCX_ARENA_MASK) != 0) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
- *arena = arena_get(arena_ind, true);
+ *arena = arena_get(tsd, arena_ind, true);
if (unlikely(*arena == NULL))
return (true);
} else
@@ -2076,7 +2096,7 @@ imallocx_prof_sample(tsd_t *tsd, size_t usize, size_t alignment, bool zero,
arena);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else
p = imallocx_flags(tsd, usize, alignment, zero, tcache, arena);
@@ -2108,7 +2128,7 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize)
prof_alloc_rollback(tsd, tctx, true);
return (NULL);
}
- prof_malloc(p, *usize, tctx);
+ prof_malloc(tsd, p, *usize, tctx);
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
return (p);
@@ -2154,9 +2174,12 @@ je_mallocx(size_t size, int flags)
assert(size != 0);
- if (unlikely(malloc_init()))
+ if (unlikely(malloc_init())) {
+ tsd = NULL;
goto label_oom;
+ }
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
if (config_prof && opt_prof)
p = imallocx_prof(tsd, size, flags, &usize);
@@ -2166,11 +2189,12 @@ je_mallocx(size_t size, int flags)
goto label_oom;
if (config_stats) {
- assert(usize == isalloc(p, config_prof));
+ assert(usize == isalloc(tsd, p, config_prof));
*tsd_thread_allocatedp_get(tsd) += usize;
}
UTRACE(0, size, p);
- JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags));
+ JEMALLOC_VALGRIND_MALLOC(true, tsd, p, usize, MALLOCX_ZERO_GET(flags));
+ witness_assert_lockless(tsd);
return (p);
label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) {
@@ -2178,6 +2202,7 @@ label_oom:
abort();
}
UTRACE(0, size, 0);
+ witness_assert_lockless(tsd);
return (NULL);
}
@@ -2195,7 +2220,7 @@ irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize,
zero, tcache, arena);
if (p == NULL)
return (NULL);
- arena_prof_promoted(p, usize);
+ arena_prof_promoted(tsd, p, usize);
} else {
p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
tcache, arena);
@@ -2214,7 +2239,7 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(old_ptr);
+ old_tctx = prof_tctx_get(tsd, old_ptr);
tctx = prof_alloc_prep(tsd, *usize, prof_active, true);
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
@@ -2237,7 +2262,7 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
* be the same as the current usize because of in-place large
* reallocation. Therefore, query the actual value of usize.
*/
- *usize = isalloc(p, config_prof);
+ *usize = isalloc(tsd, p, config_prof);
}
prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr,
old_usize, old_tctx);
@@ -2265,10 +2290,11 @@ je_rallocx(void *ptr, size_t size, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
- arena = arena_get(arena_ind, true);
+ arena = arena_get(tsd, arena_ind, true);
if (unlikely(arena == NULL))
goto label_oom;
} else
@@ -2282,7 +2308,7 @@ je_rallocx(void *ptr, size_t size, int flags)
} else
tcache = tcache_get(tsd, true);
- old_usize = isalloc(ptr, config_prof);
+ old_usize = isalloc(tsd, ptr, config_prof);
if (config_valgrind && unlikely(in_valgrind))
old_rzsize = u2rz(old_usize);
@@ -2300,7 +2326,7 @@ je_rallocx(void *ptr, size_t size, int flags)
if (unlikely(p == NULL))
goto label_oom;
if (config_stats || (config_valgrind && unlikely(in_valgrind)))
- usize = isalloc(p, config_prof);
+ usize = isalloc(tsd, p, config_prof);
}
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
@@ -2309,8 +2335,9 @@ je_rallocx(void *ptr, size_t size, int flags)
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
UTRACE(ptr, size, p);
- JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize,
+ JEMALLOC_VALGRIND_REALLOC(true, tsd, p, usize, false, ptr, old_usize,
old_rzsize, false, zero);
+ witness_assert_lockless(tsd);
return (p);
label_oom:
if (config_xmalloc && unlikely(opt_xmalloc)) {
@@ -2318,6 +2345,7 @@ label_oom:
abort();
}
UTRACE(ptr, size, 0);
+ witness_assert_lockless(tsd);
return (NULL);
}
@@ -2329,7 +2357,7 @@ ixallocx_helper(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
if (ixalloc(tsd, ptr, old_usize, size, extra, alignment, zero))
return (old_usize);
- usize = isalloc(ptr, config_prof);
+ usize = isalloc(tsd, ptr, config_prof);
return (usize);
}
@@ -2357,7 +2385,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
prof_tctx_t *old_tctx, *tctx;
prof_active = prof_active_get_unlocked();
- old_tctx = prof_tctx_get(ptr);
+ old_tctx = prof_tctx_get(tsd, ptr);
/*
* usize isn't knowable before ixalloc() returns when extra is non-zero.
* Therefore, compute its maximum possible value and use that in
@@ -2413,8 +2441,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
- old_usize = isalloc(ptr, config_prof);
+ old_usize = isalloc(tsd, ptr, config_prof);
/*
* The API explicitly absolves itself of protecting against (size +
@@ -2449,10 +2478,11 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
*tsd_thread_allocatedp_get(tsd) += usize;
*tsd_thread_deallocatedp_get(tsd) += old_usize;
}
- JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize,
+ JEMALLOC_VALGRIND_REALLOC(false, tsd, ptr, usize, false, ptr, old_usize,
old_rzsize, false, zero);
label_not_resized:
UTRACE(ptr, size, ptr);
+ witness_assert_lockless(tsd);
return (usize);
}
@@ -2461,15 +2491,20 @@ JEMALLOC_ATTR(pure)
je_sallocx(const void *ptr, int flags)
{
size_t usize;
+ tsd_t *tsd;
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+
if (config_ivsalloc)
- usize = ivsalloc(ptr, config_prof);
+ usize = ivsalloc(tsd, ptr, config_prof);
else
- usize = isalloc(ptr, config_prof);
+ usize = isalloc(tsd, ptr, config_prof);
+ witness_assert_lockless(tsd);
return (usize);
}
@@ -2483,6 +2518,7 @@ je_dallocx(void *ptr, int flags)
assert(malloc_initialized() || IS_INITIALIZER);
tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
tcache = NULL;
@@ -2493,17 +2529,21 @@ je_dallocx(void *ptr, int flags)
UTRACE(ptr, 0, 0);
ifree(tsd_fetch(), ptr, tcache, true);
+ witness_assert_lockless(tsd);
}
JEMALLOC_ALWAYS_INLINE_C size_t
-inallocx(size_t size, int flags)
+inallocx(tsd_t *tsd, size_t size, int flags)
{
size_t usize;
+ witness_assert_lockless(tsd);
+
if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0))
usize = s2u(size);
else
usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
+ witness_assert_lockless(tsd);
return (usize);
}
@@ -2516,10 +2556,11 @@ je_sdallocx(void *ptr, size_t size, int flags)
assert(ptr != NULL);
assert(malloc_initialized() || IS_INITIALIZER);
- usize = inallocx(size, flags);
- assert(usize == isalloc(ptr, config_prof));
-
tsd = tsd_fetch();
+ usize = inallocx(tsd, size, flags);
+ assert(usize == isalloc(tsd, ptr, config_prof));
+
+ witness_assert_lockless(tsd);
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE)
tcache = NULL;
@@ -2530,6 +2571,7 @@ je_sdallocx(void *ptr, size_t size, int flags)
UTRACE(ptr, 0, 0);
isfree(tsd, ptr, usize, tcache);
+ witness_assert_lockless(tsd);
}
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
@@ -2537,16 +2579,21 @@ JEMALLOC_ATTR(pure)
je_nallocx(size_t size, int flags)
{
size_t usize;
+ tsd_t *tsd;
assert(size != 0);
if (unlikely(malloc_init()))
return (0);
- usize = inallocx(size, flags);
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+
+ usize = inallocx(tsd, size, flags);
if (unlikely(usize > HUGE_MAXCLASS))
return (0);
+ witness_assert_lockless(tsd);
return (usize);
}
@@ -2554,55 +2601,82 @@ JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t newlen)
{
+ int ret;
+ tsd_t *tsd;
if (unlikely(malloc_init()))
return (EAGAIN);
- return (ctl_byname(name, oldp, oldlenp, newp, newlen));
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+ ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen);
+ witness_assert_lockless(tsd);
+ return (ret);
}
JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
{
+ int ret;
+ tsd_t *tsd;
if (unlikely(malloc_init()))
return (EAGAIN);
- return (ctl_nametomib(name, mibp, miblenp));
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+ ret = ctl_nametomib(tsd, name, mibp, miblenp);
+ witness_assert_lockless(tsd);
+ return (ret);
}
JEMALLOC_EXPORT int JEMALLOC_NOTHROW
je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen)
{
+ int ret;
+ tsd_t *tsd;
if (unlikely(malloc_init()))
return (EAGAIN);
- return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen));
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+ ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
+ witness_assert_lockless(tsd);
+ return (ret);
}
JEMALLOC_EXPORT void JEMALLOC_NOTHROW
je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
const char *opts)
{
+ tsd_t *tsd;
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
stats_print(write_cb, cbopaque, opts);
+ witness_assert_lockless(tsd);
}
JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
{
size_t ret;
+ tsd_t *tsd;
assert(malloc_initialized() || IS_INITIALIZER);
malloc_thread_init();
+ tsd = tsd_fetch();
+ witness_assert_lockless(tsd);
+
if (config_ivsalloc)
- ret = ivsalloc(ptr, config_prof);
+ ret = ivsalloc(tsd, ptr, config_prof);
else
- ret = (ptr == NULL) ? 0 : isalloc(ptr, config_prof);
+ ret = (ptr == NULL) ? 0 : isalloc(tsd, ptr, config_prof);
+ witness_assert_lockless(tsd);
return (ret);
}
@@ -2644,6 +2718,7 @@ JEMALLOC_EXPORT void
_malloc_prefork(void)
#endif
{
+ tsd_t *tsd;
unsigned i, narenas;
#ifdef JEMALLOC_MUTEX_INIT_CB
@@ -2652,18 +2727,20 @@ _malloc_prefork(void)
#endif
assert(malloc_initialized());
+ tsd = tsd_fetch();
+
/* Acquire all mutexes in a safe order. */
- ctl_prefork();
- prof_prefork();
- malloc_mutex_prefork(&arenas_lock);
+ ctl_prefork(tsd);
+ prof_prefork(tsd);
+ malloc_mutex_prefork(tsd, &arenas_lock);
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
- if ((arena = arena_get(i, false)) != NULL)
- arena_prefork(arena);
+ if ((arena = arena_get(tsd, i, false)) != NULL)
+ arena_prefork(tsd, arena);
}
- chunk_prefork();
- base_prefork();
+ chunk_prefork(tsd);
+ base_prefork(tsd);
}
#ifndef JEMALLOC_MUTEX_INIT_CB
@@ -2674,6 +2751,7 @@ JEMALLOC_EXPORT void
_malloc_postfork(void)
#endif
{
+ tsd_t *tsd;
unsigned i, narenas;
#ifdef JEMALLOC_MUTEX_INIT_CB
@@ -2682,39 +2760,44 @@ _malloc_postfork(void)
#endif
assert(malloc_initialized());
+ tsd = tsd_fetch();
+
/* Release all mutexes, now that fork() has completed. */
- base_postfork_parent();
- chunk_postfork_parent();
+ base_postfork_parent(tsd);
+ chunk_postfork_parent(tsd);
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
- if ((arena = arena_get(i, false)) != NULL)
- arena_postfork_parent(arena);
+ if ((arena = arena_get(tsd, i, false)) != NULL)
+ arena_postfork_parent(tsd, arena);
}
- malloc_mutex_postfork_parent(&arenas_lock);
- prof_postfork_parent();
- ctl_postfork_parent();
+ malloc_mutex_postfork_parent(tsd, &arenas_lock);
+ prof_postfork_parent(tsd);
+ ctl_postfork_parent(tsd);
}
void
jemalloc_postfork_child(void)
{
+ tsd_t *tsd;
unsigned i, narenas;
assert(malloc_initialized());
+ tsd = tsd_fetch();
+
/* Release all mutexes, now that fork() has completed. */
- base_postfork_child();
- chunk_postfork_child();
+ base_postfork_child(tsd);
+ chunk_postfork_child(tsd);
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
- if ((arena = arena_get(i, false)) != NULL)
- arena_postfork_child(arena);
+ if ((arena = arena_get(tsd, i, false)) != NULL)
+ arena_postfork_child(tsd, arena);
}
- malloc_mutex_postfork_child(&arenas_lock);
- prof_postfork_child();
- ctl_postfork_child();
+ malloc_mutex_postfork_child(tsd, &arenas_lock);
+ prof_postfork_child(tsd);
+ ctl_postfork_child(tsd);
}
/******************************************************************************/
diff --git a/src/mutex.c b/src/mutex.c
index 2d47af9..4174f42 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -69,7 +69,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
#endif
bool
-malloc_mutex_init(malloc_mutex_t *mutex)
+malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
{
#ifdef _WIN32
@@ -103,31 +103,34 @@ malloc_mutex_init(malloc_mutex_t *mutex)
}
pthread_mutexattr_destroy(&attr);
#endif
+ if (config_debug)
+ witness_init(&mutex->witness, name, rank, NULL);
return (false);
}
void
-malloc_mutex_prefork(malloc_mutex_t *mutex)
+malloc_mutex_prefork(tsd_t *tsd, malloc_mutex_t *mutex)
{
- malloc_mutex_lock(mutex);
+ malloc_mutex_lock(tsd, mutex);
}
void
-malloc_mutex_postfork_parent(malloc_mutex_t *mutex)
+malloc_mutex_postfork_parent(tsd_t *tsd, malloc_mutex_t *mutex)
{
- malloc_mutex_unlock(mutex);
+ malloc_mutex_unlock(tsd, mutex);
}
void
-malloc_mutex_postfork_child(malloc_mutex_t *mutex)
+malloc_mutex_postfork_child(tsd_t *tsd, malloc_mutex_t *mutex)
{
#ifdef JEMALLOC_MUTEX_INIT_CB
- malloc_mutex_unlock(mutex);
+ malloc_mutex_unlock(tsd, mutex);
#else
- if (malloc_mutex_init(mutex)) {
+ if (malloc_mutex_init(mutex, mutex->witness.name,
+ mutex->witness.rank)) {
malloc_printf("<jemalloc>: Error re-initializing mutex in "
"child\n");
if (opt_abort)
@@ -137,7 +140,7 @@ malloc_mutex_postfork_child(malloc_mutex_t *mutex)
}
bool
-mutex_boot(void)
+malloc_mutex_boot(void)
{
#ifdef JEMALLOC_MUTEX_INIT_CB
diff --git a/src/prof.c b/src/prof.c
index b387227..520bf90 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -121,9 +121,9 @@ static bool prof_booted = false;
* definition.
*/
-static bool prof_tctx_should_destroy(prof_tctx_t *tctx);
+static bool prof_tctx_should_destroy(tsd_t *tsd, prof_tctx_t *tctx);
static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx);
-static bool prof_tdata_should_destroy(prof_tdata_t *tdata,
+static bool prof_tdata_should_destroy(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached);
static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached);
@@ -213,22 +213,23 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
}
if ((uintptr_t)tctx > (uintptr_t)1U) {
- malloc_mutex_lock(tctx->tdata->lock);
+ malloc_mutex_lock(tsd, tctx->tdata->lock);
tctx->prepared = false;
- if (prof_tctx_should_destroy(tctx))
+ if (prof_tctx_should_destroy(tsd, tctx))
prof_tctx_destroy(tsd, tctx);
else
- malloc_mutex_unlock(tctx->tdata->lock);
+ malloc_mutex_unlock(tsd, tctx->tdata->lock);
}
}
void
-prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx)
+prof_malloc_sample_object(tsd_t *tsd, const void *ptr, size_t usize,
+ prof_tctx_t *tctx)
{
- prof_tctx_set(ptr, usize, tctx);
+ prof_tctx_set(tsd, ptr, usize, tctx);
- malloc_mutex_lock(tctx->tdata->lock);
+ malloc_mutex_lock(tsd, tctx->tdata->lock);
tctx->cnts.curobjs++;
tctx->cnts.curbytes += usize;
if (opt_prof_accum) {
@@ -236,23 +237,23 @@ prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx)
tctx->cnts.accumbytes += usize;
}
tctx->prepared = false;
- malloc_mutex_unlock(tctx->tdata->lock);
+ malloc_mutex_unlock(tsd, tctx->tdata->lock);
}
void
prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx)
{
- malloc_mutex_lock(tctx->tdata->lock);
+ malloc_mutex_lock(tsd, tctx->tdata->lock);
assert(tctx->cnts.curobjs > 0);
assert(tctx->cnts.curbytes >= usize);
tctx->cnts.curobjs--;
tctx->cnts.curbytes -= usize;
- if (prof_tctx_should_destroy(tctx))
+ if (prof_tctx_should_destroy(tsd, tctx))
prof_tctx_destroy(tsd, tctx);
else
- malloc_mutex_unlock(tctx->tdata->lock);
+ malloc_mutex_unlock(tsd, tctx->tdata->lock);
}
void
@@ -277,7 +278,7 @@ prof_enter(tsd_t *tsd, prof_tdata_t *tdata)
tdata->enq = true;
}
- malloc_mutex_lock(&bt2gctx_mtx);
+ malloc_mutex_lock(tsd, &bt2gctx_mtx);
}
JEMALLOC_INLINE_C void
@@ -287,7 +288,7 @@ prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
cassert(config_prof);
assert(tdata == prof_tdata_get(tsd, false));
- malloc_mutex_unlock(&bt2gctx_mtx);
+ malloc_mutex_unlock(tsd, &bt2gctx_mtx);
if (tdata != NULL) {
bool idump, gdump;
@@ -300,9 +301,9 @@ prof_leave(tsd_t *tsd, prof_tdata_t *tdata)
tdata->enq_gdump = false;
if (idump)
- prof_idump();
+ prof_idump(tsd);
if (gdump)
- prof_gdump();
+ prof_gdump(tsd);
}
}
@@ -585,7 +586,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
* into this function.
*/
prof_enter(tsd, tdata_self);
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(tsd, gctx->lock);
assert(gctx->nlimbo != 0);
if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) {
/* Remove gctx from bt2gctx. */
@@ -593,7 +594,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
not_reached();
prof_leave(tsd, tdata_self);
/* Destroy gctx. */
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
idalloctm(tsd, gctx, tcache_get(tsd, false), true, true);
} else {
/*
@@ -601,16 +602,17 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
* prof_lookup().
*/
gctx->nlimbo--;
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
prof_leave(tsd, tdata_self);
}
}
-/* tctx->tdata->lock must be held. */
static bool
-prof_tctx_should_destroy(prof_tctx_t *tctx)
+prof_tctx_should_destroy(tsd_t *tsd, prof_tctx_t *tctx)
{
+ malloc_mutex_assert_owner(tsd, tctx->tdata->lock);
+
if (opt_prof_accum)
return (false);
if (tctx->cnts.curobjs != 0)
@@ -633,7 +635,6 @@ prof_gctx_should_destroy(prof_gctx_t *gctx)
return (true);
}
-/* tctx->tdata->lock is held upon entry, and released before return. */
static void
prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
{
@@ -641,6 +642,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
prof_gctx_t *gctx = tctx->gctx;
bool destroy_tdata, destroy_tctx, destroy_gctx;
+ malloc_mutex_assert_owner(tsd, tctx->tdata->lock);
+
assert(tctx->cnts.curobjs == 0);
assert(tctx->cnts.curbytes == 0);
assert(!opt_prof_accum);
@@ -648,10 +651,10 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
assert(tctx->cnts.accumbytes == 0);
ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL);
- destroy_tdata = prof_tdata_should_destroy(tdata, false);
- malloc_mutex_unlock(tdata->lock);
+ destroy_tdata = prof_tdata_should_destroy(tsd, tdata, false);
+ malloc_mutex_unlock(tsd, tdata->lock);
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(tsd, gctx->lock);
switch (tctx->state) {
case prof_tctx_state_nominal:
tctx_tree_remove(&gctx->tctxs, tctx);
@@ -691,12 +694,14 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
destroy_tctx = false;
destroy_gctx = false;
}
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
if (destroy_gctx) {
prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx,
tdata);
}
+ malloc_mutex_assert_not_owner(tsd, tctx->tdata->lock);
+
if (destroy_tdata)
prof_tdata_destroy(tsd, tdata, false);
@@ -740,9 +745,9 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
* Increment nlimbo, in order to avoid a race condition with
* prof_tctx_destroy()/prof_gctx_try_destroy().
*/
- malloc_mutex_lock(gctx.p->lock);
+ malloc_mutex_lock(tsd, gctx.p->lock);
gctx.p->nlimbo++;
- malloc_mutex_unlock(gctx.p->lock);
+ malloc_mutex_unlock(tsd, gctx.p->lock);
new_gctx = false;
}
prof_leave(tsd, tdata);
@@ -769,11 +774,11 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
if (tdata == NULL)
return (NULL);
- malloc_mutex_lock(tdata->lock);
+ malloc_mutex_lock(tsd, tdata->lock);
not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v);
if (!not_found) /* Note double negative! */
ret.p->prepared = true;
- malloc_mutex_unlock(tdata->lock);
+ malloc_mutex_unlock(tsd, tdata->lock);
if (not_found) {
tcache_t *tcache;
void *btkey;
@@ -806,20 +811,20 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
ret.p->tctx_uid = tdata->tctx_uid_next++;
ret.p->prepared = true;
ret.p->state = prof_tctx_state_initializing;
- malloc_mutex_lock(tdata->lock);
+ malloc_mutex_lock(tsd, tdata->lock);
error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v);
- malloc_mutex_unlock(tdata->lock);
+ malloc_mutex_unlock(tsd, tdata->lock);
if (error) {
if (new_gctx)
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
idalloctm(tsd, ret.v, tcache, true, true);
return (NULL);
}
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(tsd, gctx->lock);
ret.p->state = prof_tctx_state_nominal;
tctx_tree_insert(&gctx->tctxs, ret.p);
gctx->nlimbo--;
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
}
return (ret.p);
@@ -894,11 +899,13 @@ size_t
prof_tdata_count(void)
{
size_t tdata_count = 0;
+ tsd_t *tsd;
- malloc_mutex_lock(&tdatas_mtx);
+ tsd = tsd_fetch();
+ malloc_mutex_lock(tsd, &tdatas_mtx);
tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter,
(void *)&tdata_count);
- malloc_mutex_unlock(&tdatas_mtx);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
return (tdata_count);
}
@@ -917,9 +924,9 @@ prof_bt_count(void)
if (tdata == NULL)
return (0);
- malloc_mutex_lock(&bt2gctx_mtx);
+ malloc_mutex_lock(tsd, &bt2gctx_mtx);
bt_count = ckh_count(&bt2gctx);
- malloc_mutex_unlock(&bt2gctx_mtx);
+ malloc_mutex_unlock(tsd, &bt2gctx_mtx);
return (bt_count);
}
@@ -1032,20 +1039,21 @@ prof_dump_printf(bool propagate_err, const char *format, ...)
return (ret);
}
-/* tctx->tdata->lock is held. */
static void
-prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
+prof_tctx_merge_tdata(tsd_t *tsd, prof_tctx_t *tctx, prof_tdata_t *tdata)
{
- malloc_mutex_lock(tctx->gctx->lock);
+ malloc_mutex_assert_owner(tsd, tctx->tdata->lock);
+
+ malloc_mutex_lock(tsd, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_initializing:
- malloc_mutex_unlock(tctx->gctx->lock);
+ malloc_mutex_unlock(tsd, tctx->gctx->lock);
return;
case prof_tctx_state_nominal:
tctx->state = prof_tctx_state_dumping;
- malloc_mutex_unlock(tctx->gctx->lock);
+ malloc_mutex_unlock(tsd, tctx->gctx->lock);
memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t));
@@ -1064,11 +1072,12 @@ prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata)
}
}
-/* gctx->lock is held. */
static void
-prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
+prof_tctx_merge_gctx(tsd_t *tsd, prof_tctx_t *tctx, prof_gctx_t *gctx)
{
+ malloc_mutex_assert_owner(tsd, gctx->lock);
+
gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs;
gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes;
if (opt_prof_accum) {
@@ -1077,10 +1086,12 @@ prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx)
}
}
-/* tctx->gctx is held. */
static prof_tctx_t *
prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
{
+ tsd_t *tsd = (tsd_t *)arg;
+
+ malloc_mutex_assert_owner(tsd, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_nominal:
@@ -1088,7 +1099,7 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
break;
case prof_tctx_state_dumping:
case prof_tctx_state_purgatory:
- prof_tctx_merge_gctx(tctx, tctx->gctx);
+ prof_tctx_merge_gctx(tsd, tctx, tctx->gctx);
break;
default:
not_reached();
@@ -1097,11 +1108,18 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
return (NULL);
}
-/* gctx->lock is held. */
+struct prof_tctx_dump_iter_arg_s {
+ tsd_t *tsd;
+ bool propagate_err;
+};
+
static prof_tctx_t *
-prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
+prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *opaque)
{
- bool propagate_err = *(bool *)arg;
+ struct prof_tctx_dump_iter_arg_s *arg =
+ (struct prof_tctx_dump_iter_arg_s *)opaque;
+
+ malloc_mutex_assert_owner(arg->tsd, tctx->gctx->lock);
switch (tctx->state) {
case prof_tctx_state_initializing:
@@ -1110,7 +1128,7 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
break;
case prof_tctx_state_dumping:
case prof_tctx_state_purgatory:
- if (prof_dump_printf(propagate_err,
+ if (prof_dump_printf(arg->propagate_err,
" t%"FMTu64": %"FMTu64": %"FMTu64" [%"FMTu64": "
"%"FMTu64"]\n", tctx->thr_uid, tctx->dump_cnts.curobjs,
tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs,
@@ -1123,12 +1141,14 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
return (NULL);
}
-/* tctx->gctx is held. */
static prof_tctx_t *
prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg)
{
+ tsd_t *tsd = (tsd_t *)arg;
prof_tctx_t *ret;
+ malloc_mutex_assert_owner(tsd, tctx->gctx->lock);
+
switch (tctx->state) {
case prof_tctx_state_nominal:
/* New since dumping started; ignore. */
@@ -1149,12 +1169,12 @@ label_return:
}
static void
-prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
+prof_dump_gctx_prep(tsd_t *tsd, prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
{
cassert(config_prof);
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(tsd, gctx->lock);
/*
* Increment nlimbo so that gctx won't go away before dump.
@@ -1166,19 +1186,26 @@ prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs)
memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t));
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
}
+struct prof_gctx_merge_iter_arg_s {
+ tsd_t *tsd;
+ size_t leak_ngctx;
+};
+
static prof_gctx_t *
-prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
+prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque)
{
- size_t *leak_ngctx = (size_t *)arg;
+ struct prof_gctx_merge_iter_arg_s *arg =
+ (struct prof_gctx_merge_iter_arg_s *)opaque;
- malloc_mutex_lock(gctx->lock);
- tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL);
+ malloc_mutex_lock(arg->tsd, gctx->lock);
+ tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter,
+ (void *)arg->tsd);
if (gctx->cnt_summed.curobjs != 0)
- (*leak_ngctx)++;
- malloc_mutex_unlock(gctx->lock);
+ arg->leak_ngctx++;
+ malloc_mutex_unlock(arg->tsd, gctx->lock);
return (NULL);
}
@@ -1197,7 +1224,7 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
*/
while ((gctx = gctx_tree_first(gctxs)) != NULL) {
gctx_tree_remove(gctxs, gctx);
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(tsd, gctx->lock);
{
prof_tctx_t *next;
@@ -1205,7 +1232,7 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
do {
prof_tctx_t *to_destroy =
tctx_tree_iter(&gctx->tctxs, next,
- prof_tctx_finish_iter, NULL);
+ prof_tctx_finish_iter, (void *)tsd);
if (to_destroy != NULL) {
next = tctx_tree_next(&gctx->tctxs,
to_destroy);
@@ -1220,19 +1247,26 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
gctx->nlimbo--;
if (prof_gctx_should_destroy(gctx)) {
gctx->nlimbo++;
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
} else
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(tsd, gctx->lock);
}
}
+struct prof_tdata_merge_iter_arg_s {
+ tsd_t *tsd;
+ prof_cnt_t cnt_all;
+};
+
static prof_tdata_t *
-prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
+prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata,
+ void *opaque)
{
- prof_cnt_t *cnt_all = (prof_cnt_t *)arg;
+ struct prof_tdata_merge_iter_arg_s *arg =
+ (struct prof_tdata_merge_iter_arg_s *)opaque;
- malloc_mutex_lock(tdata->lock);
+ malloc_mutex_lock(arg->tsd, tdata->lock);
if (!tdata->expired) {
size_t tabind;
union {
@@ -1244,17 +1278,17 @@ prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t));
for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL,
&tctx.v);)
- prof_tctx_merge_tdata(tctx.p, tdata);
+ prof_tctx_merge_tdata(arg->tsd, tctx.p, tdata);
- cnt_all->curobjs += tdata->cnt_summed.curobjs;
- cnt_all->curbytes += tdata->cnt_summed.curbytes;
+ arg->cnt_all.curobjs += tdata->cnt_summed.curobjs;
+ arg->cnt_all.curbytes += tdata->cnt_summed.curbytes;
if (opt_prof_accum) {
- cnt_all->accumobjs += tdata->cnt_summed.accumobjs;
- cnt_all->accumbytes += tdata->cnt_summed.accumbytes;
+ arg->cnt_all.accumobjs += tdata->cnt_summed.accumobjs;
+ arg->cnt_all.accumbytes += tdata->cnt_summed.accumbytes;
}
} else
tdata->dumping = false;
- malloc_mutex_unlock(tdata->lock);
+ malloc_mutex_unlock(arg->tsd, tdata->lock);
return (NULL);
}
@@ -1283,7 +1317,7 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
#define prof_dump_header JEMALLOC_N(prof_dump_header_impl)
#endif
static bool
-prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
+prof_dump_header(tsd_t *tsd, bool propagate_err, const prof_cnt_t *cnt_all)
{
bool ret;
@@ -1294,10 +1328,10 @@ prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes))
return (true);
- malloc_mutex_lock(&tdatas_mtx);
+ malloc_mutex_lock(tsd, &tdatas_mtx);
ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter,
(void *)&propagate_err) != NULL);
- malloc_mutex_unlock(&tdatas_mtx);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
return (ret);
}
#ifdef JEMALLOC_JET
@@ -1306,15 +1340,16 @@ prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all)
prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl);
#endif
-/* gctx->lock is held. */
static bool
-prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
- prof_gctx_tree_t *gctxs)
+prof_dump_gctx(tsd_t *tsd, bool propagate_err, prof_gctx_t *gctx,
+ const prof_bt_t *bt, prof_gctx_tree_t *gctxs)
{
bool ret;
unsigned i;
+ struct prof_tctx_dump_iter_arg_s prof_tctx_dump_iter_arg;
cassert(config_prof);
+ malloc_mutex_assert_owner(tsd, gctx->lock);
/* Avoid dumping such gctx's that have no useful data. */
if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) ||
@@ -1348,8 +1383,10 @@ prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt,
goto label_return;
}
+ prof_tctx_dump_iter_arg.tsd = tsd;
+ prof_tctx_dump_iter_arg.propagate_err = propagate_err;
if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter,
- (void *)&propagate_err) != NULL) {
+ (void *)&prof_tctx_dump_iter_arg) != NULL) {
ret = true;
goto label_return;
}
@@ -1459,22 +1496,29 @@ prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx,
}
}
+struct prof_gctx_dump_iter_arg_s {
+ tsd_t *tsd;
+ bool propagate_err;
+};
+
static prof_gctx_t *
-prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg)
+prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *opaque)
{
prof_gctx_t *ret;
- bool propagate_err = *(bool *)arg;
+ struct prof_gctx_dump_iter_arg_s *arg =
+ (struct prof_gctx_dump_iter_arg_s *)opaque;
- malloc_mutex_lock(gctx->lock);
+ malloc_mutex_lock(arg->tsd, gctx->lock);
- if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) {
+ if (prof_dump_gctx(arg->tsd, arg->propagate_err, gctx, &gctx->bt,
+ gctxs)) {
ret = gctx;
goto label_return;
}
ret = NULL;
label_return:
- malloc_mutex_unlock(gctx->lock);
+ malloc_mutex_unlock(arg->tsd, gctx->lock);
return (ret);
}
@@ -1482,13 +1526,14 @@ static bool
prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
{
prof_tdata_t *tdata;
- prof_cnt_t cnt_all;
+ struct prof_tdata_merge_iter_arg_s prof_tdata_merge_iter_arg;
size_t tabind;
union {
prof_gctx_t *p;
void *v;
} gctx;
- size_t leak_ngctx;
+ struct prof_gctx_merge_iter_arg_s prof_gctx_merge_iter_arg;
+ struct prof_gctx_dump_iter_arg_s prof_gctx_dump_iter_arg;
prof_gctx_tree_t gctxs;
cassert(config_prof);
@@ -1497,7 +1542,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
if (tdata == NULL)
return (true);
- malloc_mutex_lock(&prof_dump_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_mtx);
prof_enter(tsd, tdata);
/*
@@ -1506,20 +1551,24 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
*/
gctx_tree_new(&gctxs);
for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);)
- prof_dump_gctx_prep(gctx.p, &gctxs);
+ prof_dump_gctx_prep(tsd, gctx.p, &gctxs);
/*
* Iterate over tdatas, and for the non-expired ones snapshot their tctx
* stats and merge them into the associated gctx's.
*/
- memset(&cnt_all, 0, sizeof(prof_cnt_t));
- malloc_mutex_lock(&tdatas_mtx);
- tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all);
- malloc_mutex_unlock(&tdatas_mtx);
+ prof_tdata_merge_iter_arg.tsd = tsd;
+ memset(&prof_tdata_merge_iter_arg.cnt_all, 0, sizeof(prof_cnt_t));
+ malloc_mutex_lock(tsd, &tdatas_mtx);
+ tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter,
+ (void *)&prof_tdata_merge_iter_arg);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
/* Merge tctx stats into gctx's. */
- leak_ngctx = 0;
- gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx);
+ prof_gctx_merge_iter_arg.tsd = tsd;
+ prof_gctx_merge_iter_arg.leak_ngctx = 0;
+ gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter,
+ (void *)&prof_gctx_merge_iter_arg);
prof_leave(tsd, tdata);
@@ -1528,12 +1577,15 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
goto label_open_close_error;
/* Dump profile header. */
- if (prof_dump_header(propagate_err, &cnt_all))
+ if (prof_dump_header(tsd, propagate_err,
+ &prof_tdata_merge_iter_arg.cnt_all))
goto label_write_error;
/* Dump per gctx profile stats. */
+ prof_gctx_dump_iter_arg.tsd = tsd;
+ prof_gctx_dump_iter_arg.propagate_err = propagate_err;
if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter,
- (void *)&propagate_err) != NULL)
+ (void *)&prof_gctx_dump_iter_arg) != NULL)
goto label_write_error;
/* Dump /proc/<pid>/maps if possible. */
@@ -1544,17 +1596,18 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck)
goto label_open_close_error;
prof_gctx_finish(tsd, &gctxs);
- malloc_mutex_unlock(&prof_dump_mtx);
-
- if (leakcheck)
- prof_leakcheck(&cnt_all, leak_ngctx, filename);
+ malloc_mutex_unlock(tsd, &prof_dump_mtx);
+ if (leakcheck) {
+ prof_leakcheck(&prof_tdata_merge_iter_arg.cnt_all,
+ prof_gctx_merge_iter_arg.leak_ngctx, filename);
+ }
return (false);
label_write_error:
prof_dump_close(propagate_err);
label_open_close_error:
prof_gctx_finish(tsd, &gctxs);
- malloc_mutex_unlock(&prof_dump_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_mtx);
return (true);
}
@@ -1594,23 +1647,21 @@ prof_fdump(void)
return;
tsd = tsd_fetch();
- malloc_mutex_lock(&prof_dump_seq_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_seq_mtx);
prof_dump_filename(filename, 'f', VSEQ_INVALID);
- malloc_mutex_unlock(&prof_dump_seq_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, opt_prof_leak);
}
void
-prof_idump(void)
+prof_idump(tsd_t *tsd)
{
- tsd_t *tsd;
prof_tdata_t *tdata;
cassert(config_prof);
- if (!prof_booted)
+ if (!prof_booted || tsd == NULL)
return;
- tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return;
@@ -1621,50 +1672,46 @@ prof_idump(void)
if (opt_prof_prefix[0] != '\0') {
char filename[PATH_MAX + 1];
- malloc_mutex_lock(&prof_dump_seq_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_seq_mtx);
prof_dump_filename(filename, 'i', prof_dump_iseq);
prof_dump_iseq++;
- malloc_mutex_unlock(&prof_dump_seq_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, false);
}
}
bool
-prof_mdump(const char *filename)
+prof_mdump(tsd_t *tsd, const char *filename)
{
- tsd_t *tsd;
char filename_buf[DUMP_FILENAME_BUFSIZE];
cassert(config_prof);
if (!opt_prof || !prof_booted)
return (true);
- tsd = tsd_fetch();
if (filename == NULL) {
/* No filename specified, so automatically generate one. */
if (opt_prof_prefix[0] == '\0')
return (true);
- malloc_mutex_lock(&prof_dump_seq_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_seq_mtx);
prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
prof_dump_mseq++;
- malloc_mutex_unlock(&prof_dump_seq_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_seq_mtx);
filename = filename_buf;
}
return (prof_dump(tsd, true, filename, false));
}
void
-prof_gdump(void)
+prof_gdump(tsd_t *tsd)
{
- tsd_t *tsd;
prof_tdata_t *tdata;
cassert(config_prof);
- if (!prof_booted)
+ if (!prof_booted || tsd == NULL)
return;
- tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, false);
if (tdata == NULL)
return;
@@ -1675,10 +1722,10 @@ prof_gdump(void)
if (opt_prof_prefix[0] != '\0') {
char filename[DUMP_FILENAME_BUFSIZE];
- malloc_mutex_lock(&prof_dump_seq_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_seq_mtx);
prof_dump_filename(filename, 'u', prof_dump_useq);
prof_dump_useq++;
- malloc_mutex_unlock(&prof_dump_seq_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_seq_mtx);
prof_dump(tsd, false, filename, false);
}
}
@@ -1707,14 +1754,14 @@ prof_bt_keycomp(const void *k1, const void *k2)
}
JEMALLOC_INLINE_C uint64_t
-prof_thr_uid_alloc(void)
+prof_thr_uid_alloc(tsd_t *tsd)
{
uint64_t thr_uid;
- malloc_mutex_lock(&next_thr_uid_mtx);
+ malloc_mutex_lock(tsd, &next_thr_uid_mtx);
thr_uid = next_thr_uid;
next_thr_uid++;
- malloc_mutex_unlock(&next_thr_uid_mtx);
+ malloc_mutex_unlock(tsd, &next_thr_uid_mtx);
return (thr_uid);
}
@@ -1759,9 +1806,9 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim,
tdata->dumping = false;
tdata->active = active;
- malloc_mutex_lock(&tdatas_mtx);
+ malloc_mutex_lock(tsd, &tdatas_mtx);
tdata_tree_insert(&tdatas, tdata);
- malloc_mutex_unlock(&tdatas_mtx);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
return (tdata);
}
@@ -1770,13 +1817,13 @@ prof_tdata_t *
prof_tdata_init(tsd_t *tsd)
{
- return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL,
- prof_thread_active_init_get()));
+ return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(tsd), 0, NULL,
+ prof_thread_active_init_get(tsd)));
}
-/* tdata->lock must be held. */
static bool
-prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
+prof_tdata_should_destroy_unlocked(tsd_t *tsd, prof_tdata_t *tdata,
+ bool even_if_attached)
{
if (tdata->attached && !even_if_attached)
@@ -1786,18 +1833,32 @@ prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached)
return (true);
}
-/* tdatas_mtx must be held. */
+static bool
+prof_tdata_should_destroy(tsd_t *tsd, prof_tdata_t *tdata,
+ bool even_if_attached)
+{
+
+ malloc_mutex_assert_owner(tsd, tdata->lock);
+
+ return (prof_tdata_should_destroy_unlocked(tsd, tdata,
+ even_if_attached));
+}
+
static void
prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata,
bool even_if_attached)
{
tcache_t *tcache;
- assert(prof_tdata_should_destroy(tdata, even_if_attached));
+ malloc_mutex_assert_owner(tsd, &tdatas_mtx);
+
assert(tsd_prof_tdata_get(tsd) != tdata);
tdata_tree_remove(&tdatas, tdata);
+ assert(prof_tdata_should_destroy_unlocked(tsd, tdata,
+ even_if_attached));
+
tcache = tcache_get(tsd, false);
if (tdata->thread_name != NULL)
idalloctm(tsd, tdata->thread_name, tcache, true, true);
@@ -1809,9 +1870,9 @@ static void
prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached)
{
- malloc_mutex_lock(&tdatas_mtx);
+ malloc_mutex_lock(tsd, &tdatas_mtx);
prof_tdata_destroy_locked(tsd, tdata, even_if_attached);
- malloc_mutex_unlock(&tdatas_mtx);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
}
static void
@@ -1819,9 +1880,9 @@ prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
{
bool destroy_tdata;
- malloc_mutex_lock(tdata->lock);
+ malloc_mutex_lock(tsd, tdata->lock);
if (tdata->attached) {
- destroy_tdata = prof_tdata_should_destroy(tdata, true);
+ destroy_tdata = prof_tdata_should_destroy(tsd, tdata, true);
/*
* Only detach if !destroy_tdata, because detaching would allow
* another thread to win the race to destroy tdata.
@@ -1831,7 +1892,7 @@ prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata)
tsd_prof_tdata_set(tsd, NULL);
} else
destroy_tdata = false;
- malloc_mutex_unlock(tdata->lock);
+ malloc_mutex_unlock(tsd, tdata->lock);
if (destroy_tdata)
prof_tdata_destroy(tsd, tdata, true);
}
@@ -1851,18 +1912,18 @@ prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata)
}
static bool
-prof_tdata_expire(prof_tdata_t *tdata)
+prof_tdata_expire(tsd_t *tsd, prof_tdata_t *tdata)
{
bool destroy_tdata;
- malloc_mutex_lock(tdata->lock);
+ malloc_mutex_lock(tsd, tdata->lock);
if (!tdata->expired) {
tdata->expired = true;
destroy_tdata = tdata->attached ? false :
- prof_tdata_should_destroy(tdata, false);
+ prof_tdata_should_destroy(tsd, tdata, false);
} else
destroy_tdata = false;
- malloc_mutex_unlock(tdata->lock);
+ malloc_mutex_unlock(tsd, tdata->lock);
return (destroy_tdata);
}
@@ -1870,8 +1931,9 @@ prof_tdata_expire(prof_tdata_t *tdata)
static prof_tdata_t *
prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg)
{
+ tsd_t *tsd = (tsd_t *)arg;
- return (prof_tdata_expire(tdata) ? tdata : NULL);
+ return (prof_tdata_expire(tsd, tdata) ? tdata : NULL);
}
void
@@ -1881,15 +1943,15 @@ prof_reset(tsd_t *tsd, size_t lg_sample)
assert(lg_sample < (sizeof(uint64_t) << 3));
- malloc_mutex_lock(&prof_dump_mtx);
- malloc_mutex_lock(&tdatas_mtx);
+ malloc_mutex_lock(tsd, &prof_dump_mtx);
+ malloc_mutex_lock(tsd, &tdatas_mtx);
lg_prof_sample = lg_sample;
next = NULL;
do {
prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next,
- prof_tdata_reset_iter, NULL);
+ prof_tdata_reset_iter, (void *)tsd);
if (to_destroy != NULL) {
next = tdata_tree_next(&tdatas, to_destroy);
prof_tdata_destroy_locked(tsd, to_destroy, false);
@@ -1897,8 +1959,8 @@ prof_reset(tsd_t *tsd, size_t lg_sample)
next = NULL;
} while (next != NULL);
- malloc_mutex_unlock(&tdatas_mtx);
- malloc_mutex_unlock(&prof_dump_mtx);
+ malloc_mutex_unlock(tsd, &tdatas_mtx);
+ malloc_mutex_unlock(tsd, &prof_dump_mtx);
}
void
@@ -1915,35 +1977,33 @@ prof_tdata_cleanup(tsd_t *tsd)
}
bool
-prof_active_get(void)
+prof_active_get(tsd_t *tsd)
{
bool prof_active_current;
- malloc_mutex_lock(&prof_active_mtx);
+ malloc_mutex_lock(tsd, &prof_active_mtx);
prof_active_current = prof_active;
- malloc_mutex_unlock(&prof_active_mtx);
+ malloc_mutex_unlock(tsd, &prof_active_mtx);
return (prof_active_current);
}
bool
-prof_active_set(bool active)
+prof_active_set(tsd_t *tsd, bool active)
{
bool prof_active_old;
- malloc_mutex_lock(&prof_active_mtx);
+ malloc_mutex_lock(tsd, &prof_active_mtx);
prof_active_old = prof_active;
prof_active = active;
- malloc_mutex_unlock(&prof_active_mtx);
+ malloc_mutex_unlock(tsd, &prof_active_mtx);
return (prof_active_old);
}
const char *
-prof_thread_name_get(void)
+prof_thread_name_get(tsd_t *tsd)
{
- tsd_t *tsd;
prof_tdata_t *tdata;
- tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return ("");
@@ -2006,12 +2066,10 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name)
}
bool
-prof_thread_active_get(void)
+prof_thread_active_get(tsd_t *tsd)
{
- tsd_t *tsd;
prof_tdata_t *tdata;
- tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (false);
@@ -2019,12 +2077,10 @@ prof_thread_active_get(void)
}
bool
-prof_thread_active_set(bool active)
+prof_thread_active_set(tsd_t *tsd, bool active)
{
- tsd_t *tsd;
prof_tdata_t *tdata;
- tsd = tsd_fetch();
tdata = prof_tdata_get(tsd, true);
if (tdata == NULL)
return (true);
@@ -2033,48 +2089,48 @@ prof_thread_active_set(bool active)
}
bool
-prof_thread_active_init_get(void)
+prof_thread_active_init_get(tsd_t *tsd)
{
bool active_init;
- malloc_mutex_lock(&prof_thread_active_init_mtx);
+ malloc_mutex_lock(tsd, &prof_thread_active_init_mtx);
active_init = prof_thread_active_init;
- malloc_mutex_unlock(&prof_thread_active_init_mtx);
+ malloc_mutex_unlock(tsd, &prof_thread_active_init_mtx);
return (active_init);
}
bool
-prof_thread_active_init_set(bool active_init)
+prof_thread_active_init_set(tsd_t *tsd, bool active_init)
{
bool active_init_old;
- malloc_mutex_lock(&prof_thread_active_init_mtx);
+ malloc_mutex_lock(tsd, &prof_thread_active_init_mtx);
active_init_old = prof_thread_active_init;
prof_thread_active_init = active_init;
- malloc_mutex_unlock(&prof_thread_active_init_mtx);
+ malloc_mutex_unlock(tsd, &prof_thread_active_init_mtx);
return (active_init_old);
}
bool
-prof_gdump_get(void)
+prof_gdump_get(tsd_t *tsd)
{
bool prof_gdump_current;
- malloc_mutex_lock(&prof_gdump_mtx);
+ malloc_mutex_lock(tsd, &prof_gdump_mtx);
prof_gdump_current = prof_gdump_val;
- malloc_mutex_unlock(&prof_gdump_mtx);
+ malloc_mutex_unlock(tsd, &prof_gdump_mtx);
return (prof_gdump_current);
}
bool
-prof_gdump_set(bool gdump)
+prof_gdump_set(tsd_t *tsd, bool gdump)
{
bool prof_gdump_old;
- malloc_mutex_lock(&prof_gdump_mtx);
+ malloc_mutex_lock(tsd, &prof_gdump_mtx);
prof_gdump_old = prof_gdump_val;
prof_gdump_val = gdump;
- malloc_mutex_unlock(&prof_gdump_mtx);
+ malloc_mutex_unlock(tsd, &prof_gdump_mtx);
return (prof_gdump_old);
}
@@ -2115,47 +2171,54 @@ prof_boot1(void)
}
bool
-prof_boot2(void)
+prof_boot2(tsd_t *tsd)
{
cassert(config_prof);
if (opt_prof) {
- tsd_t *tsd;
unsigned i;
lg_prof_sample = opt_lg_prof_sample;
prof_active = opt_prof_active;
- if (malloc_mutex_init(&prof_active_mtx))
+ if (malloc_mutex_init(&prof_active_mtx, "prof_active",
+ WITNESS_RANK_PROF_ACTIVE))
return (true);
prof_gdump_val = opt_prof_gdump;
- if (malloc_mutex_init(&prof_gdump_mtx))
+ if (malloc_mutex_init(&prof_gdump_mtx, "prof_gdump",
+ WITNESS_RANK_PROF_GDUMP))
return (true);
prof_thread_active_init = opt_prof_thread_active_init;
- if (malloc_mutex_init(&prof_thread_active_init_mtx))
+ if (malloc_mutex_init(&prof_thread_active_init_mtx,
+ "prof_thread_active_init",
+ WITNESS_RANK_PROF_THREAD_ACTIVE_INIT))
return (true);
- tsd = tsd_fetch();
if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash,
prof_bt_keycomp))
return (true);
- if (malloc_mutex_init(&bt2gctx_mtx))
+ if (malloc_mutex_init(&bt2gctx_mtx, "prof_bt2gctx",
+ WITNESS_RANK_PROF_BT2GCTX))
return (true);
tdata_tree_new(&tdatas);
- if (malloc_mutex_init(&tdatas_mtx))
+ if (malloc_mutex_init(&tdatas_mtx, "prof_tdatas",
+ WITNESS_RANK_PROF_TDATAS))
return (true);
next_thr_uid = 0;
- if (malloc_mutex_init(&next_thr_uid_mtx))
+ if (malloc_mutex_init(&next_thr_uid_mtx, "prof_next_thr_uid",
+ WITNESS_RANK_PROF_NEXT_THR_UID))
return (true);
- if (malloc_mutex_init(&prof_dump_seq_mtx))
+ if (malloc_mutex_init(&prof_dump_seq_mtx, "prof_dump_seq",
+ WITNESS_RANK_PROF_DUMP_SEQ))
return (true);
- if (malloc_mutex_init(&prof_dump_mtx))
+ if (malloc_mutex_init(&prof_dump_mtx, "prof_dump",
+ WITNESS_RANK_PROF_DUMP))
return (true);
if (opt_prof_final && opt_prof_prefix[0] != '\0' &&
@@ -2165,21 +2228,23 @@ prof_boot2(void)
abort();
}
- gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS *
+ gctx_locks = (malloc_mutex_t *)base_alloc(tsd, PROF_NCTX_LOCKS *
sizeof(malloc_mutex_t));
if (gctx_locks == NULL)
return (true);
for (i = 0; i < PROF_NCTX_LOCKS; i++) {
- if (malloc_mutex_init(&gctx_locks[i]))
+ if (malloc_mutex_init(&gctx_locks[i], "prof_gctx",
+ WITNESS_RANK_PROF_GCTX))
return (true);
}
- tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS *
- sizeof(malloc_mutex_t));
+ tdata_locks = (malloc_mutex_t *)base_alloc(tsd,
+ PROF_NTDATA_LOCKS * sizeof(malloc_mutex_t));
if (tdata_locks == NULL)
return (true);
for (i = 0; i < PROF_NTDATA_LOCKS; i++) {
- if (malloc_mutex_init(&tdata_locks[i]))
+ if (malloc_mutex_init(&tdata_locks[i], "prof_tdata",
+ WITNESS_RANK_PROF_TDATA))
return (true);
}
}
@@ -2198,56 +2263,56 @@ prof_boot2(void)
}
void
-prof_prefork(void)
+prof_prefork(tsd_t *tsd)
{
if (opt_prof) {
unsigned i;
- malloc_mutex_prefork(&tdatas_mtx);
- malloc_mutex_prefork(&bt2gctx_mtx);
- malloc_mutex_prefork(&next_thr_uid_mtx);
- malloc_mutex_prefork(&prof_dump_seq_mtx);
+ malloc_mutex_prefork(tsd, &tdatas_mtx);
+ malloc_mutex_prefork(tsd, &bt2gctx_mtx);
+ malloc_mutex_prefork(tsd, &next_thr_uid_mtx);
+ malloc_mutex_prefork(tsd, &prof_dump_seq_mtx);
for (i = 0; i < PROF_NCTX_LOCKS; i++)
- malloc_mutex_prefork(&gctx_locks[i]);
+ malloc_mutex_prefork(tsd, &gctx_locks[i]);
for (i = 0; i < PROF_NTDATA_LOCKS; i++)
- malloc_mutex_prefork(&tdata_locks[i]);
+ malloc_mutex_prefork(tsd, &tdata_locks[i]);
}
}
void
-prof_postfork_parent(void)
+prof_postfork_parent(tsd_t *tsd)
{
if (opt_prof) {
unsigned i;
for (i = 0; i < PROF_NTDATA_LOCKS; i++)
- malloc_mutex_postfork_parent(&tdata_locks[i]);
+ malloc_mutex_postfork_parent(tsd, &tdata_locks[i]);
for (i = 0; i < PROF_NCTX_LOCKS; i++)
- malloc_mutex_postfork_parent(&gctx_locks[i]);
- malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
- malloc_mutex_postfork_parent(&next_thr_uid_mtx);
- malloc_mutex_postfork_parent(&bt2gctx_mtx);
- malloc_mutex_postfork_parent(&tdatas_mtx);
+ malloc_mutex_postfork_parent(tsd, &gctx_locks[i]);
+ malloc_mutex_postfork_parent(tsd, &prof_dump_seq_mtx);
+ malloc_mutex_postfork_parent(tsd, &next_thr_uid_mtx);
+ malloc_mutex_postfork_parent(tsd, &bt2gctx_mtx);
+ malloc_mutex_postfork_parent(tsd, &tdatas_mtx);
}
}
void
-prof_postfork_child(void)
+prof_postfork_child(tsd_t *tsd)
{
if (opt_prof) {
unsigned i;
for (i = 0; i < PROF_NTDATA_LOCKS; i++)
- malloc_mutex_postfork_child(&tdata_locks[i]);
+ malloc_mutex_postfork_child(tsd, &tdata_locks[i]);
for (i = 0; i < PROF_NCTX_LOCKS; i++)
- malloc_mutex_postfork_child(&gctx_locks[i]);
- malloc_mutex_postfork_child(&prof_dump_seq_mtx);
- malloc_mutex_postfork_child(&next_thr_uid_mtx);
- malloc_mutex_postfork_child(&bt2gctx_mtx);
- malloc_mutex_postfork_child(&tdatas_mtx);
+ malloc_mutex_postfork_child(tsd, &gctx_locks[i]);
+ malloc_mutex_postfork_child(tsd, &prof_dump_seq_mtx);
+ malloc_mutex_postfork_child(tsd, &next_thr_uid_mtx);
+ malloc_mutex_postfork_child(tsd, &bt2gctx_mtx);
+ malloc_mutex_postfork_child(tsd, &tdatas_mtx);
}
}
diff --git a/src/quarantine.c b/src/quarantine.c
index c024dea..6cb74b3 100644
--- a/src/quarantine.c
+++ b/src/quarantine.c
@@ -99,7 +99,7 @@ static void
quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine)
{
quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
- assert(obj->usize == isalloc(obj->ptr, config_prof));
+ assert(obj->usize == isalloc(tsd, obj->ptr, config_prof));
idalloctm(tsd, obj->ptr, NULL, false, true);
quarantine->curbytes -= obj->usize;
quarantine->curobjs--;
@@ -119,7 +119,7 @@ void
quarantine(tsd_t *tsd, void *ptr)
{
quarantine_t *quarantine;
- size_t usize = isalloc(ptr, config_prof);
+ size_t usize = isalloc(tsd, ptr, config_prof);
cassert(config_fill);
assert(opt_quarantine);
diff --git a/src/tcache.c b/src/tcache.c
index a8620c3..a9539f6 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -24,10 +24,10 @@ static tcaches_t *tcaches_avail;
/******************************************************************************/
size_t
-tcache_salloc(const void *ptr)
+tcache_salloc(tsd_t *tsd, const void *ptr)
{
- return (arena_salloc(ptr, false));
+ return (arena_salloc(tsd, ptr, false));
}
void
@@ -107,12 +107,13 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
arena_bin_t *bin = &bin_arena->bins[binind];
if (config_prof && bin_arena == arena) {
- if (arena_prof_accum(arena, tcache->prof_accumbytes))
- prof_idump();
+ if (arena_prof_accum(tsd, arena,
+ tcache->prof_accumbytes))
+ prof_idump(tsd);
tcache->prof_accumbytes = 0;
}
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
if (config_stats && bin_arena == arena) {
assert(!merged_stats);
merged_stats = true;
@@ -130,8 +131,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
(uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_bits_t *bitselm =
arena_bitselm_get_mutable(chunk, pageind);
- arena_dalloc_bin_junked_locked(bin_arena, chunk,
- ptr, bitselm);
+ arena_dalloc_bin_junked_locked(tsd, bin_arena,
+ chunk, ptr, bitselm);
} else {
/*
* This object was allocated via a different
@@ -143,7 +144,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
ndeferred++;
}
}
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
arena_decay_ticks(tsd, bin_arena, nflush - ndeferred);
}
if (config_stats && !merged_stats) {
@@ -152,11 +153,11 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
* arena, so the stats didn't get merged. Manually do so now.
*/
arena_bin_t *bin = &arena->bins[binind];
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
bin->stats.nflushes++;
bin->stats.nrequests += tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
}
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
@@ -189,7 +190,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
if (config_prof)
idump = false;
- malloc_mutex_lock(&locked_arena->lock);
+ malloc_mutex_lock(tsd, &locked_arena->lock);
if ((config_prof || config_stats) && locked_arena == arena) {
if (config_prof) {
idump = arena_prof_accum_locked(arena,
@@ -212,8 +213,8 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (extent_node_arena_get(&chunk->node) ==
locked_arena) {
- arena_dalloc_large_junked_locked(locked_arena,
- chunk, ptr);
+ arena_dalloc_large_junked_locked(tsd,
+ locked_arena, chunk, ptr);
} else {
/*
* This object was allocated via a different
@@ -225,9 +226,9 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
ndeferred++;
}
}
- malloc_mutex_unlock(&locked_arena->lock);
+ malloc_mutex_unlock(tsd, &locked_arena->lock);
if (config_prof && idump)
- prof_idump();
+ prof_idump(tsd);
arena_decay_ticks(tsd, locked_arena, nflush - ndeferred);
}
if (config_stats && !merged_stats) {
@@ -235,12 +236,12 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
* The flush loop didn't happen to flush to this thread's
* arena, so the stats didn't get merged. Manually do so now.
*/
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena->stats.nrequests_large += tbin->tstats.nrequests;
arena->stats.lstats[binind - NBINS].nrequests +=
tbin->tstats.nrequests;
tbin->tstats.nrequests = 0;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
@@ -251,33 +252,34 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
}
void
-tcache_arena_associate(tcache_t *tcache, arena_t *arena)
+tcache_arena_associate(tsd_t *tsd, tcache_t *tcache, arena_t *arena)
{
if (config_stats) {
/* Link into list of extant tcaches. */
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
ql_elm_new(tcache, link);
ql_tail_insert(&arena->tcache_ql, tcache, link);
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
}
void
-tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena, arena_t *newarena)
+tcache_arena_reassociate(tsd_t *tsd, tcache_t *tcache, arena_t *oldarena,
+ arena_t *newarena)
{
- tcache_arena_dissociate(tcache, oldarena);
- tcache_arena_associate(tcache, newarena);
+ tcache_arena_dissociate(tsd, tcache, oldarena);
+ tcache_arena_associate(tsd, tcache, newarena);
}
void
-tcache_arena_dissociate(tcache_t *tcache, arena_t *arena)
+tcache_arena_dissociate(tsd_t *tsd, tcache_t *tcache, arena_t *arena)
{
if (config_stats) {
/* Unlink from list of extant tcaches. */
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
if (config_debug) {
bool in_ql = false;
tcache_t *iter;
@@ -290,8 +292,8 @@ tcache_arena_dissociate(tcache_t *tcache, arena_t *arena)
assert(in_ql);
}
ql_remove(&arena->tcache_ql, tcache, link);
- tcache_stats_merge(tcache, arena);
- malloc_mutex_unlock(&arena->lock);
+ tcache_stats_merge(tsd, tcache, arena);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
}
@@ -327,11 +329,11 @@ tcache_create(tsd_t *tsd, arena_t *arena)
size = sa2u(size, CACHELINE);
tcache = ipallocztm(tsd, size, CACHELINE, true, false, true,
- arena_get(0, false));
+ arena_get(tsd, 0, false));
if (tcache == NULL)
return (NULL);
- tcache_arena_associate(tcache, arena);
+ tcache_arena_associate(tsd, tcache, arena);
ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
@@ -358,7 +360,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
unsigned i;
arena = arena_choose(tsd, NULL);
- tcache_arena_dissociate(tcache, arena);
+ tcache_arena_dissociate(tsd, tcache, arena);
for (i = 0; i < NBINS; i++) {
tcache_bin_t *tbin = &tcache->tbins[i];
@@ -366,9 +368,9 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
if (config_stats && tbin->tstats.nrequests != 0) {
arena_bin_t *bin = &arena->bins[i];
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
bin->stats.nrequests += tbin->tstats.nrequests;
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
}
}
@@ -377,17 +379,17 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
if (config_stats && tbin->tstats.nrequests != 0) {
- malloc_mutex_lock(&arena->lock);
+ malloc_mutex_lock(tsd, &arena->lock);
arena->stats.nrequests_large += tbin->tstats.nrequests;
arena->stats.lstats[i - NBINS].nrequests +=
tbin->tstats.nrequests;
- malloc_mutex_unlock(&arena->lock);
+ malloc_mutex_unlock(tsd, &arena->lock);
}
}
if (config_prof && tcache->prof_accumbytes > 0 &&
- arena_prof_accum(arena, tcache->prof_accumbytes))
- prof_idump();
+ arena_prof_accum(tsd, arena, tcache->prof_accumbytes))
+ prof_idump(tsd);
idalloctm(tsd, tcache, false, true, true);
}
@@ -413,21 +415,22 @@ tcache_enabled_cleanup(tsd_t *tsd)
/* Do nothing. */
}
-/* Caller must own arena->lock. */
void
-tcache_stats_merge(tcache_t *tcache, arena_t *arena)
+tcache_stats_merge(tsd_t *tsd, tcache_t *tcache, arena_t *arena)
{
unsigned i;
cassert(config_stats);
+ malloc_mutex_assert_owner(tsd, &arena->lock);
+
/* Merge and reset tcache stats. */
for (i = 0; i < NBINS; i++) {
arena_bin_t *bin = &arena->bins[i];
tcache_bin_t *tbin = &tcache->tbins[i];
- malloc_mutex_lock(&bin->lock);
+ malloc_mutex_lock(tsd, &bin->lock);
bin->stats.nrequests += tbin->tstats.nrequests;
- malloc_mutex_unlock(&bin->lock);
+ malloc_mutex_unlock(tsd, &bin->lock);
tbin->tstats.nrequests = 0;
}
@@ -447,7 +450,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
tcaches_t *elm;
if (tcaches == NULL) {
- tcaches = base_alloc(sizeof(tcache_t *) *
+ tcaches = base_alloc(tsd, sizeof(tcache_t *) *
(MALLOCX_TCACHE_MAX+1));
if (tcaches == NULL)
return (true);
@@ -455,7 +458,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
return (true);
- tcache = tcache_create(tsd, arena_get(0, false));
+ tcache = tcache_create(tsd, arena_get(tsd, 0, false));
if (tcache == NULL)
return (true);
@@ -501,7 +504,7 @@ tcaches_destroy(tsd_t *tsd, unsigned ind)
}
bool
-tcache_boot(void)
+tcache_boot(tsd_t *tsd)
{
unsigned i;
@@ -519,7 +522,7 @@ tcache_boot(void)
nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
- tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins *
+ tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsd, nhbins *
sizeof(tcache_bin_info_t));
if (tcache_bin_info == NULL)
return (true);
diff --git a/src/tsd.c b/src/tsd.c
index 34c1573..38d8bde 100644
--- a/src/tsd.c
+++ b/src/tsd.c
@@ -106,15 +106,17 @@ MALLOC_TSD
}
}
-bool
+tsd_t *
malloc_tsd_boot0(void)
{
+ tsd_t *tsd;
ncleanups = 0;
if (tsd_boot0())
- return (true);
- *tsd_arenas_tdata_bypassp_get(tsd_fetch()) = true;
- return (false);
+ return (NULL);
+ tsd = tsd_fetch();
+ *tsd_arenas_tdata_bypassp_get(tsd) = true;
+ return (tsd);
}
void
@@ -169,10 +171,10 @@ tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
tsd_init_block_t *iter;
/* Check whether this thread has already inserted into the list. */
- malloc_mutex_lock(&head->lock);
+ malloc_mutex_lock(NULL, &head->lock);
ql_foreach(iter, &head->blocks, link) {
if (iter->thread == self) {
- malloc_mutex_unlock(&head->lock);
+ malloc_mutex_unlock(NULL, &head->lock);
return (iter->data);
}
}
@@ -180,7 +182,7 @@ tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block)
ql_elm_new(block, link);
block->thread = self;
ql_tail_insert(&head->blocks, block, link);
- malloc_mutex_unlock(&head->lock);
+ malloc_mutex_unlock(NULL, &head->lock);
return (NULL);
}
@@ -188,8 +190,8 @@ void
tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block)
{
- malloc_mutex_lock(&head->lock);
+ malloc_mutex_lock(NULL, &head->lock);
ql_remove(&head->blocks, block, link);
- malloc_mutex_unlock(&head->lock);
+ malloc_mutex_unlock(NULL, &head->lock);
}
#endif
diff --git a/src/witness.c b/src/witness.c
new file mode 100644
index 0000000..b7b91ac
--- /dev/null
+++ b/src/witness.c
@@ -0,0 +1,206 @@
+#define JEMALLOC_WITNESS_C_
+#include "jemalloc/internal/jemalloc_internal.h"
+
+void
+witness_init(witness_t *witness, const char *name, witness_rank_t rank,
+ witness_comp_t *comp)
+{
+
+ witness->name = name;
+ witness->rank = rank;
+ witness->comp = comp;
+}
+
+#ifdef JEMALLOC_JET
+#undef witness_lock_error
+#define witness_lock_error JEMALLOC_N(witness_lock_error_impl)
+#endif
+static void
+witness_lock_error(const witness_list_t *witnesses, const witness_t *witness)
+{
+ witness_t *w;
+
+ malloc_printf("<jemalloc>: Lock rank order reversal:");
+ ql_foreach(w, witnesses, link) {
+ malloc_printf(" %s(%u)", w->name, w->rank);
+ }
+ malloc_printf(" %s(%u)\n", witness->name, witness->rank);
+ abort();
+}
+#ifdef JEMALLOC_JET
+#undef witness_lock_error
+#define witness_lock_error JEMALLOC_N(witness_lock_error)
+witness_lock_error_t *witness_lock_error = JEMALLOC_N(witness_lock_error_impl);
+#endif
+
+void
+witness_lock(tsd_t *tsd, witness_t *witness)
+{
+ witness_list_t *witnesses;
+ witness_t *w;
+
+ cassert(config_debug);
+
+ if (tsd == NULL)
+ return;
+ if (witness->rank == WITNESS_RANK_OMIT)
+ return;
+
+ witness_assert_not_owner(tsd, witness);
+
+ witnesses = tsd_witnessesp_get(tsd);
+ w = ql_last(witnesses, link);
+ if (w != NULL && w->rank >= witness->rank && (w->comp == NULL ||
+ w->comp != witness->comp || w->comp(w, witness) > 0))
+ witness_lock_error(witnesses, witness);
+
+ ql_elm_new(witness, link);
+ ql_tail_insert(witnesses, witness, link);
+}
+
+void
+witness_unlock(tsd_t *tsd, witness_t *witness)
+{
+ witness_list_t *witnesses;
+
+ cassert(config_debug);
+
+ if (tsd == NULL)
+ return;
+ if (witness->rank == WITNESS_RANK_OMIT)
+ return;
+
+ witness_assert_owner(tsd, witness);
+
+ witnesses = tsd_witnessesp_get(tsd);
+ ql_remove(witnesses, witness, link);
+}
+
+#ifdef JEMALLOC_JET
+#undef witness_owner_error
+#define witness_owner_error JEMALLOC_N(witness_owner_error_impl)
+#endif
+static void
+witness_owner_error(const witness_t *witness)
+{
+
+ malloc_printf("<jemalloc>: Should own %s(%u)\n", witness->name,
+ witness->rank);
+ abort();
+}
+#ifdef JEMALLOC_JET
+#undef witness_owner_error
+#define witness_owner_error JEMALLOC_N(witness_owner_error)
+witness_owner_error_t *witness_owner_error =
+ JEMALLOC_N(witness_owner_error_impl);
+#endif
+
+void
+witness_assert_owner(tsd_t *tsd, const witness_t *witness)
+{
+ witness_list_t *witnesses;
+ witness_t *w;
+
+ cassert(config_debug);
+
+ if (tsd == NULL)
+ return;
+ if (witness->rank == WITNESS_RANK_OMIT)
+ return;
+
+ witnesses = tsd_witnessesp_get(tsd);
+ ql_foreach(w, witnesses, link) {
+ if (w == witness)
+ return;
+ }
+ witness_owner_error(witness);
+}
+
+#ifdef JEMALLOC_JET
+#undef witness_not_owner_error
+#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error_impl)
+#endif
+static void
+witness_not_owner_error(const witness_t *witness)
+{
+
+ malloc_printf("<jemalloc>: Should not own %s(%u)\n", witness->name,
+ witness->rank);
+ abort();
+}
+#ifdef JEMALLOC_JET
+#undef witness_not_owner_error
+#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error)
+witness_not_owner_error_t *witness_not_owner_error =
+ JEMALLOC_N(witness_not_owner_error_impl);
+#endif
+
+void
+witness_assert_not_owner(tsd_t *tsd, const witness_t *witness)
+{
+ witness_list_t *witnesses;
+ witness_t *w;
+
+ cassert(config_debug);
+
+ if (tsd == NULL)
+ return;
+ if (witness->rank == WITNESS_RANK_OMIT)
+ return;
+
+ witnesses = tsd_witnessesp_get(tsd);
+ ql_foreach(w, witnesses, link) {
+ if (w == witness)
+ witness_not_owner_error(witness);
+ }
+}
+
+#ifdef JEMALLOC_JET
+#undef witness_lockless_error
+#define witness_lockless_error JEMALLOC_N(witness_lockless_error_impl)
+#endif
+static void
+witness_lockless_error(const witness_list_t *witnesses)
+{
+ witness_t *w;
+
+ malloc_printf("<jemalloc>: Should not own any locks:");
+ ql_foreach(w, witnesses, link) {
+ malloc_printf(" %s(%u)", w->name, w->rank);
+ }
+ malloc_printf("\n");
+ abort();
+}
+#ifdef JEMALLOC_JET
+#undef witness_lockless_error
+#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
+witness_lockless_error_t *witness_lockless_error =
+ JEMALLOC_N(witness_lockless_error_impl);
+#endif
+
+void
+witness_assert_lockless(tsd_t *tsd)
+{
+ witness_list_t *witnesses;
+ witness_t *w;
+
+ cassert(config_debug);
+
+ if (tsd == NULL)
+ return;
+
+ witnesses = tsd_witnessesp_get(tsd);
+ w = ql_last(witnesses, link);
+ if (w != NULL) {
+ witness_lockless_error(witnesses);
+ }
+}
+
+void
+witnesses_cleanup(tsd_t *tsd)
+{
+
+ witness_assert_lockless(tsd);
+
+ /* Do nothing. */
+}
diff --git a/src/zone.c b/src/zone.c
index 6859b3f..8f25051 100644
--- a/src/zone.c
+++ b/src/zone.c
@@ -56,7 +56,7 @@ zone_size(malloc_zone_t *zone, void *ptr)
* not work in practice, we must check all pointers to assure that they
* reside within a mapped chunk before determining size.
*/
- return (ivsalloc(ptr, config_prof));
+ return (ivsalloc(tsd_fetch(), ptr, config_prof));
}
static void *
@@ -87,7 +87,7 @@ static void
zone_free(malloc_zone_t *zone, void *ptr)
{
- if (ivsalloc(ptr, config_prof) != 0) {
+ if (ivsalloc(tsd_fetch(), ptr, config_prof) != 0) {
je_free(ptr);
return;
}
@@ -99,7 +99,7 @@ static void *
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
{
- if (ivsalloc(ptr, config_prof) != 0)
+ if (ivsalloc(tsd_fetch(), ptr, config_prof) != 0)
return (je_realloc(ptr, size));
return (realloc(ptr, size));
@@ -123,7 +123,7 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
{
size_t alloc_size;
- alloc_size = ivsalloc(ptr, config_prof);
+ alloc_size = ivsalloc(tsd_fetch(), ptr, config_prof);
if (alloc_size != 0) {
assert(alloc_size == size);
je_free(ptr);