diff options
| -rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/tcache_externs.h | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness_types.h | 31 | ||||
| -rw-r--r-- | src/ctl.c | 2 | ||||
| -rw-r--r-- | src/jemalloc.c | 3 | ||||
| -rw-r--r-- | src/tcache.c | 88 |
6 files changed, 101 insertions, 29 deletions
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 2567f56..36bcda2 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -469,6 +469,9 @@ tcache_flush tcache_get tcache_get_hard tcache_maxclass +tcache_prefork +tcache_postfork_child +tcache_postfork_parent tcache_salloc tcache_stats_merge tcaches diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index ead90af..3e4a751 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -43,5 +43,8 @@ bool tcaches_create(tsd_t *tsd, unsigned *r_ind); void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(tsdn_t *tsdn); +void tcache_prefork(tsdn_t *tsdn); +void tcache_postfork_parent(tsdn_t *tsdn); +void tcache_postfork_child(tsdn_t *tsdn); #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */ diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h index f765d7b..dfcf162 100644 --- a/include/jemalloc/internal/witness_types.h +++ b/include/jemalloc/internal/witness_types.h @@ -17,21 +17,22 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *, #define WITNESS_RANK_INIT 1U #define WITNESS_RANK_CTL 1U -#define WITNESS_RANK_ARENAS 2U - -#define WITNESS_RANK_PROF_DUMP 3U -#define WITNESS_RANK_PROF_BT2GCTX 4U -#define WITNESS_RANK_PROF_TDATAS 5U -#define WITNESS_RANK_PROF_TDATA 6U -#define WITNESS_RANK_PROF_GCTX 7U - -#define WITNESS_RANK_ARENA 8U -#define WITNESS_RANK_ARENA_EXTENTS 9U -#define WITNESS_RANK_ARENA_EXTENT_CACHE 10 - -#define WITNESS_RANK_RTREE_ELM 11U -#define WITNESS_RANK_RTREE 12U -#define WITNESS_RANK_BASE 13U +#define WITNESS_RANK_TCACHES 2U +#define WITNESS_RANK_ARENAS 3U + +#define WITNESS_RANK_PROF_DUMP 4U +#define WITNESS_RANK_PROF_BT2GCTX 5U +#define WITNESS_RANK_PROF_TDATAS 6U +#define WITNESS_RANK_PROF_TDATA 7U +#define WITNESS_RANK_PROF_GCTX 8U + +#define WITNESS_RANK_ARENA 9U +#define WITNESS_RANK_ARENA_EXTENTS 10U +#define WITNESS_RANK_ARENA_EXTENT_CACHE 11U + +#define WITNESS_RANK_RTREE_ELM 12U +#define WITNESS_RANK_RTREE 13U +#define WITNESS_RANK_BASE 14U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF @@ -1477,7 +1477,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, return ENOENT; } - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); READONLY(); if (tcaches_create(tsd, &tcache_ind)) { ret = EFAULT; @@ -1487,7 +1486,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, ret = 0; label_return: - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); return ret; } diff --git a/src/jemalloc.c b/src/jemalloc.c index 28759bc..45e9aea 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2765,6 +2765,7 @@ _malloc_prefork(void) witness_prefork(tsd); /* Acquire all mutexes in a safe order. */ ctl_prefork(tsd_tsdn(tsd)); + tcache_prefork(tsd_tsdn(tsd)); malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); prof_prefork0(tsd_tsdn(tsd)); for (i = 0; i < 3; i++) { @@ -2825,6 +2826,7 @@ _malloc_postfork(void) } prof_postfork_parent(tsd_tsdn(tsd)); malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); + tcache_postfork_parent(tsd_tsdn(tsd)); ctl_postfork_parent(tsd_tsdn(tsd)); } @@ -2848,6 +2850,7 @@ jemalloc_postfork_child(void) { } prof_postfork_child(tsd_tsdn(tsd)); malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); + tcache_postfork_child(tsd_tsdn(tsd)); ctl_postfork_child(tsd_tsdn(tsd)); } diff --git a/src/tcache.c b/src/tcache.c index 96a42ad..76277f0 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -21,6 +21,9 @@ static unsigned tcaches_past; /* Head of singly linked list tracking available tcaches elements. */ static tcaches_t *tcaches_avail; +/* Protects tcaches{,_past,_avail}. */ +static malloc_mutex_t tcaches_mtx; + /******************************************************************************/ size_t @@ -422,32 +425,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { } } -bool -tcaches_create(tsd_t *tsd, unsigned *r_ind) { - arena_t *arena; - tcache_t *tcache; - tcaches_t *elm; +static bool +tcaches_create_prep(tsd_t *tsd) { + bool err; + + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); if (tcaches == NULL) { tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *) * (MALLOCX_TCACHE_MAX+1), CACHELINE); if (tcaches == NULL) { - return true; + err = true; + goto label_return; } } if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) { - return true; + err = true; + goto label_return; } - arena = arena_ichoose(tsd, NULL); + + err = false; +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); + return err; +} + +bool +tcaches_create(tsd_t *tsd, unsigned *r_ind) { + witness_assert_depth(tsd_tsdn(tsd), 0); + + bool err; + + if (tcaches_create_prep(tsd)) { + err = true; + goto label_return; + } + + arena_t *arena = arena_ichoose(tsd, NULL); if (unlikely(arena == NULL)) { - return true; + err = true; + goto label_return; } - tcache = tcache_create(tsd_tsdn(tsd), arena); + tcache_t *tcache = tcache_create(tsd_tsdn(tsd), arena); if (tcache == NULL) { - return true; + err = true; + goto label_return; } + tcaches_t *elm; + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); if (tcaches_avail != NULL) { elm = tcaches_avail; tcaches_avail = tcaches_avail->next; @@ -459,12 +486,18 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) { *r_ind = tcaches_past; tcaches_past++; } + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); - return false; + err = false; +label_return: + witness_assert_depth(tsd_tsdn(tsd), 0); + return err; } static void tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) { + malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); + if (elm->tcache == NULL) { return; } @@ -474,19 +507,25 @@ tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) { void tcaches_flush(tsd_t *tsd, unsigned ind) { + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_elm_flush(tsd, &tcaches[ind]); + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); } void tcaches_destroy(tsd_t *tsd, unsigned ind) { + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; tcaches_elm_flush(tsd, elm); elm->next = tcaches_avail; tcaches_avail = elm; + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); } bool tcache_boot(tsdn_t *tsdn) { + cassert(config_tcache); + unsigned i; /* If necessary, clamp opt_lg_tcache_max. */ @@ -497,6 +536,10 @@ tcache_boot(tsdn_t *tsdn) { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } + if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) { + return true; + } + nhbins = size2index(tcache_maxclass) + 1; /* Initialize tcache_bin_info. */ @@ -527,3 +570,24 @@ tcache_boot(tsdn_t *tsdn) { return false; } + +void +tcache_prefork(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_prefork(tsdn, &tcaches_mtx); + } +} + +void +tcache_postfork_parent(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_postfork_parent(tsdn, &tcaches_mtx); + } +} + +void +tcache_postfork_child(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_postfork_child(tsdn, &tcaches_mtx); + } +} |