diff options
| -rw-r--r-- | doc/jemalloc.xml.in | 47 | ||||
| -rw-r--r-- | include/jemalloc/internal/arena.h | 34 | ||||
| -rw-r--r-- | include/jemalloc/internal/base.h | 1 | ||||
| -rw-r--r-- | include/jemalloc/internal/ctl.h | 1 | ||||
| -rw-r--r-- | include/jemalloc/internal/huge.h | 1 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal.h.in | 156 | ||||
| -rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 10 | ||||
| -rw-r--r-- | include/jemalloc/internal/stats.h | 7 | ||||
| -rw-r--r-- | src/arena.c | 10 | ||||
| -rw-r--r-- | src/base.c | 15 | ||||
| -rw-r--r-- | src/ctl.c | 30 | ||||
| -rw-r--r-- | src/huge.c | 113 | ||||
| -rw-r--r-- | src/jemalloc.c | 37 | ||||
| -rw-r--r-- | src/prof.c | 30 | ||||
| -rw-r--r-- | src/quarantine.c | 11 | ||||
| -rw-r--r-- | src/stats.c | 14 | ||||
| -rw-r--r-- | src/tcache.c | 4 | ||||
| -rw-r--r-- | src/tsd.c | 2 |
18 files changed, 356 insertions, 167 deletions
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 858572d..08fd4eb 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1773,6 +1773,21 @@ malloc_conf = "xmalloc:true";]]></programlisting> entirely devoted to allocator metadata.</para></listitem> </varlistentry> + <varlistentry id="stats.metadata"> + <term> + <mallctl>stats.metadata</mallctl> + (<type>size_t</type>) + <literal>r-</literal> + [<option>--enable-stats</option>] + </term> + <listitem><para>Total number of bytes dedicated to metadata, which + comprise base allocations used for bootstrap-sensitive internal + allocator data structures, arena chunk headers (see <link + linkend="stats.arenas.i.metadata.mapped"><mallctl>stats.arenas.<i>.metadata.mapped</mallctl></link>), + and internal allocations (see <link + linkend="stats.arenas.i.metadata.allocated"><mallctl>stats.arenas.<i>.metadata.allocated</mallctl></link>).</para></listitem> + </varlistentry> + <varlistentry id="stats.mapped"> <term> <mallctl>stats.mapped</mallctl> @@ -1875,6 +1890,38 @@ malloc_conf = "xmalloc:true";]]></programlisting> <listitem><para>Number of mapped bytes.</para></listitem> </varlistentry> + <varlistentry id="stats.arenas.i.metadata.mapped"> + <term> + <mallctl>stats.arenas.<i>.metadata.mapped</mallctl> + (<type>size_t</type>) + <literal>r-</literal> + [<option>--enable-stats</option>] + </term> + <listitem><para>Number of mapped bytes in arena chunk headers, which + track the states of the non-metadata pages.</para></listitem> + </varlistentry> + + <varlistentry id="stats.arenas.i.metadata.allocated"> + <term> + <mallctl>stats.arenas.<i>.metadata.allocated</mallctl> + (<type>size_t</type>) + <literal>r-</literal> + [<option>--enable-stats</option>] + </term> + <listitem><para>Number of bytes dedicated to internal allocations. + Internal allocations differ from application-originated allocations in + that they are for internal use, and that they are omitted from heap + profiles. This statistic is reported separately from <link + linkend="stats.metadata"><mallctl>stats.metadata</mallctl></link> and + <link + linkend="stats.arenas.i.metadata.mapped"><mallctl>stats.arenas.<i>.metadata.mapped</mallctl></link> + because it overlaps with e.g. the <link + linkend="stats.allocated"><mallctl>stats.allocated</mallctl></link> and + <link linkend="stats.active"><mallctl>stats.active</mallctl></link> + statistics, whereas the other metadata statistics do + not.</para></listitem> + </varlistentry> + <varlistentry id="stats.arenas.i.npurge"> <term> <mallctl>stats.arenas.<i>.npurge</mallctl> diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 1e19023..46367f6 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -437,6 +437,9 @@ void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, index_t binind, size_t flags); void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, size_t unzeroed); +void arena_metadata_allocated_add(arena_t *arena, size_t size); +void arena_metadata_allocated_sub(arena_t *arena, size_t size); +size_t arena_metadata_allocated_get(arena_t *arena); bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); @@ -448,6 +451,7 @@ prof_tctx_t *arena_prof_tctx_get(const void *ptr); void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache); +arena_t *arena_aalloc(const void *ptr); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache); @@ -699,6 +703,27 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, unzeroed); } +JEMALLOC_INLINE void +arena_metadata_allocated_add(arena_t *arena, size_t size) +{ + + atomic_add_z(&arena->stats.metadata_allocated, size); +} + +JEMALLOC_INLINE void +arena_metadata_allocated_sub(arena_t *arena, size_t size) +{ + + atomic_sub_z(&arena->stats.metadata_allocated, size); +} + +JEMALLOC_INLINE size_t +arena_metadata_allocated_get(arena_t *arena) +{ + + return (atomic_read_z(&arena->stats.metadata_allocated)); +} + JEMALLOC_INLINE bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) { @@ -952,6 +977,15 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, } } +JEMALLOC_ALWAYS_INLINE arena_t * +arena_aalloc(const void *ptr) +{ + arena_chunk_t *chunk; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + return (chunk->arena); +} + /* Return the size of the allocation pointed to by ptr. */ JEMALLOC_ALWAYS_INLINE size_t arena_salloc(const void *ptr, bool demote) diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index 3fb80b9..18b7a72 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -13,6 +13,7 @@ void *base_alloc(size_t size); void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dalloc(extent_node_t *node); +size_t base_allocated_get(void); bool base_boot(void); void base_prefork(void); void base_postfork_parent(void); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index a3e899e..65617bc 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -52,6 +52,7 @@ struct ctl_arena_stats_s { struct ctl_stats_s { size_t allocated; size_t active; + size_t metadata; size_t mapped; struct { size_t current; /* stats_chunks.curchunks */ diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 39d8aa5..decb024 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -23,6 +23,7 @@ typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache); +arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 4107860..a477855 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -404,8 +404,9 @@ extern size_t const index2size_tab[NSIZES]; extern uint8_t const size2index_tab[]; arena_t *a0get(void); -void *a0malloc(size_t size, bool zero); +void *a0malloc(size_t size); void a0dalloc(void *ptr); +size_t a0allocated(void); arena_t *arenas_extend(unsigned ind); arena_t *arena_init(unsigned ind); unsigned narenas_total_get(void); @@ -776,21 +777,27 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, #include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE +arena_t *iaalloc(const void *ptr); +size_t isalloc(const void *ptr, bool demote); +void *iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, + bool is_metadata, arena_t *arena); void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); void *imalloc(tsd_t *tsd, size_t size); void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); void *icalloc(tsd_t *tsd, size_t size); +void *ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, bool is_metadata, arena_t *arena); void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena); void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr, bool demote); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); +void idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata); void idalloct(tsd_t *tsd, void *ptr, bool try_tcache); -void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void idalloc(tsd_t *tsd, void *ptr); void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache); +void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, @@ -805,45 +812,97 @@ bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_ALWAYS_INLINE arena_t * +iaalloc(const void *ptr) +{ + arena_t *arena; + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) + arena = arena_aalloc(ptr); + else + arena = huge_aalloc(ptr); + + return (arena); +} + +/* + * Typical usage: + * void *ptr = [...] + * size_t sz = isalloc(ptr, config_prof); + */ +JEMALLOC_ALWAYS_INLINE size_t +isalloc(const void *ptr, bool demote) +{ + size_t ret; + arena_chunk_t *chunk; + + assert(ptr != NULL); + /* Demotion only makes sense if config_prof is true. */ + assert(config_prof || !demote); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) + ret = arena_salloc(ptr, demote); + else + ret = huge_salloc(ptr); + + return (ret); +} + JEMALLOC_ALWAYS_INLINE void * -imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, bool is_metadata, + arena_t *arena) { + void *ret; assert(size != 0); if (likely(size <= arena_maxclass)) - return (arena_malloc(tsd, arena, size, false, try_tcache)); + ret = arena_malloc(tsd, arena, size, zero, try_tcache); else - return (huge_malloc(tsd, arena, size, false, try_tcache)); + ret = huge_malloc(tsd, arena, size, zero, try_tcache); + if (config_stats && is_metadata && likely(ret != NULL)) { + arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, + config_prof)); + } + return (ret); +} + +JEMALLOC_ALWAYS_INLINE void * +imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +{ + + return (iallocztm(tsd, size, false, try_tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * imalloc(tsd_t *tsd, size_t size) { - return (imalloct(tsd, size, true, NULL)); + return (iallocztm(tsd, size, false, true, false, NULL)); } JEMALLOC_ALWAYS_INLINE void * icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) { - if (likely(size <= arena_maxclass)) - return (arena_malloc(tsd, arena, size, true, try_tcache)); - else - return (huge_malloc(tsd, arena, size, true, try_tcache)); + return (iallocztm(tsd, size, true, try_tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * icalloc(tsd_t *tsd, size_t size) { - return (icalloct(tsd, size, true, NULL)); + return (iallocztm(tsd, size, true, true, false, NULL)); } JEMALLOC_ALWAYS_INLINE void * -ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena) +ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, bool is_metadata, arena_t *arena) { void *ret; @@ -865,40 +924,28 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, try_tcache); } } - assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); + if (config_stats && is_metadata && likely(ret != NULL)) { + arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, + config_prof)); + } return (ret); } JEMALLOC_ALWAYS_INLINE void * -ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) +ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { - return (ipalloct(tsd, usize, alignment, zero, true, NULL)); + return (ipallocztm(tsd, usize, alignment, zero, try_tcache, false, + arena)); } -/* - * Typical usage: - * void *ptr = [...] - * size_t sz = isalloc(ptr, config_prof); - */ -JEMALLOC_ALWAYS_INLINE size_t -isalloc(const void *ptr, bool demote) +JEMALLOC_ALWAYS_INLINE void * +ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { - size_t ret; - arena_chunk_t *chunk; - - assert(ptr != NULL); - /* Demotion only makes sense if config_prof is true. */ - assert(config_prof || !demote); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - ret = arena_salloc(ptr, demote); - else - ret = huge_salloc(ptr); - return (ret); + return (ipallocztm(tsd, usize, alignment, zero, true, false, NULL)); } JEMALLOC_ALWAYS_INLINE size_t @@ -935,11 +982,15 @@ p2rz(const void *ptr) } JEMALLOC_ALWAYS_INLINE void -idalloct(tsd_t *tsd, void *ptr, bool try_tcache) +idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata) { arena_chunk_t *chunk; assert(ptr != NULL); + if (config_stats && is_metadata) { + arena_metadata_allocated_sub(iaalloc(ptr), isalloc(ptr, + config_prof)); + } chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) @@ -949,24 +1000,17 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache) } JEMALLOC_ALWAYS_INLINE void -isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) +idalloct(tsd_t *tsd, void *ptr, bool try_tcache) { - arena_chunk_t *chunk; - - assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_sdalloc(tsd, chunk, ptr, size, try_tcache); - else - huge_dalloc(tsd, ptr, try_tcache); + idalloctm(tsd, ptr, try_tcache, false); } JEMALLOC_ALWAYS_INLINE void idalloc(tsd_t *tsd, void *ptr) { - idalloct(tsd, ptr, true); + idalloctm(tsd, ptr, true, false); } JEMALLOC_ALWAYS_INLINE void @@ -976,7 +1020,21 @@ iqalloc(tsd_t *tsd, void *ptr, bool try_tcache) if (config_fill && unlikely(opt_quarantine)) quarantine(tsd, ptr); else - idalloct(tsd, ptr, try_tcache); + idalloctm(tsd, ptr, try_tcache, false); +} + +JEMALLOC_ALWAYS_INLINE void +isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) +{ + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) + arena_sdalloc(tsd, chunk, ptr, size, try_tcache); + else + huge_dalloc(tsd, ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE void diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1aaf80b..dfa8755 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -1,6 +1,7 @@ a0dalloc a0get a0malloc +arena_aalloc arena_get arena_get_hard arena_alloc_junk_small @@ -50,6 +51,9 @@ arena_mapbitsp_read arena_mapbitsp_write arena_maxclass arena_maxrun +arena_metadata_allocated_add +arena_metadata_allocated_get +arena_metadata_allocated_sub arena_migrate arena_miscelm_get arena_miscelm_to_pageind @@ -90,6 +94,7 @@ atomic_sub_uint32 atomic_sub_uint64 atomic_sub_z base_alloc +base_allocated_get base_boot base_calloc base_node_alloc @@ -205,6 +210,7 @@ hash_rotl_64 hash_x64_128 hash_x86_128 hash_x86_32 +huge_aalloc huge_allocated huge_boot huge_dalloc @@ -221,10 +227,13 @@ huge_prof_tctx_set huge_ralloc huge_ralloc_no_move huge_salloc +iaalloc +iallocztm icalloc icalloct idalloc idalloct +idalloctm imalloc imalloct in_valgrind @@ -234,6 +243,7 @@ index2size_lookup index2size_tab ipalloc ipalloct +ipallocztm iqalloc iralloc iralloct diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index d8600ed..7cba77b 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -111,6 +111,13 @@ struct arena_stats_s { uint64_t nmadvise; uint64_t purged; + /* + * Number of bytes currently mapped purely for metadata purposes, and + * number of bytes currently allocated for internal metadata. + */ + size_t metadata_mapped; + size_t metadata_allocated; /* Protected via atomic_*_z(). */ + /* Per-size-category statistics. */ size_t allocated_large; uint64_t nmalloc_large; diff --git a/src/arena.c b/src/arena.c index 1eb4000..984b8ad 100644 --- a/src/arena.c +++ b/src/arena.c @@ -405,8 +405,10 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, size, alignment, zero); malloc_mutex_lock(&arena->lock); - if (config_stats && chunk != NULL) + if (config_stats && chunk != NULL) { arena->stats.mapped += chunksize; + arena->stats.metadata_mapped += (map_bias << LG_PAGE); + } return (chunk); } @@ -514,8 +516,10 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) malloc_mutex_unlock(&arena->lock); chunk_dalloc((void *)spare, chunksize, arena->ind); malloc_mutex_lock(&arena->lock); - if (config_stats) + if (config_stats) { arena->stats.mapped -= chunksize; + arena->stats.metadata_mapped -= (map_bias << LG_PAGE); + } } else arena->spare = chunk; } @@ -2273,6 +2277,8 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, astats->npurge += arena->stats.npurge; astats->nmadvise += arena->stats.nmadvise; astats->purged += arena->stats.purged; + astats->metadata_mapped += arena->stats.metadata_mapped; + astats->metadata_allocated += arena_metadata_allocated_get(arena); astats->allocated_large += arena->stats.allocated_large; astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; @@ -16,6 +16,8 @@ static void *base_next_addr; static void *base_past_addr; /* Addr immediately past base_pages. */ static extent_node_t *base_nodes; +static size_t base_allocated; + /******************************************************************************/ static bool @@ -54,6 +56,8 @@ base_alloc(size_t size) /* Allocate. */ ret = base_next_addr; base_next_addr = (void *)((uintptr_t)base_next_addr + csize); + if (config_stats) + base_allocated += csize; malloc_mutex_unlock(&base_mtx); JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); @@ -102,6 +106,17 @@ base_node_dalloc(extent_node_t *node) malloc_mutex_unlock(&base_mtx); } +size_t +base_allocated_get(void) +{ + size_t ret; + + malloc_mutex_lock(&base_mtx); + ret = base_allocated; + malloc_mutex_unlock(&base_mtx); + return (ret); +} + bool base_boot(void) { @@ -183,10 +183,13 @@ CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_npurge) CTL_PROTO(stats_arenas_i_nmadvise) CTL_PROTO(stats_arenas_i_purged) +CTL_PROTO(stats_arenas_i_metadata_mapped) +CTL_PROTO(stats_arenas_i_metadata_allocated) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_cactive) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) +CTL_PROTO(stats_metadata) CTL_PROTO(stats_mapped) /******************************************************************************/ @@ -355,6 +358,11 @@ static const ctl_named_node_t stats_chunks_node[] = { {NAME("high"), CTL(stats_chunks_high)} }; +static const ctl_named_node_t stats_arenas_i_metadata_node[] = { + {NAME("mapped"), CTL(stats_arenas_i_metadata_mapped)}, + {NAME("allocated"), CTL(stats_arenas_i_metadata_allocated)} +}; + static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, @@ -432,6 +440,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("purged"), CTL(stats_arenas_i_purged)}, + {NAME("metadata"), CHILD(named, stats_arenas_i_metadata)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, @@ -451,6 +460,7 @@ static const ctl_named_node_t stats_node[] = { {NAME("cactive"), CTL(stats_cactive)}, {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, + {NAME("metadata"), CTL(stats_metadata)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("chunks"), CHILD(named, stats_chunks)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} @@ -484,14 +494,14 @@ ctl_arena_init(ctl_arena_stats_t *astats) if (astats->lstats == NULL) { astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses * - sizeof(malloc_large_stats_t), false); + sizeof(malloc_large_stats_t)); if (astats->lstats == NULL) return (true); } if (astats->hstats == NULL) { astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses * - sizeof(malloc_huge_stats_t), false); + sizeof(malloc_huge_stats_t)); if (astats->hstats == NULL) return (true); } @@ -551,6 +561,9 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->astats.nmadvise += astats->astats.nmadvise; sstats->astats.purged += astats->astats.purged; + sstats->astats.metadata_mapped += astats->astats.metadata_mapped; + sstats->astats.metadata_allocated += astats->astats.metadata_allocated; + sstats->allocated_small += astats->allocated_small; sstats->nmalloc_small += astats->nmalloc_small; sstats->ndalloc_small += astats->ndalloc_small; @@ -627,7 +640,7 @@ ctl_grow(void) /* Allocate extended arena stats. */ astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) * - sizeof(ctl_arena_stats_t), false); + sizeof(ctl_arena_stats_t)); if (astats == NULL) return (true); @@ -704,6 +717,10 @@ ctl_refresh(void) + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; ctl_stats.active = (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); + ctl_stats.metadata = base_allocated_get() + + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + + ctl_stats.arenas[ctl_stats.narenas].astats + .metadata_allocated; ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); } @@ -723,7 +740,7 @@ ctl_init(void) */ ctl_stats.narenas = narenas_total_get(); ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( - (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t), false); + (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; goto label_return; @@ -1806,6 +1823,7 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) +CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, @@ -1825,6 +1843,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_mapped, + ctl_stats.arenas[mib[2]].astats.metadata_mapped, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_allocated, + ctl_stats.arenas[mib[2]].astats.metadata_allocated, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, ctl_stats.arenas[mib[2]].allocated_small, size_t) @@ -37,8 +37,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Allocate one or more contiguous chunks for this request. */ /* Allocate an extent node with which to track the chunk. */ - node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), - CACHELINE, false, try_tcache, NULL); + node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), + CACHELINE, false, try_tcache, true, arena); if (node == NULL) return (NULL); @@ -50,7 +50,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, arena = arena_choose(tsd, arena); if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed)) == NULL) { - idalloct(tsd, node, try_tcache); + idalloctm(tsd, node, try_tcache, true); return (NULL); } @@ -73,6 +73,33 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (ret); } +static extent_node_t * +huge_node_locked(const void *ptr) +{ + extent_node_t *node, key; + + /* Extract from tree of huge allocations. */ + key.addr = __DECONST(void *, ptr); + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + malloc_mutex_unlock(&huge_mtx); + + return (node); +} + +static extent_node_t * +huge_node(const void *ptr) +{ + extent_node_t *node; + + malloc_mutex_lock(&huge_mtx); + node = huge_node_locked(ptr); + malloc_mutex_unlock(&huge_mtx); + + return (node); +} + #ifdef JEMALLOC_JET #undef huge_dalloc_junk #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) @@ -102,7 +129,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, { size_t usize_next; bool zeroed; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; /* Increase usize to incorporate extra. */ @@ -126,10 +153,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, zeroed = true; malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; /* Update the size of the huge allocation. */ assert(node->size != usize); @@ -159,7 +183,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { size_t sdiff; bool zeroed; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; sdiff = CHUNK_CEILING(usize) - usize; @@ -172,10 +196,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) } malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; /* Update the size of the huge allocation. */ node->size = usize; @@ -190,7 +211,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) static bool huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { size_t usize; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; bool is_zeroed_subchunk, is_zeroed_chunk; @@ -201,10 +222,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { } malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; is_zeroed_subchunk = node->zeroed; malloc_mutex_unlock(&huge_mtx); @@ -342,77 +360,44 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) { - extent_node_t *node, key; + extent_node_t *node; malloc_mutex_lock(&huge_mtx); - /* Extract from tree of huge allocations. */ - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); extent_tree_ad_remove(&huge, node); malloc_mutex_unlock(&huge_mtx); huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, node->size); - idalloct(tsd, node, try_tcache); + idalloctm(tsd, node, try_tcache, true); } -size_t -huge_salloc(const void *ptr) +arena_t * +huge_aalloc(const void *ptr) { - size_t ret; - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - ret = node->size; + return (huge_node(ptr)->arena); +} - malloc_mutex_unlock(&huge_mtx); +size_t +huge_salloc(const void *ptr) +{ - return (ret); + return (huge_node(ptr)->size); } prof_tctx_t * huge_prof_tctx_get(const void *ptr) { - prof_tctx_t *ret; - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - ret = node->prof_tctx; - - malloc_mutex_unlock(&huge_mtx); - - return (ret); + return (huge_node(ptr)->prof_tctx); } void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { - extent_node_t *node, key; - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - node->prof_tctx = tctx; - - malloc_mutex_unlock(&huge_mtx); + huge_node(ptr)->prof_tctx = tctx; } bool diff --git a/src/jemalloc.c b/src/jemalloc.c index 632c8d3..d1fa674 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -289,45 +289,34 @@ a0get(void) } static void * -a0imalloc(size_t size, bool zero) +a0ialloc(size_t size, bool zero, bool is_metadata) { - void *ret; if (unlikely(malloc_init_a0())) return (NULL); - if (likely(size <= arena_maxclass)) - ret = arena_malloc(NULL, a0get(), size, zero, false); - else - ret = huge_malloc(NULL, a0get(), size, zero, false); - - return (ret); + return (iallocztm(NULL, size, zero, false, is_metadata, a0get())); } static void -a0idalloc(void *ptr) +a0idalloc(void *ptr, bool is_metadata) { - arena_chunk_t *chunk; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_dalloc(NULL, chunk, ptr, false); - else - huge_dalloc(NULL, ptr, false); + idalloctm(NULL, ptr, false, is_metadata); } void * -a0malloc(size_t size, bool zero) +a0malloc(size_t size) { - return (a0imalloc(size, zero)); + return (a0ialloc(size, false, true)); } void a0dalloc(void *ptr) { - a0idalloc(ptr); + a0idalloc(ptr, true); } /* @@ -343,7 +332,7 @@ bootstrap_malloc(size_t size) if (unlikely(size == 0)) size = 1; - return (a0imalloc(size, false)); + return (a0ialloc(size, false, false)); } void * @@ -357,7 +346,7 @@ bootstrap_calloc(size_t num, size_t size) num_size = 1; } - return (a0imalloc(num_size, true)); + return (a0ialloc(num_size, true, false)); } void @@ -367,7 +356,7 @@ bootstrap_free(void *ptr) if (unlikely(ptr == NULL)) return; - a0idalloc(ptr); + a0idalloc(ptr, false); } /* Create a new arena and insert it into the arenas array at index ind. */ @@ -382,7 +371,7 @@ arena_init_locked(unsigned ind) unsigned narenas_new = narenas_total + 1; arena_t **arenas_new = (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * - sizeof(arena_t *)), false); + sizeof(arena_t *))); if (arenas_new == NULL) return (NULL); memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); @@ -519,7 +508,7 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) if (!*arenas_cache_bypassp) { *arenas_cache_bypassp = true; arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * - narenas_cache, false); + narenas_cache); *arenas_cache_bypassp = false; } else arenas_cache = NULL; @@ -1202,6 +1191,8 @@ malloc_init_hard_a0_locked(void) arena_boot(); if (config_tcache && tcache_boot()) return (true); + if (config_tcache && tcache_boot()) + malloc_mutex_unlock(&init_lock); if (huge_boot()) return (true); if (malloc_mutex_init(&arenas_lock)) @@ -532,8 +532,8 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) /* * Create a single allocation that has space for vec of length bt->len. */ - prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t, - vec) + (bt->len * sizeof(void *))); + prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t, + vec) + (bt->len * sizeof(void *)), false, true, true, NULL); if (gctx == NULL) return (NULL); gctx->lock = prof_gctx_mutex_choose(); @@ -574,7 +574,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, prof_leave(tsd, tdata_self); /* Destroy gctx. */ malloc_mutex_unlock(gctx->lock); - idalloc(tsd, gctx); + idalloctm(tsd, gctx, true, true); } else { /* * Compensate for increment in prof_tctx_destroy() or @@ -674,7 +674,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) prof_tdata_destroy(tsd, tdata, false); if (destroy_tctx) - idalloc(tsd, tctx); + idalloctm(tsd, tctx, true, true); } static bool @@ -703,7 +703,7 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { /* OOM. */ prof_leave(tsd, tdata); - idalloc(tsd, gctx.v); + idalloctm(tsd, gctx.v, true, true); return (true); } new_gctx = true; @@ -760,7 +760,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) return (NULL); /* Link a prof_tctx_t into gctx for this thread. */ - ret.v = imalloc(tsd, sizeof(prof_tctx_t)); + ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, true, true, + NULL); if (ret.p == NULL) { if (new_gctx) prof_gctx_try_destroy(tsd, tdata, gctx, tdata); @@ -778,7 +779,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) if (error) { if (new_gctx) prof_gctx_try_destroy(tsd, tdata, gctx, tdata); - idalloc(tsd, ret.v); + idalloctm(tsd, ret.v, true, true); return (NULL); } malloc_mutex_lock(gctx->lock); @@ -1158,7 +1159,7 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) to_destroy); tctx_tree_remove(&gctx->tctxs, to_destroy); - idalloc(tsd, to_destroy); + idalloctm(tsd, to_destroy, true, true); } else next = NULL; } while (next != NULL); @@ -1640,7 +1641,8 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, cassert(config_prof); /* Initialize an empty cache for this thread. */ - tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t)); + tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false, + true, true, NULL); if (tdata == NULL) return (NULL); @@ -1653,7 +1655,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { - idalloc(tsd, tdata); + idalloctm(tsd, tdata, true, true); return (NULL); } @@ -1706,9 +1708,9 @@ prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, tdata_tree_remove(&tdatas, tdata); if (tdata->thread_name != NULL) - idalloc(tsd, tdata->thread_name); + idalloctm(tsd, tdata->thread_name, true, true); ckh_delete(tsd, &tdata->bt2tctx); - idalloc(tsd, tdata); + idalloctm(tsd, tdata, true, true); } static void @@ -1869,7 +1871,7 @@ prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) if (size == 1) return (""); - ret = imalloc(tsd, size); + ret = iallocztm(tsd, size, false, true, true, NULL); if (ret == NULL) return (NULL); memcpy(ret, thread_name, size); @@ -1901,7 +1903,7 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name) return (EAGAIN); if (tdata->thread_name != NULL) { - idalloc(tsd, tdata->thread_name); + idalloctm(tsd, tdata->thread_name, true, true); tdata->thread_name = NULL; } if (strlen(s) > 0) diff --git a/src/quarantine.c b/src/quarantine.c index 12c37e0..094b44d 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -26,8 +26,9 @@ quarantine_init(tsd_t *tsd, size_t lg_maxobjs) assert(tsd_nominal(tsd)); - quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) + - ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); + quarantine = (quarantine_t *)iallocztm(tsd, offsetof(quarantine_t, objs) + + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)), false, true, + true, NULL); if (quarantine == NULL) return (NULL); quarantine->curbytes = 0; @@ -54,7 +55,7 @@ quarantine_alloc_hook_work(tsd_t *tsd) if (tsd_quarantine_get(tsd) == NULL) tsd_quarantine_set(tsd, quarantine); else - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); } static quarantine_t * @@ -86,7 +87,7 @@ quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * sizeof(quarantine_obj_t)); } - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); tsd_quarantine_set(tsd, ret); return (ret); @@ -176,7 +177,7 @@ quarantine_cleanup(tsd_t *tsd) quarantine = tsd_quarantine_get(tsd); if (quarantine != NULL) { quarantine_drain(tsd, quarantine, 0); - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); tsd_quarantine_set(tsd, NULL); } } diff --git a/src/stats.c b/src/stats.c index 2b3da64..865f775 100644 --- a/src/stats.c +++ b/src/stats.c @@ -265,6 +265,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned nthreads; const char *dss; size_t page, pactive, pdirty, mapped; + size_t metadata_mapped, metadata_allocated; uint64_t npurge, nmadvise, purged; size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests; @@ -331,6 +332,12 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); + CTL_I_GET("stats.arenas.0.metadata.mapped", &metadata_mapped, size_t); + CTL_I_GET("stats.arenas.0.metadata.allocated", &metadata_allocated, + size_t); + malloc_cprintf(write_cb, cbopaque, + "metadata: mapped: %zu, allocated: %zu\n", metadata_mapped, + metadata_allocated); if (bins) stats_arena_bins_print(write_cb, cbopaque, i); @@ -539,17 +546,18 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_stats) { size_t *cactive; - size_t allocated, active, mapped; + size_t allocated, active, metadata, mapped; size_t chunks_current, chunks_high; uint64_t chunks_total; CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); + CTL_GET("stats.metadata", &metadata, size_t); CTL_GET("stats.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, - "Allocated: %zu, active: %zu, mapped: %zu\n", - allocated, active, mapped); + "Allocated: %zu, active: %zu, metadata: %zu, mapped: %zu\n", + allocated, active, metadata, mapped); malloc_cprintf(write_cb, cbopaque, "Current active ceiling: %zu\n", atomic_read_z(cactive)); diff --git a/src/tcache.c b/src/tcache.c index 34224ec..d638015 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -298,7 +298,7 @@ tcache_create(tsd_t *tsd, arena_t *arena) /* Avoid false cacheline sharing. */ size = sa2u(size, CACHELINE); - tcache = ipalloct(tsd, size, CACHELINE, true, false, arena); + tcache = ipallocztm(tsd, size, CACHELINE, true, false, true, arena); if (tcache == NULL) return (NULL); @@ -353,7 +353,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) prof_idump(); - idalloct(tsd, tcache, false); + idalloctm(tsd, tcache, false, true); } void @@ -15,7 +15,7 @@ void * malloc_tsd_malloc(size_t size) { - return (a0malloc(CACHELINE_CEILING(size), false)); + return (a0malloc(CACHELINE_CEILING(size))); } void |