From 5ef33a9f2b9f4fb56553529f7b31f4f5f57ce014 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 19 Aug 2015 14:12:05 -0700 Subject: Don't bitshift by negative amounts. Don't bitshift by negative amounts when encoding/decoding run sizes in chunk header maps. This affected systems with page sizes greater than 8 KiB. Reported by Ingvar Hagelund . --- ChangeLog | 6 ++++ include/jemalloc/internal/arena.h | 48 ++++++++++++++++++++++----- include/jemalloc/internal/private_symbols.txt | 2 ++ src/arena.c | 7 ++-- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0cf887c..c98179c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,12 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc +* 4.x.x (XXX) + + Bug fixes: + - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk + header maps. This affected systems with page sizes greater than 8 KiB. + * 4.0.0 (August 17, 2015) This version contains many speed and space optimizations, both minor and diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index cb015ee..2347213 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -519,6 +519,7 @@ arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_size_decode(size_t mapbits); size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); @@ -530,6 +531,7 @@ size_t arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); +size_t arena_mapbits_size_encode(size_t size); void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags); void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, @@ -653,13 +655,28 @@ arena_mapbits_get(arena_chunk_t *chunk, size_t pageind) } JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_size_decode(size_t mapbits) +{ + size_t size; + + if (CHUNK_MAP_SIZE_SHIFT > 0) + size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; + else if (CHUNK_MAP_SIZE_SHIFT == 0) + size = mapbits & CHUNK_MAP_SIZE_MASK; + else + size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; + + return (size); +} + +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); - return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT); + return (arena_mapbits_size_decode(mapbits)); } JEMALLOC_ALWAYS_INLINE size_t @@ -670,7 +687,7 @@ arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); - return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT); + return (arena_mapbits_size_decode(mapbits)); } JEMALLOC_ALWAYS_INLINE size_t @@ -754,6 +771,22 @@ arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) *mapbitsp = mapbits; } +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbits_size_encode(size_t size) +{ + size_t mapbits; + + if (CHUNK_MAP_SIZE_SHIFT > 0) + mapbits = size << CHUNK_MAP_SIZE_SHIFT; + else if (CHUNK_MAP_SIZE_SHIFT == 0) + mapbits = size; + else + mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; + + assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); + return (mapbits); +} + JEMALLOC_ALWAYS_INLINE void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags) @@ -761,11 +794,10 @@ arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); assert((size & PAGE_MASK) == 0); - assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0); assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) | + arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | CHUNK_MAP_BININD_INVALID | flags); } @@ -777,10 +809,9 @@ arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, size_t mapbits = arena_mapbitsp_read(mapbitsp); assert((size & PAGE_MASK) == 0); - assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0); assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); - arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) | (mapbits - & ~CHUNK_MAP_SIZE_MASK)); + arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | + (mapbits & ~CHUNK_MAP_SIZE_MASK)); } JEMALLOC_ALWAYS_INLINE void @@ -799,11 +830,10 @@ arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); assert((size & PAGE_MASK) == 0); - assert(((size << CHUNK_MAP_SIZE_SHIFT) & ~CHUNK_MAP_SIZE_MASK) == 0); assert((flags & CHUNK_MAP_FLAGS_MASK) == flags); assert((flags & CHUNK_MAP_DECOMMITTED) == 0 || (flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == 0); - arena_mapbitsp_write(mapbitsp, (size << CHUNK_MAP_SIZE_SHIFT) | + arena_mapbitsp_write(mapbitsp, arena_mapbits_size_encode(size) | CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); } diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index dbf6aa7..ed1f6c2 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -50,6 +50,8 @@ arena_mapbits_large_size_get arena_mapbitsp_get arena_mapbitsp_read arena_mapbitsp_write +arena_mapbits_size_decode +arena_mapbits_size_encode arena_mapbits_small_runind_get arena_mapbits_small_set arena_mapbits_unallocated_set diff --git a/src/arena.c b/src/arena.c index af48b39..bd76e96 100644 --- a/src/arena.c +++ b/src/arena.c @@ -39,7 +39,7 @@ JEMALLOC_INLINE_C arena_chunk_map_misc_t * arena_miscelm_key_create(size_t size) { - return ((arena_chunk_map_misc_t *)((size << CHUNK_MAP_SIZE_SHIFT) | + return ((arena_chunk_map_misc_t *)(arena_mapbits_size_encode(size) | CHUNK_MAP_KEY)); } @@ -58,8 +58,7 @@ arena_miscelm_key_size_get(const arena_chunk_map_misc_t *miscelm) assert(arena_miscelm_is_key(miscelm)); - return (((uintptr_t)miscelm & CHUNK_MAP_SIZE_MASK) >> - CHUNK_MAP_SIZE_SHIFT); + return (arena_mapbits_size_decode((uintptr_t)miscelm)); } JEMALLOC_INLINE_C size_t @@ -73,7 +72,7 @@ arena_miscelm_size_get(arena_chunk_map_misc_t *miscelm) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); pageind = arena_miscelm_to_pageind(miscelm); mapbits = arena_mapbits_get(chunk, pageind); - return ((mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT); + return (arena_mapbits_size_decode(mapbits)); } JEMALLOC_INLINE_C int -- cgit v0.12 From d01fd19755bc0c2f5be3143349016dd0d7de7b36 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 19 Aug 2015 15:21:32 -0700 Subject: Rename index_t to szind_t to avoid an existing type on Solaris. This resolves #256. --- ChangeLog | 1 + include/jemalloc/internal/arena.h | 38 ++++++++++---------- include/jemalloc/internal/jemalloc_internal.h.in | 28 +++++++-------- include/jemalloc/internal/tcache.h | 18 +++++----- src/arena.c | 46 ++++++++++++------------ src/tcache.c | 8 ++--- test/unit/size_classes.c | 2 +- 7 files changed, 71 insertions(+), 70 deletions(-) diff --git a/ChangeLog b/ChangeLog index c98179c..284d6d2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,7 @@ brevity. Much more detail can be found in the git revision history: Bug fixes: - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk header maps. This affected systems with page sizes greater than 8 KiB. + - Rename index_t to szind_t to avoid an existing type on Solaris. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 2347213..62a9a85 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -39,7 +39,7 @@ typedef struct arena_s arena_t; #ifdef JEMALLOC_ARENA_STRUCTS_A struct arena_run_s { /* Index of bin this run is associated with. */ - index_t binind; + szind_t binind; /* Number of free regions in run. */ unsigned nfree; @@ -448,7 +448,7 @@ bool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult); void arena_maybe_purge(arena_t *arena); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, - index_t binind, uint64_t prof_accumbytes); + szind_t binind, uint64_t prof_accumbytes); void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero); #ifdef JEMALLOC_JET @@ -524,7 +524,7 @@ size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); -index_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); +szind_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_decommitted_get(arena_chunk_t *chunk, size_t pageind); @@ -541,17 +541,17 @@ void arena_mapbits_internal_set(arena_chunk_t *chunk, size_t pageind, void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags); void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - index_t binind); + szind_t binind); void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, - size_t runind, index_t binind, size_t flags); + size_t runind, szind_t binind, size_t flags); void arena_metadata_allocated_add(arena_t *arena, size_t size); void arena_metadata_allocated_sub(arena_t *arena, size_t size); size_t arena_metadata_allocated_get(arena_t *arena); bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); -index_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); -index_t arena_bin_index(arena_t *arena, arena_bin_t *bin); +szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); +szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_tctx_t *arena_prof_tctx_get(const void *ptr); @@ -701,11 +701,11 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) return (mapbits >> CHUNK_MAP_RUNIND_SHIFT); } -JEMALLOC_ALWAYS_INLINE index_t +JEMALLOC_ALWAYS_INLINE szind_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; - index_t binind; + szind_t binind; mapbits = arena_mapbits_get(chunk, pageind); binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; @@ -840,7 +840,7 @@ arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - index_t binind) + szind_t binind) { size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); size_t mapbits = arena_mapbitsp_read(mapbitsp); @@ -854,7 +854,7 @@ arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, JEMALLOC_ALWAYS_INLINE void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, - index_t binind, size_t flags) + szind_t binind, size_t flags) { size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); @@ -931,10 +931,10 @@ arena_prof_accum(arena_t *arena, uint64_t accumbytes) } } -JEMALLOC_ALWAYS_INLINE index_t +JEMALLOC_ALWAYS_INLINE szind_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits) { - index_t binind; + szind_t binind; binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; @@ -946,7 +946,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) size_t rpages_ind; arena_run_t *run; arena_bin_t *bin; - index_t run_binind, actual_binind; + szind_t run_binind, actual_binind; arena_bin_info_t *bin_info; arena_chunk_map_misc_t *miscelm; void *rpages; @@ -980,10 +980,10 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) # endif /* JEMALLOC_ARENA_INLINE_A */ # ifdef JEMALLOC_ARENA_INLINE_B -JEMALLOC_INLINE index_t +JEMALLOC_INLINE szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { - index_t binind = bin - arena->bins; + szind_t binind = bin - arena->bins; assert(binind < NBINS); return (binind); } @@ -1161,7 +1161,7 @@ arena_salloc(const void *ptr, bool demote) size_t ret; arena_chunk_t *chunk; size_t pageind; - index_t binind; + szind_t binind; assert(ptr != NULL); @@ -1220,7 +1220,7 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { /* Small allocation. */ if (likely(tcache != NULL)) { - index_t binind = arena_ptr_small_binind_get(ptr, + szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tsd, tcache, ptr, binind); } else { @@ -1272,7 +1272,7 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ if (likely(tcache != NULL)) { - index_t binind = size2index(size); + szind_t binind = size2index(size); tcache_dalloc_small(tsd, tcache, ptr, binind); } else { size_t pageind = ((uintptr_t)ptr - diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 7a137b6..f6e464e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -184,7 +184,7 @@ static const bool config_cache_oblivious = #include "jemalloc/internal/jemalloc_internal_macros.h" /* Size class index type. */ -typedef unsigned index_t; +typedef unsigned szind_t; /* * Flags bits: @@ -511,12 +511,12 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE -index_t size2index_compute(size_t size); -index_t size2index_lookup(size_t size); -index_t size2index(size_t size); -size_t index2size_compute(index_t index); -size_t index2size_lookup(index_t index); -size_t index2size(index_t index); +szind_t size2index_compute(size_t size); +szind_t size2index_lookup(size_t size); +szind_t size2index(size_t size); +size_t index2size_compute(szind_t index); +size_t index2size_lookup(szind_t index); +size_t index2size(szind_t index); size_t s2u_compute(size_t size); size_t s2u_lookup(size_t size); size_t s2u(size_t size); @@ -527,7 +527,7 @@ arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_INLINE index_t +JEMALLOC_INLINE szind_t size2index_compute(size_t size) { @@ -558,7 +558,7 @@ size2index_compute(size_t size) } } -JEMALLOC_ALWAYS_INLINE index_t +JEMALLOC_ALWAYS_INLINE szind_t size2index_lookup(size_t size) { @@ -571,7 +571,7 @@ size2index_lookup(size_t size) } } -JEMALLOC_ALWAYS_INLINE index_t +JEMALLOC_ALWAYS_INLINE szind_t size2index(size_t size) { @@ -582,7 +582,7 @@ size2index(size_t size) } JEMALLOC_INLINE size_t -index2size_compute(index_t index) +index2size_compute(szind_t index) { #if (NTBINS > 0) @@ -609,7 +609,7 @@ index2size_compute(index_t index) } JEMALLOC_ALWAYS_INLINE size_t -index2size_lookup(index_t index) +index2size_lookup(szind_t index) { size_t ret = (size_t)index2size_tab[index]; assert(ret == index2size_compute(index)); @@ -617,7 +617,7 @@ index2size_lookup(index_t index) } JEMALLOC_ALWAYS_INLINE size_t -index2size(index_t index) +index2size(szind_t index) { assert(index < NSIZES); @@ -976,7 +976,7 @@ u2rz(size_t usize) size_t ret; if (usize <= SMALL_MAXCLASS) { - index_t binind = size2index(usize); + szind_t binind = size2index(usize); ret = arena_bin_info[binind].redzone_size; } else ret = 0; diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 493f457..5079cd2 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -77,7 +77,7 @@ struct tcache_s { ql_elm(tcache_t) link; /* Used for aggregating stats. */ uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ unsigned ev_cnt; /* Event count since incremental GC. */ - index_t next_gc_bin; /* Next bin to GC. */ + szind_t next_gc_bin; /* Next bin to GC. */ tcache_bin_t tbins[1]; /* Dynamically sized. */ /* * The pointer stacks associated with tbins follow as a contiguous @@ -126,10 +126,10 @@ extern tcaches_t *tcaches; size_t tcache_salloc(const void *ptr); void tcache_event_hard(tsd_t *tsd, tcache_t *tcache); void *tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, index_t binind); + tcache_bin_t *tbin, szind_t binind); void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, - index_t binind, unsigned rem); -void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, + szind_t binind, unsigned rem); +void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); void tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena, @@ -161,7 +161,7 @@ void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, bool zero); void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, - index_t binind); + szind_t binind); void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size); tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); @@ -267,7 +267,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, bool zero) { void *ret; - index_t binind; + szind_t binind; size_t usize; tcache_bin_t *tbin; @@ -312,7 +312,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, bool zero) { void *ret; - index_t binind; + szind_t binind; size_t usize; tcache_bin_t *tbin; @@ -360,7 +360,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind) +tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind) { tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; @@ -386,7 +386,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind) JEMALLOC_ALWAYS_INLINE void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size) { - index_t binind; + szind_t binind; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; diff --git a/src/arena.c b/src/arena.c index bd76e96..b1bb9db 100644 --- a/src/arena.c +++ b/src/arena.c @@ -314,7 +314,7 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t mapbits = arena_mapbits_get(chunk, pageind); - index_t binind = arena_ptr_small_binind_get(ptr, mapbits); + szind_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); @@ -507,7 +507,7 @@ arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) static bool arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, - index_t binind) + szind_t binind) { arena_chunk_t *chunk; arena_chunk_map_misc_t *miscelm; @@ -779,7 +779,7 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) static void arena_huge_malloc_stats_update(arena_t *arena, size_t usize) { - index_t index = size2index(usize) - nlclasses - NBINS; + szind_t index = size2index(usize) - nlclasses - NBINS; cassert(config_stats); @@ -792,7 +792,7 @@ arena_huge_malloc_stats_update(arena_t *arena, size_t usize) static void arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) { - index_t index = size2index(usize) - nlclasses - NBINS; + szind_t index = size2index(usize) - nlclasses - NBINS; cassert(config_stats); @@ -805,7 +805,7 @@ arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) static void arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) { - index_t index = size2index(usize) - nlclasses - NBINS; + szind_t index = size2index(usize) - nlclasses - NBINS; cassert(config_stats); @@ -818,7 +818,7 @@ arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) static void arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) { - index_t index = size2index(usize) - nlclasses - NBINS; + szind_t index = size2index(usize) - nlclasses - NBINS; cassert(config_stats); @@ -1124,7 +1124,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero) } static arena_run_t * -arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) +arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind) { arena_run_t *run = arena_run_first_best_fit(arena, size); if (run != NULL) { @@ -1135,7 +1135,7 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) } static arena_run_t * -arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) +arena_run_alloc_small(arena_t *arena, size_t size, szind_t binind) { arena_chunk_t *chunk; arena_run_t *run; @@ -1888,7 +1888,7 @@ static arena_run_t * arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) { arena_run_t *run; - index_t binind; + szind_t binind; arena_bin_info_t *bin_info; /* Look for a usable run. */ @@ -1939,7 +1939,7 @@ static void * arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) { void *ret; - index_t binind; + szind_t binind; arena_bin_info_t *bin_info; arena_run_t *run; @@ -1985,7 +1985,7 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) } void -arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, +arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { unsigned i, nfill; @@ -2130,7 +2130,7 @@ arena_dalloc_junk_small_t *arena_dalloc_junk_small = void arena_quarantine_junk_small(void *ptr, size_t usize) { - index_t binind; + szind_t binind; arena_bin_info_t *bin_info; cassert(config_fill); assert(opt_junk_free); @@ -2148,7 +2148,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) void *ret; arena_bin_t *bin; arena_run_t *run; - index_t binind; + szind_t binind; binind = size2index(size); assert(binind < NBINS); @@ -2232,7 +2232,7 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) ret = (void *)((uintptr_t)arena_miscelm_to_rpages(miscelm) + random_offset); if (config_stats) { - index_t index = size2index(usize) - NBINS; + szind_t index = size2index(usize) - NBINS; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; @@ -2325,7 +2325,7 @@ arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, ret = arena_miscelm_to_rpages(miscelm); if (config_stats) { - index_t index = size2index(usize) - NBINS; + szind_t index = size2index(usize) - NBINS; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; @@ -2384,7 +2384,7 @@ arena_prof_promoted(const void *ptr, size_t size) { arena_chunk_t *chunk; size_t pageind; - index_t binind; + szind_t binind; cassert(config_prof); assert(ptr != NULL); @@ -2412,7 +2412,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, if (run == bin->runcur) bin->runcur = NULL; else { - index_t binind = arena_bin_index(extent_node_arena_get( + szind_t binind = arena_bin_index(extent_node_arena_get( &chunk->node), bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; @@ -2476,7 +2476,7 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_run_t *run; arena_bin_t *bin; arena_bin_info_t *bin_info; - index_t binind; + szind_t binind; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); @@ -2573,7 +2573,7 @@ arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk, if (!junked) arena_dalloc_junk_large(ptr, usize); if (config_stats) { - index_t index = size2index(usize) - NBINS; + szind_t index = size2index(usize) - NBINS; arena->stats.ndalloc_large++; arena->stats.allocated_large -= usize; @@ -2620,8 +2620,8 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_run_trim_tail(arena, chunk, run, oldsize + large_pad, size + large_pad, true); if (config_stats) { - index_t oldindex = size2index(oldsize) - NBINS; - index_t index = size2index(size) - NBINS; + szind_t oldindex = size2index(oldsize) - NBINS; + szind_t index = size2index(size) - NBINS; arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; @@ -2699,8 +2699,8 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, pageind+npages-1))); if (config_stats) { - index_t oldindex = size2index(oldsize) - NBINS; - index_t index = size2index(size) - NBINS; + szind_t oldindex = size2index(oldsize) - NBINS; + szind_t index = size2index(size) - NBINS; arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; diff --git a/src/tcache.c b/src/tcache.c index 3814365..f1a30d5 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -32,7 +32,7 @@ size_t tcache_salloc(const void *ptr) void tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { - index_t binind = tcache->next_gc_bin; + szind_t binind = tcache->next_gc_bin; tcache_bin_t *tbin = &tcache->tbins[binind]; tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; @@ -72,7 +72,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) void * tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, index_t binind) + tcache_bin_t *tbin, szind_t binind) { void *ret; @@ -87,7 +87,7 @@ tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache, void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, - index_t binind, unsigned rem) + szind_t binind, unsigned rem) { arena_t *arena; void *ptr; @@ -166,7 +166,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, } void -tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, +tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache) { arena_t *arena; diff --git a/test/unit/size_classes.c b/test/unit/size_classes.c index d791834..d3aaebd 100644 --- a/test/unit/size_classes.c +++ b/test/unit/size_classes.c @@ -26,7 +26,7 @@ get_max_size_class(void) TEST_BEGIN(test_size_classes) { size_t size_class, max_size_class; - index_t index, max_index; + szind_t index, max_index; max_size_class = get_max_size_class(); max_index = size2index(max_size_class); -- cgit v0.12 From b5c2a347d7cbf1154181ccb3adc599c8bd2094c9 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 19 Aug 2015 23:28:34 -0700 Subject: Silence compiler warnings for unreachable code. Reported by Ingvar Hagelund. --- include/jemalloc/internal/arena.h | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 62a9a85..f2954b3 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -659,12 +659,13 @@ arena_mapbits_size_decode(size_t mapbits) { size_t size; - if (CHUNK_MAP_SIZE_SHIFT > 0) - size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; - else if (CHUNK_MAP_SIZE_SHIFT == 0) - size = mapbits & CHUNK_MAP_SIZE_MASK; - else - size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; +#if CHUNK_MAP_SIZE_SHIFT > 0 + size = (mapbits & CHUNK_MAP_SIZE_MASK) >> CHUNK_MAP_SIZE_SHIFT; +#elif CHUNK_MAP_SIZE_SHIFT == 0 + size = mapbits & CHUNK_MAP_SIZE_MASK; +#else + size = (mapbits & CHUNK_MAP_SIZE_MASK) << -CHUNK_MAP_SIZE_SHIFT; +#endif return (size); } @@ -776,12 +777,13 @@ arena_mapbits_size_encode(size_t size) { size_t mapbits; - if (CHUNK_MAP_SIZE_SHIFT > 0) - mapbits = size << CHUNK_MAP_SIZE_SHIFT; - else if (CHUNK_MAP_SIZE_SHIFT == 0) - mapbits = size; - else - mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; +#if CHUNK_MAP_SIZE_SHIFT > 0 + mapbits = size << CHUNK_MAP_SIZE_SHIFT; +#elif CHUNK_MAP_SIZE_SHIFT == 0 + mapbits = size; +#else + mapbits = size >> -CHUNK_MAP_SIZE_SHIFT; +#endif assert((mapbits & ~CHUNK_MAP_SIZE_MASK) == 0); return (mapbits); -- cgit v0.12 From 45e9f66c280e1ba8bebf7bed387a43bc9e45536d Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 21 Aug 2015 12:23:06 -0700 Subject: Fix arenas_cache_cleanup(). Fix arenas_cache_cleanup() to handle allocation/deallocation within the application's thread-specific data cleanup functions even after arenas_cache is torn down. --- ChangeLog | 5 ++++- src/jemalloc.c | 6 +++++- test/unit/tsd.c | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 284d6d2..9bcf299 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,9 +4,12 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc -* 4.x.x (XXX) +* 4.0.1 (XXX) Bug fixes: + - Fix arenas_cache_cleanup() to handle allocation/deallocation within the + application's thread-specific data cleanup functions even after + arenas_cache is torn down. - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk header maps. This affected systems with page sizes greater than 8 KiB. - Rename index_t to szind_t to avoid an existing type on Solaris. diff --git a/src/jemalloc.c b/src/jemalloc.c index ed7863b..0361913 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -649,8 +649,12 @@ arenas_cache_cleanup(tsd_t *tsd) arena_t **arenas_cache; arenas_cache = tsd_arenas_cache_get(tsd); - if (arenas_cache != NULL) + if (arenas_cache != NULL) { + bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); + *arenas_cache_bypassp = true; + tsd_arenas_cache_set(tsd, NULL); a0dalloc(arenas_cache); + } } void diff --git a/test/unit/tsd.c b/test/unit/tsd.c index b031c48..8be787f 100644 --- a/test/unit/tsd.c +++ b/test/unit/tsd.c @@ -56,9 +56,14 @@ static void * thd_start(void *arg) { data_t d = (data_t)(uintptr_t)arg; + void *p; + assert_x_eq(*data_tsd_get(), DATA_INIT, "Initial tsd get should return initialization value"); + p = malloc(1); + assert_ptr_not_null(p, "Unexpected malloc() failure"); + data_tsd_set(&d); assert_x_eq(*data_tsd_get(), d, "After tsd set, tsd get should return value that was set"); @@ -67,6 +72,7 @@ thd_start(void *arg) assert_x_eq(*data_tsd_get(), (data_t)(uintptr_t)arg, "Resetting local data should have no effect on tsd"); + free(p); return (NULL); } -- cgit v0.12 From 5d2e875ac9283cb99ff714c5cb56e1fc98a7f007 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 26 Aug 2015 13:47:20 -0700 Subject: Add JEMALLOC_CXX_THROW to the memalign() function prototype. Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to match glibc and avoid compilation errors when including both jemalloc/jemalloc.h and malloc.h in C++ code. This change was unintentionally omitted from ae93d6bf364e9db9f9ee69c3e5f9df110d8685a4 (Avoid function prototype incompatibilities.). --- ChangeLog | 3 +++ include/jemalloc/jemalloc_protos.h.in | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 9bcf299..97728b0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,9 @@ brevity. Much more detail can be found in the git revision history: - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk header maps. This affected systems with page sizes greater than 8 KiB. - Rename index_t to szind_t to avoid an existing type on Solaris. + - Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to + match glibc and avoid compilation errors when including both + jemalloc/jemalloc.h and malloc.h in C++ code. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 317ffdb..a78414b 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -56,7 +56,7 @@ JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_usable_size( #ifdef JEMALLOC_OVERRIDE_MEMALIGN JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW *@je_@memalign(size_t alignment, size_t size) - JEMALLOC_ATTR(malloc); + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); #endif #ifdef JEMALLOC_OVERRIDE_VALLOC -- cgit v0.12 From 30949da601f7405c294a71d30bd67be29cfbc2a5 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 25 Aug 2015 16:13:59 -0700 Subject: Fix arenas_cache_cleanup() and arena_get_hard(). Fix arenas_cache_cleanup() and arena_get_hard() to handle allocation/deallocation within the application's thread-specific data cleanup functions even after arenas_cache is torn down. This is a more general fix that complements 45e9f66c280e1ba8bebf7bed387a43bc9e45536d (Fix arenas_cache_cleanup().). --- ChangeLog | 6 +++--- src/jemalloc.c | 11 +++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/ChangeLog b/ChangeLog index 97728b0..17f6538 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,9 +7,9 @@ brevity. Much more detail can be found in the git revision history: * 4.0.1 (XXX) Bug fixes: - - Fix arenas_cache_cleanup() to handle allocation/deallocation within the - application's thread-specific data cleanup functions even after - arenas_cache is torn down. + - Fix arenas_cache_cleanup() and arena_get_hard() to handle + allocation/deallocation within the application's thread-specific data + cleanup functions even after arenas_cache is torn down. - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk header maps. This affected systems with page sizes greater than 8 KiB. - Rename index_t to szind_t to avoid an existing type on Solaris. diff --git a/src/jemalloc.c b/src/jemalloc.c index 0361913..df962c6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -510,17 +510,17 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) assert(ind < narenas_actual || !init_if_missing); narenas_cache = (ind < narenas_actual) ? narenas_actual : ind+1; - if (!*arenas_cache_bypassp) { + if (tsd_nominal(tsd) && !*arenas_cache_bypassp) { *arenas_cache_bypassp = true; arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * narenas_cache); *arenas_cache_bypassp = false; - } else - arenas_cache = NULL; + } if (arenas_cache == NULL) { /* * This function must always tell the truth, even if - * it's slow, so don't let OOM or recursive allocation + * it's slow, so don't let OOM, thread cleanup (note + * tsd_nominal check), nor recursive allocation * avoidance (note arenas_cache_bypass check) get in the * way. */ @@ -531,6 +531,7 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) malloc_mutex_unlock(&arenas_lock); return (arena); } + assert(tsd_nominal(tsd) && !*arenas_cache_bypassp); tsd_arenas_cache_set(tsd, arenas_cache); tsd_narenas_cache_set(tsd, narenas_cache); } @@ -650,8 +651,6 @@ arenas_cache_cleanup(tsd_t *tsd) arenas_cache = tsd_arenas_cache_get(tsd); if (arenas_cache != NULL) { - bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); - *arenas_cache_bypassp = true; tsd_arenas_cache_set(tsd, NULL); a0dalloc(arenas_cache); } -- cgit v0.12 From 6d8075f1e6e72ee274832f1164c164f713788e34 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 27 Aug 2015 20:30:15 -0700 Subject: Fix chunk purge hook calls for in-place huge shrinking reallocation. Fix chunk purge hook calls for in-place huge shrinking reallocation to specify the old chunk size rather than the new chunk size. This bug caused no correctness issues for the default chunk purge function, but was visible to custom functions set via the "arena..chunk_hooks" mallctl. This resolves #264. --- ChangeLog | 4 ++++ src/huge.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 17f6538..53d9caf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -16,6 +16,10 @@ brevity. Much more detail can be found in the git revision history: - Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to match glibc and avoid compilation errors when including both jemalloc/jemalloc.h and malloc.h in C++ code. + - Fix chunk purge hook calls for in-place huge shrinking reallocation to + specify the old chunk size rather than the new chunk size. This bug caused + no correctness issues for the default chunk purge function, but was + visible to custom functions set via the "arena..chunk_hooks" mallctl. * 4.0.0 (August 17, 2015) diff --git a/src/huge.c b/src/huge.c index 54c2114..f49a937 100644 --- a/src/huge.c +++ b/src/huge.c @@ -149,7 +149,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, if (oldsize > usize) { size_t sdiff = oldsize - usize; zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr, - CHUNK_CEILING(usize), usize, sdiff); + CHUNK_CEILING(oldsize), usize, sdiff); if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); zeroed = false; @@ -204,7 +204,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) size_t sdiff = oldsize - usize; zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr + usize), - CHUNK_CEILING(usize), CHUNK_ADDR2OFFSET((uintptr_t)ptr + + CHUNK_CEILING(oldsize), CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); if (config_fill && unlikely(opt_junk_free)) { huge_dalloc_junk((void *)((uintptr_t)ptr + usize), -- cgit v0.12 From 4a2a3c9a6e5f8fda3536f48095d68ddaa13cd977 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Fri, 28 Aug 2015 13:45:51 +0900 Subject: Don't purge junk filled chunks when shrinking huge allocations When junk filling is enabled, shrinking an allocation fills the bytes that were previously allocated but now aren't. Purging the chunk before doing that is just a waste of time. This resolves #260. --- src/huge.c | 14 ++++++++------ test/integration/chunk.c | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/huge.c b/src/huge.c index f49a937..4d5887c 100644 --- a/src/huge.c +++ b/src/huge.c @@ -148,11 +148,12 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, /* Fill if necessary (shrinking). */ if (oldsize > usize) { size_t sdiff = oldsize - usize; - zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr, - CHUNK_CEILING(oldsize), usize, sdiff); if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); zeroed = false; + } else { + zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr, + CHUNK_CEILING(oldsize), usize, sdiff); } } else zeroed = true; @@ -202,14 +203,15 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) if (oldsize > usize) { size_t sdiff = oldsize - usize; - zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, - CHUNK_ADDR2BASE((uintptr_t)ptr + usize), - CHUNK_CEILING(oldsize), CHUNK_ADDR2OFFSET((uintptr_t)ptr + - usize), sdiff); if (config_fill && unlikely(opt_junk_free)) { huge_dalloc_junk((void *)((uintptr_t)ptr + usize), sdiff); zeroed = false; + } else { + zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, + CHUNK_ADDR2BASE((uintptr_t)ptr + usize), + CHUNK_CEILING(oldsize), + CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); } } else zeroed = true; diff --git a/test/integration/chunk.c b/test/integration/chunk.c index 7eb1b6d..af1c9a5 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -1,5 +1,9 @@ #include "test/jemalloc_test.h" +#ifdef JEMALLOC_FILL +const char *malloc_conf = "junk:false"; +#endif + static chunk_hooks_t orig_hooks; static chunk_hooks_t old_hooks; -- cgit v0.12 From c0f43b65500a78c87ef16579a33661103ec0a4fb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 2 Sep 2015 12:46:35 -0700 Subject: Fix TLS configuration. Fix TLS configuration such that it is enabled by default for platforms on which it works correctly. This regression was introduced by ac5db02034c01357a4ce90504886046a58117921 (Make --enable-tls and --enable-lazy-lock take precedence over configure.ac-hardcoded defaults). --- ChangeLog | 2 ++ configure.ac | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 53d9caf..dba05eb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,8 @@ brevity. Much more detail can be found in the git revision history: specify the old chunk size rather than the new chunk size. This bug caused no correctness issues for the default chunk purge function, but was visible to custom functions set via the "arena..chunk_hooks" mallctl. + - Fix TLS configuration such that it is enabled by default for platforms on + which it works correctly. * 4.0.0 (August 17, 2015) diff --git a/configure.ac b/configure.ac index f7c7f3c..5e77b68 100644 --- a/configure.ac +++ b/configure.ac @@ -1272,13 +1272,16 @@ fi , enable_tls="" ) -if test "x${enable_tls}" = "x" -a "x${force_tls}" = "x1" ; then - AC_MSG_RESULT([Forcing TLS to avoid allocator/threading bootstrap issues]) - enable_tls="1" -fi -if test "x${enable_tls}" = "x" -a "x${force_tls}" = "x0" ; then - AC_MSG_RESULT([Forcing no TLS to avoid allocator/threading bootstrap issues]) - enable_tls="0" +if test "x${enable_tls}" = "x" ; then + if test "x${force_tls}" = "x1" ; then + AC_MSG_RESULT([Forcing TLS to avoid allocator/threading bootstrap issues]) + enable_tls="1" + elif test "x${force_tls}" = "x0" ; then + AC_MSG_RESULT([Forcing no TLS to avoid allocator/threading bootstrap issues]) + enable_tls="0" + else + enable_tls="1" + fi fi if test "x${enable_tls}" = "x1" ; then AC_MSG_CHECKING([for TLS]) @@ -1298,9 +1301,12 @@ else fi AC_SUBST([enable_tls]) if test "x${enable_tls}" = "x1" ; then + if test "x${force_tls}" = "x0" ; then + AC_MSG_WARN([TLS enabled despite being marked unusable on this platform]) + fi AC_DEFINE_UNQUOTED([JEMALLOC_TLS], [ ]) elif test "x${force_tls}" = "x1" ; then - AC_MSG_ERROR([Failed to configure TLS, which is mandatory for correct function]) + AC_MSG_WARN([TLS disabled despite being marked critical on this platform]) fi dnl ============================================================================ -- cgit v0.12 From 594c759f37c301d0245dc2accf4d4aaf9d202819 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 2 Sep 2015 14:52:24 -0700 Subject: Optimize arena_prof_tctx_set(). Optimize arena_prof_tctx_set() to avoid reading run metadata when deciding whether it's actually necessary to write. --- include/jemalloc/internal/arena.h | 23 ++++++++++++++---- include/jemalloc/internal/prof.h | 10 ++++---- src/prof.c | 2 +- test/unit/prof_reset.c | 49 +++++++++++++++++++++++++-------------- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f2954b3..76c5b93 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -555,7 +555,7 @@ szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_tctx_t *arena_prof_tctx_get(const void *ptr); -void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); +void arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_t *tcache); arena_t *arena_aalloc(const void *ptr); @@ -1092,7 +1092,7 @@ arena_prof_tctx_get(const void *ptr) } JEMALLOC_INLINE void -arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) +arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) { arena_chunk_t *chunk; @@ -1102,12 +1102,25 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) { - arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, - pageind); + if (unlikely(usize > SMALL_MAXCLASS || tctx > + (prof_tctx_t *)(uintptr_t)1U)) { + arena_chunk_map_misc_t *elm; + + assert(arena_mapbits_large_get(chunk, pageind) != 0); + + elm = arena_miscelm_get(chunk, pageind); atomic_write_p(&elm->prof_tctx_pun, tctx); + } else { + /* + * tctx must always be initialized for large runs. + * Assert that the surrounding conditional logic is + * equivalent to checking whether ptr refers to a large + * run. + */ + assert(arena_mapbits_large_get(chunk, pageind) == 0); } } else huge_prof_tctx_set(ptr, tctx); diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 2e22711..fe89828 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -332,7 +332,7 @@ bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, prof_tdata_t **tdata_out); prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update); prof_tctx_t *prof_tctx_get(const void *ptr); -void prof_tctx_set(const void *ptr, prof_tctx_t *tctx); +void prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); @@ -402,13 +402,13 @@ prof_tctx_get(const void *ptr) } JEMALLOC_ALWAYS_INLINE void -prof_tctx_set(const void *ptr, prof_tctx_t *tctx) +prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); - arena_prof_tctx_set(ptr, tctx); + arena_prof_tctx_set(ptr, usize, tctx); } JEMALLOC_ALWAYS_INLINE bool @@ -473,7 +473,7 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_malloc_sample_object(ptr, usize, tctx); else - prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); + prof_tctx_set(ptr, usize, (prof_tctx_t *)(uintptr_t)1U); } JEMALLOC_ALWAYS_INLINE void @@ -503,7 +503,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_malloc_sample_object(ptr, usize, tctx); else - prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); + prof_tctx_set(ptr, usize, (prof_tctx_t *)(uintptr_t)1U); } JEMALLOC_ALWAYS_INLINE void diff --git a/src/prof.c b/src/prof.c index a05792f..b79eba6 100644 --- a/src/prof.c +++ b/src/prof.c @@ -219,7 +219,7 @@ void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { - prof_tctx_set(ptr, tctx); + prof_tctx_set(ptr, usize, tctx); malloc_mutex_lock(tctx->tdata->lock); tctx->cnts.curobjs++; diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c index 3af1964..da34d70 100644 --- a/test/unit/prof_reset.c +++ b/test/unit/prof_reset.c @@ -16,6 +16,27 @@ prof_dump_open_intercept(bool propagate_err, const char *filename) return (fd); } +static size_t +get_lg_prof_sample(void) +{ + size_t lg_prof_sample; + size_t sz = sizeof(size_t); + + assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0, + "Unexpected mallctl failure while reading profiling sample rate"); + return (lg_prof_sample); +} + +static void +do_prof_reset(size_t lg_prof_sample) +{ + assert_d_eq(mallctl("prof.reset", NULL, NULL, + &lg_prof_sample, sizeof(size_t)), 0, + "Unexpected mallctl failure while resetting profile data"); + assert_zu_eq(lg_prof_sample, get_lg_prof_sample(), + "Expected profile sample rate change"); +} + TEST_BEGIN(test_prof_reset_basic) { size_t lg_prof_sample_orig, lg_prof_sample, lg_prof_sample_next; @@ -30,9 +51,7 @@ TEST_BEGIN(test_prof_reset_basic) "Unexpected mallctl failure while reading profiling sample rate"); assert_zu_eq(lg_prof_sample_orig, 0, "Unexpected profiling sample rate"); - sz = sizeof(size_t); - assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0, - "Unexpected mallctl failure while reading profiling sample rate"); + lg_prof_sample = get_lg_prof_sample(); assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, "Unexpected disagreement between \"opt.lg_prof_sample\" and " "\"prof.lg_sample\""); @@ -41,10 +60,7 @@ TEST_BEGIN(test_prof_reset_basic) for (i = 0; i < 2; i++) { assert_d_eq(mallctl("prof.reset", NULL, NULL, NULL, 0), 0, "Unexpected mallctl failure while resetting profile data"); - sz = sizeof(size_t); - assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, - NULL, 0), 0, "Unexpected mallctl failure while reading " - "profiling sample rate"); + lg_prof_sample = get_lg_prof_sample(); assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, "Unexpected profile sample rate change"); } @@ -52,22 +68,15 @@ TEST_BEGIN(test_prof_reset_basic) /* Test resets with prof.lg_sample changes. */ lg_prof_sample_next = 1; for (i = 0; i < 2; i++) { - assert_d_eq(mallctl("prof.reset", NULL, NULL, - &lg_prof_sample_next, sizeof(size_t)), 0, - "Unexpected mallctl failure while resetting profile data"); - sz = sizeof(size_t); - assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, - NULL, 0), 0, "Unexpected mallctl failure while reading " - "profiling sample rate"); + do_prof_reset(lg_prof_sample_next); + lg_prof_sample = get_lg_prof_sample(); assert_zu_eq(lg_prof_sample, lg_prof_sample_next, "Expected profile sample rate change"); lg_prof_sample_next = lg_prof_sample_orig; } /* Make sure the test code restored prof.lg_sample. */ - sz = sizeof(size_t); - assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0, - "Unexpected mallctl failure while reading profiling sample rate"); + lg_prof_sample = get_lg_prof_sample(); assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, "Unexpected disagreement between \"opt.lg_prof_sample\" and " "\"prof.lg_sample\""); @@ -182,6 +191,7 @@ thd_start(void *varg) TEST_BEGIN(test_prof_reset) { + size_t lg_prof_sample_orig; bool active; thd_t thds[NTHREADS]; unsigned thd_args[NTHREADS]; @@ -195,6 +205,9 @@ TEST_BEGIN(test_prof_reset) "Unexpected pre-existing tdata structures"); tdata_count = prof_tdata_count(); + lg_prof_sample_orig = get_lg_prof_sample(); + do_prof_reset(5); + active = true; assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), 0, "Unexpected mallctl failure while activating profiling"); @@ -214,6 +227,8 @@ TEST_BEGIN(test_prof_reset) active = false; assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), 0, "Unexpected mallctl failure while deactivating profiling"); + + do_prof_reset(lg_prof_sample_orig); } TEST_END #undef NTHREADS -- cgit v0.12 From b4330b02a8a909aed71c46d2c661d69545628fb4 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 4 Sep 2015 10:31:41 -0700 Subject: Fix pointer comparision with undefined behavior. This didn't cause bad code generation in the one case spot-checked (gcc 4.8.1), but had the potential to to so. This bug was introduced by 594c759f37c301d0245dc2accf4d4aaf9d202819 (Optimize arena_prof_tctx_set().). --- include/jemalloc/internal/arena.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 76c5b93..4c1a471 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1105,8 +1105,8 @@ arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (unlikely(usize > SMALL_MAXCLASS || tctx > - (prof_tctx_t *)(uintptr_t)1U)) { + if (unlikely(usize > SMALL_MAXCLASS || (uintptr_t)tctx > + (uintptr_t)1U)) { arena_chunk_map_misc_t *elm; assert(arena_mapbits_large_get(chunk, pageind) != 0); -- cgit v0.12 From 0a116faf95ba8541ce75448bb9b6fba0efdde69a Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 3 Sep 2015 15:48:48 +0900 Subject: Force initialization of the init_lock in malloc_init_hard on Windows XP This resolves #269. --- src/jemalloc.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index df962c6..7cf1487 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -179,13 +179,24 @@ static bool malloc_initializer = NO_INITIALIZER; static malloc_mutex_t init_lock = SRWLOCK_INIT; #else static malloc_mutex_t init_lock; +static bool init_lock_initialized = false; JEMALLOC_ATTR(constructor) static void WINAPI _init_init_lock(void) { - malloc_mutex_init(&init_lock); + /* If another constructor in the same binary is using mallctl to + * e.g. setup chunk hooks, it may end up running before this one, + * and malloc_init_hard will crash trying to lock the uninitialized + * lock. So we force an initialization of the lock in + * malloc_init_hard as well. We don't try to care about atomicity + * of the accessed to the init_lock_initialized boolean, since it + * really only matters early in the process creation, before any + * separate thread normally starts doing anything. */ + if (!init_lock_initialized) + malloc_mutex_init(&init_lock); + init_lock_initialized = true; } #ifdef _MSC_VER @@ -1300,6 +1311,9 @@ static bool malloc_init_hard(void) { +#if defined(_WIN32) && _WIN32_WINNT < 0x0600 + _init_init_lock(); +#endif malloc_mutex_lock(&init_lock); if (!malloc_init_hard_needed()) { malloc_mutex_unlock(&init_lock); -- cgit v0.12 From a306a60651db0bd835d4009271e0be236b450fb3 Mon Sep 17 00:00:00 2001 From: Dmitry-Me Date: Fri, 4 Sep 2015 13:15:28 +0300 Subject: Reduce variables scope --- src/arena.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/arena.c b/src/arena.c index b1bb9db..949fc5b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -425,7 +425,7 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, { arena_chunk_t *chunk; arena_chunk_map_misc_t *miscelm; - size_t flag_dirty, flag_decommitted, run_ind, need_pages, i; + size_t flag_dirty, flag_decommitted, run_ind, need_pages; size_t flag_unzeroed_mask; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); @@ -459,6 +459,7 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, * The run is clean, so some pages may be zeroed (i.e. * never before touched). */ + size_t i; for (i = 0; i < need_pages; i++) { if (arena_mapbits_unzeroed_get(chunk, run_ind+i) != 0) @@ -1938,7 +1939,6 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) static void * arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) { - void *ret; szind_t binind; arena_bin_info_t *bin_info; arena_run_t *run; @@ -1952,6 +1952,7 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) * Another thread updated runcur while this one ran without the * bin lock in arena_bin_nonfull_run_get(). */ + void *ret; assert(bin->runcur->nfree > 0); ret = arena_run_reg_alloc(bin->runcur, bin_info); if (run != NULL) { @@ -1990,8 +1991,6 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind, { unsigned i, nfill; arena_bin_t *bin; - arena_run_t *run; - void *ptr; assert(tbin->ncached == 0); @@ -2001,6 +2000,8 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, szind_t binind, malloc_mutex_lock(&bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> tbin->lg_fill_div); i < nfill; i++) { + arena_run_t *run; + void *ptr; if ((run = bin->runcur) != NULL && run->nfree > 0) ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); else @@ -2075,12 +2076,13 @@ arena_redzone_corruption_t *arena_redzone_corruption = static void arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) { - size_t size = bin_info->reg_size; - size_t redzone_size = bin_info->redzone_size; - size_t i; bool error = false; if (opt_junk_alloc) { + size_t size = bin_info->reg_size; + size_t redzone_size = bin_info->redzone_size; + size_t i; + for (i = 1; i <= redzone_size; i++) { uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); if (*byte != 0xa5) { @@ -3240,7 +3242,6 @@ small_run_size_init(void) bool arena_boot(void) { - size_t header_size; unsigned i; arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); @@ -3259,7 +3260,7 @@ arena_boot(void) */ map_bias = 0; for (i = 0; i < 3; i++) { - header_size = offsetof(arena_chunk_t, map_bits) + + size_t header_size = offsetof(arena_chunk_t, map_bits) + ((sizeof(arena_chunk_map_bits_t) + sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); map_bias = (header_size + PAGE_MASK) >> LG_PAGE; -- cgit v0.12 From a00b10735a80f7070714b278c8acdad4473bea69 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 9 Sep 2015 23:16:10 -0700 Subject: Fix "prof.reset" mallctl-related corruption. Fix heap profiling to distinguish among otherwise identical sample sites with interposed resets (triggered via the "prof.reset" mallctl). This bug could cause data structure corruption that would most likely result in a segfault. --- ChangeLog | 4 ++ include/jemalloc/internal/prof.h | 5 ++- src/prof.c | 14 +++++-- test/unit/prof_reset.c | 81 ++++++++++++++++++++++++++++++++-------- 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/ChangeLog b/ChangeLog index dba05eb..63c9d56 100644 --- a/ChangeLog +++ b/ChangeLog @@ -22,6 +22,10 @@ brevity. Much more detail can be found in the git revision history: visible to custom functions set via the "arena..chunk_hooks" mallctl. - Fix TLS configuration such that it is enabled by default for platforms on which it works correctly. + - Fix heap profiling to distinguish among otherwise identical sample sites + with interposed resets (triggered via the "prof.reset" mallctl). This bug + could cause data structure corruption that would most likely result in a + segfault. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index fe89828..eca8aa8 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -90,10 +90,11 @@ struct prof_tctx_s { prof_tdata_t *tdata; /* - * Copy of tdata->thr_uid, necessary because tdata may be defunct during - * teardown. + * Copy of tdata->thr_{uid,discrim}, necessary because tdata may be + * defunct during teardown. */ uint64_t thr_uid; + uint64_t thr_discrim; /* Profiling counters, protected by tdata->lock. */ prof_cnt_t cnts; diff --git a/src/prof.c b/src/prof.c index b79eba6..7427bf5 100644 --- a/src/prof.c +++ b/src/prof.c @@ -139,9 +139,16 @@ prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) uint64_t b_thr_uid = b->thr_uid; int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); if (ret == 0) { - uint64_t a_tctx_uid = a->tctx_uid; - uint64_t b_tctx_uid = b->tctx_uid; - ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < b_tctx_uid); + uint64_t a_thr_discrim = a->thr_discrim; + uint64_t b_thr_discrim = b->thr_discrim; + ret = (a_thr_discrim > b_thr_discrim) - (a_thr_discrim < + b_thr_discrim); + if (ret == 0) { + uint64_t a_tctx_uid = a->tctx_uid; + uint64_t b_tctx_uid = b->tctx_uid; + ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < + b_tctx_uid); + } } return (ret); } @@ -791,6 +798,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) } ret.p->tdata = tdata; ret.p->thr_uid = tdata->thr_uid; + ret.p->thr_discrim = tdata->thr_discrim; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); ret.p->gctx = gctx; ret.p->tctx_uid = tdata->tctx_uid_next++; diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c index da34d70..69983e5 100644 --- a/test/unit/prof_reset.c +++ b/test/unit/prof_reset.c @@ -16,6 +16,14 @@ prof_dump_open_intercept(bool propagate_err, const char *filename) return (fd); } +static void +set_prof_active(bool active) +{ + + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure"); +} + static size_t get_lg_prof_sample(void) { @@ -97,15 +105,12 @@ prof_dump_header_intercept(bool propagate_err, const prof_cnt_t *cnt_all) TEST_BEGIN(test_prof_reset_cleanup) { - bool active; void *p; prof_dump_header_t *prof_dump_header_orig; test_skip_if(!config_prof); - active = true; - assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), - 0, "Unexpected mallctl failure while activating profiling"); + set_prof_active(true); assert_zu_eq(prof_bt_count(), 0, "Expected 0 backtraces"); p = mallocx(1, 0); @@ -133,9 +138,7 @@ TEST_BEGIN(test_prof_reset_cleanup) dallocx(p, 0); assert_zu_eq(prof_bt_count(), 0, "Expected 0 backtraces"); - active = false; - assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), - 0, "Unexpected mallctl failure while deactivating profiling"); + set_prof_active(false); } TEST_END @@ -192,7 +195,6 @@ thd_start(void *varg) TEST_BEGIN(test_prof_reset) { size_t lg_prof_sample_orig; - bool active; thd_t thds[NTHREADS]; unsigned thd_args[NTHREADS]; unsigned i; @@ -208,9 +210,7 @@ TEST_BEGIN(test_prof_reset) lg_prof_sample_orig = get_lg_prof_sample(); do_prof_reset(5); - active = true; - assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), - 0, "Unexpected mallctl failure while activating profiling"); + set_prof_active(true); for (i = 0; i < NTHREADS; i++) { thd_args[i] = i; @@ -224,9 +224,7 @@ TEST_BEGIN(test_prof_reset) assert_zu_eq(prof_tdata_count(), tdata_count, "Unexpected remaining tdata structures"); - active = false; - assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), - 0, "Unexpected mallctl failure while deactivating profiling"); + set_prof_active(false); do_prof_reset(lg_prof_sample_orig); } @@ -237,6 +235,58 @@ TEST_END #undef RESET_INTERVAL #undef DUMP_INTERVAL +/* Test sampling at the same allocation site across resets. */ +#define NITER 10 +TEST_BEGIN(test_xallocx) +{ + size_t lg_prof_sample_orig; + unsigned i; + void *ptrs[NITER]; + + test_skip_if(!config_prof); + + lg_prof_sample_orig = get_lg_prof_sample(); + set_prof_active(true); + + /* Reset profiling. */ + do_prof_reset(0); + + for (i = 0; i < NITER; i++) { + void *p; + size_t sz, nsz; + + /* Reset profiling. */ + do_prof_reset(0); + + /* Allocate small object (which will be promoted). */ + p = ptrs[i] = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + /* Reset profiling. */ + do_prof_reset(0); + + /* Perform successful xallocx(). */ + sz = sallocx(p, 0); + assert_zu_eq(xallocx(p, sz, 0, 0), sz, + "Unexpected xallocx() failure"); + + /* Perform unsuccessful xallocx(). */ + nsz = nallocx(sz+1, 0); + assert_zu_eq(xallocx(p, nsz, 0, 0), sz, + "Unexpected xallocx() success"); + } + + for (i = 0; i < NITER; i++) { + /* dallocx. */ + dallocx(ptrs[i], 0); + } + + set_prof_active(false); + do_prof_reset(lg_prof_sample_orig); +} +TEST_END +#undef NITER + int main(void) { @@ -247,5 +297,6 @@ main(void) return (test( test_prof_reset_basic, test_prof_reset_cleanup, - test_prof_reset)); + test_prof_reset, + test_xallocx)); } -- cgit v0.12 From 560a4e1e01d3733c2f107cdb3cc3580f3ed84442 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 11 Sep 2015 16:18:53 -0700 Subject: Fix xallocx() bugs. Fix xallocx() bugs related to the 'extra' parameter when specified as non-zero. --- ChangeLog | 2 + include/jemalloc/internal/arena.h | 2 +- include/jemalloc/internal/huge.h | 7 +- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/internal/size_classes.sh | 5 + src/arena.c | 202 +++++++++---------- src/huge.c | 109 +++++----- test/integration/rallocx.c | 2 +- test/integration/xallocx.c | 242 ++++++++++++++++++++++- 9 files changed, 394 insertions(+), 179 deletions(-) diff --git a/ChangeLog b/ChangeLog index 63c9d56..18d72eb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,6 +26,8 @@ brevity. Much more detail can be found in the git revision history: with interposed resets (triggered via the "prof.reset" mallctl). This bug could cause data structure corruption that would most likely result in a segfault. + - Fix xallocx() bugs related to the 'extra' parameter when specified as + non-zero. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 4c1a471..f77f257 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -488,7 +488,7 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache); + size_t size, size_t alignment, bool zero, tcache_t *tcache); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_lg_dirty_mult_default_get(void); diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 8b6c6ce..328eeed 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -13,11 +13,10 @@ void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_t *tcache); void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, bool zero, tcache_t *tcache); -bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); +bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min, + size_t usize_max, bool zero); void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t extra, size_t alignment, bool zero, - tcache_t *tcache); + size_t usize, size_t alignment, bool zero, tcache_t *tcache); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index f6e464e..a341b25 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -1096,7 +1096,7 @@ iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, zero, tcache, arena)); } - return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, alignment, zero, + return (arena_ralloc(tsd, arena, ptr, oldsize, size, alignment, zero, tcache)); } diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 1c2d681..fc82036 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -167,6 +167,8 @@ size_classes() { lg_large_minclass=$((${lg_grp} + 2)) fi fi + # Final written value is correct: + huge_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" index=$((${index} + 1)) ndelta=$((${ndelta} + 1)) done @@ -185,6 +187,7 @@ size_classes() { # - lookup_maxclass # - small_maxclass # - lg_large_minclass + # - huge_maxclass } cat <> LG_PAGE; size_t npages = (oldsize + large_pad) >> LG_PAGE; size_t followsize; - size_t usize_min = s2u(size); assert(oldsize == arena_mapbits_large_size_get(chunk, pageind) - large_pad); /* Try to extend the run. */ - assert(usize_min > oldsize); malloc_mutex_lock(&arena->lock); - if (pageind+npages < chunk_npages && - arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && - (followsize = arena_mapbits_unallocated_size_get(chunk, - pageind+npages)) >= usize_min - oldsize) { + if (pageind+npages >= chunk_npages || arena_mapbits_allocated_get(chunk, + pageind+npages) != 0) + goto label_fail; + followsize = arena_mapbits_unallocated_size_get(chunk, pageind+npages); + if (oldsize + followsize >= usize_min) { /* * The next run is available and sufficiently large. Split the * following run, then merge the first part with the existing * allocation. */ arena_run_t *run; - size_t flag_dirty, flag_unzeroed_mask, splitsize, usize; + size_t usize, splitsize, size, flag_dirty, flag_unzeroed_mask; - usize = s2u(size + extra); + usize = usize_max; while (oldsize + followsize < usize) usize = index2size(size2index(usize)-1); assert(usize >= usize_min); + assert(usize >= oldsize); splitsize = usize - oldsize; + if (splitsize == 0) + goto label_fail; run = &arena_miscelm_get(chunk, pageind+npages)->run; - if (arena_run_split_large(arena, run, splitsize, zero)) { - malloc_mutex_unlock(&arena->lock); - return (true); - } + if (arena_run_split_large(arena, run, splitsize, zero)) + goto label_fail; size = oldsize + splitsize; npages = (size + large_pad) >> LG_PAGE; @@ -2719,8 +2719,8 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, malloc_mutex_unlock(&arena->lock); return (false); } +label_fail: malloc_mutex_unlock(&arena->lock); - return (true); } @@ -2749,98 +2749,114 @@ arena_ralloc_junk_large_t *arena_ralloc_junk_large = * always fail if growing an object, and the following run is already in use. */ static bool -arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, - bool zero) +arena_ralloc_large(void *ptr, size_t oldsize, size_t usize_min, + size_t usize_max, bool zero) { - size_t usize; - - /* Make sure extra can't cause size_t overflow. */ - if (unlikely(extra >= arena_maxclass)) - return (true); + arena_chunk_t *chunk; + arena_t *arena; - usize = s2u(size + extra); - if (usize == oldsize) { - /* Same size class. */ + if (oldsize == usize_max) { + /* Current size class is compatible and maximal. */ return (false); - } else { - arena_chunk_t *chunk; - arena_t *arena; + } - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = extent_node_arena_get(&chunk->node); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = extent_node_arena_get(&chunk->node); - if (usize < oldsize) { - /* Fill before shrinking in order avoid a race. */ - arena_ralloc_junk_large(ptr, oldsize, usize); - arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, - usize); - return (false); - } else { - bool ret = arena_ralloc_large_grow(arena, chunk, ptr, - oldsize, size, extra, zero); - if (config_fill && !ret && !zero) { - if (unlikely(opt_junk_alloc)) { - memset((void *)((uintptr_t)ptr + - oldsize), 0xa5, isalloc(ptr, - config_prof) - oldsize); - } else if (unlikely(opt_zero)) { - memset((void *)((uintptr_t)ptr + - oldsize), 0, isalloc(ptr, - config_prof) - oldsize); - } + if (oldsize < usize_max) { + bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, + usize_min, usize_max, zero); + if (config_fill && !ret && !zero) { + if (unlikely(opt_junk_alloc)) { + memset((void *)((uintptr_t)ptr + oldsize), 0xa5, + isalloc(ptr, config_prof) - oldsize); + } else if (unlikely(opt_zero)) { + memset((void *)((uintptr_t)ptr + oldsize), 0, + isalloc(ptr, config_prof) - oldsize); } - return (ret); } + return (ret); } + + assert(oldsize > usize_max); + /* Fill before shrinking in order avoid a race. */ + arena_ralloc_junk_large(ptr, oldsize, usize_max); + arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, usize_max); + return (false); } bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { + size_t usize_min, usize_max; - if (likely(size <= arena_maxclass)) { + /* Check for size overflow. */ + if (unlikely(size > HUGE_MAXCLASS)) + return (true); + usize_min = s2u(size); + /* Clamp extra if necessary to avoid (size + extra) overflow. */ + if (unlikely(size + extra > HUGE_MAXCLASS)) + extra = HUGE_MAXCLASS - size; + usize_max = s2u(size + extra); + + if (likely(oldsize <= arena_maxclass && usize_min <= arena_maxclass)) { /* * Avoid moving the allocation if the size class can be left the * same. */ - if (likely(oldsize <= arena_maxclass)) { - if (oldsize <= SMALL_MAXCLASS) { - assert( - arena_bin_info[size2index(oldsize)].reg_size - == oldsize); - if ((size + extra <= SMALL_MAXCLASS && - size2index(size + extra) == - size2index(oldsize)) || (size <= oldsize && - size + extra >= oldsize)) + if (oldsize <= SMALL_MAXCLASS) { + assert(arena_bin_info[size2index(oldsize)].reg_size == + oldsize); + if ((usize_max <= SMALL_MAXCLASS && + size2index(usize_max) == size2index(oldsize)) || + (size <= oldsize && usize_max >= oldsize)) + return (false); + } else { + if (usize_max > SMALL_MAXCLASS) { + if (!arena_ralloc_large(ptr, oldsize, usize_min, + usize_max, zero)) return (false); - } else { - assert(size <= arena_maxclass); - if (size + extra > SMALL_MAXCLASS) { - if (!arena_ralloc_large(ptr, oldsize, - size, extra, zero)) - return (false); - } } } /* Reallocation would require a move. */ return (true); - } else - return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); + } else { + return (huge_ralloc_no_move(ptr, oldsize, usize_min, usize_max, + zero)); + } +} + +static void * +arena_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize, + size_t alignment, bool zero, tcache_t *tcache) +{ + + if (alignment == 0) + return (arena_malloc(tsd, arena, usize, zero, tcache)); + usize = sa2u(usize, alignment); + if (usize == 0) + return (NULL); + return (ipalloct(tsd, usize, alignment, zero, tcache, arena)); } void * arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache) + size_t alignment, bool zero, tcache_t *tcache) { void *ret; + size_t usize; + + usize = s2u(size); + if (usize == 0) + return (NULL); - if (likely(size <= arena_maxclass)) { + if (likely(usize <= arena_maxclass)) { size_t copysize; /* Try to avoid moving the allocation. */ - if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero)) + if (!arena_ralloc_no_move(ptr, oldsize, usize, 0, zero)) return (ptr); /* @@ -2848,53 +2864,23 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, * the object. In that case, fall back to allocating new space * and copying. */ - if (alignment != 0) { - size_t usize = sa2u(size + extra, alignment); - if (usize == 0) - return (NULL); - ret = ipalloct(tsd, usize, alignment, zero, tcache, - arena); - } else { - ret = arena_malloc(tsd, arena, size + extra, zero, - tcache); - } - - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, this time without extra. */ - if (alignment != 0) { - size_t usize = sa2u(size, alignment); - if (usize == 0) - return (NULL); - ret = ipalloct(tsd, usize, alignment, zero, - tcache, arena); - } else { - ret = arena_malloc(tsd, arena, size, zero, - tcache); - } - - if (ret == NULL) - return (NULL); - } + ret = arena_ralloc_move_helper(tsd, arena, usize, alignment, + zero, tcache); + if (ret == NULL) + return (NULL); /* * Junk/zero-filling were already done by * ipalloc()/arena_malloc(). */ - /* - * Copy at most size bytes (not size+extra), since the caller - * has no expectation that the extra bytes will be reliably - * preserved. - */ - copysize = (size < oldsize) ? size : oldsize; + copysize = (usize < oldsize) ? usize : oldsize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); isqalloc(tsd, ptr, oldsize, tcache); } else { - ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra, - alignment, zero, tcache); + ret = huge_ralloc(tsd, arena, ptr, oldsize, usize, alignment, + zero, tcache); } return (ret); } diff --git a/src/huge.c b/src/huge.c index 4d5887c..187bdaa 100644 --- a/src/huge.c +++ b/src/huge.c @@ -126,18 +126,19 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); #endif static void -huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, - size_t size, size_t extra, bool zero) +huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min, + size_t usize_max, bool zero) { - size_t usize_next; + size_t usize, usize_next; extent_node_t *node; arena_t *arena; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; bool zeroed; /* Increase usize to incorporate extra. */ - while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize) - usize = usize_next; + for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) + <= oldsize; usize = usize_next) + ; /* Do nothing. */ if (oldsize == usize) return; @@ -195,6 +196,8 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) arena = extent_node_arena_get(node); chunk_hooks = chunk_hooks_get(arena); + assert(oldsize > usize); + /* Split excess chunks. */ cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize), @@ -230,18 +233,11 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) } static bool -huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { - size_t usize; +huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) { extent_node_t *node; arena_t *arena; bool is_zeroed_subchunk, is_zeroed_chunk; - usize = s2u(size); - if (usize == 0) { - /* size_t overflow. */ - return (true); - } - node = huge_node_get(ptr); arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); @@ -282,89 +278,76 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { } bool -huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, - bool zero) +huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min, + size_t usize_max, bool zero) { - size_t usize; + + assert(s2u(oldsize) == oldsize); /* Both allocations must be huge to avoid a move. */ - if (oldsize < chunksize) + if (oldsize < chunksize || usize_max < chunksize) return (true); - assert(s2u(oldsize) == oldsize); - usize = s2u(size); - if (usize == 0) { - /* size_t overflow. */ - return (true); + if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) { + /* Attempt to expand the allocation in-place. */ + if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, zero)) + return (false); + /* Try again, this time with usize_min. */ + if (usize_min < usize_max && CHUNK_CEILING(usize_min) > + CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr, + oldsize, usize_min, zero)) + return (false); } /* * Avoid moving the allocation if the existing chunk size accommodates * the new size. */ - if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize) - && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(s2u(size+extra))) { - huge_ralloc_no_move_similar(ptr, oldsize, usize, size, extra, + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min) + && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) { + huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max, zero); return (false); } /* Attempt to shrink the allocation in-place. */ - if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) - return (huge_ralloc_no_move_shrink(ptr, oldsize, usize)); + if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) + return (huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)); + return (true); +} - /* Attempt to expand the allocation in-place. */ - if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) { - if (extra == 0) - return (true); +static void * +huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize, + size_t alignment, bool zero, tcache_t *tcache) +{ - /* Try again, this time without extra. */ - return (huge_ralloc_no_move_expand(ptr, oldsize, size, zero)); - } - return (false); + if (alignment <= chunksize) + return (huge_malloc(tsd, arena, usize, zero, tcache)); + return (huge_palloc(tsd, arena, usize, alignment, zero, tcache)); } void * -huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache) +huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize, + size_t alignment, bool zero, tcache_t *tcache) { void *ret; size_t copysize; /* Try to avoid moving the allocation. */ - if (!huge_ralloc_no_move(ptr, oldsize, size, extra, zero)) + if (!huge_ralloc_no_move(ptr, oldsize, usize, usize, zero)) return (ptr); /* - * size and oldsize are different enough that we need to use a + * usize and oldsize are different enough that we need to use a * different size class. In that case, fall back to allocating new * space and copying. */ - if (alignment > chunksize) { - ret = huge_palloc(tsd, arena, size + extra, alignment, zero, - tcache); - } else - ret = huge_malloc(tsd, arena, size + extra, zero, tcache); - - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, this time without extra. */ - if (alignment > chunksize) { - ret = huge_palloc(tsd, arena, size, alignment, zero, - tcache); - } else - ret = huge_malloc(tsd, arena, size, zero, tcache); - - if (ret == NULL) - return (NULL); - } + ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero, + tcache); + if (ret == NULL) + return (NULL); - /* - * Copy at most size bytes (not size+extra), since the caller has no - * expectation that the extra bytes will be reliably preserved. - */ - copysize = (size < oldsize) ? size : oldsize; + copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); isqalloc(tsd, ptr, oldsize, tcache); return (ret); diff --git a/test/integration/rallocx.c b/test/integration/rallocx.c index 8b6cde3..be1b27b 100644 --- a/test/integration/rallocx.c +++ b/test/integration/rallocx.c @@ -22,7 +22,7 @@ TEST_BEGIN(test_grow_and_shrink) szs[j-1], szs[j-1]+1); szs[j] = sallocx(q, 0); assert_zu_ne(szs[j], szs[j-1]+1, - "Expected size to at least: %zu", szs[j-1]+1); + "Expected size to be at least: %zu", szs[j-1]+1); p = q; } diff --git a/test/integration/xallocx.c b/test/integration/xallocx.c index ab4cf94..8f0de63 100644 --- a/test/integration/xallocx.c +++ b/test/integration/xallocx.c @@ -48,6 +48,243 @@ TEST_BEGIN(test_no_move_fail) } TEST_END +static unsigned +get_nsizes_impl(const char *cmd) +{ + unsigned ret; + size_t z; + + z = sizeof(unsigned); + assert_d_eq(mallctl(cmd, &ret, &z, NULL, 0), 0, + "Unexpected mallctl(\"%s\", ...) failure", cmd); + + return (ret); +} + +static unsigned +get_nsmall(void) +{ + + return (get_nsizes_impl("arenas.nbins")); +} + +static unsigned +get_nlarge(void) +{ + + return (get_nsizes_impl("arenas.nlruns")); +} + +static unsigned +get_nhuge(void) +{ + + return (get_nsizes_impl("arenas.nhchunks")); +} + +static size_t +get_size_impl(const char *cmd, size_t ind) +{ + size_t ret; + size_t z; + size_t mib[4]; + size_t miblen = 4; + + z = sizeof(size_t); + assert_d_eq(mallctlnametomib(cmd, mib, &miblen), + 0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd); + mib[2] = ind; + z = sizeof(size_t); + assert_d_eq(mallctlbymib(mib, miblen, &ret, &z, NULL, 0), + 0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind); + + return (ret); +} + +static size_t +get_small_size(size_t ind) +{ + + return (get_size_impl("arenas.bin.0.size", ind)); +} + +static size_t +get_large_size(size_t ind) +{ + + return (get_size_impl("arenas.lrun.0.size", ind)); +} + +static size_t +get_huge_size(size_t ind) +{ + + return (get_size_impl("arenas.hchunk.0.size", ind)); +} + +TEST_BEGIN(test_extra_small) +{ + size_t small0, small1, hugemax; + void *p; + + /* Get size classes. */ + small0 = get_small_size(0); + small1 = get_small_size(1); + hugemax = get_huge_size(get_nhuge()-1); + + p = mallocx(small0, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + assert_zu_eq(xallocx(p, small1, 0, 0), small0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, small1, 0, 0), small0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, small0, small1 - small0, 0), small0, + "Unexpected xallocx() behavior"); + + /* Test size+extra overflow. */ + assert_zu_eq(xallocx(p, small0, hugemax - small0 + 1, 0), small0, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, small0, SIZE_T_MAX - small0, 0), small0, + "Unexpected xallocx() behavior"); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_extra_large) +{ + size_t smallmax, large0, large1, large2, huge0, hugemax; + void *p; + + /* Get size classes. */ + smallmax = get_small_size(get_nsmall()-1); + large0 = get_large_size(0); + large1 = get_large_size(1); + large2 = get_large_size(2); + huge0 = get_huge_size(0); + hugemax = get_huge_size(get_nhuge()-1); + + p = mallocx(large2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + assert_zu_eq(xallocx(p, large2, 0, 0), large2, + "Unexpected xallocx() behavior"); + /* Test size decrease with zero extra. */ + assert_zu_eq(xallocx(p, large0, 0, 0), large0, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, smallmax, 0, 0), large0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, large2, 0, 0), large2, + "Unexpected xallocx() behavior"); + /* Test size decrease with non-zero extra. */ + assert_zu_eq(xallocx(p, large0, large2 - large0, 0), large2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, large1, large2 - large1, 0), large2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, large0, large1 - large0, 0), large1, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, smallmax, large0 - smallmax, 0), large0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, large0, 0, 0), large0, + "Unexpected xallocx() behavior"); + /* Test size increase with zero extra. */ + assert_zu_eq(xallocx(p, large2, 0, 0), large2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, huge0, 0, 0), large2, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, large0, 0, 0), large0, + "Unexpected xallocx() behavior"); + /* Test size increase with non-zero extra. */ + assert_zu_lt(xallocx(p, large0, huge0 - large0, 0), huge0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, large0, 0, 0), large0, + "Unexpected xallocx() behavior"); + /* Test size increase with non-zero extra. */ + assert_zu_eq(xallocx(p, large0, large2 - large0, 0), large2, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, large2, 0, 0), large2, + "Unexpected xallocx() behavior"); + /* Test size+extra overflow. */ + assert_zu_lt(xallocx(p, large2, hugemax - large2 + 1, 0), huge0, + "Unexpected xallocx() behavior"); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_extra_huge) +{ + size_t largemax, huge0, huge1, huge2, hugemax; + void *p; + + /* Get size classes. */ + largemax = get_large_size(get_nlarge()-1); + huge0 = get_huge_size(0); + huge1 = get_huge_size(1); + huge2 = get_huge_size(2); + hugemax = get_huge_size(get_nhuge()-1); + + p = mallocx(huge2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, + "Unexpected xallocx() behavior"); + /* Test size decrease with zero extra. */ + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, largemax, 0, 0), huge0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, + "Unexpected xallocx() behavior"); + /* Test size decrease with non-zero extra. */ + assert_zu_eq(xallocx(p, huge0, huge2 - huge0, 0), huge2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, huge1, huge2 - huge1, 0), huge2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, huge0, huge1 - huge0, 0), huge1, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, largemax, huge0 - largemax, 0), huge0, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() behavior"); + /* Test size increase with zero extra. */ + assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, + "Unexpected xallocx() behavior"); + assert_zu_eq(xallocx(p, hugemax+1, 0, 0), huge2, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() behavior"); + /* Test size increase with non-zero extra. */ + assert_zu_le(xallocx(p, huge0, SIZE_T_MAX - huge0, 0), hugemax, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() behavior"); + /* Test size increase with non-zero extra. */ + assert_zu_eq(xallocx(p, huge0, huge2 - huge0, 0), huge2, + "Unexpected xallocx() behavior"); + + assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, + "Unexpected xallocx() behavior"); + /* Test size+extra overflow. */ + assert_zu_le(xallocx(p, huge2, hugemax - huge2 + 1, 0), hugemax, + "Unexpected xallocx() behavior"); + + dallocx(p, 0); +} +TEST_END + int main(void) { @@ -55,5 +292,8 @@ main(void) return (test( test_same_size, test_extra_no_move, - test_no_move_fail)); + test_no_move_fail, + test_extra_small, + test_extra_large, + test_extra_huge)); } -- cgit v0.12 From 676df88e48ae5ab77b05d78cb511cfa2e57d277f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 11 Sep 2015 20:50:20 -0700 Subject: Rename arena_maxclass to large_maxclass. arena_maxclass is no longer an appropriate name, because arenas also manage huge allocations. --- include/jemalloc/internal/arena.h | 4 ++-- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/internal/private_symbols.txt | 2 +- src/arena.c | 20 ++++++++++---------- src/tcache.c | 6 +++--- test/unit/junk.c | 12 ++++++------ test/unit/stats.c | 6 +++--- test/unit/zero.c | 4 ++-- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f77f257..9712c1c 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -424,7 +424,7 @@ extern arena_bin_info_t arena_bin_info[NBINS]; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t map_misc_offset; extern size_t arena_maxrun; /* Max run size for arenas. */ -extern size_t arena_maxclass; /* Max size class for arenas. */ +extern size_t large_maxclass; /* Max large size class. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ @@ -1143,7 +1143,7 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, zero)); } else return (arena_malloc_small(arena, size, zero)); - } else if (likely(size <= arena_maxclass)) { + } else if (likely(size <= large_maxclass)) { /* * Initialize tcache after checking size in order to avoid * infinite recursion during tcache initialization. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index a341b25..e2959f1 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -705,7 +705,7 @@ sa2u(size_t size, size_t alignment) } /* Try for a large size class. */ - if (likely(size <= arena_maxclass) && likely(alignment < chunksize)) { + if (likely(size <= large_maxclass) && likely(alignment < chunksize)) { /* * We can't achieve subpage alignment, so round up alignment * to the minimum that can actually be supported. diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index ed1f6c2..9d21a80 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -58,7 +58,6 @@ arena_mapbits_unallocated_set arena_mapbits_unallocated_size_get arena_mapbits_unallocated_size_set arena_mapbits_unzeroed_get -arena_maxclass arena_maxrun arena_maybe_purge arena_metadata_allocated_add @@ -285,6 +284,7 @@ ixalloc jemalloc_postfork_child jemalloc_postfork_parent jemalloc_prefork +large_maxclass lg_floor malloc_cprintf malloc_mutex_init diff --git a/src/arena.c b/src/arena.c index b41f0ce..a119d26 100644 --- a/src/arena.c +++ b/src/arena.c @@ -11,7 +11,7 @@ arena_bin_info_t arena_bin_info[NBINS]; size_t map_bias; size_t map_misc_offset; size_t arena_maxrun; /* Max run size for arenas. */ -size_t arena_maxclass; /* Max size class for arenas. */ +size_t large_maxclass; /* Max large size class. */ static size_t small_maxrun; /* Max run size used for small size classes. */ static bool *small_run_tab; /* Valid small run page multiples. */ unsigned nlclasses; /* Number of large size classes. */ @@ -2357,7 +2357,7 @@ arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, && (usize & PAGE_MASK) == 0))) { /* Small; alignment doesn't require special run placement. */ ret = arena_malloc(tsd, arena, usize, zero, tcache); - } else if (usize <= arena_maxclass && alignment <= PAGE) { + } else if (usize <= large_maxclass && alignment <= PAGE) { /* * Large; alignment doesn't require special run placement. * However, the cached pointer may be at a random offset from @@ -2368,7 +2368,7 @@ arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, if (config_cache_oblivious) ret = (void *)((uintptr_t)ret & ~PAGE_MASK); } else { - if (likely(usize <= arena_maxclass)) { + if (likely(usize <= large_maxclass)) { ret = arena_palloc_large(tsd, arena, usize, alignment, zero); } else if (likely(alignment <= chunksize)) @@ -2800,7 +2800,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, extra = HUGE_MAXCLASS - size; usize_max = s2u(size + extra); - if (likely(oldsize <= arena_maxclass && usize_min <= arena_maxclass)) { + if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { /* * Avoid moving the allocation if the size class can be left the * same. @@ -2852,7 +2852,7 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, if (usize == 0) return (NULL); - if (likely(usize <= arena_maxclass)) { + if (likely(usize <= large_maxclass)) { size_t copysize; /* Try to avoid moving the allocation. */ @@ -3258,17 +3258,17 @@ arena_boot(void) arena_maxrun = chunksize - (map_bias << LG_PAGE); assert(arena_maxrun > 0); - arena_maxclass = index2size(size2index(chunksize)-1); - if (arena_maxclass > arena_maxrun) { + large_maxclass = index2size(size2index(chunksize)-1); + if (large_maxclass > arena_maxrun) { /* * For small chunk sizes it's possible for there to be fewer * non-header pages available than are necessary to serve the * size classes just below chunksize. */ - arena_maxclass = arena_maxrun; + large_maxclass = arena_maxrun; } - assert(arena_maxclass > 0); - nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS); + assert(large_maxclass > 0); + nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); nhclasses = NSIZES - nlclasses - NBINS; bin_info_init(); diff --git a/src/tcache.c b/src/tcache.c index f1a30d5..fdafd0c 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -496,13 +496,13 @@ tcache_boot(void) unsigned i; /* - * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is + * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is * known. */ if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS) tcache_maxclass = SMALL_MAXCLASS; - else if ((1U << opt_lg_tcache_max) > arena_maxclass) - tcache_maxclass = arena_maxclass; + else if ((1U << opt_lg_tcache_max) > large_maxclass) + tcache_maxclass = large_maxclass; else tcache_maxclass = (1U << opt_lg_tcache_max); diff --git a/test/unit/junk.c b/test/unit/junk.c index 01d314b..b23dd1e 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -140,7 +140,7 @@ TEST_BEGIN(test_junk_large) { test_skip_if(!config_fill); - test_junk(SMALL_MAXCLASS+1, arena_maxclass); + test_junk(SMALL_MAXCLASS+1, large_maxclass); } TEST_END @@ -148,7 +148,7 @@ TEST_BEGIN(test_junk_huge) { test_skip_if(!config_fill); - test_junk(arena_maxclass+1, chunksize*2); + test_junk(large_maxclass+1, chunksize*2); } TEST_END @@ -172,8 +172,8 @@ arena_ralloc_junk_large_intercept(void *ptr, size_t old_usize, size_t usize) { arena_ralloc_junk_large_orig(ptr, old_usize, usize); - assert_zu_eq(old_usize, arena_maxclass, "Unexpected old_usize"); - assert_zu_eq(usize, shrink_size(arena_maxclass), "Unexpected usize"); + assert_zu_eq(old_usize, large_maxclass, "Unexpected old_usize"); + assert_zu_eq(usize, shrink_size(large_maxclass), "Unexpected usize"); most_recently_trimmed = ptr; } @@ -181,13 +181,13 @@ TEST_BEGIN(test_junk_large_ralloc_shrink) { void *p1, *p2; - p1 = mallocx(arena_maxclass, 0); + p1 = mallocx(large_maxclass, 0); assert_ptr_not_null(p1, "Unexpected mallocx() failure"); arena_ralloc_junk_large_orig = arena_ralloc_junk_large; arena_ralloc_junk_large = arena_ralloc_junk_large_intercept; - p2 = rallocx(p1, shrink_size(arena_maxclass), 0); + p2 = rallocx(p1, shrink_size(large_maxclass), 0); assert_ptr_eq(p1, p2, "Unexpected move during shrink"); arena_ralloc_junk_large = arena_ralloc_junk_large_orig; diff --git a/test/unit/stats.c b/test/unit/stats.c index 81ef0b7..8e4bc63 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -42,7 +42,7 @@ TEST_BEGIN(test_stats_huge) size_t sz; int expected = config_stats ? 0 : ENOENT; - p = mallocx(arena_maxclass+1, 0); + p = mallocx(large_maxclass+1, 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, @@ -88,7 +88,7 @@ TEST_BEGIN(test_stats_arenas_summary) little = mallocx(SMALL_MAXCLASS, 0); assert_ptr_not_null(little, "Unexpected mallocx() failure"); - large = mallocx(arena_maxclass, 0); + large = mallocx(large_maxclass, 0); assert_ptr_not_null(large, "Unexpected mallocx() failure"); huge = mallocx(chunksize, 0); assert_ptr_not_null(huge, "Unexpected mallocx() failure"); @@ -200,7 +200,7 @@ TEST_BEGIN(test_stats_arenas_large) assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), 0, "Unexpected mallctl() failure"); - p = mallocx(arena_maxclass, 0); + p = mallocx(large_maxclass, 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, diff --git a/test/unit/zero.c b/test/unit/zero.c index 65a8f0c..93afc2b 100644 --- a/test/unit/zero.c +++ b/test/unit/zero.c @@ -55,7 +55,7 @@ TEST_BEGIN(test_zero_large) { test_skip_if(!config_fill); - test_zero(SMALL_MAXCLASS+1, arena_maxclass); + test_zero(SMALL_MAXCLASS+1, large_maxclass); } TEST_END @@ -63,7 +63,7 @@ TEST_BEGIN(test_zero_huge) { test_skip_if(!config_fill); - test_zero(arena_maxclass+1, chunksize*2); + test_zero(large_maxclass+1, chunksize*2); } TEST_END -- cgit v0.12 From 8f57e3f1aeb86021b3d078b825bc8c42b2a9af6f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 11 Sep 2015 20:59:00 -0700 Subject: Remove check_stress from check target's dependencies. Prior to this change the debug build/test command needed to look like: make all tests && make check_unit && make check_integration && \ make check_integration_prof This is now simply: make check Rename the check_stress target to stress. --- Makefile.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5084b1a..9b2d0a1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -343,9 +343,9 @@ check_unit_dir: @mkdir -p $(objroot)test/unit check_integration_dir: @mkdir -p $(objroot)test/integration -check_stress_dir: +stress_dir: @mkdir -p $(objroot)test/stress -check_dir: check_unit_dir check_integration_dir check_stress_dir +check_dir: check_unit_dir check_integration_dir check_unit: tests_unit check_unit_dir $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) @@ -355,7 +355,7 @@ ifeq ($(enable_prof), 1) endif check_integration: tests_integration check_integration_dir $(SHELL) $(objroot)test/test.sh $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) -check_stress: tests_stress check_stress_dir +stress: tests_stress stress_dir $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%) check: tests check_dir check_integration_prof $(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%) @@ -372,7 +372,7 @@ coverage_integration: check_integration $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src integration $(C_TESTLIB_INTEGRATION_OBJS) $(SHELL) $(srcroot)coverage.sh $(srcroot)test/integration integration $(TESTS_INTEGRATION_OBJS) -coverage_stress: check_stress +coverage_stress: stress $(SHELL) $(srcroot)coverage.sh $(srcroot)src pic $(C_PIC_OBJS) $(SHELL) $(srcroot)coverage.sh $(srcroot)src jet $(C_JET_OBJS) $(SHELL) $(srcroot)coverage.sh $(srcroot)test/src stress $(C_TESTLIB_STRESS_OBJS) -- cgit v0.12 From 4acb6c7ff3411ddc4d180b0cbdba4fd2c3651ef0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 22:31:32 -0700 Subject: Fix ixallocx_prof() size+extra overflow. Fix ixallocx_prof() to clamp the extra parameter if size+extra would overflow HUGE_MAXCLASS. --- src/jemalloc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/jemalloc.c b/src/jemalloc.c index 7cf1487..6ed3d4e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2275,6 +2275,9 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(ptr); + /* Clamp extra if necessary to avoid (size + extra) overflow. */ + if (unlikely(size + extra > HUGE_MAXCLASS)) + extra = HUGE_MAXCLASS - size; /* * usize isn't knowable before ixalloc() returns when extra is non-zero. * Therefore, compute its maximum possible value and use that in -- cgit v0.12 From 46ff0491280635e51c9771d56a2d64dc0c4d6bd2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 22:40:42 -0700 Subject: Optimize irallocx_prof() to optimistically update the sampler state. --- src/jemalloc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 6ed3d4e..74fab0f 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2124,7 +2124,7 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(tsd, *usize, false); + tctx = prof_alloc_prep(tsd, *usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd, oldptr, old_usize, size, alignment, *usize, zero, tcache, arena, tctx); @@ -2133,7 +2133,7 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, tcache, arena); } if (unlikely(p == NULL)) { - prof_alloc_rollback(tsd, tctx, false); + prof_alloc_rollback(tsd, tctx, true); return (NULL); } @@ -2148,7 +2148,7 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, */ *usize = isalloc(p, config_prof); } - prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx); + prof_realloc(tsd, p, *usize, tctx, true, old_usize, old_tctx); return (p); } -- cgit v0.12 From ef363de7010b5e13f4e1c0d7b3a109362bda7aa7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 22:45:31 -0700 Subject: Fix irealloc_prof() to prof_alloc_rollback() on OOM. --- ChangeLog | 1 + src/jemalloc.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 18d72eb..1625776 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ brevity. Much more detail can be found in the git revision history: segfault. - Fix xallocx() bugs related to the 'extra' parameter when specified as non-zero. + - Fix irealloc_prof() to prof_alloc_rollback() on OOM. * 4.0.0 (August 17, 2015) diff --git a/src/jemalloc.c b/src/jemalloc.c index 74fab0f..6801702 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1712,8 +1712,10 @@ irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) p = irealloc_prof_sample(tsd, oldptr, old_usize, usize, tctx); else p = iralloc(tsd, oldptr, old_usize, usize, 0, false); - if (p == NULL) + if (unlikely(p == NULL)) { + prof_alloc_rollback(tsd, tctx, true); return (NULL); + } prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); return (p); -- cgit v0.12 From cec0d63d8bc46205d38456024176a0ece590253e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:17:25 -0700 Subject: Make one call to prof_active_get_unlocked() per allocation event. Make one call to prof_active_get_unlocked() per allocation event, and use the result throughout the relevant functions that handle an allocation event. Also add a missing check in prof_realloc(). These fixes protect allocation events against concurrent prof_active changes. --- ChangeLog | 4 ++++ include/jemalloc/internal/prof.h | 18 ++++++++++-------- src/jemalloc.c | 29 +++++++++++++++++++---------- 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1625776..21edac0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -29,6 +29,10 @@ brevity. Much more detail can be found in the git revision history: - Fix xallocx() bugs related to the 'extra' parameter when specified as non-zero. - Fix irealloc_prof() to prof_alloc_rollback() on OOM. + - Make one call to prof_active_get_unlocked() per allocation event, and use + the result throughout the relevant functions that handle an allocation + event. Also add a missing check in prof_realloc(). These fixes protect + allocation events against concurrent prof_active changes. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index eca8aa8..c66611c 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -331,14 +331,16 @@ bool prof_gdump_get_unlocked(void); prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, prof_tdata_t **tdata_out); -prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update); +prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, + bool update); prof_tctx_t *prof_tctx_get(const void *ptr); void prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, - prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx); + prof_tctx_t *tctx, bool prof_active, bool updated, + size_t old_usize, prof_tctx_t *old_tctx); void prof_free(tsd_t *tsd, const void *ptr, size_t usize); #endif @@ -443,7 +445,7 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, } JEMALLOC_ALWAYS_INLINE prof_tctx_t * -prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) +prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update) { prof_tctx_t *ret; prof_tdata_t *tdata; @@ -451,8 +453,8 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) assert(usize == s2u(usize)); - if (!prof_active_get_unlocked() || likely(prof_sample_accum_update(tsd, - usize, update, &tdata))) + if (!prof_active || likely(prof_sample_accum_update(tsd, usize, update, + &tdata))) ret = (prof_tctx_t *)(uintptr_t)1U; else { bt_init(&bt, tdata->vec); @@ -479,17 +481,17 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) JEMALLOC_ALWAYS_INLINE void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, - bool updated, size_t old_usize, prof_tctx_t *old_tctx) + bool prof_active, bool updated, size_t old_usize, prof_tctx_t *old_tctx) { cassert(config_prof); assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); - if (!updated && ptr != NULL) { + if (prof_active && !updated && ptr != NULL) { assert(usize == isalloc(ptr, true)); if (prof_sample_accum_update(tsd, usize, true, NULL)) { /* - * Don't sample. The usize passed to PROF_ALLOC_PREP() + * Don't sample. The usize passed to prof_alloc_prep() * was larger than what actually got allocated, so a * backtrace was captured for this allocation, even * though its actual usize was insufficient to cross the diff --git a/src/jemalloc.c b/src/jemalloc.c index 6801702..5a32baf 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1378,7 +1378,7 @@ imalloc_prof(tsd_t *tsd, size_t usize) void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(tsd, usize, true); + tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = imalloc_prof_sample(tsd, usize, tctx); else @@ -1468,7 +1468,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(tsd, usize, true); + tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = imemalign_prof_sample(tsd, alignment, usize, tctx); else @@ -1599,7 +1599,7 @@ icalloc_prof(tsd_t *tsd, size_t usize) void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(tsd, usize, true); + tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = icalloc_prof_sample(tsd, usize, tctx); else @@ -1704,10 +1704,12 @@ JEMALLOC_ALWAYS_INLINE_C void * irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) { void *p; + bool prof_active; prof_tctx_t *old_tctx, *tctx; + prof_active = prof_active_get_unlocked(); old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(tsd, usize, true); + tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = irealloc_prof_sample(tsd, oldptr, old_usize, usize, tctx); else @@ -1716,7 +1718,8 @@ irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) prof_alloc_rollback(tsd, tctx, true); return (NULL); } - prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); + prof_realloc(tsd, p, usize, tctx, prof_active, true, old_usize, + old_tctx); return (p); } @@ -2014,7 +2017,7 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, &zero, &tcache, &arena))) return (NULL); - tctx = prof_alloc_prep(tsd, *usize, true); + tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true); if (likely((uintptr_t)tctx == (uintptr_t)1U)) { p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, zero, tcache, arena); @@ -2123,10 +2126,12 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, arena_t *arena) { void *p; + bool prof_active; prof_tctx_t *old_tctx, *tctx; + prof_active = prof_active_get_unlocked(); old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(tsd, *usize, true); + tctx = prof_alloc_prep(tsd, *usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd, oldptr, old_usize, size, alignment, *usize, zero, tcache, arena, tctx); @@ -2150,7 +2155,8 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, */ *usize = isalloc(p, config_prof); } - prof_realloc(tsd, p, *usize, tctx, true, old_usize, old_tctx); + prof_realloc(tsd, p, *usize, tctx, prof_active, true, + old_usize, old_tctx); return (p); } @@ -2274,8 +2280,10 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { size_t max_usize, usize; + bool prof_active; prof_tctx_t *old_tctx, *tctx; + prof_active = prof_active_get_unlocked(); old_tctx = prof_tctx_get(ptr); /* Clamp extra if necessary to avoid (size + extra) overflow. */ if (unlikely(size + extra > HUGE_MAXCLASS)) @@ -2288,7 +2296,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, */ max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - tctx = prof_alloc_prep(tsd, max_usize, false); + tctx = prof_alloc_prep(tsd, max_usize, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, alignment, zero, max_usize, tctx); @@ -2300,7 +2308,8 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_alloc_rollback(tsd, tctx, false); return (usize); } - prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx); + prof_realloc(tsd, ptr, usize, tctx, prof_active, false, old_usize, + old_tctx); return (usize); } -- cgit v0.12 From d9704042ee436cdb329a0777ad350b9fef78891d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:28:32 -0700 Subject: s/oldptr/old_ptr/g --- src/jemalloc.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 5a32baf..0c66f4b 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1682,7 +1682,7 @@ label_return: } static void * -irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize, +irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, prof_tctx_t *tctx) { void *p; @@ -1690,30 +1690,30 @@ irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize, if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloc(tsd, oldptr, old_usize, LARGE_MINCLASS, 0, false); + p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = iralloc(tsd, oldptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); return (p); } JEMALLOC_ALWAYS_INLINE_C void * -irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) +irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; prof_active = prof_active_get_unlocked(); - old_tctx = prof_tctx_get(oldptr); + old_tctx = prof_tctx_get(old_ptr); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = irealloc_prof_sample(tsd, oldptr, old_usize, usize, tctx); + p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); else - p = iralloc(tsd, oldptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); return (NULL); @@ -2098,7 +2098,7 @@ label_oom: } static void * -irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, +irallocx_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t usize, bool zero, tcache_t *tcache, arena_t *arena, prof_tctx_t *tctx) { @@ -2107,13 +2107,13 @@ irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloct(tsd, oldptr, old_usize, LARGE_MINCLASS, alignment, + p = iralloct(tsd, old_ptr, old_usize, LARGE_MINCLASS, alignment, zero, tcache, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { - p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, + p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, tcache, arena); } @@ -2121,7 +2121,7 @@ irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, } JEMALLOC_ALWAYS_INLINE_C void * -irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, +irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, arena_t *arena) { @@ -2130,13 +2130,13 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, prof_tctx_t *old_tctx, *tctx; prof_active = prof_active_get_unlocked(); - old_tctx = prof_tctx_get(oldptr); + old_tctx = prof_tctx_get(old_ptr); tctx = prof_alloc_prep(tsd, *usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irallocx_prof_sample(tsd, oldptr, old_usize, size, + p = irallocx_prof_sample(tsd, old_ptr, old_usize, size, alignment, *usize, zero, tcache, arena, tctx); } else { - p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, + p = iralloct(tsd, old_ptr, old_usize, size, alignment, zero, tcache, arena); } if (unlikely(p == NULL)) { @@ -2144,7 +2144,7 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, return (NULL); } - if (p == oldptr && alignment != 0) { + if (p == old_ptr && alignment != 0) { /* * The allocation did not move, so it is possible that the size * class is smaller than would guarantee the requested -- cgit v0.12 From ce9a4e34795a22838b97a5f10cd8090ab21f22fd Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:31:02 -0700 Subject: s/max_usize/usize_max/g --- src/jemalloc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 0c66f4b..a7c7a03 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2251,7 +2251,7 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, static size_t ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, - size_t alignment, size_t max_usize, bool zero, prof_tctx_t *tctx) + size_t alignment, size_t usize_max, bool zero, prof_tctx_t *tctx) { size_t usize; @@ -2265,7 +2265,7 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, (SMALL_MAXCLASS+1), alignment, zero)) return (old_usize); usize = isalloc(ptr, config_prof); - if (max_usize < LARGE_MINCLASS) + if (usize_max < LARGE_MINCLASS) arena_prof_promoted(ptr, usize); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, @@ -2279,7 +2279,7 @@ JEMALLOC_ALWAYS_INLINE_C size_t ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { - size_t max_usize, usize; + size_t usize_max, usize; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2294,12 +2294,12 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, * prof_alloc_prep() to decide whether to capture a backtrace. * prof_realloc() will use the actual usize to decide whether to sample. */ - max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, + usize_max = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - tctx = prof_alloc_prep(tsd, max_usize, prof_active, false); + tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, - alignment, zero, max_usize, tctx); + alignment, zero, usize_max, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero); -- cgit v0.12 From 23f6e103c871b4db5b315caf1a8d5c46d9675691 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:32:26 -0700 Subject: Fix ixallocx_prof_sample() argument order reversal. Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() in the correct order. --- ChangeLog | 2 ++ src/jemalloc.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 21edac0..95a4ac5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,6 +33,8 @@ brevity. Much more detail can be found in the git revision history: the result throughout the relevant functions that handle an allocation event. Also add a missing check in prof_realloc(). These fixes protect allocation events against concurrent prof_active changes. + - Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() in + the correct order. * 4.0.0 (August 17, 2015) diff --git a/src/jemalloc.c b/src/jemalloc.c index a7c7a03..2566f9b 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2299,7 +2299,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, - alignment, zero, usize_max, tctx); + alignment, usize_max, zero, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero); -- cgit v0.12 From ea8d97b8978a0c0423f0ed64332463a25b787c3d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:44:37 -0700 Subject: Fix prof_{malloc,free}_sample_object() call order in prof_realloc(). Fix prof_realloc() to call prof_free_sampled_object() after calling prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were the same, the tctx could have been prematurely destroyed. --- ChangeLog | 3 +++ include/jemalloc/internal/prof.h | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 95a4ac5..269d089 100644 --- a/ChangeLog +++ b/ChangeLog @@ -35,6 +35,9 @@ brevity. Much more detail can be found in the git revision history: allocation events against concurrent prof_active changes. - Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() in the correct order. + - Fix prof_realloc() to call prof_free_sampled_object() after calling + prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were + the same, the tctx could have been prematurely destroyed. * 4.0.0 (August 17, 2015) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index c66611c..5eb5926 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -483,6 +483,7 @@ JEMALLOC_ALWAYS_INLINE void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, bool prof_active, bool updated, size_t old_usize, prof_tctx_t *old_tctx) { + bool sampled, old_sampled; cassert(config_prof); assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); @@ -501,12 +502,16 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, } } - if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U)) - prof_free_sampled_object(tsd, old_usize, old_tctx); - if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) + sampled = ((uintptr_t)tctx > (uintptr_t)1U); + old_sampled = ((uintptr_t)old_tctx > (uintptr_t)1U); + + if (unlikely(sampled)) prof_malloc_sample_object(ptr, usize, tctx); else prof_tctx_set(ptr, usize, (prof_tctx_t *)(uintptr_t)1U); + + if (unlikely(old_sampled)) + prof_free_sampled_object(tsd, old_usize, old_tctx); } JEMALLOC_ALWAYS_INLINE void -- cgit v0.12 From 708ed79834fc3b8e5b14dbb0128a0ebfce63a38f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Sep 2015 23:48:11 -0700 Subject: Resolve an unsupported special case in arena_prof_tctx_set(). Add arena_prof_tctx_reset() and use it instead of arena_prof_tctx_set() when resetting the tctx pointer during reallocation, which happens whenever an originally sampled reallocated object is not sampled during reallocation. This regression was introduced by 594c759f37c301d0245dc2accf4d4aaf9d202819 (Optimize arena_prof_tctx_set().) --- include/jemalloc/internal/arena.h | 31 +++++++++++++++++++++++++++ include/jemalloc/internal/huge.h | 1 + include/jemalloc/internal/private_symbols.txt | 3 +++ include/jemalloc/internal/prof.h | 20 ++++++++++++++--- src/huge.c | 7 ++++++ src/jemalloc.c | 6 +++--- 6 files changed, 62 insertions(+), 6 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 9712c1c..12c6179 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -556,6 +556,8 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_tctx_t *arena_prof_tctx_get(const void *ptr); void arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); +void arena_prof_tctx_reset(const void *ptr, size_t usize, + const void *old_ptr, prof_tctx_t *old_tctx); void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_t *tcache); arena_t *arena_aalloc(const void *ptr); @@ -1126,6 +1128,35 @@ arena_prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) huge_prof_tctx_set(ptr, tctx); } +JEMALLOC_INLINE void +arena_prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr, + prof_tctx_t *old_tctx) +{ + + cassert(config_prof); + assert(ptr != NULL); + + if (unlikely(usize > SMALL_MAXCLASS || (ptr == old_ptr && + (uintptr_t)old_tctx > (uintptr_t)1U))) { + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) { + size_t pageind; + arena_chunk_map_misc_t *elm; + + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> + LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != + 0); + assert(arena_mapbits_large_get(chunk, pageind) != 0); + + elm = arena_miscelm_get(chunk, pageind); + atomic_write_p(&elm->prof_tctx_pun, + (prof_tctx_t *)(uintptr_t)1U); + } else + huge_prof_tctx_reset(ptr); + } +} + JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_t *tcache) diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 328eeed..ece7af9 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -26,6 +26,7 @@ arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); +void huge_prof_tctx_reset(const void *ptr); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 9d21a80..a90021a 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -80,6 +80,7 @@ arena_prof_accum_impl arena_prof_accum_locked arena_prof_promoted arena_prof_tctx_get +arena_prof_tctx_reset arena_prof_tctx_set arena_ptr_small_binind_get arena_purge_all @@ -250,6 +251,7 @@ huge_dalloc_junk huge_malloc huge_palloc huge_prof_tctx_get +huge_prof_tctx_reset huge_prof_tctx_set huge_ralloc huge_ralloc_no_move @@ -379,6 +381,7 @@ prof_reset prof_sample_accum_update prof_sample_threshold_update prof_tctx_get +prof_tctx_reset prof_tctx_set prof_tdata_cleanup prof_tdata_get diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 5eb5926..e5198c3 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -335,11 +335,13 @@ prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, bool update); prof_tctx_t *prof_tctx_get(const void *ptr); void prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx); +void prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr, + prof_tctx_t *tctx); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, - prof_tctx_t *tctx, bool prof_active, bool updated, + prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, size_t old_usize, prof_tctx_t *old_tctx); void prof_free(tsd_t *tsd, const void *ptr, size_t usize); #endif @@ -414,6 +416,17 @@ prof_tctx_set(const void *ptr, size_t usize, prof_tctx_t *tctx) arena_prof_tctx_set(ptr, usize, tctx); } +JEMALLOC_ALWAYS_INLINE void +prof_tctx_reset(const void *ptr, size_t usize, const void *old_ptr, + prof_tctx_t *old_tctx) +{ + + cassert(config_prof); + assert(ptr != NULL); + + arena_prof_tctx_reset(ptr, usize, old_ptr, old_tctx); +} + JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, prof_tdata_t **tdata_out) @@ -481,7 +494,8 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) JEMALLOC_ALWAYS_INLINE void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, - bool prof_active, bool updated, size_t old_usize, prof_tctx_t *old_tctx) + bool prof_active, bool updated, const void *old_ptr, size_t old_usize, + prof_tctx_t *old_tctx) { bool sampled, old_sampled; @@ -508,7 +522,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, if (unlikely(sampled)) prof_malloc_sample_object(ptr, usize, tctx); else - prof_tctx_set(ptr, usize, (prof_tctx_t *)(uintptr_t)1U); + prof_tctx_reset(ptr, usize, old_ptr, old_tctx); if (unlikely(old_sampled)) prof_free_sampled_object(tsd, old_usize, old_tctx); diff --git a/src/huge.c b/src/huge.c index 187bdaa..f8778db 100644 --- a/src/huge.c +++ b/src/huge.c @@ -424,3 +424,10 @@ huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) extent_node_prof_tctx_set(node, tctx); malloc_mutex_unlock(&arena->huge_mtx); } + +void +huge_prof_tctx_reset(const void *ptr) +{ + + huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); +} diff --git a/src/jemalloc.c b/src/jemalloc.c index 2566f9b..a29e613 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1718,7 +1718,7 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize) prof_alloc_rollback(tsd, tctx, true); return (NULL); } - prof_realloc(tsd, p, usize, tctx, prof_active, true, old_usize, + prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize, old_tctx); return (p); @@ -2155,7 +2155,7 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, */ *usize = isalloc(p, config_prof); } - prof_realloc(tsd, p, *usize, tctx, prof_active, true, + prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr, old_usize, old_tctx); return (p); @@ -2308,7 +2308,7 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_alloc_rollback(tsd, tctx, false); return (usize); } - prof_realloc(tsd, ptr, usize, tctx, prof_active, false, old_usize, + prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize, old_tctx); return (usize); -- cgit v0.12 From 0108b1fd0411db444d4074582013879f757fa1df Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 00:08:05 -0700 Subject: Don't run stress tests as part of check target. This change was intended as part of 8f57e3f1aeb86021b3d078b825bc8c42b2a9af6f (Remove check_stress from check target's dependencies.). --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 9b2d0a1..a836e74 100644 --- a/Makefile.in +++ b/Makefile.in @@ -358,7 +358,7 @@ check_integration: tests_integration check_integration_dir stress: tests_stress stress_dir $(SHELL) $(objroot)test/test.sh $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%) check: tests check_dir check_integration_prof - $(SHELL) $(objroot)test/test.sh $(TESTS:$(srcroot)%.c=$(objroot)%) + $(SHELL) $(objroot)test/test.sh $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%) $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%) ifeq ($(enable_code_coverage), 1) coverage_unit: check_unit -- cgit v0.12 From 8c485b02a61ab96d4d248e234302edf57577b77d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 00:49:09 -0700 Subject: Fix ixallocx_prof() to check for size greater than HUGE_MAXCLASS. --- ChangeLog | 3 +-- src/jemalloc.c | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 269d089..e4da638 100644 --- a/ChangeLog +++ b/ChangeLog @@ -26,8 +26,7 @@ brevity. Much more detail can be found in the git revision history: with interposed resets (triggered via the "prof.reset" mallctl). This bug could cause data structure corruption that would most likely result in a segfault. - - Fix xallocx() bugs related to the 'extra' parameter when specified as - non-zero. + - Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS. - Fix irealloc_prof() to prof_alloc_rollback() on OOM. - Make one call to prof_active_get_unlocked() per allocation event, and use the result throughout the relevant functions that handle an allocation diff --git a/src/jemalloc.c b/src/jemalloc.c index a29e613..f403306 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2286,8 +2286,12 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_active = prof_active_get_unlocked(); old_tctx = prof_tctx_get(ptr); /* Clamp extra if necessary to avoid (size + extra) overflow. */ - if (unlikely(size + extra > HUGE_MAXCLASS)) + if (unlikely(size + extra > HUGE_MAXCLASS)) { + /* Check for size overflow. */ + if (size > HUGE_MAXCLASS) + return (old_usize); extra = HUGE_MAXCLASS - size; + } /* * usize isn't knowable before ixalloc() returns when extra is non-zero. * Therefore, compute its maximum possible value and use that in -- cgit v0.12 From 6d91929e52ba87c222f4f92bd1c9ddc9e7c2c083 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 10:42:36 -0700 Subject: Address portability issues on Solaris. Don't assume Bourne shell is in /bin/sh when running size_classes.sh . Consider __sparcv9 a synonym for __sparc64__ when defining LG_QUANTUM. This resolves #275. --- configure.ac | 3 ++- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 5e77b68..2308b99 100644 --- a/configure.ac +++ b/configure.ac @@ -1621,8 +1621,9 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ ]) AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ mkdir -p "${objroot}include/jemalloc/internal" - "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" ${LG_TINY_MIN} "${LG_PAGE_SIZES}" ${LG_SIZE_CLASS_GROUP} > "${objroot}include/jemalloc/internal/size_classes.h" + "${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" ${LG_TINY_MIN} "${LG_PAGE_SIZES}" ${LG_SIZE_CLASS_GROUP} > "${objroot}include/jemalloc/internal/size_classes.h" ], [ + SHELL="${SHELL}" srcdir="${srcdir}" objroot="${objroot}" LG_QUANTA="${LG_QUANTA}" diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index e2959f1..8536a3e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -232,7 +232,7 @@ typedef unsigned szind_t; # ifdef __alpha__ # define LG_QUANTUM 4 # endif -# ifdef __sparc64__ +# if (defined(__sparc64__) || defined(__sparcv9)) # define LG_QUANTUM 4 # endif # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -- cgit v0.12 From 78ae1ac486ffd7953536786c9a5f9dc2bda78858 Mon Sep 17 00:00:00 2001 From: Dmitry-Me Date: Tue, 8 Sep 2015 15:09:20 +0300 Subject: Reduce variable scope. This resolves #274. --- src/chunk_dss.c | 8 +++----- src/chunk_mmap.c | 6 ++++-- src/prof.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/chunk_dss.c b/src/chunk_dss.c index de0546d..61fc916 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -69,8 +69,6 @@ void * chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { - void *ret; - cassert(have_dss); assert(size > 0 && (size & chunksize_mask) == 0); assert(alignment > 0 && (alignment & chunksize_mask) == 0); @@ -84,9 +82,6 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, malloc_mutex_lock(&dss_mtx); if (dss_prev != (void *)-1) { - size_t gap_size, cpad_size; - void *cpad, *dss_next; - intptr_t incr; /* * The loop is necessary to recover from races with other @@ -94,6 +89,9 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, * malloc. */ do { + void *ret, *cpad, *dss_next; + size_t gap_size, cpad_size; + intptr_t incr; /* Avoid an unnecessary system call. */ if (new_addr != NULL && dss_max != new_addr) break; diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index 36eb075..b9ba741 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -6,14 +6,16 @@ static void * chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero, bool *commit) { - void *ret, *pages; - size_t alloc_size, leadsize; + void *ret; + size_t alloc_size; alloc_size = size + alignment - PAGE; /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); do { + void *pages; + size_t leadsize; pages = pages_map(NULL, alloc_size); if (pages == NULL) return (NULL); diff --git a/src/prof.c b/src/prof.c index 7427bf5..d68478f 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1577,7 +1577,6 @@ prof_idump(void) { tsd_t *tsd; prof_tdata_t *tdata; - char filename[PATH_MAX + 1]; cassert(config_prof); @@ -1593,6 +1592,7 @@ prof_idump(void) } if (opt_prof_prefix[0] != '\0') { + char filename[PATH_MAX + 1]; malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'i', prof_dump_iseq); prof_dump_iseq++; @@ -1631,7 +1631,6 @@ prof_gdump(void) { tsd_t *tsd; prof_tdata_t *tdata; - char filename[DUMP_FILENAME_BUFSIZE]; cassert(config_prof); @@ -1647,6 +1646,7 @@ prof_gdump(void) } if (opt_prof_prefix[0] != '\0') { + char filename[DUMP_FILENAME_BUFSIZE]; malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'u', prof_dump_useq); prof_dump_useq++; -- cgit v0.12 From aca490f004bffa619319aec718fc74e9855b45ae Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 14:39:29 -0700 Subject: Add more xallocx() overflow tests. --- test/integration/xallocx.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/test/integration/xallocx.c b/test/integration/xallocx.c index 8f0de63..76c91e9 100644 --- a/test/integration/xallocx.c +++ b/test/integration/xallocx.c @@ -122,6 +122,68 @@ get_huge_size(size_t ind) return (get_size_impl("arenas.hchunk.0.size", ind)); } +TEST_BEGIN(test_size) +{ + size_t small0, hugemax; + void *p; + + /* Get size classes. */ + small0 = get_small_size(0); + hugemax = get_huge_size(get_nhuge()-1); + + p = mallocx(small0, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + /* Test smallest supported size. */ + assert_zu_eq(xallocx(p, 1, 0, 0), small0, + "Unexpected xallocx() behavior"); + + /* Test largest supported size. */ + assert_zu_le(xallocx(p, hugemax, 0, 0), hugemax, + "Unexpected xallocx() behavior"); + + /* Test size overflow. */ + assert_zu_le(xallocx(p, hugemax+1, 0, 0), hugemax, + "Unexpected xallocx() behavior"); + assert_zu_le(xallocx(p, SIZE_T_MAX, 0, 0), hugemax, + "Unexpected xallocx() behavior"); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_size_extra_overflow) +{ + size_t small0, hugemax; + void *p; + + /* Get size classes. */ + small0 = get_small_size(0); + hugemax = get_huge_size(get_nhuge()-1); + + p = mallocx(small0, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + /* Test overflows that can be resolved by clamping extra. */ + assert_zu_le(xallocx(p, hugemax-1, 2, 0), hugemax, + "Unexpected xallocx() behavior"); + assert_zu_le(xallocx(p, hugemax, 1, 0), hugemax, + "Unexpected xallocx() behavior"); + + /* Test overflow such that hugemax-size underflows. */ + assert_zu_le(xallocx(p, hugemax+1, 2, 0), hugemax, + "Unexpected xallocx() behavior"); + assert_zu_le(xallocx(p, hugemax+2, 3, 0), hugemax, + "Unexpected xallocx() behavior"); + assert_zu_le(xallocx(p, SIZE_T_MAX-2, 2, 0), hugemax, + "Unexpected xallocx() behavior"); + assert_zu_le(xallocx(p, SIZE_T_MAX-1, 1, 0), hugemax, + "Unexpected xallocx() behavior"); + + dallocx(p, 0); +} +TEST_END + TEST_BEGIN(test_extra_small) { size_t small0, small1, hugemax; @@ -293,6 +355,8 @@ main(void) test_same_size, test_extra_no_move, test_no_move_fail, + test_size, + test_size_extra_overflow, test_extra_small, test_extra_large, test_extra_huge)); -- cgit v0.12 From 9a505b768cd50bffbfaa3a993df9117e7454134e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 14:39:58 -0700 Subject: Centralize xallocx() size[+extra] overflow checks. --- src/arena.c | 7 ------- src/jemalloc.c | 18 +++++++++++------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/arena.c b/src/arena.c index a119d26..2e888ea 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2791,15 +2791,8 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, { size_t usize_min, usize_max; - /* Check for size overflow. */ - if (unlikely(size > HUGE_MAXCLASS)) - return (true); usize_min = s2u(size); - /* Clamp extra if necessary to avoid (size + extra) overflow. */ - if (unlikely(size + extra > HUGE_MAXCLASS)) - extra = HUGE_MAXCLASS - size; usize_max = s2u(size + extra); - if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) { /* * Avoid moving the allocation if the size class can be left the diff --git a/src/jemalloc.c b/src/jemalloc.c index f403306..ab7cf02 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2285,13 +2285,6 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, prof_active = prof_active_get_unlocked(); old_tctx = prof_tctx_get(ptr); - /* Clamp extra if necessary to avoid (size + extra) overflow. */ - if (unlikely(size + extra > HUGE_MAXCLASS)) { - /* Check for size overflow. */ - if (size > HUGE_MAXCLASS) - return (old_usize); - extra = HUGE_MAXCLASS - size; - } /* * usize isn't knowable before ixalloc() returns when extra is non-zero. * Therefore, compute its maximum possible value and use that in @@ -2335,6 +2328,17 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) tsd = tsd_fetch(); old_usize = isalloc(ptr, config_prof); + + /* Clamp extra if necessary to avoid (size + extra) overflow. */ + if (unlikely(size + extra > HUGE_MAXCLASS)) { + /* Check for size overflow. */ + if (unlikely(size > HUGE_MAXCLASS)) { + usize = old_usize; + goto label_not_resized; + } + extra = HUGE_MAXCLASS - size; + } + if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); -- cgit v0.12 From 345c1b0eeeac333c3da8baa46e9b96c3b2aca443 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 14:59:56 -0700 Subject: Link test to librt if it contains clock_gettime(2). This resolves #257. --- Makefile.in | 7 ++++--- configure.ac | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile.in b/Makefile.in index a836e74..01285af 100644 --- a/Makefile.in +++ b/Makefile.in @@ -28,6 +28,7 @@ CFLAGS := @CFLAGS@ LDFLAGS := @LDFLAGS@ EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ LIBS := @LIBS@ +TESTLIBS := @TESTLIBS@ RPATH_EXTRA := @RPATH_EXTRA@ SO := @so@ IMPORTLIB := @importlib@ @@ -265,15 +266,15 @@ $(STATIC_LIBS): $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(TESTS_UNIT_LINK_OBJS) $(C_JET_OBJS) $(C_TESTLIB_UNIT_OBJS) @mkdir -p $(@D) - $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(TESTLIBS) $(EXTRA_LDFLAGS) $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) - $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(EXTRA_LDFLAGS) + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(filter -lpthread,$(LIBS))) -lm $(TESTLIBS) $(EXTRA_LDFLAGS) $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TESTLIB_STRESS_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) - $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(TESTLIBS) $(EXTRA_LDFLAGS) build_lib_shared: $(DSOS) build_lib_static: $(STATIC_LIBS) diff --git a/configure.ac b/configure.ac index 2308b99..7a1290e 100644 --- a/configure.ac +++ b/configure.ac @@ -1190,6 +1190,14 @@ fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" +dnl Check whether clock_gettime(2) is in libc or librt. This function is only +dnl used in test code, so save the result to TESTLIBS to avoid poluting LIBS. +SAVED_LIBS="${LIBS}" +LIBS= +AC_SEARCH_LIBS([clock_gettime], [rt], [TESTLIBS="${LIBS}"]) +AC_SUBST([TESTLIBS]) +LIBS="${SAVED_LIBS}" + dnl Check if the GNU-specific secure_getenv function exists. AC_CHECK_FUNC([secure_getenv], [have_secure_getenv="1"], @@ -1694,6 +1702,7 @@ AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) AC_MSG_RESULT([EXTRA_LDFLAGS : ${EXTRA_LDFLAGS}]) AC_MSG_RESULT([LIBS : ${LIBS}]) +AC_MSG_RESULT([TESTLIBS : ${TESTLIBS}]) AC_MSG_RESULT([RPATH_EXTRA : ${RPATH_EXTRA}]) AC_MSG_RESULT([]) AC_MSG_RESULT([XSLTPROC : ${XSLTPROC}]) -- cgit v0.12 From 65b940a3c5b7467d79f757e98aa5ba1810129c3e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 15:48:42 -0700 Subject: Loosen expected xallocx() results. Systems that do not support chunk split/merge cannot shrink/grow huge allocations in place. --- test/integration/xallocx.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/integration/xallocx.c b/test/integration/xallocx.c index 76c91e9..058e27c 100644 --- a/test/integration/xallocx.c +++ b/test/integration/xallocx.c @@ -300,9 +300,9 @@ TEST_BEGIN(test_extra_huge) assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, "Unexpected xallocx() behavior"); /* Test size decrease with zero extra. */ - assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + assert_zu_ge(xallocx(p, huge0, 0, 0), huge0, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, largemax, 0, 0), huge0, + assert_zu_ge(xallocx(p, largemax, 0, 0), huge0, "Unexpected xallocx() behavior"); assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, @@ -314,27 +314,27 @@ TEST_BEGIN(test_extra_huge) "Unexpected xallocx() behavior"); assert_zu_eq(xallocx(p, huge0, huge1 - huge0, 0), huge1, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, largemax, huge0 - largemax, 0), huge0, + assert_zu_ge(xallocx(p, largemax, huge0 - largemax, 0), huge0, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + assert_zu_ge(xallocx(p, huge0, 0, 0), huge0, "Unexpected xallocx() behavior"); /* Test size increase with zero extra. */ - assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, + assert_zu_le(xallocx(p, huge2, 0, 0), huge2, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, hugemax+1, 0, 0), huge2, + assert_zu_le(xallocx(p, hugemax+1, 0, 0), huge2, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + assert_zu_ge(xallocx(p, huge0, 0, 0), huge0, "Unexpected xallocx() behavior"); /* Test size increase with non-zero extra. */ assert_zu_le(xallocx(p, huge0, SIZE_T_MAX - huge0, 0), hugemax, "Unexpected xallocx() behavior"); - assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + assert_zu_ge(xallocx(p, huge0, 0, 0), huge0, "Unexpected xallocx() behavior"); /* Test size increase with non-zero extra. */ - assert_zu_eq(xallocx(p, huge0, huge2 - huge0, 0), huge2, + assert_zu_le(xallocx(p, huge0, huge2 - huge0, 0), huge2, "Unexpected xallocx() behavior"); assert_zu_eq(xallocx(p, huge2, 0, 0), huge2, -- cgit v0.12 From 1d7540c9d71ee8a85ea97c9459698e090ee04719 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Sep 2015 15:26:23 -0700 Subject: Update ChangeLog for 4.0.1. --- ChangeLog | 71 ++++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/ChangeLog b/ChangeLog index e4da638..4498683 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,39 +4,62 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc -* 4.0.1 (XXX) +* 4.0.1 (September 15, 2015) + + This is a bugfix release that is somewhat high risk due to the amount of + refactoring required to address deep xallocx() problems. As a side effect of + these fixes, xallocx() now tries harder to partially fulfill requests for + optional extra space. Note that a couple of minor heap profiling + optimizations are included, but these are better thought of as performance + fixes that were integral to disovering most of the other bugs. + + Optimizations: + - Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the + fast path when heap profiling is enabled. Additionally, split a special + case out into arena_prof_tctx_reset(), which also avoids chunk metadata + reads. + - Optimize irallocx_prof() to optimistically update the sampler state. The + prior implementation appears to have been a holdover from when + rallocx()/xallocx() functionality was combined as rallocm(). Bug fixes: + - Fix TLS configuration such that it is enabled by default for platforms on + which it works correctly. - Fix arenas_cache_cleanup() and arena_get_hard() to handle allocation/deallocation within the application's thread-specific data cleanup functions even after arenas_cache is torn down. - - Don't bitshift by negative amounts when encoding/decoding run sizes in chunk - header maps. This affected systems with page sizes greater than 8 KiB. - - Rename index_t to szind_t to avoid an existing type on Solaris. - - Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to - match glibc and avoid compilation errors when including both - jemalloc/jemalloc.h and malloc.h in C++ code. + - Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS. - Fix chunk purge hook calls for in-place huge shrinking reallocation to specify the old chunk size rather than the new chunk size. This bug caused no correctness issues for the default chunk purge function, but was visible to custom functions set via the "arena..chunk_hooks" mallctl. - - Fix TLS configuration such that it is enabled by default for platforms on - which it works correctly. - - Fix heap profiling to distinguish among otherwise identical sample sites - with interposed resets (triggered via the "prof.reset" mallctl). This bug - could cause data structure corruption that would most likely result in a - segfault. - - Fix xallocx() bugs related to size+extra exceeding HUGE_MAXCLASS. - - Fix irealloc_prof() to prof_alloc_rollback() on OOM. - - Make one call to prof_active_get_unlocked() per allocation event, and use - the result throughout the relevant functions that handle an allocation - event. Also add a missing check in prof_realloc(). These fixes protect - allocation events against concurrent prof_active changes. - - Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() in - the correct order. - - Fix prof_realloc() to call prof_free_sampled_object() after calling - prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were - the same, the tctx could have been prematurely destroyed. + - Fix heap profiling bugs: + + Fix heap profiling to distinguish among otherwise identical sample sites + with interposed resets (triggered via the "prof.reset" mallctl). This bug + could cause data structure corruption that would most likely result in a + segfault. + + Fix irealloc_prof() to prof_alloc_rollback() on OOM. + + Make one call to prof_active_get_unlocked() per allocation event, and use + the result throughout the relevant functions that handle an allocation + event. Also add a missing check in prof_realloc(). These fixes protect + allocation events against concurrent prof_active changes. + + Fix ixallocx_prof() to pass usize_max and zero to ixallocx_prof_sample() + in the correct order. + + Fix prof_realloc() to call prof_free_sampled_object() after calling + prof_malloc_sample_object(). Prior to this fix, if tctx and old_tctx were + the same, the tctx could have been prematurely destroyed. + - Fix portability bugs: + + Don't bitshift by negative amounts when encoding/decoding run sizes in + chunk header maps. This affected systems with page sizes greater than 8 + KiB. + + Rename index_t to szind_t to avoid an existing type on Solaris. + + Add JEMALLOC_CXX_THROW to the memalign() function prototype, in order to + match glibc and avoid compilation errors when including both + jemalloc/jemalloc.h and malloc.h in C++ code. + + Don't assume that /bin/sh is appropriate when running size_classes.sh + during configuration. + + Consider __sparcv9 a synonym for __sparc64__ when defining LG_QUANTUM. + + Link tests to librt if it contains clock_gettime(2). * 4.0.0 (August 17, 2015) -- cgit v0.12