diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/arena.c | 44 | ||||
| -rw-r--r-- | src/chunk.c | 2 | ||||
| -rw-r--r-- | src/ctl.c | 82 | ||||
| -rw-r--r-- | src/jemalloc.c | 34 |
4 files changed, 95 insertions, 67 deletions
diff --git a/src/arena.c b/src/arena.c index 05a787f..d28b629 100644 --- a/src/arena.c +++ b/src/arena.c @@ -369,13 +369,20 @@ arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) } static inline void +arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) +{ + + VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << + LG_PAGE)), PAGE); +} + +static inline void arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) { size_t i; UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); - VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), PAGE); + arena_run_page_mark_zeroed(chunk, run_ind); for (i = 0; i < PAGE / sizeof(size_t); i++) assert(p[i] == 0); } @@ -458,6 +465,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, } else if (config_debug) { arena_run_page_validate_zeroed( chunk, run_ind+i); + } else { + arena_run_page_mark_zeroed( + chunk, run_ind+i); } } } else { @@ -467,6 +477,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, */ arena_run_zero(chunk, run_ind, need_pages); } + } else { + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } /* @@ -508,9 +521,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); } + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } - VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), (need_pages << LG_PAGE)); } static arena_chunk_t * @@ -569,17 +582,24 @@ arena_chunk_alloc(arena_t *arena) * unless the chunk is not zeroed. */ if (zero == false) { + VALGRIND_MAKE_MEM_UNDEFINED( + (void *)arena_mapp_get(chunk, map_bias+1), + (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); for (i = map_bias+1; i < chunk_npages-1; i++) arena_mapbits_unzeroed_set(chunk, i, unzeroed); - } else if (config_debug) { + } else { VALGRIND_MAKE_MEM_DEFINED( (void *)arena_mapp_get(chunk, map_bias+1), - (void *)((uintptr_t) - arena_mapp_get(chunk, chunk_npages-1) - - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); - for (i = map_bias+1; i < chunk_npages-1; i++) { - assert(arena_mapbits_unzeroed_get(chunk, i) == - unzeroed); + (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); + if (config_debug) { + for (i = map_bias+1; i < chunk_npages-1; i++) { + assert(arena_mapbits_unzeroed_get(chunk, + i) == unzeroed); + } } } arena_mapbits_unallocated_set(chunk, chunk_npages-1, @@ -1458,6 +1478,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) } else if (opt_zero) memset(ret, 0, size); } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], @@ -1466,7 +1487,6 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); return (ret); } diff --git a/src/chunk.c b/src/chunk.c index aef3fed..b17f43f 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -294,7 +294,7 @@ label_return: if (xnode != NULL) base_node_dealloc(xnode); if (xprev != NULL) - base_node_dealloc(prev); + base_node_dealloc(xprev); } void @@ -546,43 +546,30 @@ ctl_arena_refresh(arena_t *arena, unsigned i) static bool ctl_grow(void) { - size_t astats_size; ctl_arena_stats_t *astats; arena_t **tarenas; - /* Extend arena stats and arenas arrays. */ - astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t); - if (ctl_stats.narenas == narenas_auto) { - /* ctl_stats.arenas and arenas came from base_alloc(). */ - astats = (ctl_arena_stats_t *)imalloc(astats_size); - if (astats == NULL) - return (true); - memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * - sizeof(ctl_arena_stats_t)); - - tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * - sizeof(arena_t *)); - if (tarenas == NULL) { - idalloc(astats); - return (true); - } - memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *)); - } else { - astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas, - astats_size, 0, 0, false, false); - if (astats == NULL) - return (true); - - tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) * - sizeof(arena_t *), 0, 0, false, false); - if (tarenas == NULL) - return (true); + /* Allocate extended arena stats and arenas arrays. */ + astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) * + sizeof(ctl_arena_stats_t)); + if (astats == NULL) + return (true); + tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * + sizeof(arena_t *)); + if (tarenas == NULL) { + idalloc(astats); + return (true); } - /* Initialize the new astats and arenas elements. */ + + /* Initialize the new astats element. */ + memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * + sizeof(ctl_arena_stats_t)); memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); - if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) + if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { + idalloc(tarenas); + idalloc(astats); return (true); - tarenas[ctl_stats.narenas] = NULL; + } /* Swap merged stats to their new location. */ { ctl_arena_stats_t tstats; @@ -593,13 +580,34 @@ ctl_grow(void) memcpy(&astats[ctl_stats.narenas + 1], &tstats, sizeof(ctl_arena_stats_t)); } + /* Initialize the new arenas element. */ + tarenas[ctl_stats.narenas] = NULL; + { + arena_t **arenas_old = arenas; + /* + * Swap extended arenas array into place. Although ctl_mtx + * protects this function from other threads extending the + * array, it does not protect from other threads mutating it + * (i.e. initializing arenas and setting array elements to + * point to them). Therefore, array copying must happen under + * the protection of arenas_lock. + */ + malloc_mutex_lock(&arenas_lock); + arenas = tarenas; + memcpy(arenas, arenas_old, ctl_stats.narenas * + sizeof(arena_t *)); + narenas_total++; + arenas_extend(narenas_total - 1); + malloc_mutex_unlock(&arenas_lock); + /* + * Deallocate arenas_old only if it came from imalloc() (not + * base_alloc()). + */ + if (ctl_stats.narenas != narenas_auto) + idalloc(arenas_old); + } ctl_stats.arenas = astats; ctl_stats.narenas++; - malloc_mutex_lock(&arenas_lock); - arenas = tarenas; - narenas_total++; - arenas_extend(narenas_total - 1); - malloc_mutex_unlock(&arenas_lock); return (false); } @@ -1109,7 +1117,7 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - uint64_t newval; + UNUSED uint64_t newval; malloc_mutex_lock(&ctl_mtx); WRITE(newval, uint64_t); diff --git a/src/jemalloc.c b/src/jemalloc.c index bc350ed..ae56db6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -282,7 +282,7 @@ arenas_cleanup(void *arg) malloc_mutex_unlock(&arenas_lock); } -static JEMALLOC_ATTR(always_inline) void +JEMALLOC_ALWAYS_INLINE_C void malloc_thread_init(void) { @@ -299,7 +299,7 @@ malloc_thread_init(void) quarantine_alloc_hook(); } -static JEMALLOC_ATTR(always_inline) bool +JEMALLOC_ALWAYS_INLINE_C bool malloc_init(void) { @@ -436,8 +436,9 @@ malloc_conf_init(void) } break; case 1: { + int linklen = 0; #ifndef _WIN32 - int linklen; + int saved_errno = errno; const char *linkname = # ifdef JEMALLOC_PREFIX "/etc/"JEMALLOC_PREFIX"malloc.conf" @@ -446,21 +447,20 @@ malloc_conf_init(void) # endif ; - if ((linklen = readlink(linkname, buf, - sizeof(buf) - 1)) != -1) { - /* - * Use the contents of the "/etc/malloc.conf" - * symbolic link's name. - */ - buf[linklen] = '\0'; - opts = buf; - } else -#endif - { + /* + * Try to use the contents of the "/etc/malloc.conf" + * symbolic link's name. + */ + linklen = readlink(linkname, buf, sizeof(buf) - 1); + if (linklen == -1) { /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; + linklen = 0; + /* restore errno */ + set_errno(saved_errno); } +#endif + buf[linklen] = '\0'; + opts = buf; break; } case 2: { const char *envname = @@ -1402,7 +1402,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, */ #ifdef JEMALLOC_EXPERIMENTAL -static JEMALLOC_ATTR(always_inline) void * +JEMALLOC_ALWAYS_INLINE_C void * iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) { |