summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/arena.c157
-rw-r--r--src/chunk.c14
-rw-r--r--src/chunk_dss.c45
-rw-r--r--src/ctl.c10
-rw-r--r--src/extent.c37
-rw-r--r--src/huge.c45
-rw-r--r--src/jemalloc.c43
-rw-r--r--src/pages.c4
-rw-r--r--[-rwxr-xr-x]src/stats.c97
-rw-r--r--[-rwxr-xr-x]src/tcache.c114
-rw-r--r--[-rwxr-xr-x]src/util.c0
-rw-r--r--src/witness.c18
-rw-r--r--src/zone.c240
13 files changed, 604 insertions, 220 deletions
diff --git a/src/arena.c b/src/arena.c
index 648a8da..a9dff0b 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -4,6 +4,8 @@
/******************************************************************************/
/* Data. */
+bool opt_thp = true;
+static bool thp_initially_huge;
purge_mode_t opt_purge = PURGE_DEFAULT;
const char *purge_mode_names[] = {
"ratio",
@@ -568,8 +570,8 @@ arena_chunk_init_spare(arena_t *arena)
}
static bool
-arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
- size_t sn, bool zero)
+arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, size_t sn, bool zero,
+ bool *gdump)
{
/*
@@ -580,7 +582,7 @@ arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
*/
extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true);
extent_node_achunk_set(&chunk->node, true);
- return (chunk_register(tsdn, chunk, &chunk->node));
+ return (chunk_register(chunk, &chunk->node, gdump));
}
static arena_chunk_t *
@@ -591,6 +593,8 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena,
size_t sn;
malloc_mutex_unlock(tsdn, &arena->lock);
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks,
NULL, chunksize, chunksize, &sn, zero, commit);
@@ -603,16 +607,20 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena,
chunk = NULL;
}
}
- if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn,
- *zero)) {
- if (!*commit) {
- /* Undo commit of header. */
- chunk_hooks->decommit(chunk, chunksize, 0, map_bias <<
- LG_PAGE, arena->ind);
+ if (chunk != NULL) {
+ bool gdump;
+ if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) {
+ if (!*commit) {
+ /* Undo commit of header. */
+ chunk_hooks->decommit(chunk, chunksize, 0,
+ map_bias << LG_PAGE, arena->ind);
+ }
+ chunk_dalloc_wrapper(tsdn, arena, chunk_hooks,
+ (void *)chunk, chunksize, sn, *zero, *commit);
+ chunk = NULL;
}
- chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk,
- chunksize, sn, *zero, *commit);
- chunk = NULL;
+ if (config_prof && opt_prof && gdump)
+ prof_gdump(tsdn);
}
malloc_mutex_lock(tsdn, &arena->lock);
@@ -627,14 +635,24 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero,
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
size_t sn;
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1);
+ malloc_mutex_assert_owner(tsdn, &arena->lock);
+
chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize,
chunksize, &sn, zero, commit, true);
if (chunk != NULL) {
- if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) {
+ bool gdump;
+ if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) {
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk,
chunksize, sn, true);
return (NULL);
}
+ if (config_prof && opt_prof && gdump) {
+ malloc_mutex_unlock(tsdn, &arena->lock);
+ prof_gdump(tsdn);
+ malloc_mutex_lock(tsdn, &arena->lock);
+ }
}
if (chunk == NULL) {
chunk = arena_chunk_alloc_internal_hard(tsdn, arena,
@@ -664,7 +682,9 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena)
if (chunk == NULL)
return (NULL);
- chunk->hugepage = true;
+ if (config_thp && opt_thp) {
+ chunk->hugepage = thp_initially_huge;
+ }
/*
* Initialize the map to contain one maximal free untouched run. Mark
@@ -729,14 +749,17 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena)
static void
arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
{
- size_t sn, hugepage;
+ size_t sn;
+ UNUSED bool hugepage JEMALLOC_CC_SILENCE_INIT(false);
bool committed;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
chunk_deregister(chunk, &chunk->node);
sn = extent_node_sn_get(&chunk->node);
- hugepage = chunk->hugepage;
+ if (config_thp && opt_thp) {
+ hugepage = chunk->hugepage;
+ }
committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0);
if (!committed) {
/*
@@ -749,13 +772,16 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk)
chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE,
arena->ind);
}
- if (!hugepage) {
+ if (config_thp && opt_thp && hugepage != thp_initially_huge) {
/*
- * Convert chunk back to the default state, so that all
- * subsequent chunk allocations start out with chunks that can
- * be backed by transparent huge pages.
+ * Convert chunk back to initial THP state, so that all
+ * subsequent chunk allocations start out in a consistent state.
*/
- pages_huge(chunk, chunksize);
+ if (thp_initially_huge) {
+ pages_huge(chunk, chunksize);
+ } else {
+ pages_nohuge(chunk, chunksize);
+ }
}
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize,
@@ -1695,13 +1721,13 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/*
* If this is the first run purged within chunk, mark
- * the chunk as non-huge. This will prevent all use of
- * transparent huge pages for this chunk until the chunk
- * as a whole is deallocated.
+ * the chunk as non-THP-capable. This will prevent all
+ * use of THPs for this chunk until the chunk as a whole
+ * is deallocated.
*/
- if (chunk->hugepage) {
- pages_nohuge(chunk, chunksize);
- chunk->hugepage = false;
+ if (config_thp && opt_thp && chunk->hugepage) {
+ chunk->hugepage = pages_nohuge(chunk,
+ chunksize);
}
assert(pageind + npages <= chunk_npages);
@@ -2694,6 +2720,7 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
return (arena_malloc_small(tsdn, arena, ind, zero));
if (likely(size <= large_maxclass))
return (arena_malloc_large(tsdn, arena, ind, zero));
+ assert(index2size(ind) >= chunksize);
return (huge_malloc(tsdn, arena, index2size(ind), zero));
}
@@ -3755,11 +3782,78 @@ bin_info_init(void)
#undef SC
}
+static void
+init_thp_initially_huge(void) {
+ int fd;
+ char buf[sizeof("[always] madvise never\n")];
+ ssize_t nread;
+ static const char *enabled_states[] = {
+ "[always] madvise never\n",
+ "always [madvise] never\n",
+ "always madvise [never]\n"
+ };
+ static const bool thp_initially_huge_states[] = {
+ true,
+ false,
+ false
+ };
+ unsigned i;
+
+ if (config_debug) {
+ for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
+ i++) {
+ assert(sizeof(buf) > strlen(enabled_states[i]));
+ }
+ }
+ assert(sizeof(enabled_states)/sizeof(const char *) ==
+ sizeof(thp_initially_huge_states)/sizeof(bool));
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open)
+ fd = (int)syscall(SYS_open,
+ "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+#else
+ fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY);
+#endif
+ if (fd == -1) {
+ goto label_error;
+ }
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read)
+ nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
+#else
+ nread = read(fd, &buf, sizeof(buf));
+#endif
+
+#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close)
+ syscall(SYS_close, fd);
+#else
+ close(fd);
+#endif
+
+ if (nread < 1) {
+ goto label_error;
+ }
+ for (i = 0; i < sizeof(enabled_states)/sizeof(const char *);
+ i++) {
+ if (strncmp(buf, enabled_states[i], (size_t)nread) == 0) {
+ thp_initially_huge = thp_initially_huge_states[i];
+ return;
+ }
+ }
+
+label_error:
+ thp_initially_huge = false;
+}
+
void
arena_boot(void)
{
unsigned i;
+ if (config_thp && opt_thp) {
+ init_thp_initially_huge();
+ }
+
arena_lg_dirty_mult_default_set(opt_lg_dirty_mult);
arena_decay_time_default_set(opt_decay_time);
@@ -3790,15 +3884,8 @@ arena_boot(void)
arena_maxrun = chunksize - (map_bias << LG_PAGE);
assert(arena_maxrun > 0);
large_maxclass = index2size(size2index(chunksize)-1);
- if (large_maxclass > arena_maxrun) {
- /*
- * For small chunk sizes it's possible for there to be fewer
- * non-header pages available than are necessary to serve the
- * size classes just below chunksize.
- */
- large_maxclass = arena_maxrun;
- }
assert(large_maxclass > 0);
+ assert(large_maxclass + large_pad <= arena_maxrun);
nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS);
nhclasses = NSIZES - nlclasses - NBINS;
diff --git a/src/chunk.c b/src/chunk.c
index c1c514a..94f28f2 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -141,7 +141,7 @@ chunk_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena,
}
bool
-chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node)
+chunk_register(const void *chunk, const extent_node_t *node, bool *gdump)
{
assert(extent_node_addr_get(node) == chunk);
@@ -160,8 +160,7 @@ chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node)
*/
high = atomic_read_z(&highchunks);
}
- if (cur > high && prof_gdump_get_unlocked())
- prof_gdump(tsdn);
+ *gdump = (cur > high && prof_gdump_get_unlocked());
}
return (false);
@@ -189,12 +188,17 @@ chunk_deregister(const void *chunk, const extent_node_t *node)
static extent_node_t *
chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szsnad, size_t size)
{
+ extent_node_t *node;
+ size_t qsize;
extent_node_t key;
assert(size == CHUNK_CEILING(size));
- extent_node_init(&key, arena, NULL, size, 0, false, false);
- return (extent_tree_szsnad_nsearch(chunks_szsnad, &key));
+ qsize = extent_size_quantize_ceil(size);
+ extent_node_init(&key, arena, NULL, qsize, 0, false, false);
+ node = extent_tree_szsnad_nsearch(chunks_szsnad, &key);
+ assert(node == NULL || extent_node_size_get(node) >= size);
+ return node;
}
static void *
diff --git a/src/chunk_dss.c b/src/chunk_dss.c
index ee3f838..8c67939 100644
--- a/src/chunk_dss.c
+++ b/src/chunk_dss.c
@@ -115,8 +115,9 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
* malloc.
*/
while (true) {
- void *ret, *cpad, *max_cur, *dss_next, *dss_prev;
- size_t gap_size, cpad_size;
+ void *ret, *max_cur, *dss_next, *dss_prev;
+ void *gap_addr_chunk, *gap_addr_subchunk;
+ size_t gap_size_chunk, gap_size_subchunk;
intptr_t incr;
max_cur = chunk_dss_max_update(new_addr);
@@ -124,25 +125,32 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
goto label_oom;
/*
- * Calculate how much padding is necessary to
- * chunk-align the end of the DSS.
- */
- gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) &
- chunksize_mask;
- /*
- * Compute how much chunk-aligned pad space (if any) is
+ * Compute how much chunk-aligned gap space (if any) is
* necessary to satisfy alignment. This space can be
* recycled for later use.
*/
- cpad = (void *)((uintptr_t)dss_max + gap_size);
- ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max,
- alignment);
- cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
+ gap_addr_chunk = (void *)(CHUNK_CEILING(
+ (uintptr_t)max_cur));
+ ret = (void *)ALIGNMENT_CEILING(
+ (uintptr_t)gap_addr_chunk, alignment);
+ gap_size_chunk = (uintptr_t)ret -
+ (uintptr_t)gap_addr_chunk;
+ /*
+ * Compute the address just past the end of the desired
+ * allocation space.
+ */
dss_next = (void *)((uintptr_t)ret + size);
- if ((uintptr_t)ret < (uintptr_t)dss_max ||
- (uintptr_t)dss_next < (uintptr_t)dss_max)
+ if ((uintptr_t)ret < (uintptr_t)max_cur ||
+ (uintptr_t)dss_next < (uintptr_t)max_cur)
goto label_oom; /* Wrap-around. */
- incr = gap_size + cpad_size + size;
+ /* Compute the increment, including subchunk bytes. */
+ gap_addr_subchunk = max_cur;
+ gap_size_subchunk = (uintptr_t)ret -
+ (uintptr_t)gap_addr_subchunk;
+ incr = gap_size_subchunk + size;
+
+ assert((uintptr_t)max_cur + incr == (uintptr_t)ret +
+ size);
/*
* Optimistically update dss_max, and roll back below if
@@ -157,11 +165,12 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
dss_prev = chunk_dss_sbrk(incr);
if (dss_prev == max_cur) {
/* Success. */
- if (cpad_size != 0) {
+ if (gap_size_chunk != 0) {
chunk_hooks_t chunk_hooks =
CHUNK_HOOKS_INITIALIZER;
chunk_dalloc_wrapper(tsdn, arena,
- &chunk_hooks, cpad, cpad_size,
+ &chunk_hooks, gap_addr_chunk,
+ gap_size_chunk,
arena_extent_sn_next(arena), false,
true);
}
diff --git a/src/ctl.c b/src/ctl.c
index bc78b20..56bc4f4 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -84,6 +84,7 @@ CTL_PROTO(config_prof_libgcc)
CTL_PROTO(config_prof_libunwind)
CTL_PROTO(config_stats)
CTL_PROTO(config_tcache)
+CTL_PROTO(config_thp)
CTL_PROTO(config_tls)
CTL_PROTO(config_utrace)
CTL_PROTO(config_valgrind)
@@ -104,6 +105,7 @@ CTL_PROTO(opt_utrace)
CTL_PROTO(opt_xmalloc)
CTL_PROTO(opt_tcache)
CTL_PROTO(opt_lg_tcache_max)
+CTL_PROTO(opt_thp)
CTL_PROTO(opt_prof)
CTL_PROTO(opt_prof_prefix)
CTL_PROTO(opt_prof_active)
@@ -258,6 +260,7 @@ static const ctl_named_node_t config_node[] = {
{NAME("prof_libunwind"), CTL(config_prof_libunwind)},
{NAME("stats"), CTL(config_stats)},
{NAME("tcache"), CTL(config_tcache)},
+ {NAME("thp"), CTL(config_thp)},
{NAME("tls"), CTL(config_tls)},
{NAME("utrace"), CTL(config_utrace)},
{NAME("valgrind"), CTL(config_valgrind)},
@@ -281,6 +284,7 @@ static const ctl_named_node_t opt_node[] = {
{NAME("xmalloc"), CTL(opt_xmalloc)},
{NAME("tcache"), CTL(opt_tcache)},
{NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)},
+ {NAME("thp"), CTL(opt_thp)},
{NAME("prof"), CTL(opt_prof)},
{NAME("prof_prefix"), CTL(opt_prof_prefix)},
{NAME("prof_active"), CTL(opt_prof_active)},
@@ -1268,6 +1272,7 @@ CTL_RO_CONFIG_GEN(config_prof_libgcc, bool)
CTL_RO_CONFIG_GEN(config_prof_libunwind, bool)
CTL_RO_CONFIG_GEN(config_stats, bool)
CTL_RO_CONFIG_GEN(config_tcache, bool)
+CTL_RO_CONFIG_GEN(config_thp, bool)
CTL_RO_CONFIG_GEN(config_tls, bool)
CTL_RO_CONFIG_GEN(config_utrace, bool)
CTL_RO_CONFIG_GEN(config_valgrind, bool)
@@ -1291,6 +1296,7 @@ CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool)
CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool)
CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool)
CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t)
+CTL_RO_NL_CGEN(config_thp, opt_thp, opt_thp, bool)
CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool)
CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *)
CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool)
@@ -1476,7 +1482,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
if (!config_tcache)
return (ENOENT);
- malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT;
@@ -1486,8 +1491,7 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
ret = 0;
label_return:
- malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
- return (ret);
+ return ret;
}
static int
diff --git a/src/extent.c b/src/extent.c
index 218156c..ff8de2f 100644
--- a/src/extent.c
+++ b/src/extent.c
@@ -3,13 +3,11 @@
/******************************************************************************/
-/*
- * Round down to the nearest chunk size that can actually be requested during
- * normal huge allocation.
- */
-JEMALLOC_INLINE_C size_t
-extent_quantize(size_t size)
-{
+#ifndef JEMALLOC_JET
+static
+#endif
+size_t
+extent_size_quantize_floor(size_t size) {
size_t ret;
szind_t ind;
@@ -25,11 +23,32 @@ extent_quantize(size_t size)
return (ret);
}
+size_t
+extent_size_quantize_ceil(size_t size) {
+ size_t ret;
+
+ assert(size > 0);
+
+ ret = extent_size_quantize_floor(size);
+ if (ret < size) {
+ /*
+ * Skip a quantization that may have an adequately large extent,
+ * because under-sized extents may be mixed in. This only
+ * happens when an unusual size is requested, i.e. for aligned
+ * allocation, and is just one of several places where linear
+ * search would potentially find sufficiently aligned available
+ * memory somewhere lower.
+ */
+ ret = index2size(size2index(ret + 1));
+ }
+ return ret;
+}
+
JEMALLOC_INLINE_C int
extent_sz_comp(const extent_node_t *a, const extent_node_t *b)
{
- size_t a_qsize = extent_quantize(extent_node_size_get(a));
- size_t b_qsize = extent_quantize(extent_node_size_get(b));
+ size_t a_qsize = extent_size_quantize_floor(extent_node_size_get(a));
+ size_t b_qsize = extent_size_quantize_floor(extent_node_size_get(b));
return ((a_qsize > b_qsize) - (a_qsize < b_qsize));
}
diff --git a/src/huge.c b/src/huge.c
index 8abd8c0..0fbaa41 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -15,20 +15,20 @@ huge_node_get(const void *ptr)
}
static bool
-huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
+huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump)
{
assert(extent_node_addr_get(node) == ptr);
assert(!extent_node_achunk_get(node));
- return (chunk_register(tsdn, ptr, node));
+ return (chunk_register(ptr, node, gdump));
}
static void
-huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node)
+huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump)
{
bool err;
- err = huge_node_set(tsdn, ptr, node);
+ err = huge_node_set(tsdn, ptr, node, gdump);
assert(!err);
}
@@ -57,11 +57,13 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
arena_t *iarena;
extent_node_t *node;
size_t sn;
- bool is_zeroed;
+ bool is_zeroed, gdump;
/* Allocate one or more contiguous chunks for this request. */
assert(!tsdn_null(tsdn) || arena != NULL);
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
ausize = sa2u(usize, alignment);
if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
@@ -91,11 +93,13 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
extent_node_init(node, arena, ret, usize, sn, is_zeroed, true);
- if (huge_node_set(tsdn, ret, node)) {
+ if (huge_node_set(tsdn, ret, node, &gdump)) {
arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn);
idalloctm(tsdn, node, NULL, true, true);
return (NULL);
}
+ if (config_prof && opt_prof && gdump)
+ prof_gdump(tsdn);
/* Insert node into huge. */
malloc_mutex_lock(tsdn, &arena->huge_mtx);
@@ -144,7 +148,10 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
extent_node_t *node;
arena_t *arena;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
- bool pre_zeroed, post_zeroed;
+ bool pre_zeroed, post_zeroed, gdump;
+
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
/* Increase usize to incorporate extra. */
for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
@@ -178,10 +185,13 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
huge_node_unset(ptr, node);
assert(extent_node_size_get(node) != usize);
extent_node_size_set(node, usize);
- huge_node_reset(tsdn, ptr, node);
+ huge_node_reset(tsdn, ptr, node, &gdump);
/* Update zeroed. */
extent_node_zeroed_set(node, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ /* gdump without any locks held. */
+ if (config_prof && opt_prof && gdump)
+ prof_gdump(tsdn);
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
@@ -207,7 +217,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
arena_t *arena;
chunk_hooks_t chunk_hooks;
size_t cdiff;
- bool pre_zeroed, post_zeroed;
+ bool pre_zeroed, post_zeroed, gdump;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
@@ -215,6 +225,8 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
chunk_hooks = chunk_hooks_get(tsdn, arena);
assert(oldsize > usize);
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
/* Split excess chunks. */
cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
@@ -241,10 +253,13 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
/* Update the size of the huge allocation. */
huge_node_unset(ptr, node);
extent_node_size_set(node, usize);
- huge_node_reset(tsdn, ptr, node);
+ huge_node_reset(tsdn, ptr, node, &gdump);
/* Update zeroed. */
extent_node_zeroed_set(node, post_zeroed);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ /* gdump without any locks held. */
+ if (config_prof && opt_prof && gdump)
+ prof_gdump(tsdn);
/* Zap the excess chunks. */
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize,
@@ -258,7 +273,7 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
size_t usize, bool zero) {
extent_node_t *node;
arena_t *arena;
- bool is_zeroed_subchunk, is_zeroed_chunk;
+ bool is_zeroed_subchunk, is_zeroed_chunk, gdump;
node = huge_node_get(ptr);
arena = extent_node_arena_get(node);
@@ -266,6 +281,9 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
is_zeroed_subchunk = extent_node_zeroed_get(node);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ /* prof_gdump() requirement. */
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0);
+
/*
* Use is_zeroed_chunk to detect whether the trailing memory is zeroed,
* update extent's zeroed field, and zero as necessary.
@@ -280,8 +298,11 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
extent_node_size_set(node, usize);
extent_node_zeroed_set(node, extent_node_zeroed_get(node) &&
is_zeroed_chunk);
- huge_node_reset(tsdn, ptr, node);
+ huge_node_reset(tsdn, ptr, node, &gdump);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
+ /* gdump without any locks held. */
+ if (config_prof && opt_prof && gdump)
+ prof_gdump(tsdn);
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed_subchunk) {
diff --git a/src/jemalloc.c b/src/jemalloc.c
index baead66..f73a26c 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -790,18 +790,19 @@ stats_print_atexit(void)
* Begin initialization functions.
*/
-#ifndef JEMALLOC_HAVE_SECURE_GETENV
static char *
-secure_getenv(const char *name)
+jemalloc_secure_getenv(const char *name)
{
-
+#ifdef JEMALLOC_HAVE_SECURE_GETENV
+ return secure_getenv(name);
+#else
# ifdef JEMALLOC_HAVE_ISSETUGID
if (issetugid() != 0)
return (NULL);
# endif
return (getenv(name));
-}
#endif
+}
static unsigned
malloc_ncpus(void)
@@ -1018,7 +1019,7 @@ malloc_conf_init(void)
#endif
;
- if ((opts = secure_getenv(envname)) != NULL) {
+ if ((opts = jemalloc_secure_getenv(envname)) != NULL) {
/*
* Do nothing; opts is already initialized to
* the value of the MALLOC_CONF environment
@@ -1074,18 +1075,18 @@ malloc_conf_init(void)
k, klen, v, vlen); \
} else if (clip) { \
if (CONF_MIN_##check_min(um, \
- (min))) \
+ (t)(min))) \
o = (t)(min); \
else if (CONF_MAX_##check_max( \
- um, (max))) \
+ um, (t)(max))) \
o = (t)(max); \
else \
o = (t)um; \
} else { \
if (CONF_MIN_##check_min(um, \
- (min)) || \
+ (t)(min)) || \
CONF_MAX_##check_max(um, \
- (max))) { \
+ (t)(max))) { \
malloc_conf_error( \
"Out-of-range " \
"conf value", \
@@ -1135,16 +1136,18 @@ malloc_conf_init(void)
CONF_HANDLE_BOOL(opt_abort, "abort", true)
/*
- * Chunks always require at least one header page,
- * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and
- * possibly an additional page in the presence of
- * redzones. In order to simplify options processing,
- * use a conservative bound that accommodates all these
- * constraints.
+ * Chunks always require at least one header page, as
+ * many as 2^(LG_SIZE_CLASS_GROUP+1) data pages (plus an
+ * additional page in the presence of cache-oblivious
+ * large), and possibly an additional page in the
+ * presence of redzones. In order to simplify options
+ * processing, use a conservative bound that
+ * accommodates all these constraints.
*/
CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
- LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1),
- (sizeof(size_t) << 3) - 1, yes, yes, true)
+ LG_SIZE_CLASS_GROUP + 1 + ((config_cache_oblivious
+ || config_fill) ? 1 : 0), (sizeof(size_t) << 3) - 1,
+ yes, yes, true)
if (strncmp("dss", k, klen) == 0) {
int i;
bool match = false;
@@ -1269,6 +1272,9 @@ malloc_conf_init(void)
"lg_tcache_max", -1,
(sizeof(size_t) << 3) - 1)
}
+ if (config_thp) {
+ CONF_HANDLE_BOOL(opt_thp, "thp", true)
+ }
if (config_prof) {
CONF_HANDLE_BOOL(opt_prof, "prof", true)
CONF_HANDLE_CHAR_P(opt_prof_prefix,
@@ -2827,6 +2833,7 @@ _malloc_prefork(void)
witness_prefork(tsd);
/* Acquire all mutexes in a safe order. */
ctl_prefork(tsd_tsdn(tsd));
+ tcache_prefork(tsd_tsdn(tsd));
malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock);
prof_prefork0(tsd_tsdn(tsd));
for (i = 0; i < 3; i++) {
@@ -2886,6 +2893,7 @@ _malloc_postfork(void)
}
prof_postfork_parent(tsd_tsdn(tsd));
malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock);
+ tcache_postfork_parent(tsd_tsdn(tsd));
ctl_postfork_parent(tsd_tsdn(tsd));
}
@@ -2910,6 +2918,7 @@ jemalloc_postfork_child(void)
}
prof_postfork_child(tsd_tsdn(tsd));
malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock);
+ tcache_postfork_child(tsd_tsdn(tsd));
ctl_postfork_child(tsd_tsdn(tsd));
}
diff --git a/src/pages.c b/src/pages.c
index 5f0c966..7698e49 100644
--- a/src/pages.c
+++ b/src/pages.c
@@ -199,7 +199,7 @@ pages_huge(void *addr, size_t size)
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
-#ifdef JEMALLOC_THP
+#ifdef JEMALLOC_HAVE_MADVISE_HUGE
return (madvise(addr, size, MADV_HUGEPAGE) != 0);
#else
return (false);
@@ -213,7 +213,7 @@ pages_nohuge(void *addr, size_t size)
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
-#ifdef JEMALLOC_THP
+#ifdef JEMALLOC_HAVE_MADVISE_HUGE
return (madvise(addr, size, MADV_NOHUGEPAGE) != 0);
#else
return (false);
diff --git a/src/stats.c b/src/stats.c
index 1360f3b..b76afc5 100755..100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -39,7 +39,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
bool json, bool large, bool huge, unsigned i)
{
size_t page;
- bool config_tcache, in_gap, in_gap_prev;
+ bool in_gap, in_gap_prev;
unsigned nbins, j;
CTL_GET("arenas.page", &page, size_t);
@@ -49,7 +49,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\t\"bins\": [\n");
} else {
- CTL_GET("config.tcache", &config_tcache, bool);
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
"bins: size ind allocated nmalloc"
@@ -137,8 +136,16 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
availregs = nregs * curruns;
milli = (availregs != 0) ? (1000 * curregs) / availregs
: 1000;
- assert(milli <= 1000);
- if (milli < 10) {
+
+ if (milli > 1000) {
+ /*
+ * Race detected: the counters were read in
+ * separate mallctl calls and concurrent
+ * operations happened in between. In this case
+ * no meaningful utilization can be computed.
+ */
+ malloc_snprintf(util, sizeof(util), " race");
+ } else if (milli < 10) {
malloc_snprintf(util, sizeof(util),
"0.00%zu", milli);
} else if (milli < 100) {
@@ -147,8 +154,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque,
} else if (milli < 1000) {
malloc_snprintf(util, sizeof(util), "0.%zu",
milli);
- } else
+ } else {
+ assert(milli == 1000);
malloc_snprintf(util, sizeof(util), "1");
+ }
if (config_tcache) {
malloc_cprintf(write_cb, cbopaque,
@@ -536,7 +545,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
"\t\t\t\t\t\"allocated\": %zu\n", metadata_allocated);
malloc_cprintf(write_cb, cbopaque,
- "\t\t\t\t},\n");
+ "\t\t\t\t}%s\n", (bins || large || huge) ? "," : "");
} else {
malloc_cprintf(write_cb, cbopaque,
"metadata: mapped: %zu, allocated: %zu\n",
@@ -555,7 +564,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
static void
stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
- bool json, bool merged, bool unmerged)
+ bool json, bool more)
{
const char *cpv;
bool bv;
@@ -741,6 +750,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
OPT_WRITE_BOOL(xmalloc, ",")
OPT_WRITE_BOOL(tcache, ",")
OPT_WRITE_SSIZE_T(lg_tcache_max, ",")
+ OPT_WRITE_BOOL(thp, ",")
OPT_WRITE_BOOL(prof, ",")
OPT_WRITE_CHAR_P(prof_prefix, ",")
OPT_WRITE_BOOL_MUTABLE(prof_active, prof.active, ",")
@@ -838,9 +848,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"nbins\": %u,\n", nbins);
- CTL_GET("arenas.nhbins", &uv, unsigned);
- malloc_cprintf(write_cb, cbopaque,
- "\t\t\t\"nhbins\": %u,\n", uv);
+ if (config_tcache) {
+ CTL_GET("arenas.nhbins", &uv, unsigned);
+ malloc_cprintf(write_cb, cbopaque,
+ "\t\t\t\"nhbins\": %u,\n", uv);
+ }
malloc_cprintf(write_cb, cbopaque,
"\t\t\t\"bin\": [\n");
@@ -907,11 +919,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
"\t\t\t]\n");
malloc_cprintf(write_cb, cbopaque,
- "\t\t},\n");
+ "\t\t}%s\n", (config_prof || more) ? "," : "");
}
/* prof. */
- if (json) {
+ if (config_prof && json) {
malloc_cprintf(write_cb, cbopaque,
"\t\t\"prof\": {\n");
@@ -937,8 +949,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque,
"\t\t\t\"lg_sample\": %zd\n", ssv);
malloc_cprintf(write_cb, cbopaque,
- "\t\t}%s\n", (config_stats || merged || unmerged) ? "," :
- "");
+ "\t\t}%s\n", more ? "," : "");
}
}
@@ -1023,31 +1034,37 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque,
narenas, bins, large, huge);
if (json) {
malloc_cprintf(write_cb, cbopaque,
- "\t\t\t}%s\n", (ninitialized > 1) ?
- "," : "");
+ "\t\t\t}%s\n", unmerged ? "," :
+ "");
}
}
/* Unmerged stats. */
- for (i = j = 0; i < narenas; i++) {
- if (initialized[i]) {
- if (json) {
- j++;
- malloc_cprintf(write_cb,
- cbopaque,
- "\t\t\t\"%u\": {\n", i);
- } else {
- malloc_cprintf(write_cb,
- cbopaque, "\narenas[%u]:\n",
- i);
- }
- stats_arena_print(write_cb, cbopaque,
- json, i, bins, large, huge);
- if (json) {
- malloc_cprintf(write_cb,
- cbopaque,
- "\t\t\t}%s\n", (j <
- ninitialized) ? "," : "");
+ if (unmerged) {
+ for (i = j = 0; i < narenas; i++) {
+ if (initialized[i]) {
+ if (json) {
+ j++;
+ malloc_cprintf(write_cb,
+ cbopaque,
+ "\t\t\t\"%u\": {\n",
+ i);
+ } else {
+ malloc_cprintf(write_cb,
+ cbopaque,
+ "\narenas[%u]:\n",
+ i);
+ }
+ stats_arena_print(write_cb,
+ cbopaque, json, i, bins,
+ large, huge);
+ if (json) {
+ malloc_cprintf(write_cb,
+ cbopaque,
+ "\t\t\t}%s\n", (j <
+ ninitialized) ? ","
+ : "");
+ }
}
}
}
@@ -1069,8 +1086,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
size_t u64sz;
bool json = false;
bool general = true;
- bool merged = true;
- bool unmerged = true;
+ bool merged = config_stats;
+ bool unmerged = config_stats;
bool bins = true;
bool large = true;
bool huge = true;
@@ -1137,8 +1154,10 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
"___ Begin jemalloc statistics ___\n");
}
- if (general)
- stats_general_print(write_cb, cbopaque, json, merged, unmerged);
+ if (general) {
+ bool more = (merged || unmerged);
+ stats_general_print(write_cb, cbopaque, json, more);
+ }
if (config_stats) {
stats_print_helper(write_cb, cbopaque, json, merged, unmerged,
bins, large, huge);
diff --git a/src/tcache.c b/src/tcache.c
index 21540ff..e3b04be 100755..100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -21,6 +21,9 @@ static unsigned tcaches_past;
/* Head of singly linked list tracking available tcaches elements. */
static tcaches_t *tcaches_avail;
+/* Protects tcaches{,_past,_avail}. */
+static malloc_mutex_t tcaches_mtx;
+
/******************************************************************************/
size_t
@@ -444,29 +447,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
}
}
+static bool
+tcaches_create_prep(tsd_t *tsd) {
+ bool err;
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
+
+ if (tcaches == NULL) {
+ tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
+ (MALLOCX_TCACHE_MAX+1));
+ if (tcaches == NULL) {
+ err = true;
+ goto label_return;
+ }
+ }
+
+ if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
+ err = true;
+ goto label_return;
+ }
+
+ err = false;
+label_return:
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
+ return err;
+}
+
bool
-tcaches_create(tsd_t *tsd, unsigned *r_ind)
-{
+tcaches_create(tsd_t *tsd, unsigned *r_ind) {
+ bool err;
arena_t *arena;
tcache_t *tcache;
tcaches_t *elm;
- if (tcaches == NULL) {
- tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) *
- (MALLOCX_TCACHE_MAX+1));
- if (tcaches == NULL)
- return (true);
+ if (tcaches_create_prep(tsd)) {
+ err = true;
+ goto label_return;
}
- if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
- return (true);
arena = arena_ichoose(tsd, NULL);
- if (unlikely(arena == NULL))
- return (true);
+ if (unlikely(arena == NULL)) {
+ err = true;
+ goto label_return;
+ }
tcache = tcache_create(tsd_tsdn(tsd), arena);
- if (tcache == NULL)
- return (true);
+ if (tcache == NULL) {
+ err = true;
+ goto label_return;
+ }
+ malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
if (tcaches_avail != NULL) {
elm = tcaches_avail;
tcaches_avail = tcaches_avail->next;
@@ -478,41 +508,50 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind)
*r_ind = tcaches_past;
tcaches_past++;
}
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
- return (false);
+ err = false;
+label_return:
+ malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &tcaches_mtx);
+ return err;
}
static void
-tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm)
-{
+tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) {
+ malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
- if (elm->tcache == NULL)
+ if (elm->tcache == NULL) {
return;
+ }
tcache_destroy(tsd, elm->tcache);
elm->tcache = NULL;
}
void
-tcaches_flush(tsd_t *tsd, unsigned ind)
-{
-
+tcaches_flush(tsd_t *tsd, unsigned ind) {
+ malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
tcaches_elm_flush(tsd, &tcaches[ind]);
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
}
void
-tcaches_destroy(tsd_t *tsd, unsigned ind)
-{
- tcaches_t *elm = &tcaches[ind];
+tcaches_destroy(tsd_t *tsd, unsigned ind) {
+ tcaches_t *elm;
+
+ malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
+ elm = &tcaches[ind];
tcaches_elm_flush(tsd, elm);
elm->next = tcaches_avail;
tcaches_avail = elm;
+ malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
}
bool
-tcache_boot(tsdn_t *tsdn)
-{
+tcache_boot(tsdn_t *tsdn) {
unsigned i;
+ cassert(config_tcache);
+
/*
* If necessary, clamp opt_lg_tcache_max, now that large_maxclass is
* known.
@@ -524,6 +563,10 @@ tcache_boot(tsdn_t *tsdn)
else
tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
+ if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) {
+ return true;
+ }
+
nhbins = size2index(tcache_maxclass) + 1;
/* Initialize tcache_bin_info. */
@@ -553,3 +596,24 @@ tcache_boot(tsdn_t *tsdn)
return (false);
}
+
+void
+tcache_prefork(tsdn_t *tsdn) {
+ if (!config_prof && opt_tcache) {
+ malloc_mutex_prefork(tsdn, &tcaches_mtx);
+ }
+}
+
+void
+tcache_postfork_parent(tsdn_t *tsdn) {
+ if (!config_prof && opt_tcache) {
+ malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
+ }
+}
+
+void
+tcache_postfork_child(tsdn_t *tsdn) {
+ if (!config_prof && opt_tcache) {
+ malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
+ }
+}
diff --git a/src/util.c b/src/util.c
index dd8c236..dd8c236 100755..100644
--- a/src/util.c
+++ b/src/util.c
diff --git a/src/witness.c b/src/witness.c
index 23753f2..c3a65f7 100644
--- a/src/witness.c
+++ b/src/witness.c
@@ -71,15 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error =
#endif
#ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
#endif
void
-witness_lockless_error(const witness_list_t *witnesses)
-{
+witness_depth_error(const witness_list_t *witnesses,
+ witness_rank_t rank_inclusive, unsigned depth) {
witness_t *w;
- malloc_printf("<jemalloc>: Should not own any locks:");
+ malloc_printf("<jemalloc>: Should own %u lock%s of rank >= %u:", depth,
+ (depth != 1) ? "s" : "", rank_inclusive);
ql_foreach(w, witnesses, link) {
malloc_printf(" %s(%u)", w->name, w->rank);
}
@@ -87,10 +88,9 @@ witness_lockless_error(const witness_list_t *witnesses)
abort();
}
#ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
-witness_lockless_error_t *witness_lockless_error =
- JEMALLOC_N(n_witness_lockless_error);
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(witness_depth_error)
+witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
#endif
void
diff --git a/src/zone.c b/src/zone.c
index 0571920..6215133 100644
--- a/src/zone.c
+++ b/src/zone.c
@@ -3,6 +3,75 @@
# error "This source file is for zones on Darwin (OS X)."
#endif
+/* Definitions of the following structs in malloc/malloc.h might be too old
+ * for the built binary to run on newer versions of OSX. So use the newest
+ * possible version of those structs.
+ */
+typedef struct _malloc_zone_t {
+ void *reserved1;
+ void *reserved2;
+ size_t (*size)(struct _malloc_zone_t *, const void *);
+ void *(*malloc)(struct _malloc_zone_t *, size_t);
+ void *(*calloc)(struct _malloc_zone_t *, size_t, size_t);
+ void *(*valloc)(struct _malloc_zone_t *, size_t);
+ void (*free)(struct _malloc_zone_t *, void *);
+ void *(*realloc)(struct _malloc_zone_t *, void *, size_t);
+ void (*destroy)(struct _malloc_zone_t *);
+ const char *zone_name;
+ unsigned (*batch_malloc)(struct _malloc_zone_t *, size_t, void **, unsigned);
+ void (*batch_free)(struct _malloc_zone_t *, void **, unsigned);
+ struct malloc_introspection_t *introspect;
+ unsigned version;
+ void *(*memalign)(struct _malloc_zone_t *, size_t, size_t);
+ void (*free_definite_size)(struct _malloc_zone_t *, void *, size_t);
+ size_t (*pressure_relief)(struct _malloc_zone_t *, size_t);
+} malloc_zone_t;
+
+typedef struct {
+ vm_address_t address;
+ vm_size_t size;
+} vm_range_t;
+
+typedef struct malloc_statistics_t {
+ unsigned blocks_in_use;
+ size_t size_in_use;
+ size_t max_size_in_use;
+ size_t size_allocated;
+} malloc_statistics_t;
+
+typedef kern_return_t memory_reader_t(task_t, vm_address_t, vm_size_t, void **);
+
+typedef void vm_range_recorder_t(task_t, void *, unsigned type, vm_range_t *, unsigned);
+
+typedef struct malloc_introspection_t {
+ kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, memory_reader_t, vm_range_recorder_t);
+ size_t (*good_size)(malloc_zone_t *, size_t);
+ boolean_t (*check)(malloc_zone_t *);
+ void (*print)(malloc_zone_t *, boolean_t);
+ void (*log)(malloc_zone_t *, void *);
+ void (*force_lock)(malloc_zone_t *);
+ void (*force_unlock)(malloc_zone_t *);
+ void (*statistics)(malloc_zone_t *, malloc_statistics_t *);
+ boolean_t (*zone_locked)(malloc_zone_t *);
+ boolean_t (*enable_discharge_checking)(malloc_zone_t *);
+ boolean_t (*disable_discharge_checking)(malloc_zone_t *);
+ void (*discharge)(malloc_zone_t *, void *);
+#ifdef __BLOCKS__
+ void (*enumerate_discharged_pointers)(malloc_zone_t *, void (^)(void *, void *));
+#else
+ void *enumerate_unavailable_without_blocks;
+#endif
+ void (*reinit_lock)(malloc_zone_t *);
+} malloc_introspection_t;
+
+extern kern_return_t malloc_get_all_zones(task_t, memory_reader_t, vm_address_t **, unsigned *);
+
+extern malloc_zone_t *malloc_default_zone(void);
+
+extern void malloc_zone_register(malloc_zone_t *zone);
+
+extern void malloc_zone_unregister(malloc_zone_t *zone);
+
/*
* The malloc_default_purgeable_zone() function is only available on >= 10.6.
* We need to check whether it is present at runtime, thus the weak_import.
@@ -20,24 +89,35 @@ static struct malloc_introspection_t jemalloc_zone_introspect;
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
-static size_t zone_size(malloc_zone_t *zone, void *ptr);
+static size_t zone_size(malloc_zone_t *zone, const void *ptr);
static void *zone_malloc(malloc_zone_t *zone, size_t size);
static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
static void *zone_valloc(malloc_zone_t *zone, size_t size);
static void zone_free(malloc_zone_t *zone, void *ptr);
static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
-#if (JEMALLOC_ZONE_VERSION >= 5)
static void *zone_memalign(malloc_zone_t *zone, size_t alignment,
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 6)
size_t size);
static void zone_free_definite_size(malloc_zone_t *zone, void *ptr,
size_t size);
-#endif
-static void *zone_destroy(malloc_zone_t *zone);
+static void zone_destroy(malloc_zone_t *zone);
+static unsigned zone_batch_malloc(struct _malloc_zone_t *zone, size_t size,
+ void **results, unsigned num_requested);
+static void zone_batch_free(struct _malloc_zone_t *zone,
+ void **to_be_freed, unsigned num_to_be_freed);
+static size_t zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal);
static size_t zone_good_size(malloc_zone_t *zone, size_t size);
+static kern_return_t zone_enumerator(task_t task, void *data, unsigned type_mask,
+ vm_address_t zone_address, memory_reader_t reader,
+ vm_range_recorder_t recorder);
+static boolean_t zone_check(malloc_zone_t *zone);
+static void zone_print(malloc_zone_t *zone, boolean_t verbose);
+static void zone_log(malloc_zone_t *zone, void *address);
static void zone_force_lock(malloc_zone_t *zone);
static void zone_force_unlock(malloc_zone_t *zone);
+static void zone_statistics(malloc_zone_t *zone,
+ malloc_statistics_t *stats);
+static boolean_t zone_locked(malloc_zone_t *zone);
+static void zone_reinit_lock(malloc_zone_t *zone);
/******************************************************************************/
/*
@@ -45,7 +125,7 @@ static void zone_force_unlock(malloc_zone_t *zone);
*/
static size_t
-zone_size(malloc_zone_t *zone, void *ptr)
+zone_size(malloc_zone_t *zone, const void *ptr)
{
/*
@@ -106,7 +186,6 @@ zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
return (realloc(ptr, size));
}
-#if (JEMALLOC_ZONE_VERSION >= 5)
static void *
zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
{
@@ -116,9 +195,7 @@ zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
return (ret);
}
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 6)
static void
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
{
@@ -133,15 +210,46 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
free(ptr);
}
-#endif
-static void *
+static void
zone_destroy(malloc_zone_t *zone)
{
/* This function should never be called. */
not_reached();
- return (NULL);
+}
+
+static unsigned
+zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results,
+ unsigned num_requested)
+{
+ unsigned i;
+
+ for (i = 0; i < num_requested; i++) {
+ results[i] = je_malloc(size);
+ if (!results[i])
+ break;
+ }
+
+ return i;
+}
+
+static void
+zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed,
+ unsigned num_to_be_freed)
+{
+ unsigned i;
+
+ for (i = 0; i < num_to_be_freed; i++) {
+ zone_free(zone, to_be_freed[i]);
+ to_be_freed[i] = NULL;
+ }
+}
+
+static size_t
+zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal)
+{
+ return 0;
}
static size_t
@@ -153,6 +261,30 @@ zone_good_size(malloc_zone_t *zone, size_t size)
return (s2u(size));
}
+static kern_return_t
+zone_enumerator(task_t task, void *data, unsigned type_mask,
+ vm_address_t zone_address, memory_reader_t reader,
+ vm_range_recorder_t recorder)
+{
+ return KERN_SUCCESS;
+}
+
+static boolean_t
+zone_check(malloc_zone_t *zone)
+{
+ return true;
+}
+
+static void
+zone_print(malloc_zone_t *zone, boolean_t verbose)
+{
+}
+
+static void
+zone_log(malloc_zone_t *zone, void *address)
+{
+}
+
static void
zone_force_lock(malloc_zone_t *zone)
{
@@ -177,52 +309,68 @@ zone_force_unlock(malloc_zone_t *zone)
}
static void
+zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats)
+{
+ /* We make no effort to actually fill the values */
+ stats->blocks_in_use = 0;
+ stats->size_in_use = 0;
+ stats->max_size_in_use = 0;
+ stats->size_allocated = 0;
+}
+
+static boolean_t
+zone_locked(malloc_zone_t *zone)
+{
+ /* Pretend no lock is being held */
+ return false;
+}
+
+static void
+zone_reinit_lock(malloc_zone_t *zone)
+{
+ /* As of OSX 10.12, this function is only used when force_unlock would
+ * be used if the zone version were < 9. So just use force_unlock. */
+ zone_force_unlock(zone);
+}
+
+static void
zone_init(void)
{
- jemalloc_zone.size = (void *)zone_size;
- jemalloc_zone.malloc = (void *)zone_malloc;
- jemalloc_zone.calloc = (void *)zone_calloc;
- jemalloc_zone.valloc = (void *)zone_valloc;
- jemalloc_zone.free = (void *)zone_free;
- jemalloc_zone.realloc = (void *)zone_realloc;
- jemalloc_zone.destroy = (void *)zone_destroy;
+ jemalloc_zone.size = zone_size;
+ jemalloc_zone.malloc = zone_malloc;
+ jemalloc_zone.calloc = zone_calloc;
+ jemalloc_zone.valloc = zone_valloc;
+ jemalloc_zone.free = zone_free;
+ jemalloc_zone.realloc = zone_realloc;
+ jemalloc_zone.destroy = zone_destroy;
jemalloc_zone.zone_name = "jemalloc_zone";
- jemalloc_zone.batch_malloc = NULL;
- jemalloc_zone.batch_free = NULL;
+ jemalloc_zone.batch_malloc = zone_batch_malloc;
+ jemalloc_zone.batch_free = zone_batch_free;
jemalloc_zone.introspect = &jemalloc_zone_introspect;
- jemalloc_zone.version = JEMALLOC_ZONE_VERSION;
-#if (JEMALLOC_ZONE_VERSION >= 5)
+ jemalloc_zone.version = 9;
jemalloc_zone.memalign = zone_memalign;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 6)
jemalloc_zone.free_definite_size = zone_free_definite_size;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 8)
- jemalloc_zone.pressure_relief = NULL;
-#endif
-
- jemalloc_zone_introspect.enumerator = NULL;
- jemalloc_zone_introspect.good_size = (void *)zone_good_size;
- jemalloc_zone_introspect.check = NULL;
- jemalloc_zone_introspect.print = NULL;
- jemalloc_zone_introspect.log = NULL;
- jemalloc_zone_introspect.force_lock = (void *)zone_force_lock;
- jemalloc_zone_introspect.force_unlock = (void *)zone_force_unlock;
- jemalloc_zone_introspect.statistics = NULL;
-#if (JEMALLOC_ZONE_VERSION >= 6)
- jemalloc_zone_introspect.zone_locked = NULL;
-#endif
-#if (JEMALLOC_ZONE_VERSION >= 7)
+ jemalloc_zone.pressure_relief = zone_pressure_relief;
+
+ jemalloc_zone_introspect.enumerator = zone_enumerator;
+ jemalloc_zone_introspect.good_size = zone_good_size;
+ jemalloc_zone_introspect.check = zone_check;
+ jemalloc_zone_introspect.print = zone_print;
+ jemalloc_zone_introspect.log = zone_log;
+ jemalloc_zone_introspect.force_lock = zone_force_lock;
+ jemalloc_zone_introspect.force_unlock = zone_force_unlock;
+ jemalloc_zone_introspect.statistics = zone_statistics;
+ jemalloc_zone_introspect.zone_locked = zone_locked;
jemalloc_zone_introspect.enable_discharge_checking = NULL;
jemalloc_zone_introspect.disable_discharge_checking = NULL;
jemalloc_zone_introspect.discharge = NULL;
-# ifdef __BLOCKS__
+#ifdef __BLOCKS__
jemalloc_zone_introspect.enumerate_discharged_pointers = NULL;
-# else
+#else
jemalloc_zone_introspect.enumerate_unavailable_without_blocks = NULL;
-# endif
#endif
+ jemalloc_zone_introspect.reinit_lock = zone_reinit_lock;
}
static malloc_zone_t *
generated by cgit v0.12 at 2025-09-06 17:04:06 (GMT)