From 01b3fe55ff3ac8e4aa689f09fcb0729da8037638 Mon Sep 17 00:00:00 2001
From: Jason Evans <jasone@canonware.com>
Date: Tue, 3 Apr 2012 09:28:00 -0700
Subject: Add a0malloc(), a0calloc(), and a0free().
Add a0malloc(), a0calloc(), and a0free(), which are used by FreeBSD's
libc to allocate/deallocate TLS in static binaries.
---
include/jemalloc/internal/arena.h | 36 +++++++++-------
include/jemalloc/internal/jemalloc_internal.h.in | 19 +++++----
include/jemalloc/internal/tcache.h | 2 +-
src/arena.c | 6 +--
src/ctl.c | 2 +-
src/jemalloc.c | 53 +++++++++++++++++++++++-
6 files changed, 89 insertions(+), 29 deletions(-)
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index 41df11f..03e3f3c 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -373,7 +373,7 @@ void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero);
+ size_t alignment, bool zero, bool try_tcache);
bool arena_new(arena_t *arena, unsigned ind);
void arena_boot(void);
void arena_prefork(arena_t *arena);
@@ -390,9 +390,10 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr);
prof_ctx_t *arena_prof_ctx_get(const void *ptr);
void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
-void *arena_malloc(size_t size, bool zero);
+void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache);
void *arena_malloc_prechosen(arena_t *arena, size_t size, bool zero);
-void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr);
+void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
+ bool try_tcache);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
@@ -548,7 +549,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
}
JEMALLOC_INLINE void *
-arena_malloc(size_t size, bool zero)
+arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache)
{
tcache_t *tcache;
@@ -556,20 +557,24 @@ arena_malloc(size_t size, bool zero)
assert(size <= arena_maxclass);
if (size <= SMALL_MAXCLASS) {
- if ((tcache = tcache_get(true)) != NULL)
+ if (try_tcache && (tcache = tcache_get(true)) != NULL)
return (tcache_alloc_small(tcache, size, zero));
- else
- return (arena_malloc_small(choose_arena(), size, zero));
+ else {
+ return (arena_malloc_small(choose_arena(arena), size,
+ zero));
+ }
} else {
/*
* Initialize tcache after checking size in order to avoid
* infinite recursion during tcache initialization.
*/
- if (size <= tcache_maxclass && (tcache = tcache_get(true)) !=
- NULL)
+ if (try_tcache && size <= tcache_maxclass && (tcache =
+ tcache_get(true)) != NULL)
return (tcache_alloc_large(tcache, size, zero));
- else
- return (arena_malloc_large(choose_arena(), size, zero));
+ else {
+ return (arena_malloc_large(choose_arena(arena), size,
+ zero));
+ }
}
}
@@ -587,11 +592,11 @@ arena_malloc_prechosen(arena_t *arena, size_t size, bool zero)
}
JEMALLOC_INLINE void
-arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
+arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
{
size_t pageind;
arena_chunk_map_t *mapelm;
- tcache_t *tcache = tcache_get(false);
+ tcache_t *tcache;
assert(arena != NULL);
assert(chunk->arena == arena);
@@ -603,7 +608,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
/* Small allocation. */
- if (tcache != NULL)
+ if (try_tcache && (tcache = tcache_get(false)) != NULL)
tcache_dalloc_small(tcache, ptr);
else {
arena_run_t *run;
@@ -630,7 +635,8 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
assert(((uintptr_t)ptr & PAGE_MASK) == 0);
- if (size <= tcache_maxclass && tcache != NULL) {
+ if (try_tcache && size <= tcache_maxclass && (tcache =
+ tcache_get(false)) != NULL) {
tcache_dalloc_large(tcache, ptr, size);
} else {
malloc_mutex_lock(&arena->lock);
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 0c22bfb..c8e4019 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -399,7 +399,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment, size_t *run_size_p);
-arena_t *choose_arena(void);
+arena_t *choose_arena(arena_t *arena);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
@@ -517,10 +517,13 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p)
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
-choose_arena(void)
+choose_arena(arena_t *arena)
{
arena_t *ret;
+ if (arena != NULL)
+ return (arena);
+
if ((ret = *arenas_tsd_get()) == NULL) {
ret = choose_arena_hard();
assert(ret != NULL);
@@ -556,7 +559,7 @@ imalloc(size_t size)
assert(size != 0);
if (size <= arena_maxclass)
- return (arena_malloc(size, false));
+ return (arena_malloc(NULL, size, false, true));
else
return (huge_malloc(size, false));
}
@@ -566,7 +569,7 @@ icalloc(size_t size)
{
if (size <= arena_maxclass)
- return (arena_malloc(size, true));
+ return (arena_malloc(NULL, size, true, true));
else
return (huge_malloc(size, true));
}
@@ -580,7 +583,7 @@ ipalloc(size_t usize, size_t alignment, bool zero)
assert(usize == sa2u(usize, alignment, NULL));
if (usize <= arena_maxclass && alignment <= PAGE)
- ret = arena_malloc(usize, zero);
+ ret = arena_malloc(NULL, usize, zero, true);
else {
size_t run_size JEMALLOC_CC_SILENCE_INIT(0);
@@ -594,7 +597,7 @@ ipalloc(size_t usize, size_t alignment, bool zero)
*/
sa2u(usize, alignment, &run_size);
if (run_size <= arena_maxclass) {
- ret = arena_palloc(choose_arena(), usize, run_size,
+ ret = arena_palloc(choose_arena(NULL), usize, run_size,
alignment, zero);
} else if (alignment <= chunksize)
ret = huge_malloc(usize, zero);
@@ -647,7 +650,7 @@ idalloc(void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
- arena_dalloc(chunk->arena, chunk, ptr);
+ arena_dalloc(chunk->arena, chunk, ptr, true);
else
huge_dalloc(ptr, true);
}
@@ -711,7 +714,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
} else {
if (size + extra <= arena_maxclass) {
return (arena_ralloc(ptr, oldsize, size, extra,
- alignment, zero));
+ alignment, zero, true));
} else {
return (huge_ralloc(ptr, oldsize, size, extra,
alignment, zero));
diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h
index a1d9aae..93e721d 100644
--- a/include/jemalloc/internal/tcache.h
+++ b/include/jemalloc/internal/tcache.h
@@ -237,7 +237,7 @@ tcache_get(bool create)
tcache_enabled_set(false); /* Memoize. */
return (NULL);
}
- return (tcache_create(choose_arena()));
+ return (tcache_create(choose_arena(NULL)));
}
if (tcache == TCACHE_STATE_PURGATORY) {
/*
diff --git a/src/arena.c b/src/arena.c
index b7e1422..6444099 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -1888,7 +1888,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
void *
arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero)
+ size_t alignment, bool zero, bool try_tcache)
{
void *ret;
size_t copysize;
@@ -1909,7 +1909,7 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
return (NULL);
ret = ipalloc(usize, alignment, zero);
} else
- ret = arena_malloc(size + extra, zero);
+ ret = arena_malloc(NULL, size + extra, zero, try_tcache);
if (ret == NULL) {
if (extra == 0)
@@ -1921,7 +1921,7 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
return (NULL);
ret = ipalloc(usize, alignment, zero);
} else
- ret = arena_malloc(size, zero);
+ ret = arena_malloc(NULL, size, zero, try_tcache);
if (ret == NULL)
return (NULL);
diff --git a/src/ctl.c b/src/ctl.c
index 2afca51..6777688 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -1016,7 +1016,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
int ret;
unsigned newind, oldind;
- newind = oldind = choose_arena()->ind;
+ newind = oldind = choose_arena(NULL)->ind;
WRITE(newind, unsigned);
READ(oldind, unsigned);
if (newind != oldind) {
diff --git a/src/jemalloc.c b/src/jemalloc.c
index a6d2df5..690cf08 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -1487,7 +1487,6 @@ je_nallocm(size_t *rsize, size_t size, int flags)
* End experimental functions.
*/
/******************************************************************************/
-
/*
* The following functions are used by threading libraries for protection of
* malloc during fork().
@@ -1552,3 +1551,55 @@ jemalloc_postfork_child(void)
}
/******************************************************************************/
+/*
+ * The following functions are used for TLS allocation/deallocation in static
+ * binaries on FreeBSD. The primary difference between these and i[mcd]alloc()
+ * is that these avoid accessing TLS variables.
+ */
+
+static void *
+a0alloc(size_t size, bool zero)
+{
+
+ if (malloc_init())
+ return (NULL);
+
+ if (size == 0)
+ size = 1;
+
+ if (size <= arena_maxclass)
+ return (arena_malloc(arenas[0], size, zero, false));
+ else
+ return (huge_malloc(size, zero));
+}
+
+void *
+a0malloc(size_t size)
+{
+
+ return (a0alloc(size, false));
+}
+
+void *
+a0calloc(size_t num, size_t size)
+{
+
+ return (a0alloc(num * size, true));
+}
+
+void
+a0free(void *ptr)
+{
+ arena_chunk_t *chunk;
+
+ if (ptr == NULL)
+ return;
+
+ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+ if (chunk != ptr)
+ arena_dalloc(chunk->arena, chunk, ptr, false);
+ else
+ huge_dalloc(ptr, true);
+}
+
+/******************************************************************************/
--
cgit v0.12