diff options
| -rw-r--r-- | include/jemalloc/internal/arena.h | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/base.h | 5 | ||||
| -rw-r--r-- | include/jemalloc/internal/chunk_dss.h | 9 | ||||
| -rw-r--r-- | include/jemalloc/internal/huge.h | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal.h.in | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/mutex.h | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/private_namespace.h | 18 | ||||
| -rw-r--r-- | src/arena.c | 30 | ||||
| -rw-r--r-- | src/base.c | 23 | ||||
| -rw-r--r-- | src/chunk_dss.c | 36 | ||||
| -rw-r--r-- | src/huge.c | 21 | ||||
| -rw-r--r-- | src/jemalloc.c | 48 | ||||
| -rw-r--r-- | src/mutex.c | 26 |
13 files changed, 192 insertions, 36 deletions
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 16c2b1e..1609adc 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -376,6 +376,9 @@ void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero); bool arena_new(arena_t *arena, unsigned ind); void arena_boot(void); +void arena_prefork(arena_t *arena); +void arena_postfork_parent(arena_t *arena); +void arena_postfork_child(arena_t *arena); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index e353f30..796a283 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -9,12 +9,13 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -extern malloc_mutex_t base_mtx; - void *base_alloc(size_t size); extent_node_t *base_node_alloc(void); void base_node_dealloc(extent_node_t *node); bool base_boot(void); +void base_prefork(void); +void base_postfork_parent(void); +void base_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h index 35cd461..a39a203 100644 --- a/include/jemalloc/internal/chunk_dss.h +++ b/include/jemalloc/internal/chunk_dss.h @@ -9,16 +9,13 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -/* - * Protects sbrk() calls. This avoids malloc races among threads, though it - * does not protect against races with threads that call sbrk() directly. - */ -extern malloc_mutex_t dss_mtx; - void *chunk_alloc_dss(size_t size, bool *zero); bool chunk_in_dss(void *chunk); bool chunk_dealloc_dss(void *chunk, size_t size); bool chunk_dss_boot(void); +void chunk_dss_prefork(void); +void chunk_dss_postfork_parent(void); +void chunk_dss_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 3a6b0b8..e8513c9 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -28,6 +28,9 @@ size_t huge_salloc(const void *ptr); prof_ctx_t *huge_prof_ctx_get(const void *ptr); void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); bool huge_boot(void); +void huge_prefork(void); +void huge_postfork_parent(void); +void huge_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 5759ed5..800d72c 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -410,7 +410,8 @@ thread_allocated_t *thread_allocated_get_hard(void); arena_t *arenas_extend(unsigned ind); arena_t *choose_arena_hard(void); void jemalloc_prefork(void); -void jemalloc_postfork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 6a7b4fc..9d13658 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -29,6 +29,9 @@ extern bool isthreaded; bool malloc_mutex_init(malloc_mutex_t *mutex); void malloc_mutex_destroy(malloc_mutex_t *mutex); +void malloc_mutex_prefork(malloc_mutex_t *mutex); +void malloc_mutex_postfork_parent(malloc_mutex_t *mutex); +void malloc_mutex_postfork_child(malloc_mutex_t *mutex); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h index 89d3b5c..e7370fe 100644 --- a/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h @@ -8,6 +8,9 @@ #define arena_malloc_small JEMALLOC_N(arena_malloc_small) #define arena_new JEMALLOC_N(arena_new) #define arena_palloc JEMALLOC_N(arena_palloc) +#define arena_postfork_child JEMALLOC_N(arena_postfork_child) +#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) +#define arena_prefork JEMALLOC_N(arena_prefork) #define arena_prof_accum JEMALLOC_N(arena_prof_accum) #define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) #define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) @@ -32,6 +35,9 @@ #define base_boot JEMALLOC_N(base_boot) #define base_node_alloc JEMALLOC_N(base_node_alloc) #define base_node_dealloc JEMALLOC_N(base_node_dealloc) +#define base_postfork_child JEMALLOC_N(base_postfork_child) +#define base_postfork_parent JEMALLOC_N(base_postfork_parent) +#define base_prefork JEMALLOC_N(base_prefork) #define bitmap_full JEMALLOC_N(bitmap_full) #define bitmap_get JEMALLOC_N(bitmap_get) #define bitmap_info_init JEMALLOC_N(bitmap_info_init) @@ -54,6 +60,9 @@ #define chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss) #define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) #define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) +#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child) +#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent) +#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) #define chunk_in_dss JEMALLOC_N(chunk_in_dss) #define chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot) #define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) @@ -115,6 +124,9 @@ #define huge_dalloc JEMALLOC_N(huge_dalloc) #define huge_malloc JEMALLOC_N(huge_malloc) #define huge_palloc JEMALLOC_N(huge_palloc) +#define huge_postfork_child JEMALLOC_N(huge_postfork_child) +#define huge_postfork_parent JEMALLOC_N(huge_postfork_parent) +#define huge_prefork JEMALLOC_N(huge_prefork) #define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) #define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) #define huge_ralloc JEMALLOC_N(huge_ralloc) @@ -129,12 +141,16 @@ #define isalloc JEMALLOC_N(isalloc) #define ivsalloc JEMALLOC_N(ivsalloc) #define jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init) -#define jemalloc_postfork JEMALLOC_N(jemalloc_postfork) +#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) +#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) #define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) #define malloc_cprintf JEMALLOC_N(malloc_cprintf) #define malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy) #define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) #define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) +#define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) +#define malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent) +#define malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork) #define malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock) #define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) #define malloc_printf JEMALLOC_N(malloc_printf) diff --git a/src/arena.c b/src/arena.c index c14cb2c..898f8c7 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2169,3 +2169,33 @@ arena_boot(void) bin_info_init(); } + +void +arena_prefork(arena_t *arena) +{ + unsigned i; + + malloc_mutex_prefork(&arena->lock); + for (i = 0; i < NBINS; i++) + malloc_mutex_prefork(&arena->bins[i].lock); +} + +void +arena_postfork_parent(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_parent(&arena->bins[i].lock); + malloc_mutex_postfork_parent(&arena->lock); +} + +void +arena_postfork_child(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_child(&arena->bins[i].lock); + malloc_mutex_postfork_child(&arena->lock); +} @@ -4,7 +4,7 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t base_mtx; +static malloc_mutex_t base_mtx; /* * Current pages that are being used for internal memory allocations. These @@ -104,3 +104,24 @@ base_boot(void) return (false); } + +void +base_prefork(void) +{ + + malloc_mutex_prefork(&base_mtx); +} + +void +base_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&base_mtx); +} + +void +base_postfork_child(void) +{ + + malloc_mutex_postfork_child(&base_mtx); +} diff --git a/src/chunk_dss.c b/src/chunk_dss.c index c25baea..405dc29 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -3,14 +3,18 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t dss_mtx; +/* + * Protects sbrk() calls. This avoids malloc races among threads, though it + * does not protect against races with threads that call sbrk() directly. + */ +static malloc_mutex_t dss_mtx; /* Base address of the DSS. */ -static void *dss_base; +static void *dss_base; /* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ -static void *dss_prev; +static void *dss_prev; /* Current upper limit on DSS addresses. */ -static void *dss_max; +static void *dss_max; /* * Trees of chunks that were previously allocated (trees differ only in node @@ -291,4 +295,28 @@ chunk_dss_boot(void) return (false); } +void +chunk_dss_prefork(void) +{ + + if (config_dss) + malloc_mutex_prefork(&dss_mtx); +} + +void +chunk_dss_postfork_parent(void) +{ + + if (config_dss) + malloc_mutex_postfork_parent(&dss_mtx); +} + +void +chunk_dss_postfork_child(void) +{ + + if (config_dss) + malloc_mutex_postfork_child(&dss_mtx); +} + /******************************************************************************/ @@ -359,3 +359,24 @@ huge_boot(void) return (false); } + +void +huge_prefork(void) +{ + + malloc_mutex_prefork(&huge_mtx); +} + +void +huge_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&huge_mtx); +} + +void +huge_postfork_child(void) +{ + + malloc_mutex_postfork_child(&huge_mtx); +} diff --git a/src/jemalloc.c b/src/jemalloc.c index 2f3f372..385eb03 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -610,8 +610,8 @@ malloc_init_hard(void) malloc_conf_init(); /* Register fork handlers. */ - if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, - jemalloc_postfork) != 0) { + if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, + jemalloc_postfork_child) != 0) { malloc_write("<jemalloc>: Error in pthread_atfork()\n"); if (opt_abort) abort(); @@ -1593,40 +1593,46 @@ jemalloc_prefork(void) unsigned i; /* Acquire all mutexes in a safe order. */ - - malloc_mutex_lock(&arenas_lock); + malloc_mutex_prefork(&arenas_lock); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) - malloc_mutex_lock(&arenas[i]->lock); + arena_prefork(arenas[i]); } - - malloc_mutex_lock(&base_mtx); - - malloc_mutex_lock(&huge_mtx); - - if (config_dss) - malloc_mutex_lock(&dss_mtx); + base_prefork(); + huge_prefork(); + chunk_dss_prefork(); } void -jemalloc_postfork(void) +jemalloc_postfork_parent(void) { unsigned i; /* Release all mutexes, now that fork() has completed. */ + chunk_dss_postfork_parent(); + huge_postfork_parent(); + base_postfork_parent(); + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + arena_postfork_parent(arenas[i]); + } + malloc_mutex_postfork_parent(&arenas_lock); +} - if (config_dss) - malloc_mutex_unlock(&dss_mtx); - - malloc_mutex_unlock(&huge_mtx); - - malloc_mutex_unlock(&base_mtx); +void +jemalloc_postfork_child(void) +{ + unsigned i; + /* Release all mutexes, now that fork() has completed. */ + chunk_dss_postfork_child(); + huge_postfork_child(); + base_postfork_child(); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) - malloc_mutex_unlock(&arenas[i]->lock); + arena_postfork_child(arenas[i]); } - malloc_mutex_unlock(&arenas_lock); + malloc_mutex_postfork_child(&arenas_lock); } /******************************************************************************/ diff --git a/src/mutex.c b/src/mutex.c index 0e09060..243b712 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -92,3 +92,29 @@ malloc_mutex_destroy(malloc_mutex_t *mutex) } #endif } + +void +malloc_mutex_prefork(malloc_mutex_t *mutex) +{ + + malloc_mutex_lock(mutex); +} + +void +malloc_mutex_postfork_parent(malloc_mutex_t *mutex) +{ + + malloc_mutex_unlock(mutex); +} + +void +malloc_mutex_postfork_child(malloc_mutex_t *mutex) +{ + + if (malloc_mutex_init(mutex)) { + malloc_printf("<jemalloc>: Error re-initializing mutex in " + "child\n"); + if (opt_abort) + abort(); + } +} |