diff options
| -rw-r--r-- | jemalloc/configure.ac | 13 | ||||
| -rw-r--r-- | jemalloc/include/jemalloc/jemalloc_defs.h.in | 3 | ||||
| -rw-r--r-- | jemalloc/src/arena.c | 9 | ||||
| -rw-r--r-- | jemalloc/src/prof.c | 94 |
4 files changed, 100 insertions, 19 deletions
diff --git a/jemalloc/configure.ac b/jemalloc/configure.ac index 412d3d1..26e1c41 100644 --- a/jemalloc/configure.ac +++ b/jemalloc/configure.ac @@ -29,7 +29,7 @@ dnl JE_COMPILABLE(label, hcode, mcode, rvar) AC_DEFUN([JE_COMPILABLE], [ AC_MSG_CHECKING([whether $1 is compilable]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [$2], [$3])], AC_MSG_RESULT([yes]) [$4="yes"], @@ -454,6 +454,17 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then if test "x${enable_prof_libunwind}" = "x1" ; then backtrace_method="libunwind" AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ]) + JE_COMPILABLE([libunwind frame cache], [ +#define UNW_LOCAL_ONLY +#include <libunwind.h> +], [ +unw_tdep_make_frame_cache(0, 0); +unw_tdep_free_frame_cache(0); +unw_tdep_trace(0, 0, 0, 0); +], [libunwind_cache]) + if test "x${libunwind_cache}" = "xyes" ; then + AC_DEFINE([JEMALLOC_PROF_LIBUNWIND_CACHE], [ ]) + fi fi fi diff --git a/jemalloc/include/jemalloc/jemalloc_defs.h.in b/jemalloc/include/jemalloc/jemalloc_defs.h.in index d8c81d7..88e435d 100644 --- a/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -62,6 +62,9 @@ /* Use libunwind for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBUNWIND +/* Use libunwind's backtrace caching for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBUNWIND_CACHE + /* Use libgcc for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBGCC diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c index 1954da9..9aaf47f 100644 --- a/jemalloc/src/arena.c +++ b/jemalloc/src/arena.c @@ -868,9 +868,10 @@ arena_purge(arena_t *arena, bool all) } assert(ndirty == arena->ndirty); #endif - assert(arena->ndirty > arena->npurgatory); + assert(arena->ndirty > arena->npurgatory || all); assert(arena->ndirty > chunk_npages || all); - assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); + assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - + npurgatory) || all); #ifdef JEMALLOC_STATS arena->stats.npurge++; @@ -882,8 +883,10 @@ arena_purge(arena_t *arena, bool all) * multiple threads from racing to reduce ndirty below the threshold. */ npurgatory = arena->ndirty - arena->npurgatory; - if (all == false) + if (all == false) { + assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); npurgatory -= arena->nactive >> opt_lg_dirty_mult; + } arena->npurgatory += npurgatory; while (npurgatory > 0) { diff --git a/jemalloc/src/prof.c b/jemalloc/src/prof.c index 8370042..537c5c1 100644 --- a/jemalloc/src/prof.c +++ b/jemalloc/src/prof.c @@ -44,6 +44,12 @@ pthread_key_t prof_tdata_tsd; static ckh_t bt2ctx; static malloc_mutex_t bt2ctx_mtx; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static __thread unw_tdep_cache_t *libunwind_cache_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +static pthread_key_t libunwind_cache_tsd; +#endif + static malloc_mutex_t prof_dump_seq_mtx; static uint64_t prof_dump_seq; static uint64_t prof_dump_iseq; @@ -95,6 +101,9 @@ static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2); static bool prof_bt_keycomp(const void *k1, const void *k2); static void prof_tdata_cleanup(void *arg); +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static void libunwind_cache_thread_cleanup(void *arg); +#endif /******************************************************************************/ @@ -177,6 +186,11 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_cursor_t cursor; unsigned i; int err; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + unw_tdep_cache_t *cache; + int len = nignore + 1 + max; + void* vec[len]; +#endif assert(bt->len == 0); assert(bt->vec != NULL); @@ -185,24 +199,53 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_getcontext(&uc); unw_init_local(&cursor, &uc); - /* Throw away (nignore+1) stack frames, if that many exist. */ - for (i = 0; i < nignore + 1; i++) { - err = unw_step(&cursor); - if (err <= 0) - return; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + cache = libunwind_cache_tls; + if (cache == NULL) { + cache = unw_tdep_make_frame_cache(imalloc, idalloc); + if (cache != NULL) { + libunwind_cache_tls = cache; + pthread_setspecific(libunwind_cache_tsd, cache); + } } + if (cache != NULL && unw_tdep_trace(&cursor, vec, &len, cache) >= 0) { + /* + * The trace cache successfully looked up the backtrace. + * Discard the first (nignore+1) elements when copying the + * result, since there was no way to tell unw_tdep_trace() to + * skip those frames. + */ + assert(len >= nignore + 1); + len -= nignore + 1; + if (len > 0) { + memcpy(bt->vec, &vec[nignore + 1], sizeof(void *) * + len); + bt->len = len; + } + } else { +#endif + /* Throw away (nignore+1) stack frames, if that many exist. */ + for (i = 0; i < nignore + 1; i++) { + err = unw_step(&cursor); + if (err <= 0) + return; + } - /* - * Iterate over stack frames until there are no more, or until no space - * remains in bt. - */ - for (i = 0; i < max; i++) { - unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); - bt->len++; - err = unw_step(&cursor); - if (err <= 0) - break; + /* + * Iterate over stack frames until there are no more, or until + * no space remains in bt. + */ + for (i = 0; i < max; i++) { + unw_get_reg(&cursor, UNW_REG_IP, + (unw_word_t *)&bt->vec[i]); + bt->len++; + err = unw_step(&cursor); + if (err <= 0) + break; + } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE } +#endif } #endif #ifdef JEMALLOC_PROF_LIBGCC @@ -1156,6 +1199,19 @@ prof_tdata_cleanup(void *arg) } } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static void +libunwind_cache_thread_cleanup(void *arg) +{ + unw_tdep_cache_t *cache = libunwind_cache_tls; + + if (cache != NULL) { + unw_tdep_free_frame_cache(cache); + libunwind_cache_tls = NULL; + } +} +#endif + void prof_boot0(void) { @@ -1208,6 +1264,14 @@ prof_boot2(void) "<jemalloc>: Error in pthread_key_create()\n"); abort(); } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + if (pthread_key_create(&libunwind_cache_tsd, + libunwind_cache_thread_cleanup) != 0) { + malloc_write( + "<jemalloc>: Error in pthread_key_create()\n"); + abort(); + } +#endif prof_bt_max = (1U << opt_lg_prof_bt_max); if (malloc_mutex_init(&prof_dump_seq_mtx)) |