From adc675c8ef55b59bb2facf795a3c26411cfbf3ed Mon Sep 17 00:00:00 2001 From: "je@facebook.com" Date: Fri, 4 Jun 2010 17:36:05 -0700 Subject: Add support for libunwind backtrace caching. Use libunwind's unw_tdep_trace() if it is available. --- jemalloc/configure.ac | 13 +++- jemalloc/include/jemalloc/jemalloc_defs.h.in | 3 + jemalloc/src/prof.c | 94 +++++++++++++++++++++++----- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/jemalloc/configure.ac b/jemalloc/configure.ac index 412d3d1..26e1c41 100644 --- a/jemalloc/configure.ac +++ b/jemalloc/configure.ac @@ -29,7 +29,7 @@ dnl JE_COMPILABLE(label, hcode, mcode, rvar) AC_DEFUN([JE_COMPILABLE], [ AC_MSG_CHECKING([whether $1 is compilable]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [$2], [$3])], AC_MSG_RESULT([yes]) [$4="yes"], @@ -454,6 +454,17 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then if test "x${enable_prof_libunwind}" = "x1" ; then backtrace_method="libunwind" AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ]) + JE_COMPILABLE([libunwind frame cache], [ +#define UNW_LOCAL_ONLY +#include +], [ +unw_tdep_make_frame_cache(0, 0); +unw_tdep_free_frame_cache(0); +unw_tdep_trace(0, 0, 0, 0); +], [libunwind_cache]) + if test "x${libunwind_cache}" = "xyes" ; then + AC_DEFINE([JEMALLOC_PROF_LIBUNWIND_CACHE], [ ]) + fi fi fi diff --git a/jemalloc/include/jemalloc/jemalloc_defs.h.in b/jemalloc/include/jemalloc/jemalloc_defs.h.in index d8c81d7..88e435d 100644 --- a/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -62,6 +62,9 @@ /* Use libunwind for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBUNWIND +/* Use libunwind's backtrace caching for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBUNWIND_CACHE + /* Use libgcc for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBGCC diff --git a/jemalloc/src/prof.c b/jemalloc/src/prof.c index 8370042..537c5c1 100644 --- a/jemalloc/src/prof.c +++ b/jemalloc/src/prof.c @@ -44,6 +44,12 @@ pthread_key_t prof_tdata_tsd; static ckh_t bt2ctx; static malloc_mutex_t bt2ctx_mtx; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static __thread unw_tdep_cache_t *libunwind_cache_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +static pthread_key_t libunwind_cache_tsd; +#endif + static malloc_mutex_t prof_dump_seq_mtx; static uint64_t prof_dump_seq; static uint64_t prof_dump_iseq; @@ -95,6 +101,9 @@ static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2); static bool prof_bt_keycomp(const void *k1, const void *k2); static void prof_tdata_cleanup(void *arg); +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static void libunwind_cache_thread_cleanup(void *arg); +#endif /******************************************************************************/ @@ -177,6 +186,11 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_cursor_t cursor; unsigned i; int err; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + unw_tdep_cache_t *cache; + int len = nignore + 1 + max; + void* vec[len]; +#endif assert(bt->len == 0); assert(bt->vec != NULL); @@ -185,24 +199,53 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_getcontext(&uc); unw_init_local(&cursor, &uc); - /* Throw away (nignore+1) stack frames, if that many exist. */ - for (i = 0; i < nignore + 1; i++) { - err = unw_step(&cursor); - if (err <= 0) - return; +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + cache = libunwind_cache_tls; + if (cache == NULL) { + cache = unw_tdep_make_frame_cache(imalloc, idalloc); + if (cache != NULL) { + libunwind_cache_tls = cache; + pthread_setspecific(libunwind_cache_tsd, cache); + } } + if (cache != NULL && unw_tdep_trace(&cursor, vec, &len, cache) >= 0) { + /* + * The trace cache successfully looked up the backtrace. + * Discard the first (nignore+1) elements when copying the + * result, since there was no way to tell unw_tdep_trace() to + * skip those frames. + */ + assert(len >= nignore + 1); + len -= nignore + 1; + if (len > 0) { + memcpy(bt->vec, &vec[nignore + 1], sizeof(void *) * + len); + bt->len = len; + } + } else { +#endif + /* Throw away (nignore+1) stack frames, if that many exist. */ + for (i = 0; i < nignore + 1; i++) { + err = unw_step(&cursor); + if (err <= 0) + return; + } - /* - * Iterate over stack frames until there are no more, or until no space - * remains in bt. - */ - for (i = 0; i < max; i++) { - unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); - bt->len++; - err = unw_step(&cursor); - if (err <= 0) - break; + /* + * Iterate over stack frames until there are no more, or until + * no space remains in bt. + */ + for (i = 0; i < max; i++) { + unw_get_reg(&cursor, UNW_REG_IP, + (unw_word_t *)&bt->vec[i]); + bt->len++; + err = unw_step(&cursor); + if (err <= 0) + break; + } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE } +#endif } #endif #ifdef JEMALLOC_PROF_LIBGCC @@ -1156,6 +1199,19 @@ prof_tdata_cleanup(void *arg) } } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE +static void +libunwind_cache_thread_cleanup(void *arg) +{ + unw_tdep_cache_t *cache = libunwind_cache_tls; + + if (cache != NULL) { + unw_tdep_free_frame_cache(cache); + libunwind_cache_tls = NULL; + } +} +#endif + void prof_boot0(void) { @@ -1208,6 +1264,14 @@ prof_boot2(void) ": Error in pthread_key_create()\n"); abort(); } +#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + if (pthread_key_create(&libunwind_cache_tsd, + libunwind_cache_thread_cleanup) != 0) { + malloc_write( + ": Error in pthread_key_create()\n"); + abort(); + } +#endif prof_bt_max = (1U << opt_lg_prof_bt_max); if (malloc_mutex_init(&prof_dump_seq_mtx)) -- cgit v0.12 From af8ad3ec6abe9840bd503535a604a6173ce73515 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 23 Mar 2011 20:39:02 -0700 Subject: Fix an assertion in arena_purge(). arena_purge() may be called even when there are no dirty pages, so loosen an assertion accordingly. --- jemalloc/src/arena.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jemalloc/src/arena.c b/jemalloc/src/arena.c index 1954da9..9aaf47f 100644 --- a/jemalloc/src/arena.c +++ b/jemalloc/src/arena.c @@ -868,9 +868,10 @@ arena_purge(arena_t *arena, bool all) } assert(ndirty == arena->ndirty); #endif - assert(arena->ndirty > arena->npurgatory); + assert(arena->ndirty > arena->npurgatory || all); assert(arena->ndirty > chunk_npages || all); - assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); + assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - + npurgatory) || all); #ifdef JEMALLOC_STATS arena->stats.npurge++; @@ -882,8 +883,10 @@ arena_purge(arena_t *arena, bool all) * multiple threads from racing to reduce ndirty below the threshold. */ npurgatory = arena->ndirty - arena->npurgatory; - if (all == false) + if (all == false) { + assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); npurgatory -= arena->nactive >> opt_lg_dirty_mult; + } arena->npurgatory += npurgatory; while (npurgatory > 0) { -- cgit v0.12 From 9f949f9d82e63432b99000643fe983babd6a230b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 22 Mar 2011 20:44:40 -0700 Subject: Revert "Add support for libunwind backtrace caching." This reverts commit adc675c8ef55b59bb2facf795a3c26411cfbf3ed. The original commit added support for a non-standard libunwind API, so it was not of general utility. --- jemalloc/configure.ac | 13 +--- jemalloc/include/jemalloc/jemalloc_defs.h.in | 3 - jemalloc/src/prof.c | 94 +++++----------------------- 3 files changed, 16 insertions(+), 94 deletions(-) diff --git a/jemalloc/configure.ac b/jemalloc/configure.ac index 26e1c41..412d3d1 100644 --- a/jemalloc/configure.ac +++ b/jemalloc/configure.ac @@ -29,7 +29,7 @@ dnl JE_COMPILABLE(label, hcode, mcode, rvar) AC_DEFUN([JE_COMPILABLE], [ AC_MSG_CHECKING([whether $1 is compilable]) -AC_COMPILE_IFELSE([AC_LANG_PROGRAM( +AC_RUN_IFELSE([AC_LANG_PROGRAM( [$2], [$3])], AC_MSG_RESULT([yes]) [$4="yes"], @@ -454,17 +454,6 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then if test "x${enable_prof_libunwind}" = "x1" ; then backtrace_method="libunwind" AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ]) - JE_COMPILABLE([libunwind frame cache], [ -#define UNW_LOCAL_ONLY -#include -], [ -unw_tdep_make_frame_cache(0, 0); -unw_tdep_free_frame_cache(0); -unw_tdep_trace(0, 0, 0, 0); -], [libunwind_cache]) - if test "x${libunwind_cache}" = "xyes" ; then - AC_DEFINE([JEMALLOC_PROF_LIBUNWIND_CACHE], [ ]) - fi fi fi diff --git a/jemalloc/include/jemalloc/jemalloc_defs.h.in b/jemalloc/include/jemalloc/jemalloc_defs.h.in index 88e435d..d8c81d7 100644 --- a/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -62,9 +62,6 @@ /* Use libunwind for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBUNWIND -/* Use libunwind's backtrace caching for profile backtracing if defined. */ -#undef JEMALLOC_PROF_LIBUNWIND_CACHE - /* Use libgcc for profile backtracing if defined. */ #undef JEMALLOC_PROF_LIBGCC diff --git a/jemalloc/src/prof.c b/jemalloc/src/prof.c index 537c5c1..8370042 100644 --- a/jemalloc/src/prof.c +++ b/jemalloc/src/prof.c @@ -44,12 +44,6 @@ pthread_key_t prof_tdata_tsd; static ckh_t bt2ctx; static malloc_mutex_t bt2ctx_mtx; -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE -static __thread unw_tdep_cache_t *libunwind_cache_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -static pthread_key_t libunwind_cache_tsd; -#endif - static malloc_mutex_t prof_dump_seq_mtx; static uint64_t prof_dump_seq; static uint64_t prof_dump_iseq; @@ -101,9 +95,6 @@ static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2); static bool prof_bt_keycomp(const void *k1, const void *k2); static void prof_tdata_cleanup(void *arg); -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE -static void libunwind_cache_thread_cleanup(void *arg); -#endif /******************************************************************************/ @@ -186,11 +177,6 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_cursor_t cursor; unsigned i; int err; -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE - unw_tdep_cache_t *cache; - int len = nignore + 1 + max; - void* vec[len]; -#endif assert(bt->len == 0); assert(bt->vec != NULL); @@ -199,53 +185,24 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) unw_getcontext(&uc); unw_init_local(&cursor, &uc); -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE - cache = libunwind_cache_tls; - if (cache == NULL) { - cache = unw_tdep_make_frame_cache(imalloc, idalloc); - if (cache != NULL) { - libunwind_cache_tls = cache; - pthread_setspecific(libunwind_cache_tsd, cache); - } + /* Throw away (nignore+1) stack frames, if that many exist. */ + for (i = 0; i < nignore + 1; i++) { + err = unw_step(&cursor); + if (err <= 0) + return; } - if (cache != NULL && unw_tdep_trace(&cursor, vec, &len, cache) >= 0) { - /* - * The trace cache successfully looked up the backtrace. - * Discard the first (nignore+1) elements when copying the - * result, since there was no way to tell unw_tdep_trace() to - * skip those frames. - */ - assert(len >= nignore + 1); - len -= nignore + 1; - if (len > 0) { - memcpy(bt->vec, &vec[nignore + 1], sizeof(void *) * - len); - bt->len = len; - } - } else { -#endif - /* Throw away (nignore+1) stack frames, if that many exist. */ - for (i = 0; i < nignore + 1; i++) { - err = unw_step(&cursor); - if (err <= 0) - return; - } - /* - * Iterate over stack frames until there are no more, or until - * no space remains in bt. - */ - for (i = 0; i < max; i++) { - unw_get_reg(&cursor, UNW_REG_IP, - (unw_word_t *)&bt->vec[i]); - bt->len++; - err = unw_step(&cursor); - if (err <= 0) - break; - } -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE + /* + * Iterate over stack frames until there are no more, or until no space + * remains in bt. + */ + for (i = 0; i < max; i++) { + unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); + bt->len++; + err = unw_step(&cursor); + if (err <= 0) + break; } -#endif } #endif #ifdef JEMALLOC_PROF_LIBGCC @@ -1199,19 +1156,6 @@ prof_tdata_cleanup(void *arg) } } -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE -static void -libunwind_cache_thread_cleanup(void *arg) -{ - unw_tdep_cache_t *cache = libunwind_cache_tls; - - if (cache != NULL) { - unw_tdep_free_frame_cache(cache); - libunwind_cache_tls = NULL; - } -} -#endif - void prof_boot0(void) { @@ -1264,14 +1208,6 @@ prof_boot2(void) ": Error in pthread_key_create()\n"); abort(); } -#ifdef JEMALLOC_PROF_LIBUNWIND_CACHE - if (pthread_key_create(&libunwind_cache_tsd, - libunwind_cache_thread_cleanup) != 0) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); - } -#endif prof_bt_max = (1U << opt_lg_prof_bt_max); if (malloc_mutex_init(&prof_dump_seq_mtx)) -- cgit v0.12 From 3e292475ee336daa8c34498e461141808ca13bee Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 24 Mar 2011 16:48:11 -0700 Subject: Implement atomic operations for x86/x64. Add inline assembly implementations of atomic_{add,sub}_uint{32,64}() for x86/x64, in order to support compilers that are missing the relevant gcc intrinsics. --- jemalloc/include/jemalloc/internal/atomic.h | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/jemalloc/include/jemalloc/internal/atomic.h b/jemalloc/include/jemalloc/internal/atomic.h index 821c2ef..9a29862 100644 --- a/jemalloc/include/jemalloc/internal/atomic.h +++ b/jemalloc/include/jemalloc/internal/atomic.h @@ -40,6 +40,7 @@ uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) +/******************************************************************************/ /* 64-bit operations. */ #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 JEMALLOC_INLINE uint64_t @@ -69,12 +70,40 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); } +#elif (defined(__amd64_) || defined(__x86_64__)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + asm volatile ( + "lock; xaddq %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + x = (uint64_t)(-(int64_t)x); + asm volatile ( + "lock; xaddq %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} #else # if (LG_SIZEOF_PTR == 3) # error "Missing implementation for 64-bit atomic operations" # endif #endif +/******************************************************************************/ /* 32-bit operations. */ #ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 JEMALLOC_INLINE uint32_t @@ -104,6 +133,33 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); } +#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + asm volatile ( + "lock; xaddl %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + x = (uint32_t)(-(int32_t)x); + asm volatile ( + "lock; xaddl %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} #else # error "Missing implementation for 32-bit atomic operations" #endif -- cgit v0.12 From 7d9ebea57d1d82702261cb75a1b885a709b6bcc7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 30 Mar 2011 15:01:08 -0700 Subject: Update ChangeLog for 2.2.1. --- jemalloc/ChangeLog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/jemalloc/ChangeLog b/jemalloc/ChangeLog index 6db63db..7b262c9 100644 --- a/jemalloc/ChangeLog +++ b/jemalloc/ChangeLog @@ -6,6 +6,13 @@ found in the git revision history: http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git git://canonware.com/jemalloc.git +* 2.2.1 (March 30, 2011) + + Bug fixes: + - Implement atomic operations for x86/x64. This fixes compilation failures + for versions of gcc that are still in wide use. + - Fix an assertion in arena_purge(). + * 2.2.0 (March 22, 2011) This version incorporates several improvements to algorithms and data -- cgit v0.12