From 73d3d58dc234315214c0d73d6badd4fdc53cbeff Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 11 May 2016 15:33:28 -0700 Subject: Optimize witness fast path. Short-circuit commonly called witness functions so that they only execute in debug builds, and remove equivalent guards from mutex functions. This avoids pointless code execution in witness_assert_lockless(), which is typically called twice per allocation/deallocation function invocation. Inline commonly called witness functions so that optimized builds can completely remove calls as dead code. --- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/internal/mutex.h | 13 +- include/jemalloc/internal/witness.h | 152 ++++++++++++++++++++++- src/witness.c | 122 +----------------- 4 files changed, 157 insertions(+), 132 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 69d94ec..51bf897 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -531,9 +531,9 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/witness.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 00f0b91..5221799 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -81,8 +81,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { - if (config_debug) - witness_assert_not_owner(tsdn, &mutex->witness); + witness_assert_not_owner(tsdn, &mutex->witness); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 AcquireSRWLockExclusive(&mutex->lock); @@ -94,8 +93,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) #else pthread_mutex_lock(&mutex->lock); #endif - if (config_debug) - witness_lock(tsdn, &mutex->witness); + witness_lock(tsdn, &mutex->witness); } } @@ -104,8 +102,7 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { - if (config_debug) - witness_unlock(tsdn, &mutex->witness); + witness_unlock(tsdn, &mutex->witness); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 ReleaseSRWLockExclusive(&mutex->lock); @@ -124,7 +121,7 @@ JEMALLOC_INLINE void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - if (isthreaded && config_debug) + if (isthreaded) witness_assert_owner(tsdn, &mutex->witness); } @@ -132,7 +129,7 @@ JEMALLOC_INLINE void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - if (isthreaded && config_debug) + if (isthreaded) witness_assert_not_owner(tsdn, &mutex->witness); } #endif diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index 4d312ea..d78dca2 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -74,24 +74,28 @@ void witness_init(witness_t *witness, const char *name, witness_rank_t rank, #ifdef JEMALLOC_JET typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); extern witness_lock_error_t *witness_lock_error; +#else +void witness_lock_error(const witness_list_t *witnesses, + const witness_t *witness); #endif -void witness_lock(tsdn_t *tsdn, witness_t *witness); -void witness_unlock(tsdn_t *tsdn, witness_t *witness); #ifdef JEMALLOC_JET typedef void (witness_owner_error_t)(const witness_t *); extern witness_owner_error_t *witness_owner_error; +#else +void witness_owner_error(const witness_t *witness); #endif -void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); #ifdef JEMALLOC_JET typedef void (witness_not_owner_error_t)(const witness_t *); extern witness_not_owner_error_t *witness_not_owner_error; +#else +void witness_not_owner_error(const witness_t *witness); #endif -void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); #ifdef JEMALLOC_JET typedef void (witness_lockless_error_t)(const witness_list_t *); extern witness_lockless_error_t *witness_lockless_error; +#else +void witness_lockless_error(const witness_list_t *witnesses); #endif -void witness_assert_lockless(tsdn_t *tsdn); void witnesses_cleanup(tsd_t *tsd); void witness_fork_cleanup(tsd_t *tsd); @@ -103,5 +107,143 @@ void witness_postfork_child(tsd_t *tsd); /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +#ifndef JEMALLOC_ENABLE_INLINE +void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); +void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); +void witness_assert_lockless(tsdn_t *tsdn); +void witness_lock(tsdn_t *tsdn, witness_t *witness); +void witness_unlock(tsdn_t *tsdn, witness_t *witness); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) +JEMALLOC_INLINE void +witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) +{ + tsd_t *tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) + return; + + if (tsdn_null(tsdn)) + return; + tsd = tsdn_tsd(tsdn); + if (witness->rank == WITNESS_RANK_OMIT) + return; + + witnesses = tsd_witnessesp_get(tsd); + ql_foreach(w, witnesses, link) { + if (w == witness) + return; + } + witness_owner_error(witness); +} + +JEMALLOC_INLINE void +witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) +{ + tsd_t *tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) + return; + + if (tsdn_null(tsdn)) + return; + tsd = tsdn_tsd(tsdn); + if (witness->rank == WITNESS_RANK_OMIT) + return; + + witnesses = tsd_witnessesp_get(tsd); + ql_foreach(w, witnesses, link) { + if (w == witness) + witness_not_owner_error(witness); + } +} + +JEMALLOC_INLINE void +witness_assert_lockless(tsdn_t *tsdn) +{ + tsd_t *tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) + return; + + if (tsdn_null(tsdn)) + return; + tsd = tsdn_tsd(tsdn); + + witnesses = tsd_witnessesp_get(tsd); + w = ql_last(witnesses, link); + if (w != NULL) + witness_lockless_error(witnesses); +} + +JEMALLOC_INLINE void +witness_lock(tsdn_t *tsdn, witness_t *witness) +{ + tsd_t *tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) + return; + + if (tsdn_null(tsdn)) + return; + tsd = tsdn_tsd(tsdn); + if (witness->rank == WITNESS_RANK_OMIT) + return; + + witness_assert_not_owner(tsdn, witness); + + witnesses = tsd_witnessesp_get(tsd); + w = ql_last(witnesses, link); + if (w == NULL) { + /* No other locks; do nothing. */ + } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { + /* Forking, and relaxed ranking satisfied. */ + } else if (w->rank > witness->rank) { + /* Not forking, rank order reversal. */ + witness_lock_error(witnesses, witness); + } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != + witness->comp || w->comp(w, witness) > 0)) { + /* + * Missing/incompatible comparison function, or comparison + * function indicates rank order reversal. + */ + witness_lock_error(witnesses, witness); + } + + ql_elm_new(witness, link); + ql_tail_insert(witnesses, witness, link); +} + +JEMALLOC_INLINE void +witness_unlock(tsdn_t *tsdn, witness_t *witness) +{ + tsd_t *tsd; + witness_list_t *witnesses; + + if (!config_debug) + return; + + if (tsdn_null(tsdn)) + return; + tsd = tsdn_tsd(tsdn); + if (witness->rank == WITNESS_RANK_OMIT) + return; + + witness_assert_owner(tsdn, witness); + + witnesses = tsd_witnessesp_get(tsd); + ql_remove(witnesses, witness, link); +} +#endif + #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/src/witness.c b/src/witness.c index f5176b6..2431203 100644 --- a/src/witness.c +++ b/src/witness.c @@ -15,7 +15,7 @@ witness_init(witness_t *witness, const char *name, witness_rank_t rank, #undef witness_lock_error #define witness_lock_error JEMALLOC_N(witness_lock_error_impl) #endif -static void +void witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) { witness_t *w; @@ -33,66 +33,11 @@ witness_lock_error(const witness_list_t *witnesses, const witness_t *witness) witness_lock_error_t *witness_lock_error = JEMALLOC_N(witness_lock_error_impl); #endif -void -witness_lock(tsdn_t *tsdn, witness_t *witness) -{ - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (tsdn_null(tsdn)) - return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) - return; - - witness_assert_not_owner(tsdn, witness); - - witnesses = tsd_witnessesp_get(tsd); - w = ql_last(witnesses, link); - if (w == NULL) { - /* No other locks; do nothing. */ - } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { - /* Forking, and relaxed ranking satisfied. */ - } else if (w->rank > witness->rank) { - /* Not forking, rank order reversal. */ - witness_lock_error(witnesses, witness); - } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != - witness->comp || w->comp(w, witness) > 0)) { - /* - * Missing/incompatible comparison function, or comparison - * function indicates rank order reversal. - */ - witness_lock_error(witnesses, witness); - } - - ql_elm_new(witness, link); - ql_tail_insert(witnesses, witness, link); -} - -void -witness_unlock(tsdn_t *tsdn, witness_t *witness) -{ - tsd_t *tsd; - witness_list_t *witnesses; - - if (tsdn_null(tsdn)) - return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) - return; - - witness_assert_owner(tsdn, witness); - - witnesses = tsd_witnessesp_get(tsd); - ql_remove(witnesses, witness, link); -} - #ifdef JEMALLOC_JET #undef witness_owner_error #define witness_owner_error JEMALLOC_N(witness_owner_error_impl) #endif -static void +void witness_owner_error(const witness_t *witness) { @@ -107,32 +52,11 @@ witness_owner_error_t *witness_owner_error = JEMALLOC_N(witness_owner_error_impl); #endif -void -witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) -{ - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (tsdn_null(tsdn)) - return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) - return; - - witnesses = tsd_witnessesp_get(tsd); - ql_foreach(w, witnesses, link) { - if (w == witness) - return; - } - witness_owner_error(witness); -} - #ifdef JEMALLOC_JET #undef witness_not_owner_error #define witness_not_owner_error JEMALLOC_N(witness_not_owner_error_impl) #endif -static void +void witness_not_owner_error(const witness_t *witness) { @@ -147,31 +71,11 @@ witness_not_owner_error_t *witness_not_owner_error = JEMALLOC_N(witness_not_owner_error_impl); #endif -void -witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) -{ - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (tsdn_null(tsdn)) - return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) - return; - - witnesses = tsd_witnessesp_get(tsd); - ql_foreach(w, witnesses, link) { - if (w == witness) - witness_not_owner_error(witness); - } -} - #ifdef JEMALLOC_JET #undef witness_lockless_error #define witness_lockless_error JEMALLOC_N(witness_lockless_error_impl) #endif -static void +void witness_lockless_error(const witness_list_t *witnesses) { witness_t *w; @@ -191,24 +95,6 @@ witness_lockless_error_t *witness_lockless_error = #endif void -witness_assert_lockless(tsdn_t *tsdn) -{ - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (tsdn_null(tsdn)) - return; - tsd = tsdn_tsd(tsdn); - - witnesses = tsd_witnessesp_get(tsd); - w = ql_last(witnesses, link); - if (w != NULL) { - witness_lockless_error(witnesses); - } -} - -void witnesses_cleanup(tsd_t *tsd) { -- cgit v0.12