diff options
Diffstat (limited to 'include/jemalloc/internal/witness.h')
| -rw-r--r-- | include/jemalloc/internal/witness.h | 350 |
1 files changed, 196 insertions, 154 deletions
diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index 30d8c7e..33be666 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -1,29 +1,35 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES +#ifndef JEMALLOC_INTERNAL_WITNESS_H +#define JEMALLOC_INTERNAL_WITNESS_H -typedef struct witness_s witness_t; -typedef unsigned witness_rank_t; -typedef ql_head(witness_t) witness_list_t; -typedef int witness_comp_t (const witness_t *, const witness_t *); +#include "jemalloc/internal/ql.h" + +/******************************************************************************/ +/* LOCK RANKS */ +/******************************************************************************/ /* - * Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by - * the witness machinery. + * Witnesses with rank WITNESS_RANK_OMIT are completely ignored by the witness + * machinery. */ -#define WITNESS_RANK_OMIT 0U + +#define WITNESS_RANK_OMIT 0U #define WITNESS_RANK_MIN 1U -#define WITNESS_RANK_INIT 1U -#define WITNESS_RANK_CTL 1U +#define WITNESS_RANK_INIT 1U +#define WITNESS_RANK_CTL 1U #define WITNESS_RANK_TCACHES 2U -#define WITNESS_RANK_ARENAS 3U +#define WITNESS_RANK_ARENAS 3U + +#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U + +#define WITNESS_RANK_PROF_DUMP 5U +#define WITNESS_RANK_PROF_BT2GCTX 6U +#define WITNESS_RANK_PROF_TDATAS 7U +#define WITNESS_RANK_PROF_TDATA 8U +#define WITNESS_RANK_PROF_GCTX 9U -#define WITNESS_RANK_PROF_DUMP 4U -#define WITNESS_RANK_PROF_BT2GCTX 5U -#define WITNESS_RANK_PROF_TDATAS 6U -#define WITNESS_RANK_PROF_TDATA 7U -#define WITNESS_RANK_PROF_GCTX 8U +#define WITNESS_RANK_BACKGROUND_THREAD 10U /* * Used as an argument to witness_assert_depth_to_rank() in order to validate @@ -31,29 +37,44 @@ typedef int witness_comp_t (const witness_t *, const witness_t *); * witness_assert_depth_to_rank() is inclusive rather than exclusive, this * definition can have the same value as the minimally ranked core lock. */ -#define WITNESS_RANK_CORE 9U +#define WITNESS_RANK_CORE 11U + +#define WITNESS_RANK_DECAY 11U +#define WITNESS_RANK_TCACHE_QL 12U +#define WITNESS_RANK_EXTENT_GROW 13U +#define WITNESS_RANK_EXTENTS 14U +#define WITNESS_RANK_EXTENT_AVAIL 15U + +#define WITNESS_RANK_EXTENT_POOL 16U +#define WITNESS_RANK_RTREE 17U +#define WITNESS_RANK_BASE 18U +#define WITNESS_RANK_ARENA_LARGE 19U + +#define WITNESS_RANK_LEAF 0xffffffffU +#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF +#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF +#define WITNESS_RANK_DSS WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF -#define WITNESS_RANK_ARENA 9U -#define WITNESS_RANK_ARENA_CHUNKS 10U -#define WITNESS_RANK_ARENA_NODE_CACHE 11U - -#define WITNESS_RANK_BASE 12U - -#define WITNESS_RANK_LEAF 0xffffffffU -#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF -#define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF -#define WITNESS_RANK_DSS WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF - -#define WITNESS_INITIALIZER(rank) {"initializer", rank, NULL, {NULL, NULL}} - -#endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS +/* PER-WITNESS DATA */ +/******************************************************************************/ +#if defined(JEMALLOC_DEBUG) +# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}} +#else +# define WITNESS_INITIALIZER(name, rank) +#endif + +typedef struct witness_s witness_t; +typedef unsigned witness_rank_t; +typedef ql_head(witness_t) witness_list_t; +typedef int witness_comp_t (const witness_t *, void *, const witness_t *, + void *); struct witness_s { /* Name, used for printing lock order reversal messages. */ @@ -72,143 +93,159 @@ struct witness_s { */ witness_comp_t *comp; + /* Opaque data, passed to comp(). */ + void *opaque; + /* Linkage for thread's currently owned locks. */ ql_elm(witness_t) link; }; -#endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS +/* PER-THREAD DATA */ +/******************************************************************************/ +typedef struct witness_tsd_s witness_tsd_t; +struct witness_tsd_s { + witness_list_t witnesses; + bool forking; +}; + +#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false } +#define WITNESS_TSDN_NULL ((witness_tsdn_t *)0) + +/******************************************************************************/ +/* (PER-THREAD) NULLABILITY HELPERS */ +/******************************************************************************/ +typedef struct witness_tsdn_s witness_tsdn_t; +struct witness_tsdn_s { + witness_tsd_t witness_tsd; +}; + +JEMALLOC_ALWAYS_INLINE witness_tsdn_t * +witness_tsd_tsdn(witness_tsd_t *witness_tsd) { + return (witness_tsdn_t *)witness_tsd; +} + +JEMALLOC_ALWAYS_INLINE bool +witness_tsdn_null(witness_tsdn_t *witness_tsdn) { + return witness_tsdn == NULL; +} + +JEMALLOC_ALWAYS_INLINE witness_tsd_t * +witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) { + assert(!witness_tsdn_null(witness_tsdn)); + return &witness_tsdn->witness_tsd; +} + +/******************************************************************************/ +/* API */ +/******************************************************************************/ +void witness_init(witness_t *witness, const char *name, witness_rank_t rank, + witness_comp_t *comp, void *opaque); -void witness_init(witness_t *witness, const char *name, witness_rank_t rank, - witness_comp_t *comp); -#ifdef JEMALLOC_JET typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); -extern witness_lock_error_t *witness_lock_error; -#else -void witness_lock_error(const witness_list_t *witnesses, - const witness_t *witness); -#endif -#ifdef JEMALLOC_JET +extern witness_lock_error_t *JET_MUTABLE witness_lock_error; + typedef void (witness_owner_error_t)(const witness_t *); -extern witness_owner_error_t *witness_owner_error; -#else -void witness_owner_error(const witness_t *witness); -#endif -#ifdef JEMALLOC_JET +extern witness_owner_error_t *JET_MUTABLE witness_owner_error; + typedef void (witness_not_owner_error_t)(const witness_t *); -extern witness_not_owner_error_t *witness_not_owner_error; -#else -void witness_not_owner_error(const witness_t *witness); -#endif -#ifdef JEMALLOC_JET +extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error; + typedef void (witness_depth_error_t)(const witness_list_t *, witness_rank_t rank_inclusive, unsigned depth); -extern witness_depth_error_t *witness_depth_error; -#else -void witness_depth_error(const witness_list_t *witnesses, - witness_rank_t rank_inclusive, unsigned depth); -#endif +extern witness_depth_error_t *JET_MUTABLE witness_depth_error; -void witnesses_cleanup(tsd_t *tsd); -void witness_fork_cleanup(tsd_t *tsd); -void witness_prefork(tsd_t *tsd); -void witness_postfork_parent(tsd_t *tsd); -void witness_postfork_child(tsd_t *tsd); +void witnesses_cleanup(witness_tsd_t *witness_tsd); +void witness_prefork(witness_tsd_t *witness_tsd); +void witness_postfork_parent(witness_tsd_t *witness_tsd); +void witness_postfork_child(witness_tsd_t *witness_tsd); -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -bool witness_owner(tsd_t *tsd, const witness_t *witness); -void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, - unsigned depth); -void witness_assert_depth(tsdn_t *tsdn, unsigned depth); -void witness_assert_lockless(tsdn_t *tsdn); -void witness_lock(tsdn_t *tsdn, witness_t *witness); -void witness_unlock(tsdn_t *tsdn, witness_t *witness); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) -JEMALLOC_INLINE bool -witness_owner(tsd_t *tsd, const witness_t *witness) -{ +/* Helper, not intended for direct use. */ +static inline bool +witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) { witness_list_t *witnesses; witness_t *w; cassert(config_debug); - witnesses = tsd_witnessesp_get(tsd); + witnesses = &witness_tsd->witnesses; ql_foreach(w, witnesses, link) { - if (w == witness) - return (true); + if (w == witness) { + return true; + } } - return (false); + return false; } -JEMALLOC_INLINE void -witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) -{ - tsd_t *tsd; +static inline void +witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) { + witness_tsd_t *witness_tsd; - if (!config_debug) + if (!config_debug) { return; + } - if (tsdn_null(tsdn)) + if (witness_tsdn_null(witness_tsdn)) { return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { return; + } - if (witness_owner(tsd, witness)) + if (witness_owner(witness_tsd, witness)) { return; + } witness_owner_error(witness); } -JEMALLOC_INLINE void -witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) -{ - tsd_t *tsd; +static inline void +witness_assert_not_owner(witness_tsdn_t *witness_tsdn, + const witness_t *witness) { + witness_tsd_t *witness_tsd; witness_list_t *witnesses; witness_t *w; - if (!config_debug) + if (!config_debug) { return; + } - if (tsdn_null(tsdn)) + if (witness_tsdn_null(witness_tsdn)) { return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { return; + } - witnesses = tsd_witnessesp_get(tsd); + witnesses = &witness_tsd->witnesses; ql_foreach(w, witnesses, link) { - if (w == witness) + if (w == witness) { witness_not_owner_error(witness); + } } } -JEMALLOC_INLINE void -witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, - unsigned depth) { - tsd_t *tsd; +static inline void +witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, + witness_rank_t rank_inclusive, unsigned depth) { + witness_tsd_t *witness_tsd; unsigned d; witness_list_t *witnesses; witness_t *w; - if (!config_debug) + if (!config_debug) { return; + } - if (tsdn_null(tsdn)) + if (witness_tsdn_null(witness_tsdn)) { return; - tsd = tsdn_tsd(tsdn); + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); d = 0; - witnesses = tsd_witnessesp_get(tsd); + witnesses = &witness_tsd->witnesses; w = ql_last(witnesses, link); if (w != NULL) { ql_reverse_foreach(w, witnesses, link) { @@ -218,49 +255,53 @@ witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, d++; } } - if (d != depth) + if (d != depth) { witness_depth_error(witnesses, rank_inclusive, depth); + } } -JEMALLOC_INLINE void -witness_assert_depth(tsdn_t *tsdn, unsigned depth) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth); +static inline void +witness_assert_depth(witness_tsdn_t *witness_tsdn, unsigned depth) { + witness_assert_depth_to_rank(witness_tsdn, WITNESS_RANK_MIN, depth); } -JEMALLOC_INLINE void -witness_assert_lockless(tsdn_t *tsdn) { - witness_assert_depth(tsdn, 0); +static inline void +witness_assert_lockless(witness_tsdn_t *witness_tsdn) { + witness_assert_depth(witness_tsdn, 0); } -JEMALLOC_INLINE void -witness_lock(tsdn_t *tsdn, witness_t *witness) -{ - tsd_t *tsd; +static inline void +witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { + witness_tsd_t *witness_tsd; witness_list_t *witnesses; witness_t *w; - if (!config_debug) + if (!config_debug) { return; + } - if (tsdn_null(tsdn)) + if (witness_tsdn_null(witness_tsdn)) { return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { return; + } - witness_assert_not_owner(tsdn, witness); + witness_assert_not_owner(witness_tsdn, witness); - witnesses = tsd_witnessesp_get(tsd); + witnesses = &witness_tsd->witnesses; w = ql_last(witnesses, link); if (w == NULL) { /* No other locks; do nothing. */ - } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { + } else if (witness_tsd->forking && w->rank <= witness->rank) { /* Forking, and relaxed ranking satisfied. */ } else if (w->rank > witness->rank) { /* Not forking, rank order reversal. */ witness_lock_error(witnesses, witness); } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != - witness->comp || w->comp(w, witness) > 0)) { + witness->comp || w->comp(w, w->opaque, witness, witness->opaque) > + 0)) { /* * Missing/incompatible comparison function, or comparison * function indicates rank order reversal. @@ -272,33 +313,34 @@ witness_lock(tsdn_t *tsdn, witness_t *witness) ql_tail_insert(witnesses, witness, link); } -JEMALLOC_INLINE void -witness_unlock(tsdn_t *tsdn, witness_t *witness) -{ - tsd_t *tsd; +static inline void +witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) { + witness_tsd_t *witness_tsd; witness_list_t *witnesses; - if (!config_debug) + if (!config_debug) { return; + } - if (tsdn_null(tsdn)) + if (witness_tsdn_null(witness_tsdn)) { return; - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { return; + } /* * Check whether owner before removal, rather than relying on * witness_assert_owner() to abort, so that unit tests can test this * function's failure mode without causing undefined behavior. */ - if (witness_owner(tsd, witness)) { - witnesses = tsd_witnessesp_get(tsd); + if (witness_owner(witness_tsd, witness)) { + witnesses = &witness_tsd->witnesses; ql_remove(witnesses, witness, link); - } else - witness_assert_owner(tsdn, witness); + } else { + witness_assert_owner(witness_tsdn, witness); + } } -#endif -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ +#endif /* JEMALLOC_INTERNAL_WITNESS_H */ |