diff options
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_includes.h | 4 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_inlines_c.h | 22 | ||||
| -rw-r--r-- | include/jemalloc/internal/mutex_inlines.h | 14 | ||||
| -rw-r--r-- | include/jemalloc/internal/mutex_pool_inlines.h | 1 | ||||
| -rw-r--r-- | include/jemalloc/internal/mutex_structs.h | 3 | ||||
| -rw-r--r-- | include/jemalloc/internal/tsd.h | 98 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness.h | 345 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness_externs.h | 25 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness_inlines.h | 188 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness_structs.h | 28 | ||||
| -rw-r--r-- | include/jemalloc/internal/witness_types.h | 70 | ||||
| -rw-r--r-- | src/arena.c | 18 | ||||
| -rw-r--r-- | src/extent.c | 54 | ||||
| -rw-r--r-- | src/jemalloc.c | 78 | ||||
| -rw-r--r-- | src/tcache.c | 4 | ||||
| -rw-r--r-- | src/tsd.c | 2 | ||||
| -rw-r--r-- | src/witness.c | 18 | ||||
| -rw-r--r-- | test/unit/witness.c | 172 |
18 files changed, 604 insertions, 540 deletions
diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h index 45e648b..2abc478 100644 --- a/include/jemalloc/internal/jemalloc_internal_includes.h +++ b/include/jemalloc/internal/jemalloc_internal_includes.h @@ -40,7 +40,6 @@ /* TYPES */ /******************************************************************************/ -#include "jemalloc/internal/witness_types.h" #include "jemalloc/internal/mutex_types.h" #include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_dss_types.h" @@ -54,7 +53,6 @@ /* STRUCTS */ /******************************************************************************/ -#include "jemalloc/internal/witness_structs.h" #include "jemalloc/internal/mutex_structs.h" #include "jemalloc/internal/mutex_pool_structs.h" #include "jemalloc/internal/arena_structs_a.h" @@ -72,7 +70,6 @@ /******************************************************************************/ #include "jemalloc/internal/jemalloc_internal_externs.h" -#include "jemalloc/internal/witness_externs.h" #include "jemalloc/internal/mutex_externs.h" #include "jemalloc/internal/extent_externs.h" #include "jemalloc/internal/extent_dss_externs.h" @@ -89,7 +86,6 @@ /* INLINES */ /******************************************************************************/ -#include "jemalloc/internal/witness_inlines.h" #include "jemalloc/internal/mutex_inlines.h" #include "jemalloc/internal/mutex_pool_inlines.h" #include "jemalloc/internal/jemalloc_internal_inlines_a.h" diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 415c503..80dfbef 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_INLINES_C_H #include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/witness.h" JEMALLOC_ALWAYS_INLINE arena_t * iaalloc(tsdn_t *tsdn, const void *ptr) { @@ -25,7 +26,8 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, assert(size != 0); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); if (config_stats && is_internal && likely(ret != NULL)) { @@ -49,7 +51,8 @@ ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, assert(usize == sa2u(usize, alignment)); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -82,7 +85,8 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, alloc_ctx_t *alloc_ctx, assert(ptr != NULL); assert(!is_internal || tcache == NULL); assert(!is_internal || arena_is_auto(iaalloc(tsdn, ptr))); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); if (config_stats && is_internal) { arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); } @@ -100,7 +104,8 @@ idalloc(tsd_t *tsd, void *ptr) { JEMALLOC_ALWAYS_INLINE void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, alloc_ctx_t *alloc_ctx, bool slow_path) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path); } @@ -108,7 +113,8 @@ JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; @@ -146,7 +152,8 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { assert(ptr != NULL); assert(size != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { @@ -174,7 +181,8 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero) { assert(ptr != NULL); assert(size != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h index babe8d3..b86a4ad 100644 --- a/include/jemalloc/internal/mutex_inlines.h +++ b/include/jemalloc/internal/mutex_inlines.h @@ -31,14 +31,14 @@ mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { /* Trylock: return false if the lock is successfully acquired. */ static inline bool malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { - witness_assert_not_owner(tsdn, &mutex->witness); + witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { return true; } mutex_owner_stats_update(tsdn, mutex); } - witness_lock(tsdn, &mutex->witness); + witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); return false; } @@ -69,19 +69,19 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { static inline void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { - witness_assert_not_owner(tsdn, &mutex->witness); + witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); } mutex_owner_stats_update(tsdn, mutex); } - witness_lock(tsdn, &mutex->witness); + witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); } static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { - witness_unlock(tsdn, &mutex->witness); + witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { MALLOC_MUTEX_UNLOCK(mutex); } @@ -89,12 +89,12 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - witness_assert_owner(tsdn, &mutex->witness); + witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); } static inline void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - witness_assert_not_owner(tsdn, &mutex->witness); + witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); } /* Copy the prof data from mutex for processing. */ diff --git a/include/jemalloc/internal/mutex_pool_inlines.h b/include/jemalloc/internal/mutex_pool_inlines.h index 0b667aa..bc257ea 100644 --- a/include/jemalloc/internal/mutex_pool_inlines.h +++ b/include/jemalloc/internal/mutex_pool_inlines.h @@ -4,6 +4,7 @@ #include "jemalloc/internal/hash.h" #include "jemalloc/internal/mutex_inlines.h" #include "jemalloc/internal/mutex_pool_structs.h" +#include "jemalloc/internal/witness.h" /* * This file really combines "inlines" and "externs", but only transitionally. diff --git a/include/jemalloc/internal/mutex_structs.h b/include/jemalloc/internal/mutex_structs.h index 92f4167..c1b6552 100644 --- a/include/jemalloc/internal/mutex_structs.h +++ b/include/jemalloc/internal/mutex_structs.h @@ -3,8 +3,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex_prof.h" -#include "jemalloc/internal/witness_types.h" -#include "jemalloc/internal/witness_structs.h" +#include "jemalloc/internal/witness.h" struct malloc_mutex_s { union { diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 1a26975..c192a6c 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -10,8 +10,7 @@ #include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/tcache_structs.h" #include "jemalloc/internal/util.h" -#include "jemalloc/internal/witness_types.h" -#include "jemalloc/internal/witness_structs.h" +#include "jemalloc/internal/witness.h" /* * Thread-Specific-Data layout @@ -52,30 +51,29 @@ typedef void (*test_callback_t)(int *); # define MALLOC_TSD_TEST_DATA_INIT 0x72b65c10 # define MALLOC_TEST_TSD \ - O(test_data, int) \ - O(test_callback, test_callback_t) + O(test_data, int, int) \ + O(test_callback, test_callback_t, int) # define MALLOC_TEST_TSD_INITIALIZER , MALLOC_TSD_TEST_DATA_INIT, NULL #else # define MALLOC_TEST_TSD # define MALLOC_TEST_TSD_INITIALIZER #endif +/* O(name, type, nullable type */ #define MALLOC_TSD \ -/* O(name, type) */ \ - O(tcache_enabled, bool) \ - O(arenas_tdata_bypass, bool) \ - O(reentrancy_level, int8_t) \ - O(narenas_tdata, uint32_t) \ - O(thread_allocated, uint64_t) \ - O(thread_deallocated, uint64_t) \ - O(prof_tdata, prof_tdata_t *) \ - O(rtree_ctx, rtree_ctx_t) \ - O(iarena, arena_t *) \ - O(arena, arena_t *) \ - O(arenas_tdata, arena_tdata_t *) \ - O(tcache, tcache_t) \ - O(witnesses, witness_list_t) \ - O(witness_fork, bool) \ + O(tcache_enabled, bool, bool) \ + O(arenas_tdata_bypass, bool, bool) \ + O(reentrancy_level, int8_t, int8_t) \ + O(narenas_tdata, uint32_t, uint32_t) \ + O(thread_allocated, uint64_t, uint64_t) \ + O(thread_deallocated, uint64_t, uint64_t) \ + O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ + O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ + O(iarena, arena_t *, arena_t *) \ + O(arena, arena_t *, arena_t *) \ + O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ + O(tcache, tcache_t, tcache_t) \ + O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD #define TSD_INITIALIZER { \ @@ -92,8 +90,7 @@ typedef void (*test_callback_t)(int *); NULL, \ NULL, \ TCACHE_ZERO_INITIALIZER, \ - ql_head_initializer(witnesses), \ - false \ + WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ } @@ -119,7 +116,7 @@ struct tsd_s { * setters below. */ tsd_state_t state; -#define O(n, t) \ +#define O(n, t, nt) \ t use_a_getter_or_setter_instead_##n; MALLOC_TSD #undef O @@ -135,6 +132,22 @@ struct tsdn_s { tsd_t tsd; }; #define TSDN_NULL ((tsdn_t *)0) +JEMALLOC_ALWAYS_INLINE tsdn_t * +tsd_tsdn(tsd_t *tsd) { + return (tsdn_t *)tsd; +} + +JEMALLOC_ALWAYS_INLINE bool +tsdn_null(const tsdn_t *tsdn) { + return tsdn == NULL; +} + +JEMALLOC_ALWAYS_INLINE tsd_t * +tsdn_tsd(tsdn_t *tsdn) { + assert(!tsdn_null(tsdn)); + + return &tsdn->tsd; +} void *malloc_tsd_malloc(size_t size); void malloc_tsd_dalloc(void *wrapper); @@ -166,7 +179,7 @@ void tsd_slow_update(tsd_t *tsd); * foo. This omits some safety checks, and so can be used during tsd * initialization and cleanup. */ -#define O(n, t) \ +#define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \ return &tsd->use_a_getter_or_setter_instead_##n; \ @@ -175,7 +188,7 @@ MALLOC_TSD #undef O /* tsd_foop_get(tsd) returns a pointer to the thread-local instance of foo. */ -#define O(n, t) \ +#define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ assert(tsd->state == tsd_state_nominal || \ @@ -186,8 +199,24 @@ tsd_##n##p_get(tsd_t *tsd) { \ MALLOC_TSD #undef O +/* + * tsdn_foop_get(tsdn) returns either the thread-local instance of foo (if tsdn + * isn't NULL), or NULL (if tsdn is NULL), cast to the nullable pointer type. + */ +#define O(n, t, nt) \ +JEMALLOC_ALWAYS_INLINE nt * \ +tsdn_##n##p_get(tsdn_t *tsdn) { \ + if (tsdn_null(tsdn)) { \ + return NULL; \ + } \ + tsd_t *tsd = tsdn_tsd(tsdn); \ + return (nt *)tsd_##n##p_get(tsd); \ +} +MALLOC_TSD +#undef O + /* tsd_foo_get(tsd) returns the value of the thread-local instance of foo. */ -#define O(n, t) \ +#define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t \ tsd_##n##_get(tsd_t *tsd) { \ return *tsd_##n##p_get(tsd); \ @@ -196,7 +225,7 @@ MALLOC_TSD #undef O /* tsd_foo_set(tsd, val) updates the thread-local instance of foo to be val. */ -#define O(n, t) \ +#define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t val) { \ *tsd_##n##p_get(tsd) = val; \ @@ -243,11 +272,6 @@ tsd_fetch(void) { return tsd_fetch_impl(true); } -JEMALLOC_ALWAYS_INLINE tsdn_t * -tsd_tsdn(tsd_t *tsd) { - return (tsdn_t *)tsd; -} - static inline bool tsd_nominal(tsd_t *tsd) { return (tsd->state <= tsd_state_nominal_max); @@ -262,18 +286,6 @@ tsdn_fetch(void) { return tsd_tsdn(tsd_fetch_impl(false)); } -JEMALLOC_ALWAYS_INLINE bool -tsdn_null(const tsdn_t *tsdn) { - return tsdn == NULL; -} - -JEMALLOC_ALWAYS_INLINE tsd_t * -tsdn_tsd(tsdn_t *tsdn) { - assert(!tsdn_null(tsdn)); - - return &tsdn->tsd; -} - JEMALLOC_ALWAYS_INLINE rtree_ctx_t * tsd_rtree_ctx(tsd_t *tsd) { return tsd_rtree_ctxp_get(tsd); diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h new file mode 100644 index 0000000..c71911f --- /dev/null +++ b/include/jemalloc/internal/witness.h @@ -0,0 +1,345 @@ +#ifndef JEMALLOC_INTERNAL_WITNESS_H +#define JEMALLOC_INTERNAL_WITNESS_H + +#include "jemalloc/internal/ql.h" + +/******************************************************************************/ +/* LOCK RANKS */ +/******************************************************************************/ + +/* + * Witnesses with rank WITNESS_RANK_OMIT are completely ignored by the witness + * machinery. + */ + +#define WITNESS_RANK_OMIT 0U + +#define WITNESS_RANK_MIN 1U + +#define WITNESS_RANK_INIT 1U +#define WITNESS_RANK_CTL 1U +#define WITNESS_RANK_TCACHES 2U +#define WITNESS_RANK_ARENAS 3U + +#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U + +#define WITNESS_RANK_PROF_DUMP 5U +#define WITNESS_RANK_PROF_BT2GCTX 6U +#define WITNESS_RANK_PROF_TDATAS 7U +#define WITNESS_RANK_PROF_TDATA 8U +#define WITNESS_RANK_PROF_GCTX 9U + +#define WITNESS_RANK_BACKGROUND_THREAD 10U + +/* + * Used as an argument to witness_assert_depth_to_rank() in order to validate + * depth excluding non-core locks with lower ranks. Since the rank argument to + * witness_assert_depth_to_rank() is inclusive rather than exclusive, this + * definition can have the same value as the minimally ranked core lock. + */ +#define WITNESS_RANK_CORE 11U + +#define WITNESS_RANK_DECAY 11U +#define WITNESS_RANK_TCACHE_QL 12U +#define WITNESS_RANK_EXTENTS 13U +#define WITNESS_RANK_EXTENT_FREELIST 14U + +#define WITNESS_RANK_EXTENT_POOL 15U +#define WITNESS_RANK_RTREE 16U +#define WITNESS_RANK_BASE 17U +#define WITNESS_RANK_ARENA_LARGE 18U + +#define WITNESS_RANK_LEAF 0xffffffffU +#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF +#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF +#define WITNESS_RANK_DSS WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF +#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF + +/******************************************************************************/ +/* PER-WITNESS DATA */ +/******************************************************************************/ +#if defined(JEMALLOC_DEBUG) +# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}} +#else +# define WITNESS_INITIALIZER(name, rank) +#endif + +typedef struct witness_s witness_t; +typedef unsigned witness_rank_t; +typedef ql_head(witness_t) witness_list_t; +typedef int witness_comp_t (const witness_t *, void *, const witness_t *, + void *); + +struct witness_s { + /* Name, used for printing lock order reversal messages. */ + const char *name; + + /* + * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses + * must be acquired in order of increasing rank. + */ + witness_rank_t rank; + + /* + * If two witnesses are of equal rank and they have the samp comp + * function pointer, it is called as a last attempt to differentiate + * between witnesses of equal rank. + */ + witness_comp_t *comp; + + /* Opaque data, passed to comp(). */ + void *opaque; + + /* Linkage for thread's currently owned locks. */ + ql_elm(witness_t) link; +}; + +/******************************************************************************/ +/* PER-THREAD DATA */ +/******************************************************************************/ +typedef struct witness_tsd_s witness_tsd_t; +struct witness_tsd_s { + witness_list_t witnesses; + bool forking; +}; + +#define WITNESS_TSD_INITIALIZER { ql_head_initializer(witnesses), false } +#define WITNESS_TSDN_NULL ((witness_tsdn_t *)0) + +/******************************************************************************/ +/* (PER-THREAD) NULLABILITY HELPERS */ +/******************************************************************************/ +typedef struct witness_tsdn_s witness_tsdn_t; +struct witness_tsdn_s { + witness_tsd_t witness_tsd; +}; + +JEMALLOC_ALWAYS_INLINE witness_tsdn_t * +witness_tsd_tsdn(witness_tsd_t *witness_tsd) { + return (witness_tsdn_t *)witness_tsd; +} + +JEMALLOC_ALWAYS_INLINE bool +witness_tsdn_null(witness_tsdn_t *witness_tsdn) { + return witness_tsdn == NULL; +} + +JEMALLOC_ALWAYS_INLINE witness_tsd_t * +witness_tsdn_tsd(witness_tsdn_t *witness_tsdn) { + assert(!witness_tsdn_null(witness_tsdn)); + return &witness_tsdn->witness_tsd; +} + +/******************************************************************************/ +/* API */ +/******************************************************************************/ +void witness_init(witness_t *witness, const char *name, witness_rank_t rank, + witness_comp_t *comp, void *opaque); + +typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); +extern witness_lock_error_t *JET_MUTABLE witness_lock_error; + +typedef void (witness_owner_error_t)(const witness_t *); +extern witness_owner_error_t *JET_MUTABLE witness_owner_error; + +typedef void (witness_not_owner_error_t)(const witness_t *); +extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error; + +typedef void (witness_depth_error_t)(const witness_list_t *, + witness_rank_t rank_inclusive, unsigned depth); +extern witness_depth_error_t *JET_MUTABLE witness_depth_error; + +void witnesses_cleanup(witness_tsd_t *witness_tsd); +void witness_prefork(witness_tsd_t *witness_tsd); +void witness_postfork_parent(witness_tsd_t *witness_tsd); +void witness_postfork_child(witness_tsd_t *witness_tsd); + +/* Helper, not intended for direct use. */ +static inline bool +witness_owner(witness_tsd_t *witness_tsd, const witness_t *witness) { + witness_list_t *witnesses; + witness_t *w; + + cassert(config_debug); + + witnesses = &witness_tsd->witnesses; + ql_foreach(w, witnesses, link) { + if (w == witness) { + return true; + } + } + + return false; +} + +static inline void +witness_assert_owner(witness_tsdn_t *witness_tsdn, const witness_t *witness) { + witness_tsd_t *witness_tsd; + + if (!config_debug) { + return; + } + + if (witness_tsdn_null(witness_tsdn)) { + return; + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { + return; + } + + if (witness_owner(witness_tsd, witness)) { + return; + } + witness_owner_error(witness); +} + +static inline void +witness_assert_not_owner(witness_tsdn_t *witness_tsdn, + const witness_t *witness) { + witness_tsd_t *witness_tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) { + return; + } + + if (witness_tsdn_null(witness_tsdn)) { + return; + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { + return; + } + + witnesses = &witness_tsd->witnesses; + ql_foreach(w, witnesses, link) { + if (w == witness) { + witness_not_owner_error(witness); + } + } +} + +static inline void +witness_assert_depth_to_rank(witness_tsdn_t *witness_tsdn, + witness_rank_t rank_inclusive, unsigned depth) { + witness_tsd_t *witness_tsd; + unsigned d; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) { + return; + } + + if (witness_tsdn_null(witness_tsdn)) { + return; + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + + d = 0; + witnesses = &witness_tsd->witnesses; + w = ql_last(witnesses, link); + if (w != NULL) { + ql_reverse_foreach(w, witnesses, link) { + if (w->rank < rank_inclusive) { + break; + } + d++; + } + } + if (d != depth) { + witness_depth_error(witnesses, rank_inclusive, depth); + } +} + +static inline void +witness_assert_depth(witness_tsdn_t *witness_tsdn, unsigned depth) { + witness_assert_depth_to_rank(witness_tsdn, WITNESS_RANK_MIN, depth); +} + +static inline void +witness_assert_lockless(witness_tsdn_t *witness_tsdn) { + witness_assert_depth(witness_tsdn, 0); +} + +static inline void +witness_lock(witness_tsdn_t *witness_tsdn, witness_t *witness) { + witness_tsd_t *witness_tsd; + witness_list_t *witnesses; + witness_t *w; + + if (!config_debug) { + return; + } + + if (witness_tsdn_null(witness_tsdn)) { + return; + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { + return; + } + + witness_assert_not_owner(witness_tsdn, witness); + + witnesses = &witness_tsd->witnesses; + w = ql_last(witnesses, link); + if (w == NULL) { + /* No other locks; do nothing. */ + } else if (witness_tsd->forking && w->rank <= witness->rank) { + /* Forking, and relaxed ranking satisfied. */ + } else if (w->rank > witness->rank) { + /* Not forking, rank order reversal. */ + witness_lock_error(witnesses, witness); + } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != + witness->comp || w->comp(w, w->opaque, witness, witness->opaque) > + 0)) { + /* + * Missing/incompatible comparison function, or comparison + * function indicates rank order reversal. + */ + witness_lock_error(witnesses, witness); + } + + ql_elm_new(witness, link); + ql_tail_insert(witnesses, witness, link); +} + +static inline void +witness_unlock(witness_tsdn_t *witness_tsdn, witness_t *witness) { + witness_tsd_t *witness_tsd; + witness_list_t *witnesses; + + if (!config_debug) { + return; + } + + if (witness_tsdn_null(witness_tsdn)) { + return; + } + witness_tsd = witness_tsdn_tsd(witness_tsdn); + if (witness->rank == WITNESS_RANK_OMIT) { + return; + } + + /* + * Check whether owner before removal, rather than relying on + * witness_assert_owner() to abort, so that unit tests can test this + * function's failure mode without causing undefined behavior. + */ + if (witness_owner(witness_tsd, witness)) { + witnesses = &witness_tsd->witnesses; + ql_remove(witnesses, witness, link); + } else { + witness_assert_owner(witness_tsdn, witness); + } +} + +#endif /* JEMALLOC_INTERNAL_WITNESS_H */ diff --git a/include/jemalloc/internal/witness_externs.h b/include/jemalloc/internal/witness_externs.h deleted file mode 100644 index 99df4c5..0000000 --- a/include/jemalloc/internal/witness_externs.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_WITNESS_EXTERNS_H -#define JEMALLOC_INTERNAL_WITNESS_EXTERNS_H - -void witness_init(witness_t *witness, const char *name, witness_rank_t rank, - witness_comp_t *comp, void *opaque); - -typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); -extern witness_lock_error_t *JET_MUTABLE witness_lock_error; - -typedef void (witness_owner_error_t)(const witness_t *); -extern witness_owner_error_t *JET_MUTABLE witness_owner_error; - -typedef void (witness_not_owner_error_t)(const witness_t *); -extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error; - -typedef void (witness_depth_error_t)(const witness_list_t *, - witness_rank_t rank_inclusive, unsigned depth); -extern witness_depth_error_t *JET_MUTABLE witness_depth_error; - -void witnesses_cleanup(tsd_t *tsd); -void witness_prefork(tsd_t *tsd); -void witness_postfork_parent(tsd_t *tsd); -void witness_postfork_child(tsd_t *tsd); - -#endif /* JEMALLOC_INTERNAL_WITNESS_EXTERNS_H */ diff --git a/include/jemalloc/internal/witness_inlines.h b/include/jemalloc/internal/witness_inlines.h deleted file mode 100644 index 51d1af3..0000000 --- a/include/jemalloc/internal/witness_inlines.h +++ /dev/null @@ -1,188 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_WITNESS_INLINES_H -#define JEMALLOC_INTERNAL_WITNESS_INLINES_H - -#include "jemalloc/internal/ql.h" - -/* Helper, not intended for direct use. */ -static inline bool -witness_owner(tsd_t *tsd, const witness_t *witness) { - witness_list_t *witnesses; - witness_t *w; - - cassert(config_debug); - - witnesses = tsd_witnessesp_get(tsd); - ql_foreach(w, witnesses, link) { - if (w == witness) { - return true; - } - } - - return false; -} - -static inline void -witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) { - tsd_t *tsd; - - if (!config_debug) { - return; - } - - if (tsdn_null(tsdn)) { - return; - } - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) { - return; - } - - if (witness_owner(tsd, witness)) { - return; - } - witness_owner_error(witness); -} - -static inline void -witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) { - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (!config_debug) { - return; - } - - if (tsdn_null(tsdn)) { - return; - } - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) { - return; - } - - witnesses = tsd_witnessesp_get(tsd); - ql_foreach(w, witnesses, link) { - if (w == witness) { - witness_not_owner_error(witness); - } - } -} - -static inline void -witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, - unsigned depth) { - tsd_t *tsd; - unsigned d; - witness_list_t *witnesses; - witness_t *w; - - if (!config_debug) { - return; - } - - if (tsdn_null(tsdn)) { - return; - } - tsd = tsdn_tsd(tsdn); - - d = 0; - witnesses = tsd_witnessesp_get(tsd); - w = ql_last(witnesses, link); - if (w != NULL) { - ql_reverse_foreach(w, witnesses, link) { - if (w->rank < rank_inclusive) { - break; - } - d++; - } - } - if (d != depth) { - witness_depth_error(witnesses, rank_inclusive, depth); - } -} - -static inline void -witness_assert_depth(tsdn_t *tsdn, unsigned depth) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth); -} - -static inline void -witness_assert_lockless(tsdn_t *tsdn) { - witness_assert_depth(tsdn, 0); -} - -static inline void -witness_lock(tsdn_t *tsdn, witness_t *witness) { - tsd_t *tsd; - witness_list_t *witnesses; - witness_t *w; - - if (!config_debug) { - return; - } - - if (tsdn_null(tsdn)) { - return; - } - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) { - return; - } - - witness_assert_not_owner(tsdn, witness); - - witnesses = tsd_witnessesp_get(tsd); - w = ql_last(witnesses, link); - if (w == NULL) { - /* No other locks; do nothing. */ - } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { - /* Forking, and relaxed ranking satisfied. */ - } else if (w->rank > witness->rank) { - /* Not forking, rank order reversal. */ - witness_lock_error(witnesses, witness); - } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != - witness->comp || w->comp(w, w->opaque, witness, witness->opaque) > - 0)) { - /* - * Missing/incompatible comparison function, or comparison - * function indicates rank order reversal. - */ - witness_lock_error(witnesses, witness); - } - - ql_elm_new(witness, link); - ql_tail_insert(witnesses, witness, link); -} - -static inline void -witness_unlock(tsdn_t *tsdn, witness_t *witness) { - tsd_t *tsd; - witness_list_t *witnesses; - - if (!config_debug) { - return; - } - - if (tsdn_null(tsdn)) { - return; - } - tsd = tsdn_tsd(tsdn); - if (witness->rank == WITNESS_RANK_OMIT) { - return; - } - - /* - * Check whether owner before removal, rather than relying on - * witness_assert_owner() to abort, so that unit tests can test this - * function's failure mode without causing undefined behavior. - */ - if (witness_owner(tsd, witness)) { - witnesses = tsd_witnessesp_get(tsd); - ql_remove(witnesses, witness, link); - } else { - witness_assert_owner(tsdn, witness); - } -} - -#endif /* JEMALLOC_INTERNAL_WITNESS_INLINES_H */ diff --git a/include/jemalloc/internal/witness_structs.h b/include/jemalloc/internal/witness_structs.h deleted file mode 100644 index 95d1970..0000000 --- a/include/jemalloc/internal/witness_structs.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_WITNESS_STRUCTS_H -#define JEMALLOC_INTERNAL_WITNESS_STRUCTS_H - -struct witness_s { - /* Name, used for printing lock order reversal messages. */ - const char *name; - - /* - * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses - * must be acquired in order of increasing rank. - */ - witness_rank_t rank; - - /* - * If two witnesses are of equal rank and they have the samp comp - * function pointer, it is called as a last attempt to differentiate - * between witnesses of equal rank. - */ - witness_comp_t *comp; - - /* Opaque data, passed to comp(). */ - void *opaque; - - /* Linkage for thread's currently owned locks. */ - ql_elm(witness_t) link; -}; - -#endif /* JEMALLOC_INTERNAL_WITNESS_STRUCTS_H */ diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h deleted file mode 100644 index 28ec7c8..0000000 --- a/include/jemalloc/internal/witness_types.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_WITNESS_TYPES_H -#define JEMALLOC_INTERNAL_WITNESS_TYPES_H - -#include "jemalloc/internal/ql.h" - -typedef struct witness_s witness_t; -typedef unsigned witness_rank_t; -typedef ql_head(witness_t) witness_list_t; -typedef int witness_comp_t (const witness_t *, void *, const witness_t *, - void *); - -/* - * Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by - * the witness machinery. - */ -#define WITNESS_RANK_OMIT 0U - -#define WITNESS_RANK_MIN 1U - -#define WITNESS_RANK_INIT 1U -#define WITNESS_RANK_CTL 1U -#define WITNESS_RANK_TCACHES 2U -#define WITNESS_RANK_ARENAS 3U - -#define WITNESS_RANK_BACKGROUND_THREAD_GLOBAL 4U - -#define WITNESS_RANK_PROF_DUMP 5U -#define WITNESS_RANK_PROF_BT2GCTX 6U -#define WITNESS_RANK_PROF_TDATAS 7U -#define WITNESS_RANK_PROF_TDATA 8U -#define WITNESS_RANK_PROF_GCTX 9U - -#define WITNESS_RANK_BACKGROUND_THREAD 10U - -/* - * Used as an argument to witness_assert_depth_to_rank() in order to validate - * depth excluding non-core locks with lower ranks. Since the rank argument to - * witness_assert_depth_to_rank() is inclusive rather than exclusive, this - * definition can have the same value as the minimally ranked core lock. - */ -#define WITNESS_RANK_CORE 11U - -#define WITNESS_RANK_DECAY 11U -#define WITNESS_RANK_TCACHE_QL 12U -#define WITNESS_RANK_EXTENTS 13U -#define WITNESS_RANK_EXTENT_FREELIST 14U - -#define WITNESS_RANK_EXTENT_POOL 15U -#define WITNESS_RANK_RTREE 16U -#define WITNESS_RANK_BASE 17U -#define WITNESS_RANK_ARENA_LARGE 18U - -#define WITNESS_RANK_LEAF 0xffffffffU -#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF -#define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF -#define WITNESS_RANK_DSS WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_ACCUM WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF -#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF - -#if defined(JEMALLOC_DEBUG) -# define WITNESS_INITIALIZER(name, rank) {name, rank, NULL, NULL, {NULL, NULL}} -#else -# define WITNESS_INITIALIZER(name, rank) -#endif - -#endif /* JEMALLOC_INTERNAL_WITNESS_TYPES_H */ diff --git a/src/arena.c b/src/arena.c index 48d536e..35b4e5a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -361,7 +361,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extents_dalloc(tsdn, arena, r_extent_hooks, &arena->extents_dirty, extent); @@ -497,7 +498,8 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); szind_t szind = size2index(usize); size_t mapped_add; @@ -892,7 +894,8 @@ static size_t arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit, extent_list_t *decay_extents) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); /* Stash extents according to npages_limit. */ size_t nstashed = 0; @@ -978,7 +981,8 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, extents_t *extents, bool all, size_t npages_limit) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 1); malloc_mutex_assert_owner(tsdn, &decay->mtx); if (decay->purging) { @@ -1253,7 +1257,8 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, extent_t *slab; bool zero, commit; - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); zero = false; commit = true; @@ -1271,7 +1276,8 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, static extent_t * arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, const arena_bin_info_t *bin_info) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; szind_t szind = size2index(bin_info->reg_size); diff --git a/src/extent.c b/src/extent.c index 6503f2a..ff09f7f 100644 --- a/src/extent.c +++ b/src/extent.c @@ -149,7 +149,8 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { extent_t * extent_alloc(tsdn_t *tsdn, arena_t *arena) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_t *extent = extent_avail_first(&arena->extent_avail); @@ -164,7 +165,8 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) { void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_avail_insert(&arena->extent_avail, extent); @@ -415,7 +417,8 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { assert(size + pad != 0); assert(alignment != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr, size, pad, alignment, slab, szind, zero, commit); @@ -426,7 +429,8 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_addr_set(extent, extent_base_get(extent)); extent_zeroed_set(extent, false); @@ -607,7 +611,8 @@ static void extent_gdump_add(tsdn_t *tsdn, const extent_t *extent) { cassert(config_prof); /* prof_gdump() requirement. */ - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); if (opt_prof && extent_state_get(extent) == extent_state_active) { size_t nadd = extent_size_get(extent) >> LG_PAGE; @@ -730,7 +735,8 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, bool locked, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, bool *zero, bool *commit) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, locked ? 1 : 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, locked ? 1 : 0); if (locked) { malloc_mutex_assert_owner(tsdn, &extents->mtx); } @@ -869,7 +875,8 @@ static extent_t * extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); assert(new_addr == NULL || !slab); assert(pad == 0 || !slab); assert(!*zero || !slab); @@ -1219,7 +1226,8 @@ extent_t * extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, bool *zero, bool *commit) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); @@ -1385,7 +1393,8 @@ void extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_hooks_t *extent_hooks = EXTENT_HOOKS_INITIALIZER; - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); if (extent_register(tsdn, extent)) { extents_leak(tsdn, arena, &extent_hooks, @@ -1418,7 +1427,8 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_addr_set(extent, extent_base_get(extent)); @@ -1445,7 +1455,8 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); /* * Deregister first to avoid a race with other allocating threads, and @@ -1508,7 +1519,8 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); /* Deregister first to avoid a race with other allocating threads. */ extent_deregister(tsdn, extent); @@ -1543,7 +1555,8 @@ bool extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); bool err = ((*r_extent_hooks)->commit == NULL || @@ -1566,7 +1579,8 @@ bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); @@ -1597,7 +1611,8 @@ bool extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); return ((*r_extent_hooks)->purge_lazy == NULL || @@ -1625,7 +1640,8 @@ bool extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); return ((*r_extent_hooks)->purge_forced == NULL || @@ -1649,7 +1665,8 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b) { assert(extent_size_get(extent) == size_a + size_b); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); @@ -1742,7 +1759,8 @@ extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); extent_hooks_assure_initialized(arena, r_extent_hooks); diff --git a/src/jemalloc.c b/src/jemalloc.c index b03e5f4..f083adc 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1755,7 +1755,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { */ reentrancy_level = tsd_reentrancy_level_get(tsd); if (reentrancy_level == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } if (sopts->slow && unlikely(reentrancy_level > 0)) { /* @@ -1832,7 +1832,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { /* Success! */ if (reentrancy_level == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } *dopts->result = allocation; return 0; @@ -1847,7 +1847,7 @@ label_oom: UTRACE(NULL, size, NULL); } - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); if (sopts->set_errno_on_error) { set_errno(ENOMEM); @@ -1878,7 +1878,7 @@ label_invalid_alignment: UTRACE(NULL, size, NULL); } - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); if (sopts->null_out_result_on_error) { *dopts->result = NULL; @@ -2080,7 +2080,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { tsd_assert_fast(tsd); } if (tsd_reentrancy_level_get(tsd) == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } else { assert(slow_path); } @@ -2120,7 +2120,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { tsd_assert_fast(tsd); } if (tsd_reentrancy_level_get(tsd) == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } else { assert(slow_path); } @@ -2181,7 +2181,7 @@ je_realloc(void *ptr, size_t size) { assert(malloc_initialized() || IS_INITIALIZER); tsd_t *tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); @@ -2224,7 +2224,7 @@ je_realloc(void *ptr, size_t size) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, ret); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return ret; } @@ -2234,7 +2234,8 @@ je_free(void *ptr) { if (likely(ptr != NULL)) { tsd_t *tsd = tsd_fetch(); if (tsd_reentrancy_level_get(tsd) == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn( + tsd))); } tcache_t *tcache; @@ -2252,7 +2253,8 @@ je_free(void *ptr) { ifree(tsd, ptr, tcache, true); } if (tsd_reentrancy_level_get(tsd) == 0) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn( + tsd))); } } } @@ -2513,7 +2515,7 @@ je_rallocx(void *ptr, size_t size, int flags) { assert(size != 0); assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -2569,7 +2571,7 @@ je_rallocx(void *ptr, size_t size, int flags) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, p); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); return p; label_oom: if (config_xmalloc && unlikely(opt_xmalloc)) { @@ -2577,7 +2579,7 @@ label_oom: abort(); } UTRACE(ptr, size, 0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); return NULL; } @@ -2669,7 +2671,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { assert(SIZE_T_MAX - size >= extra); assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); @@ -2712,7 +2714,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { } label_not_resized: UTRACE(ptr, size, ptr); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); return usize; } @@ -2726,7 +2728,7 @@ je_sallocx(const void *ptr, int flags) { assert(ptr != NULL); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); if (config_debug || force_ivsalloc) { usize = ivsalloc(tsdn, ptr); @@ -2735,7 +2737,7 @@ je_sallocx(const void *ptr, int flags) { usize = isalloc(tsdn, ptr); } - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return usize; } @@ -2746,7 +2748,7 @@ je_dallocx(void *ptr, int flags) { tsd_t *tsd = tsd_fetch(); bool fast = tsd_fast(tsd); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); tcache_t *tcache; if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { @@ -2777,12 +2779,12 @@ je_dallocx(void *ptr, int flags) { } else { ifree(tsd, ptr, tcache, true); } - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } JEMALLOC_ALWAYS_INLINE size_t inallocx(tsdn_t *tsdn, size_t size, int flags) { - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); size_t usize; if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) { @@ -2790,7 +2792,7 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { } else { usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); } - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return usize; } @@ -2803,7 +2805,7 @@ je_sdallocx(void *ptr, size_t size, int flags) { bool fast = tsd_fast(tsd); size_t usize = inallocx(tsd_tsdn(tsd), size, flags); assert(usize == isalloc(tsd_tsdn(tsd), ptr)); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); tcache_t *tcache; if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { @@ -2834,7 +2836,7 @@ je_sdallocx(void *ptr, size_t size, int flags) { } else { isfree(tsd, ptr, usize, tcache, true); } - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2850,14 +2852,14 @@ je_nallocx(size_t size, int flags) { } tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); usize = inallocx(tsdn, size, flags); if (unlikely(usize > LARGE_MAXCLASS)) { return 0; } - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return usize; } @@ -2872,9 +2874,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, } tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); return ret; } @@ -2888,9 +2890,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { } tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); ret = ctl_nametomib(tsdn, name, mibp, miblenp); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return ret; } @@ -2905,9 +2907,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lockless(tsdn_witness_tsdp_get(tsd_tsdn(tsd))); return ret; } @@ -2917,9 +2919,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, tsdn_t *tsdn; tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); stats_print(write_cb, cbopaque, opts); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2930,7 +2932,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { assert(malloc_initialized() || IS_INITIALIZER); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); if (unlikely(ptr == NULL)) { ret = 0; @@ -2943,7 +2945,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { } } - witness_assert_lockless(tsdn); + witness_assert_lockless(tsdn_witness_tsdp_get(tsdn)); return ret; } @@ -3000,7 +3002,7 @@ _malloc_prefork(void) narenas = narenas_total_get(); - witness_prefork(tsd); + witness_prefork(tsd_witness_tsdp_get(tsd)); /* Acquire all mutexes in a safe order. */ ctl_prefork(tsd_tsdn(tsd)); tcache_prefork(tsd_tsdn(tsd)); @@ -3067,7 +3069,7 @@ _malloc_postfork(void) tsd = tsd_fetch(); - witness_postfork_parent(tsd); + witness_postfork_parent(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { arena_t *arena; @@ -3094,7 +3096,7 @@ jemalloc_postfork_child(void) { tsd = tsd_fetch(); - witness_postfork_child(tsd); + witness_postfork_child(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { arena_t *arena; diff --git a/src/tcache.c b/src/tcache.c index d9f5e7c..4bb2fb8 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -568,7 +568,7 @@ label_return: bool tcaches_create(tsd_t *tsd, unsigned *r_ind) { - witness_assert_depth(tsd_tsdn(tsd), 0); + witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); bool err; @@ -600,7 +600,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) { err = false; label_return: - witness_assert_depth(tsd_tsdn(tsd), 0); + witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0); return err; } @@ -140,7 +140,7 @@ tsd_do_data_cleanup(tsd_t *tsd) { arena_cleanup(tsd); arenas_tdata_cleanup(tsd); tcache_cleanup(tsd); - witnesses_cleanup(tsd); + witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd)); } void diff --git a/src/witness.c b/src/witness.c index 0e910dc..f42b72a 100644 --- a/src/witness.c +++ b/src/witness.c @@ -63,38 +63,38 @@ witness_depth_error_t *JET_MUTABLE witness_depth_error = witness_depth_error_impl; void -witnesses_cleanup(tsd_t *tsd) { - witness_assert_lockless(tsd_tsdn(tsd)); +witnesses_cleanup(witness_tsd_t *witness_tsd) { + witness_assert_lockless(witness_tsd_tsdn(witness_tsd)); /* Do nothing. */ } void -witness_prefork(tsd_t *tsd) { +witness_prefork(witness_tsd_t *witness_tsd) { if (!config_debug) { return; } - tsd_witness_fork_set(tsd, true); + witness_tsd->forking = true; } void -witness_postfork_parent(tsd_t *tsd) { +witness_postfork_parent(witness_tsd_t *witness_tsd) { if (!config_debug) { return; } - tsd_witness_fork_set(tsd, false); + witness_tsd->forking = false; } void -witness_postfork_child(tsd_t *tsd) { +witness_postfork_child(witness_tsd_t *witness_tsd) { if (!config_debug) { return; } #ifndef JEMALLOC_MUTEX_INIT_CB witness_list_t *witnesses; - witnesses = tsd_witnessesp_get(tsd); + witnesses = &witness_tsd->witnesses; ql_new(witnesses); #endif - tsd_witness_fork_set(tsd, false); + witness_tsd->forking = false; } diff --git a/test/unit/witness.c b/test/unit/witness.c index de2e602..5986da4 100644 --- a/test/unit/witness.c +++ b/test/unit/witness.c @@ -55,95 +55,91 @@ witness_comp_reverse(const witness_t *a, void *oa, const witness_t *b, TEST_BEGIN(test_witness) { witness_t a, b; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0); + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 0); witness_init(&a, "a", 1, NULL, NULL); - witness_assert_not_owner(tsdn, &a); - witness_lock(tsdn, &a); - witness_assert_owner(tsdn, &a); - witness_assert_depth(tsdn, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0); + witness_assert_not_owner(&witness_tsdn, &a); + witness_lock(&witness_tsdn, &a); + witness_assert_owner(&witness_tsdn, &a); + witness_assert_depth(&witness_tsdn, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 0); witness_init(&b, "b", 2, NULL, NULL); - witness_assert_not_owner(tsdn, &b); - witness_lock(tsdn, &b); - witness_assert_owner(tsdn, &b); - witness_assert_depth(tsdn, 2); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); - - witness_unlock(tsdn, &a); - witness_assert_depth(tsdn, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); - witness_unlock(tsdn, &b); - - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); - witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0); + witness_assert_not_owner(&witness_tsdn, &b); + witness_lock(&witness_tsdn, &b); + witness_assert_owner(&witness_tsdn, &b); + witness_assert_depth(&witness_tsdn, 2); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 2); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)3U, 0); + + witness_unlock(&witness_tsdn, &a); + witness_assert_depth(&witness_tsdn, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)2U, 1); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)3U, 0); + witness_unlock(&witness_tsdn, &b); + + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); + witness_assert_depth_to_rank(&witness_tsdn, (witness_rank_t)1U, 0); } TEST_END TEST_BEGIN(test_witness_comp) { witness_t a, b, c, d; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_init(&a, "a", 1, witness_comp, &a); - witness_assert_not_owner(tsdn, &a); - witness_lock(tsdn, &a); - witness_assert_owner(tsdn, &a); - witness_assert_depth(tsdn, 1); + witness_assert_not_owner(&witness_tsdn, &a); + witness_lock(&witness_tsdn, &a); + witness_assert_owner(&witness_tsdn, &a); + witness_assert_depth(&witness_tsdn, 1); witness_init(&b, "b", 1, witness_comp, &b); - witness_assert_not_owner(tsdn, &b); - witness_lock(tsdn, &b); - witness_assert_owner(tsdn, &b); - witness_assert_depth(tsdn, 2); - witness_unlock(tsdn, &b); - witness_assert_depth(tsdn, 1); + witness_assert_not_owner(&witness_tsdn, &b); + witness_lock(&witness_tsdn, &b); + witness_assert_owner(&witness_tsdn, &b); + witness_assert_depth(&witness_tsdn, 2); + witness_unlock(&witness_tsdn, &b); + witness_assert_depth(&witness_tsdn, 1); witness_lock_error_orig = witness_lock_error; witness_lock_error = witness_lock_error_intercept; saw_lock_error = false; witness_init(&c, "c", 1, witness_comp_reverse, &c); - witness_assert_not_owner(tsdn, &c); + witness_assert_not_owner(&witness_tsdn, &c); assert_false(saw_lock_error, "Unexpected witness lock error"); - witness_lock(tsdn, &c); + witness_lock(&witness_tsdn, &c); assert_true(saw_lock_error, "Expected witness lock error"); - witness_unlock(tsdn, &c); - witness_assert_depth(tsdn, 1); + witness_unlock(&witness_tsdn, &c); + witness_assert_depth(&witness_tsdn, 1); saw_lock_error = false; witness_init(&d, "d", 1, NULL, NULL); - witness_assert_not_owner(tsdn, &d); + witness_assert_not_owner(&witness_tsdn, &d); assert_false(saw_lock_error, "Unexpected witness lock error"); - witness_lock(tsdn, &d); + witness_lock(&witness_tsdn, &d); assert_true(saw_lock_error, "Expected witness lock error"); - witness_unlock(tsdn, &d); - witness_assert_depth(tsdn, 1); + witness_unlock(&witness_tsdn, &d); + witness_assert_depth(&witness_tsdn, 1); - witness_unlock(tsdn, &a); + witness_unlock(&witness_tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_lock_error = witness_lock_error_orig; } @@ -151,7 +147,7 @@ TEST_END TEST_BEGIN(test_witness_reversal) { witness_t a, b; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); @@ -159,24 +155,22 @@ TEST_BEGIN(test_witness_reversal) { witness_lock_error = witness_lock_error_intercept; saw_lock_error = false; - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_init(&a, "a", 1, NULL, NULL); witness_init(&b, "b", 2, NULL, NULL); - witness_lock(tsdn, &b); - witness_assert_depth(tsdn, 1); + witness_lock(&witness_tsdn, &b); + witness_assert_depth(&witness_tsdn, 1); assert_false(saw_lock_error, "Unexpected witness lock error"); - witness_lock(tsdn, &a); + witness_lock(&witness_tsdn, &a); assert_true(saw_lock_error, "Expected witness lock error"); - witness_unlock(tsdn, &a); - witness_assert_depth(tsdn, 1); - witness_unlock(tsdn, &b); + witness_unlock(&witness_tsdn, &a); + witness_assert_depth(&witness_tsdn, 1); + witness_unlock(&witness_tsdn, &b); - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_lock_error = witness_lock_error_orig; } @@ -184,7 +178,7 @@ TEST_END TEST_BEGIN(test_witness_recursive) { witness_t a; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); @@ -196,22 +190,20 @@ TEST_BEGIN(test_witness_recursive) { witness_lock_error = witness_lock_error_intercept; saw_lock_error = false; - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_init(&a, "a", 1, NULL, NULL); - witness_lock(tsdn, &a); + witness_lock(&witness_tsdn, &a); assert_false(saw_lock_error, "Unexpected witness lock error"); assert_false(saw_not_owner_error, "Unexpected witness not owner error"); - witness_lock(tsdn, &a); + witness_lock(&witness_tsdn, &a); assert_true(saw_lock_error, "Expected witness lock error"); assert_true(saw_not_owner_error, "Expected witness not owner error"); - witness_unlock(tsdn, &a); + witness_unlock(&witness_tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_owner_error = witness_owner_error_orig; witness_lock_error = witness_lock_error_orig; @@ -221,7 +213,7 @@ TEST_END TEST_BEGIN(test_witness_unlock_not_owned) { witness_t a; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); @@ -229,17 +221,15 @@ TEST_BEGIN(test_witness_unlock_not_owned) { witness_owner_error = witness_owner_error_intercept; saw_owner_error = false; - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_init(&a, "a", 1, NULL, NULL); assert_false(saw_owner_error, "Unexpected owner error"); - witness_unlock(tsdn, &a); + witness_unlock(&witness_tsdn, &a); assert_true(saw_owner_error, "Expected owner error"); - witness_assert_lockless(tsdn); + witness_assert_lockless(&witness_tsdn); witness_owner_error = witness_owner_error_orig; } @@ -247,7 +237,7 @@ TEST_END TEST_BEGIN(test_witness_depth) { witness_t a; - tsdn_t *tsdn; + witness_tsdn_t witness_tsdn = { WITNESS_TSD_INITIALIZER }; test_skip_if(!config_debug); @@ -255,26 +245,24 @@ TEST_BEGIN(test_witness_depth) { witness_depth_error = witness_depth_error_intercept; saw_depth_error = false; - tsdn = tsdn_fetch(); - - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); witness_init(&a, "a", 1, NULL, NULL); assert_false(saw_depth_error, "Unexpected depth error"); - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); - witness_lock(tsdn, &a); - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); + witness_lock(&witness_tsdn, &a); + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); assert_true(saw_depth_error, "Expected depth error"); - witness_unlock(tsdn, &a); + witness_unlock(&witness_tsdn, &a); - witness_assert_lockless(tsdn); - witness_assert_depth(tsdn, 0); + witness_assert_lockless(&witness_tsdn); + witness_assert_depth(&witness_tsdn, 0); witness_depth_error = witness_depth_error_orig; } |