Add witness_assert_depth[_to_rank]().

This makes it possible to make lock state assertions about precisely
which locks are held.

6 files changed, 84 insertions, 26 deletions

diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 745220e..2567f56 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -549,13 +549,15 @@ tsdn_fetch
 tsdn_null
 tsdn_rtree_ctx
 tsdn_tsd
+witness_assert_depth
+witness_assert_depth_to_rank
 witness_assert_lockless
 witness_assert_not_owner
 witness_assert_owner
+witness_depth_error
 witness_init
 witness_lock
 witness_lock_error
-witness_lockless_error
 witness_not_owner_error
 witness_owner
 witness_owner_error
diff --git a/include/jemalloc/internal/witness_externs.h b/include/jemalloc/internal/witness_externs.h
index dcd987c..5d91fde 100644
--- a/include/jemalloc/internal/witness_externs.h
+++ b/include/jemalloc/internal/witness_externs.h
@@ -23,10 +23,12 @@ extern witness_not_owner_error_t *witness_not_owner_error;
 void	witness_not_owner_error(const witness_t *witness);
 #endif
 #ifdef JEMALLOC_JET
-typedef void (witness_lockless_error_t)(const witness_list_t *);
-extern witness_lockless_error_t *witness_lockless_error;
+typedef void (witness_depth_error_t)(const witness_list_t *,
+    witness_rank_t rank_inclusive, unsigned depth);
+extern witness_depth_error_t *witness_depth_error;
 #else
-void	witness_lockless_error(const witness_list_t *witnesses);
+void	witness_depth_error(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth);
 #endif
 
 void	witnesses_cleanup(tsd_t *tsd);
diff --git a/include/jemalloc/internal/witness_inlines.h b/include/jemalloc/internal/witness_inlines.h
index c2a2781..51f3f6e 100644
--- a/include/jemalloc/internal/witness_inlines.h
+++ b/include/jemalloc/internal/witness_inlines.h
@@ -5,6 +5,9 @@
 bool	witness_owner(tsd_t *tsd, const witness_t *witness);
 void	witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
 void	witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
+void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+    unsigned depth);
+void witness_assert_depth(tsdn_t *tsdn, unsigned depth);
 void	witness_assert_lockless(tsdn_t *tsdn);
 void	witness_lock(tsdn_t *tsdn, witness_t *witness);
 void	witness_unlock(tsdn_t *tsdn, witness_t *witness);
@@ -78,8 +81,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) {
 }
 
 JEMALLOC_INLINE void
-witness_assert_lockless(tsdn_t *tsdn) {
+witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+    unsigned depth) {
 	tsd_t *tsd;
+	unsigned d;
 	witness_list_t *witnesses;
 	witness_t *w;
 
@@ -92,14 +97,33 @@ witness_assert_lockless(tsdn_t *tsdn) {
 	}
 	tsd = tsdn_tsd(tsdn);
 
+	d = 0;
 	witnesses = tsd_witnessesp_get(tsd);
 	w = ql_last(witnesses, link);
 	if (w != NULL) {
-		witness_lockless_error(witnesses);
+		ql_reverse_foreach(w, witnesses, link) {
+			if (w->rank < rank_inclusive) {
+				break;
+			}
+			d++;
+		}
+	}
+	if (d != depth) {
+		witness_depth_error(witnesses, rank_inclusive, depth);
 	}
 }
 
 JEMALLOC_INLINE void
+witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
+	witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
+}
+
+JEMALLOC_INLINE void
+witness_assert_lockless(tsdn_t *tsdn) {
+	witness_assert_depth(tsdn, 0);
+}
+
+JEMALLOC_INLINE void
 witness_lock(tsdn_t *tsdn, witness_t *witness) {
 	tsd_t *tsd;
 	witness_list_t *witnesses;
diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h
index c2a73f2..f765d7b 100644
--- a/include/jemalloc/internal/witness_types.h
+++ b/include/jemalloc/internal/witness_types.h
@@ -13,6 +13,8 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *,
  */
 #define WITNESS_RANK_OMIT		0U
 
+#define WITNESS_RANK_MIN		1U
+
 #define WITNESS_RANK_INIT		1U
 #define WITNESS_RANK_CTL		1U
 #define WITNESS_RANK_ARENAS		2U
diff --git a/src/witness.c b/src/witness.c
index 1c03457..034ea92 100644
--- a/src/witness.c
+++ b/src/witness.c
@@ -65,14 +65,16 @@ witness_not_owner_error_t *witness_not_owner_error =
 #endif
 
 #ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
 #endif
 void
-witness_lockless_error(const witness_list_t *witnesses) {
+witness_depth_error(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth) {
 	witness_t *w;
 
-	malloc_printf("<jemalloc>: Should not own any locks:");
+	malloc_printf("<jemalloc>: Should own %u lock%s of rank >= %u:", depth,
+	    (depth != 1) ?  "s" : "", rank_inclusive);
 	ql_foreach(w, witnesses, link) {
 		malloc_printf(" %s(%u)", w->name, w->rank);
 	}
@@ -80,10 +82,9 @@ witness_lockless_error(const witness_list_t *witnesses) {
 	abort();
 }
 #ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
-witness_lockless_error_t *witness_lockless_error =
-    JEMALLOC_N(n_witness_lockless_error);
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(witness_depth_error)
+witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
 #endif
 
 void
diff --git a/test/unit/witness.c b/test/unit/witness.c
index c914e4b..de2e602 100644
--- a/test/unit/witness.c
+++ b/test/unit/witness.c
@@ -3,12 +3,12 @@
 static witness_lock_error_t *witness_lock_error_orig;
 static witness_owner_error_t *witness_owner_error_orig;
 static witness_not_owner_error_t *witness_not_owner_error_orig;
-static witness_lockless_error_t *witness_lockless_error_orig;
+static witness_depth_error_t *witness_depth_error_orig;
 
 static bool saw_lock_error;
 static bool saw_owner_error;
 static bool saw_not_owner_error;
-static bool saw_lockless_error;
+static bool saw_depth_error;
 
 static void
 witness_lock_error_intercept(const witness_list_t *witnesses,
@@ -27,8 +27,9 @@ witness_not_owner_error_intercept(const witness_t *witness) {
 }
 
 static void
-witness_lockless_error_intercept(const witness_list_t *witnesses) {
-	saw_lockless_error = true;
+witness_depth_error_intercept(const witness_list_t *witnesses,
+    witness_rank_t rank_inclusive, unsigned depth) {
+	saw_depth_error = true;
 }
 
 static int
@@ -61,21 +62,36 @@ TEST_BEGIN(test_witness) {
 	tsdn = tsdn_fetch();
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
 
 	witness_init(&a, "a", 1, NULL, NULL);
 	witness_assert_not_owner(tsdn, &a);
 	witness_lock(tsdn, &a);
 	witness_assert_owner(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0);
 
 	witness_init(&b, "b", 2, NULL, NULL);
 	witness_assert_not_owner(tsdn, &b);
 	witness_lock(tsdn, &b);
 	witness_assert_owner(tsdn, &b);
+	witness_assert_depth(tsdn, 2);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
 
 	witness_unlock(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
 	witness_unlock(tsdn, &b);
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
+	witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
 }
 TEST_END
 
@@ -93,12 +109,15 @@ TEST_BEGIN(test_witness_comp) {
 	witness_assert_not_owner(tsdn, &a);
 	witness_lock(tsdn, &a);
 	witness_assert_owner(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
 
 	witness_init(&b, "b", 1, witness_comp, &b);
 	witness_assert_not_owner(tsdn, &b);
 	witness_lock(tsdn, &b);
 	witness_assert_owner(tsdn, &b);
+	witness_assert_depth(tsdn, 2);
 	witness_unlock(tsdn, &b);
+	witness_assert_depth(tsdn, 1);
 
 	witness_lock_error_orig = witness_lock_error;
 	witness_lock_error = witness_lock_error_intercept;
@@ -110,6 +129,7 @@ TEST_BEGIN(test_witness_comp) {
 	witness_lock(tsdn, &c);
 	assert_true(saw_lock_error, "Expected witness lock error");
 	witness_unlock(tsdn, &c);
+	witness_assert_depth(tsdn, 1);
 
 	saw_lock_error = false;
 
@@ -119,6 +139,7 @@ TEST_BEGIN(test_witness_comp) {
 	witness_lock(tsdn, &d);
 	assert_true(saw_lock_error, "Expected witness lock error");
 	witness_unlock(tsdn, &d);
+	witness_assert_depth(tsdn, 1);
 
 	witness_unlock(tsdn, &a);
 
@@ -146,11 +167,13 @@ TEST_BEGIN(test_witness_reversal) {
 	witness_init(&b, "b", 2, NULL, NULL);
 
 	witness_lock(tsdn, &b);
+	witness_assert_depth(tsdn, 1);
 	assert_false(saw_lock_error, "Unexpected witness lock error");
 	witness_lock(tsdn, &a);
 	assert_true(saw_lock_error, "Expected witness lock error");
 
 	witness_unlock(tsdn, &a);
+	witness_assert_depth(tsdn, 1);
 	witness_unlock(tsdn, &b);
 
 	witness_assert_lockless(tsdn);
@@ -222,34 +245,38 @@ TEST_BEGIN(test_witness_unlock_not_owned) {
 }
 TEST_END
 
-TEST_BEGIN(test_witness_lockful) {
+TEST_BEGIN(test_witness_depth) {
 	witness_t a;
 	tsdn_t *tsdn;
 
 	test_skip_if(!config_debug);
 
-	witness_lockless_error_orig = witness_lockless_error;
-	witness_lockless_error = witness_lockless_error_intercept;
-	saw_lockless_error = false;
+	witness_depth_error_orig = witness_depth_error;
+	witness_depth_error = witness_depth_error_intercept;
+	saw_depth_error = false;
 
 	tsdn = tsdn_fetch();
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
 	witness_init(&a, "a", 1, NULL, NULL);
 
-	assert_false(saw_lockless_error, "Unexpected lockless error");
+	assert_false(saw_depth_error, "Unexpected depth error");
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
 	witness_lock(tsdn, &a);
 	witness_assert_lockless(tsdn);
-	assert_true(saw_lockless_error, "Expected lockless error");
+	witness_assert_depth(tsdn, 0);
+	assert_true(saw_depth_error, "Expected depth error");
 
 	witness_unlock(tsdn, &a);
 
 	witness_assert_lockless(tsdn);
+	witness_assert_depth(tsdn, 0);
 
-	witness_lockless_error = witness_lockless_error_orig;
+	witness_depth_error = witness_depth_error_orig;
 }
 TEST_END
 
@@ -261,5 +288,5 @@ main(void) {
 	    test_witness_reversal,
 	    test_witness_recursive,
 	    test_witness_unlock_not_owned,
-	    test_witness_lockful);
+	    test_witness_depth);
 }