Output bin lock profiling results to malloc_stats.

Two counters are included for the small bins: lock contention rate, and
max lock waiting time.

4 files changed, 40 insertions, 0 deletions

diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h
index cf0ce23..8e81fcd 100644
--- a/include/jemalloc/internal/mutex_inlines.h
+++ b/include/jemalloc/internal/mutex_inlines.h
@@ -9,6 +9,9 @@ bool	malloc_mutex_trylock(malloc_mutex_t *mutex);
 void	malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void	malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
 void	malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
+void	malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data,
+    malloc_mutex_t *mutex);
+void	malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
@@ -23,6 +26,24 @@ malloc_mutex_trylock(malloc_mutex_t *mutex) {
 	return MALLOC_MUTEX_TRYLOCK(mutex);
 }
 
+/* Aggregate lock prof data. */
+JEMALLOC_INLINE void
+malloc_lock_prof_merge(lock_prof_data_t *sum, lock_prof_data_t *data) {
+	nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
+	if (nstime_compare(&data->max_wait_time, &sum->max_wait_time)) {
+		nstime_copy(&sum->max_wait_time, &data->max_wait_time);
+	}
+	sum->n_wait_times += data->n_wait_times;
+	sum->n_spin_acquired += data->n_spin_acquired;
+
+	if (sum->max_n_thds < data->max_n_thds) {
+		sum->max_n_thds = data->max_n_thds;
+	}
+	sum->n_waiting_thds += data->n_waiting_thds;
+	sum->n_owner_switches += data->n_owner_switches;
+	sum->n_lock_ops += data->n_lock_ops;
+}
+
 JEMALLOC_INLINE void
 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	witness_assert_not_owner(tsdn, &mutex->witness);
@@ -58,6 +79,20 @@ JEMALLOC_INLINE void
 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
 	witness_assert_not_owner(tsdn, &mutex->witness);
 }
+
+/* Copy the prof data from mutex for processing. */
+JEMALLOC_INLINE void
+malloc_lock_prof_read(tsdn_t *tsdn, lock_prof_data_t *data,
+    malloc_mutex_t *mutex) {
+	lock_prof_data_t *source = &mutex->prof_data;
+	/* Can only read with the lock. */
+	malloc_mutex_assert_owner(tsdn, mutex);
+
+	*data = *source;
+	/* n_wait_thds is not reported (modified w/o locking). */
+	data->n_waiting_thds = 0;
+}
+
 #endif
 
 #endif /* JEMALLOC_INTERNAL_MUTEX_INLINES_H */
diff --git a/include/jemalloc/internal/nstime_externs.h b/include/jemalloc/internal/nstime_externs.h
index cf14ae0..1abc84d 100644
--- a/include/jemalloc/internal/nstime_externs.h
+++ b/include/jemalloc/internal/nstime_externs.h
@@ -5,6 +5,7 @@ void	nstime_init(nstime_t *time, uint64_t ns);
 void	nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
 uint64_t	nstime_ns(const nstime_t *time);
 uint64_t	nstime_sec(const nstime_t *time);
+uint64_t	nstime_msec(const nstime_t *time);
 uint64_t	nstime_nsec(const nstime_t *time);
 void	nstime_copy(nstime_t *time, const nstime_t *source);
 int	nstime_compare(const nstime_t *a, const nstime_t *b);
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index e138de0..14ecded 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -273,6 +273,7 @@ malloc_mutex_assert_owner
 malloc_mutex_boot
 malloc_mutex_init
 malloc_mutex_lock
+malloc_mutex_lock_slow
 malloc_mutex_postfork_child
 malloc_mutex_postfork_parent
 malloc_mutex_prefork
@@ -302,6 +303,7 @@ nstime_imultiply
 nstime_init
 nstime_init2
 nstime_monotonic
+nstime_msec
 nstime_ns
 nstime_nsec
 nstime_sec
diff --git a/include/jemalloc/internal/stats_structs.h b/include/jemalloc/internal/stats_structs.h
index ffcb3c1..cbc448a 100644
--- a/include/jemalloc/internal/stats_structs.h
+++ b/include/jemalloc/internal/stats_structs.h
@@ -56,6 +56,8 @@ struct malloc_bin_stats_s {
 
 	/* Current number of slabs in this bin. */
 	size_t		curslabs;
+
+	lock_prof_data_t lock_data;
 };
 
 struct malloc_large_stats_s {