Prefer old/low extent_t structures during reuse.

Rather than using a LIFO queue to track available extent_t structures,
use a red-black tree, and always choose the oldest/lowest available
during reuse.

6 files changed, 57 insertions, 15 deletions

diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h
index 1370b53..14c473c 100644
--- a/include/jemalloc/internal/arena_structs_b.h
+++ b/include/jemalloc/internal/arena_structs_b.h
@@ -233,12 +233,13 @@ struct arena_s {
 	atomic_u_t		extent_grow_next;
 
 	/*
-	 * Freelist of extent structures that were allocated via base_alloc().
+	 * Available extent structures that were allocated via
+	 * base_alloc_extent().
 	 *
-	 * Synchronization: extent_freelist_mtx.
+	 * Synchronization: extent_avail_mtx.
 	 */
-	extent_list_t		extent_freelist;
-	malloc_mutex_t		extent_freelist_mtx;
+	extent_tree_t		extent_avail;
+	malloc_mutex_t		extent_avail_mtx;
 
 	/*
 	 * bins is used to store heaps of free regions.
diff --git a/include/jemalloc/internal/ctl_types.h b/include/jemalloc/internal/ctl_types.h
index 065ccda..e798609 100644
--- a/include/jemalloc/internal/ctl_types.h
+++ b/include/jemalloc/internal/ctl_types.h
@@ -14,7 +14,7 @@ typedef enum {
 
 #define ARENA_PROF_MUTEXES						\
     OP(large)								\
-    OP(extent_freelist)							\
+    OP(extent_avail)							\
     OP(extents_dirty)							\
     OP(extents_muzzy)							\
     OP(extents_retained)						\
diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h
index 3fe4a0a..58e57e7 100644
--- a/include/jemalloc/internal/extent_externs.h
+++ b/include/jemalloc/internal/extent_externs.h
@@ -1,6 +1,7 @@
 #ifndef JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
 #define JEMALLOC_INTERNAL_EXTENT_EXTERNS_H
 
+#include "jemalloc/internal/rb.h"
 #include "jemalloc/internal/ph.h"
 
 extern rtree_t			extents_rtree;
@@ -17,6 +18,7 @@ size_t extent_size_quantize_floor(size_t size);
 size_t extent_size_quantize_ceil(size_t size);
 #endif
 
+rb_proto(, extent_avail_, extent_tree_t, extent_t)
 ph_proto(, extent_heap_, extent_heap_t, extent_t)
 
 bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state,
diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
index e1c5cea..fbe51e4 100644
--- a/include/jemalloc/internal/extent_inlines.h
+++ b/include/jemalloc/internal/extent_inlines.h
@@ -53,8 +53,10 @@ void extent_list_replace(extent_list_t *list, extent_t *to_remove,
     extent_t *to_insert);
 void extent_list_remove(extent_list_t *list, extent_t *extent);
 int extent_sn_comp(const extent_t *a, const extent_t *b);
+int extent_esn_comp(const extent_t *a, const extent_t *b);
 int extent_ad_comp(const extent_t *a, const extent_t *b);
 int extent_snad_comp(const extent_t *a, const extent_t *b);
+int extent_esnead_comp(const extent_t *a, const extent_t *b);
 #endif
 
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
@@ -379,6 +381,14 @@ extent_sn_comp(const extent_t *a, const extent_t *b) {
 }
 
 JEMALLOC_INLINE int
+extent_esn_comp(const extent_t *a, const extent_t *b) {
+	size_t a_esn = extent_esn_get(a);
+	size_t b_esn = extent_esn_get(b);
+
+	return (a_esn > b_esn) - (a_esn < b_esn);
+}
+
+JEMALLOC_INLINE int
 extent_ad_comp(const extent_t *a, const extent_t *b) {
 	uintptr_t a_addr = (uintptr_t)extent_addr_get(a);
 	uintptr_t b_addr = (uintptr_t)extent_addr_get(b);
@@ -387,6 +397,14 @@ extent_ad_comp(const extent_t *a, const extent_t *b) {
 }
 
 JEMALLOC_INLINE int
+extent_ead_comp(const extent_t *a, const extent_t *b) {
+	uintptr_t a_eaddr = (uintptr_t)a;
+	uintptr_t b_eaddr = (uintptr_t)b;
+
+	return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr);
+}
+
+JEMALLOC_INLINE int
 extent_snad_comp(const extent_t *a, const extent_t *b) {
 	int ret;
 
@@ -398,6 +416,19 @@ extent_snad_comp(const extent_t *a, const extent_t *b) {
 	ret = extent_ad_comp(a, b);
 	return ret;
 }
+
+JEMALLOC_INLINE int
+extent_esnead_comp(const extent_t *a, const extent_t *b) {
+	int ret;
+
+	ret = extent_esn_comp(a, b);
+	if (ret != 0) {
+		return ret;
+	}
+
+	ret = extent_ead_comp(a, b);
+	return ret;
+}
 #endif
 
 #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */
diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h
index 38c3c8a..7066b8f 100644
--- a/include/jemalloc/internal/extent_structs.h
+++ b/include/jemalloc/internal/extent_structs.h
@@ -2,8 +2,9 @@
 #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H
 
 #include "jemalloc/internal/atomic.h"
-#include "jemalloc/internal/ph.h"
 #include "jemalloc/internal/ql.h"
+#include "jemalloc/internal/rb.h"
+#include "jemalloc/internal/ph.h"
 
 typedef enum {
 	extent_state_active   = 0,
@@ -117,15 +118,18 @@ struct extent_s {
 		size_t			e_bsize;
 	};
 
-	/*
-	 * List linkage, used by a variety of lists:
-	 * - arena_bin_t's slabs_full
-	 * - extents_t's LRU
-	 * - stashed dirty extents
-	 * - arena's large allocations
-	 * - arena's extent structure freelist
-	 */
-	ql_elm(extent_t)	ql_link;
+	union {
+		/*
+		 * List linkage, used by a variety of lists:
+		 * - arena_bin_t's slabs_full
+		 * - extents_t's LRU
+		 * - stashed dirty extents
+		 * - arena's large allocations
+		 */
+		ql_elm(extent_t)	ql_link;
+		/* Red-black tree linkage, used by arena's extent_avail. */
+		rb_node(extent_t)	rb_link;
+	};
 
 	/* Linkage for per size class sn/address-ordered heaps. */
 	phn(extent_t)		ph_link;
@@ -142,6 +146,7 @@ struct extent_s {
 	};
 };
 typedef ql_head(extent_t) extent_list_t;
+typedef rb_tree(extent_t) extent_tree_t;
 typedef ph(extent_t) extent_heap_t;
 
 /* Quantized collection of extents, with built-in LRU queue. */
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index dd35d50..34c2789 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -160,8 +160,11 @@ extent_dss_boot
 extent_dss_mergeable
 extent_dss_prec_get
 extent_dss_prec_set
+extent_ead_comp
+extent_esn_comp
 extent_esn_get
 extent_esn_set
+extent_esnead_comp
 extent_heap_empty
 extent_heap_first
 extent_heap_insert