Add init function support to tsd members.

This will facilitate embedding tcache into tsd, which will require proper
initialization cannot be done via the static initializer.  Make tsd->rtree_ctx
to be initialized via rtree_ctx_data_init().

6 files changed, 35 insertions, 26 deletions

diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 5d03f5d..1cced60 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -424,6 +424,7 @@ prof_thread_name_set
 psz2ind
 psz2u
 rtree_clear
+rtree_ctx_data_init
 rtree_delete
 rtree_extent_read
 rtree_extent_szind_read
@@ -520,6 +521,7 @@ tsd_booted
 tsd_booted_get
 tsd_cleanup
 tsd_cleanup_wrapper
+tsd_data_init
 tsd_fetch
 tsd_fetch_impl
 tsd_get
diff --git a/include/jemalloc/internal/rtree_externs.h b/include/jemalloc/internal/rtree_externs.h
index 5145c12..482f6ba 100644
--- a/include/jemalloc/internal/rtree_externs.h
+++ b/include/jemalloc/internal/rtree_externs.h
@@ -43,5 +43,6 @@ void rtree_leaf_elm_witness_access(tsdn_t *tsdn, const rtree_t *rtree,
     const rtree_leaf_elm_t *elm);
 void rtree_leaf_elm_witness_release(tsdn_t *tsdn, const rtree_t *rtree,
     const rtree_leaf_elm_t *elm);
+bool rtree_ctx_data_init(rtree_ctx_t *ctx);
 
 #endif /* JEMALLOC_INTERNAL_RTREE_EXTERNS_H */
diff --git a/include/jemalloc/internal/rtree_types.h b/include/jemalloc/internal/rtree_types.h
index de3893b..e604156 100644
--- a/include/jemalloc/internal/rtree_types.h
+++ b/include/jemalloc/internal/rtree_types.h
@@ -38,6 +38,9 @@ typedef struct rtree_s rtree_t;
 #  define RTREE_LEAF_COMPACT
 #endif
 
+/* Needed for initialization only. */
+#define RTREE_LEAFKEY_INVALID	((uintptr_t)1)
+
 /*
  * Number of leafkey/leaf pairs to cache.  Each entry supports an entire leaf,
  * so the cache hit rate is typically high even with a small number of entries.
@@ -51,12 +54,13 @@ typedef struct rtree_s rtree_t;
  * the tree nodes, and the cache will itself suffer cache misses if made overly
  * large, not to mention the cost of linear search.
  */
-#define RTREE_CTX_NCACHE 8
+#define RTREE_CTX_NCACHE	8
 
-/* Static initializer for rtree_ctx_t. */
-#define RTREE_CTX_INITIALIZER	{					\
-	{{0, NULL} /* C initializes all trailing elements to NULL. */}	\
-}
+/*
+ * Zero initializer required for tsd initialization only.  Proper initialization
+ * done via rtree_ctx_data_init().
+ */
+#define RTREE_CTX_ZERO_INITIALIZER {{{0}}}
 
 /*
  * Maximum number of concurrently acquired elements per thread.  This controls
diff --git a/include/jemalloc/internal/tsd_externs.h b/include/jemalloc/internal/tsd_externs.h
index 87ebaf2..9b88a56 100644
--- a/include/jemalloc/internal/tsd_externs.h
+++ b/include/jemalloc/internal/tsd_externs.h
@@ -14,5 +14,6 @@ void	*tsd_init_check_recursion(tsd_init_head_t *head,
 void	tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block);
 #endif
 void	tsd_cleanup(void *arg);
+bool	tsd_data_init(void *arg);
 
 #endif /* JEMALLOC_INTERNAL_TSD_EXTERNS_H */
diff --git a/include/jemalloc/internal/tsd_inlines.h b/include/jemalloc/internal/tsd_inlines.h
index 96de406..7d57b7d 100644
--- a/include/jemalloc/internal/tsd_inlines.h
+++ b/include/jemalloc/internal/tsd_inlines.h
@@ -8,7 +8,7 @@ tsd_t *tsd_fetch_impl(bool init);
 tsd_t *tsd_fetch(void);
 tsdn_t *tsd_tsdn(tsd_t *tsd);
 bool tsd_nominal(tsd_t *tsd);
-#define O(n, t, gs, c)							\
+#define O(n, t, gs, i, c)						\
 t *tsd_##n##p_get(tsd_t *tsd);						\
 t tsd_##n##_get(tsd_t *tsd);						\
 void tsd_##n##_set(tsd_t *tsd, t n);
@@ -39,9 +39,11 @@ tsd_fetch_impl(bool init) {
 			tsd->state = tsd_state_nominal;
 			/* Trigger cleanup handler registration. */
 			tsd_set(tsd);
+			tsd_data_init(tsd);
 		} else if (tsd->state == tsd_state_purgatory) {
 			tsd->state = tsd_state_reincarnated;
 			tsd_set(tsd);
+			tsd_data_init(tsd);
 		} else {
 			assert(tsd->state == tsd_state_reincarnated);
 		}
@@ -76,7 +78,7 @@ tsd_##n##_set(tsd_t *tsd, t n) {					\
 	tsd->n = n;							\
 }
 #define MALLOC_TSD_getset_no(n, t)
-#define O(n, t, gs, c)							\
+#define O(n, t, gs, i, c)						\
 JEMALLOC_ALWAYS_INLINE t *						\
 tsd_##n##p_get(tsd_t *tsd) {						\
 	return &tsd->n;							\
@@ -121,8 +123,7 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) {
 	 * return a pointer to it.
 	 */
 	if (unlikely(tsdn_null(tsdn))) {
-		static const rtree_ctx_t rtree_ctx = RTREE_CTX_INITIALIZER;
-		memcpy(fallback, &rtree_ctx, sizeof(rtree_ctx_t));
+		rtree_ctx_data_init(fallback);
 		return fallback;
 	}
 	return tsd_rtree_ctx(tsdn_tsd(tsdn));
diff --git a/include/jemalloc/internal/tsd_structs.h b/include/jemalloc/internal/tsd_structs.h
index 722b966..b4ac09f 100644
--- a/include/jemalloc/internal/tsd_structs.h
+++ b/include/jemalloc/internal/tsd_structs.h
@@ -15,23 +15,23 @@ struct tsd_init_head_s {
 #endif
 
 #define MALLOC_TSD							\
-/*  O(name,			type,		[gs]et,	cleanup) */	\
-    O(tcache,			tcache_t *,	yes,	yes)		\
-    O(thread_allocated,		uint64_t,	yes,	no)		\
-    O(thread_deallocated,	uint64_t,	yes,	no)		\
-    O(prof_tdata,		prof_tdata_t *,	yes,	yes)		\
-    O(iarena,			arena_t *,	yes,	yes)		\
-    O(arena,			arena_t *,	yes,	yes)		\
-    O(arenas_tdata,		arena_tdata_t *,yes,	yes)		\
-    O(narenas_tdata,		unsigned,	yes,	no)		\
-    O(arenas_tdata_bypass,	bool,		no,	no)		\
+/*  O(name,			type,		[gs]et,	init,	cleanup) */ \
+    O(tcache,			tcache_t *,	yes,	no,	yes)	\
+    O(thread_allocated,		uint64_t,	yes,	no,	no)	\
+    O(thread_deallocated,	uint64_t,	yes,	no,	no)	\
+    O(prof_tdata,		prof_tdata_t *,	yes,	no,	yes)	\
+    O(iarena,			arena_t *,	yes,	no,	yes)	\
+    O(arena,			arena_t *,	yes,	no,	yes)	\
+    O(arenas_tdata,		arena_tdata_t *,yes,	no,	yes)	\
+    O(narenas_tdata,		unsigned,	yes,	no,	no)	\
+    O(arenas_tdata_bypass,	bool,		no,	no,	no)	\
     O(tcache_enabled,		tcache_enabled_t,			\
-						yes,	no)		\
-    O(rtree_ctx,		rtree_ctx_t,	no,	no)		\
-    O(witnesses,		witness_list_t,	no,	yes)		\
+						yes,	no,	no)	\
+    O(rtree_ctx,		rtree_ctx_t,	no,	yes,	no)	\
+    O(witnesses,		witness_list_t,	no,	no,	yes)	\
     O(rtree_leaf_elm_witnesses,	rtree_leaf_elm_witness_tsd_t,		\
-						no,	no)		\
-    O(witness_fork,		bool,		yes,	no)		\
+						no,	no,	no)	\
+    O(witness_fork,		bool,		yes,	no,	no)
 
 #define TSD_INITIALIZER {						\
     tsd_state_uninitialized,						\
@@ -45,7 +45,7 @@ struct tsd_init_head_s {
     0,									\
     false,								\
     tcache_enabled_default,						\
-    RTREE_CTX_INITIALIZER,						\
+    RTREE_CTX_ZERO_INITIALIZER,						\
     ql_head_initializer(witnesses),					\
     RTREE_ELM_WITNESS_TSD_INITIALIZER,					\
     false								\
@@ -53,7 +53,7 @@ struct tsd_init_head_s {
 
 struct tsd_s {
 	tsd_state_t	state;
-#define O(n, t, gs, c)							\
+#define O(n, t, gs, i, c)						\
 	t		n;
 MALLOC_TSD
 #undef O