Implement tsd.

Implement tsd, which is a TLS/TSD abstraction that uses one or both
internally.  Modify bootstrapping such that no tsd's are utilized until
allocation is safe.

Remove malloc_[v]tprintf(), and use malloc_snprintf() instead.

Fix %p argument size handling in malloc_vsnprintf().

Fix a long-standing statistics-related bug in the "thread.arena"
mallctl that could cause crashes due to linked list corruption.

8 files changed, 274 insertions, 244 deletions

diff --git a/src/chunk.c b/src/chunk.c
index b908650..f50e840 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -100,7 +100,7 @@ chunk_dealloc(void *chunk, size_t size, bool unmap)
 }
 
 bool
-chunk_boot(void)
+chunk_boot0(void)
 {
 
 	/* Set variables according to the value of opt_lg_chunk. */
@@ -114,8 +114,6 @@ chunk_boot(void)
 			return (true);
 		memset(&stats_chunks, 0, sizeof(chunk_stats_t));
 	}
-	if (chunk_mmap_boot())
-		return (true);
 	if (config_dss && chunk_dss_boot())
 		return (true);
 	if (config_ivsalloc) {
@@ -127,3 +125,13 @@ chunk_boot(void)
 
 	return (false);
 }
+
+bool
+chunk_boot1(void)
+{
+
+	if (chunk_mmap_boot())
+		return (true);
+
+	return (false);
+}
diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c
index 6ea2118..749a2da 100644
--- a/src/chunk_mmap.c
+++ b/src/chunk_mmap.c
@@ -8,20 +8,9 @@
  * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
  * potentially avoid some system calls.
  */
-#ifdef JEMALLOC_TLS
-static __thread bool	mmap_unaligned_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#define	MMAP_UNALIGNED_GET()	mmap_unaligned_tls
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	mmap_unaligned_tls = (v);					\
-} while (0)
-#else
-static pthread_key_t	mmap_unaligned_tsd;
-#define	MMAP_UNALIGNED_GET()	((bool)pthread_getspecific(mmap_unaligned_tsd))
-#define	MMAP_UNALIGNED_SET(v)	do {					\
-	pthread_setspecific(mmap_unaligned_tsd, (void *)(v));		\
-} while (0)
-#endif
+malloc_tsd_data(static, mmap_unaligned, bool, false)
+malloc_tsd_funcs(JEMALLOC_INLINE, mmap_unaligned, bool, false,
+    malloc_tsd_no_cleanup)
 
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
@@ -128,8 +117,10 @@ chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve)
 	 * the next chunk_alloc_mmap() execution tries the fast allocation
 	 * method.
 	 */
-	if (unaligned == false)
-		MMAP_UNALIGNED_SET(false);
+	if (unaligned == false && mmap_unaligned_booted) {
+		bool mu = false;
+		mmap_unaligned_tsd_set(&mu);
+	}
 
 	return (ret);
 }
@@ -167,7 +158,7 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
 	 * fast method next time.
 	 */
 
-	if (MMAP_UNALIGNED_GET() == false) {
+	if (mmap_unaligned_booted && *mmap_unaligned_tsd_get() == false) {
 		size_t offset;
 
 		ret = pages_map(NULL, size, noreserve);
@@ -176,7 +167,8 @@ chunk_alloc_mmap_internal(size_t size, bool noreserve)
 
 		offset = CHUNK_ADDR2OFFSET(ret);
 		if (offset != 0) {
-			MMAP_UNALIGNED_SET(true);
+			bool mu = true;
+			mmap_unaligned_tsd_set(&mu);
 			/* Try to extend chunk boundary. */
 			if (pages_map((void *)((uintptr_t)ret + size),
 			    chunksize - offset, noreserve) == NULL) {
@@ -225,11 +217,15 @@ bool
 chunk_mmap_boot(void)
 {
 
-#ifndef JEMALLOC_TLS
-	if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
-		malloc_write("<jemalloc>: Error in pthread_key_create()\n");
+	/*
+	 * XXX For the non-TLS implementation of tsd, the first access from
+	 * each thread causes memory allocation.  The result is a bootstrapping
+	 * problem for this particular use case, so for now just disable it by
+	 * leaving it in an unbooted state.
+	 */
+#ifdef JEMALLOC_TLS
+	if (mmap_unaligned_tsd_boot())
 		return (true);
-	}
 #endif
 
 	return (false);
diff --git a/src/ctl.c b/src/ctl.c
index 1ef84e8..e17e503 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -978,13 +978,13 @@ thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
 
 	VOID();
 
-	tcache = TCACHE_GET();
-	if (tcache == NULL) {
+	if ((tcache = *tcache_tsd_get()) == NULL) {
 		ret = 0;
 		goto RETURN;
 	}
 	tcache_destroy(tcache);
-	TCACHE_SET(NULL);
+	tcache = NULL;
+	tcache_tsd_set(&tcache);
 
 	ret = 0;
 RETURN:
@@ -1012,23 +1012,26 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
 
 		/* Initialize arena if necessary. */
 		malloc_mutex_lock(&arenas_lock);
-		if ((arena = arenas[newind]) == NULL)
-			arena = arenas_extend(newind);
-		arenas[oldind]->nthreads--;
-		arenas[newind]->nthreads++;
-		malloc_mutex_unlock(&arenas_lock);
-		if (arena == NULL) {
+		if ((arena = arenas[newind]) == NULL && (arena =
+		    arenas_extend(newind)) == NULL) {
+			malloc_mutex_unlock(&arenas_lock);
 			ret = EAGAIN;
 			goto RETURN;
 		}
+		assert(arena == arenas[newind]);
+		arenas[oldind]->nthreads--;
+		arenas[newind]->nthreads++;
+		malloc_mutex_unlock(&arenas_lock);
 
 		/* Set new arena association. */
-		ARENA_SET(arena);
 		if (config_tcache) {
-			tcache_t *tcache = TCACHE_GET();
-			if (tcache != NULL)
-				tcache->arena = arena;
+			tcache_t *tcache;
+			if ((tcache = *tcache_tsd_get()) != NULL) {
+				tcache_arena_dissociate(tcache);
+				tcache_arena_associate(tcache, arena);
+			}
 		}
+		arenas_tsd_set(&arena);
 	}
 
 	ret = 0;
@@ -1036,11 +1039,14 @@ RETURN:
 	return (ret);
 }
 
-CTL_RO_NL_CGEN(config_stats, thread_allocated, ALLOCATED_GET(), uint64_t)
-CTL_RO_NL_CGEN(config_stats, thread_allocatedp, ALLOCATEDP_GET(), uint64_t *)
-CTL_RO_NL_CGEN(config_stats, thread_deallocated, DEALLOCATED_GET(), uint64_t)
-CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, DEALLOCATEDP_GET(),
-    uint64_t *)
+CTL_RO_NL_CGEN(config_stats, thread_allocated,
+    thread_allocated_tsd_get()->allocated, uint64_t)
+CTL_RO_NL_CGEN(config_stats, thread_allocatedp,
+    &thread_allocated_tsd_get()->allocated, uint64_t *)
+CTL_RO_NL_CGEN(config_stats, thread_deallocated,
+    thread_allocated_tsd_get()->deallocated, uint64_t)
+CTL_RO_NL_CGEN(config_stats, thread_deallocatedp,
+    &thread_allocated_tsd_get()->deallocated, uint64_t *)
 
 /******************************************************************************/
 
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 2610452..331e473 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -4,36 +4,9 @@
 /******************************************************************************/
 /* Data. */
 
-malloc_mutex_t		arenas_lock;
-arena_t			**arenas;
-unsigned		narenas;
-
-pthread_key_t		arenas_tsd;
-#ifdef JEMALLOC_TLS
-__thread arena_t	*arenas_tls JEMALLOC_ATTR(tls_model("initial-exec"));
-#endif
-
-#ifdef JEMALLOC_TLS
-__thread thread_allocated_t	thread_allocated_tls;
-#endif
-pthread_key_t		thread_allocated_tsd;
-
-/* Set to true once the allocator has been initialized. */
-static bool		malloc_initialized = false;
-
-/* Used to let the initializing thread recursively allocate. */
-static pthread_t	malloc_initializer = (unsigned long)0;
-
-/* Used to avoid initialization races. */
-static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
-
-#ifdef DYNAMIC_PAGE_SHIFT
-size_t		pagesize;
-size_t		pagesize_mask;
-size_t		lg_pagesize;
-#endif
-
-unsigned	ncpus;
+malloc_tsd_data(, arenas, arena_t *, NULL)
+malloc_tsd_data(, thread_allocated, thread_allocated_t,
+    THREAD_ALLOCATED_INITIALIZER)
 
 /* Runtime configuration options. */
 const char	*je_malloc_conf JEMALLOC_ATTR(visibility("default"));
@@ -52,15 +25,32 @@ bool	opt_xmalloc = false;
 bool	opt_zero = false;
 size_t	opt_narenas = 0;
 
+#ifdef DYNAMIC_PAGE_SHIFT
+size_t		pagesize;
+size_t		pagesize_mask;
+size_t		lg_pagesize;
+#endif
+
+unsigned	ncpus;
+
+malloc_mutex_t		arenas_lock;
+arena_t			**arenas;
+unsigned		narenas;
+
+/* Set to true once the allocator has been initialized. */
+static bool		malloc_initialized = false;
+
+/* Used to let the initializing thread recursively allocate. */
+static pthread_t	malloc_initializer = (unsigned long)0;
+
+/* Used to avoid initialization races. */
+static malloc_mutex_t	init_lock = MALLOC_MUTEX_INITIALIZER;
+
 /******************************************************************************/
 /* Function prototypes for non-inline static functions. */
 
 static void	stats_print_atexit(void);
 static unsigned	malloc_ncpus(void);
-static void	arenas_cleanup(void *arg);
-#ifndef JEMALLOC_TLS
-static void	thread_allocated_cleanup(void *arg);
-#endif
 static bool	malloc_conf_next(char const **opts_p, char const **k_p,
     size_t *klen_p, char const **v_p, size_t *vlen_p);
 static void	malloc_conf_error(const char *msg, const char *k, size_t klen,
@@ -156,7 +146,7 @@ choose_arena_hard(void)
 		malloc_mutex_unlock(&arenas_lock);
 	}
 
-	ARENA_SET(ret);
+	arenas_tsd_set(&ret);
 
 	return (ret);
 }
@@ -197,26 +187,6 @@ stats_print_atexit(void)
 	je_malloc_stats_print(NULL, NULL, NULL);
 }
 
-thread_allocated_t *
-thread_allocated_get_hard(void)
-{
-	thread_allocated_t *thread_allocated = (thread_allocated_t *)
-	    imalloc(sizeof(thread_allocated_t));
-	if (thread_allocated == NULL) {
-		static thread_allocated_t static_thread_allocated = {0, 0};
-		malloc_write("<jemalloc>: Error allocating TSD;"
-		    " mallctl(\"thread.{de,}allocated[p]\", ...)"
-		    " will be inaccurate\n");
-		if (opt_abort)
-			abort();
-		return (&static_thread_allocated);
-	}
-	pthread_setspecific(thread_allocated_tsd, thread_allocated);
-	thread_allocated->allocated = 0;
-	thread_allocated->deallocated = 0;
-	return (thread_allocated);
-}
-
 /*
  * End miscellaneous support functions.
  */
@@ -241,32 +211,16 @@ malloc_ncpus(void)
 	return (ret);
 }
 
-static void
+void
 arenas_cleanup(void *arg)
 {
-	arena_t *arena = (arena_t *)arg;
+	arena_t *arena = *(arena_t **)arg;
 
 	malloc_mutex_lock(&arenas_lock);
 	arena->nthreads--;
 	malloc_mutex_unlock(&arenas_lock);
 }
 
-#ifndef JEMALLOC_TLS
-static void
-thread_allocated_cleanup(void *arg)
-{
-	uint64_t *allocated = (uint64_t *)arg;
-
-	if (allocated != NULL)
-		idalloc(allocated);
-}
-#endif
-
-/*
- * FreeBSD's pthreads implementation calls malloc(3), so the malloc
- * implementation has to take pains to avoid infinite recursion during
- * initialization.
- */
 static inline bool
 malloc_init(void)
 {
@@ -604,6 +558,7 @@ malloc_init_hard(void)
 	}
 #endif
 
+	malloc_tsd_boot();
 	if (config_prof)
 		prof_boot0();
 
@@ -631,7 +586,7 @@ malloc_init_hard(void)
 		}
 	}
 
-	if (chunk_boot()) {
+	if (chunk_boot0()) {
 		malloc_mutex_unlock(&init_lock);
 		return (true);
 	}
@@ -646,7 +601,7 @@ malloc_init_hard(void)
 
 	arena_boot();
 
-	if (config_tcache && tcache_boot()) {
+	if (config_tcache && tcache_boot0()) {
 		malloc_mutex_unlock(&init_lock);
 		return (true);
 	}
@@ -656,23 +611,9 @@ malloc_init_hard(void)
 		return (true);
 	}
 
-#ifndef JEMALLOC_TLS
-	/* Initialize allocation counters before any allocations can occur. */
-	if (config_stats && pthread_key_create(&thread_allocated_tsd,
-	    thread_allocated_cleanup) != 0) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-#endif
-
 	if (malloc_mutex_init(&arenas_lock))
 		return (true);
 
-	if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) {
-		malloc_mutex_unlock(&init_lock);
-		return (true);
-	}
-
 	/*
 	 * Create enough scaffolding to allow recursive allocation in
 	 * malloc_ncpus().
@@ -691,25 +632,38 @@ malloc_init_hard(void)
 		return (true);
 	}
 
-	/*
-	 * Assign the initial arena to the initial thread, in order to avoid
-	 * spurious creation of an extra arena if the application switches to
-	 * threaded mode.
-	 */
-	ARENA_SET(arenas[0]);
-	arenas[0]->nthreads++;
+	/* Initialize allocation counters before any allocations can occur. */
+	if (config_stats && thread_allocated_tsd_boot()) {
+		malloc_mutex_unlock(&init_lock);
+		return (true);
+	}
 
 	if (config_prof && prof_boot2()) {
 		malloc_mutex_unlock(&init_lock);
 		return (true);
 	}
 
+	if (arenas_tsd_boot()) {
+		malloc_mutex_unlock(&init_lock);
+		return (true);
+	}
+
+	if (config_tcache && tcache_boot1()) {
+		malloc_mutex_unlock(&init_lock);
+		return (true);
+	}
+
 	/* Get number of CPUs. */
 	malloc_initializer = pthread_self();
 	malloc_mutex_unlock(&init_lock);
 	ncpus = malloc_ncpus();
 	malloc_mutex_lock(&init_lock);
 
+	if (chunk_boot1()) {
+		malloc_mutex_unlock(&init_lock);
+		return (true);
+	}
+
 	if (opt_narenas == 0) {
 		/*
 		 * For SMP systems, create more than one arena per CPU by
@@ -844,7 +798,7 @@ OOM:
 		prof_malloc(ret, usize, cnt);
 	if (config_stats && ret != NULL) {
 		assert(usize == isalloc(ret));
-		ALLOCATED_ADD(usize, 0);
+		thread_allocated_tsd_get()->allocated += usize;
 	}
 	return (ret);
 }
@@ -939,7 +893,7 @@ imemalign(void **memptr, size_t alignment, size_t size,
 RETURN:
 	if (config_stats && result != NULL) {
 		assert(usize == isalloc(result));
-		ALLOCATED_ADD(usize, 0);
+		thread_allocated_tsd_get()->allocated += usize;
 	}
 	if (config_prof && opt_prof && result != NULL)
 		prof_malloc(result, usize, cnt);
@@ -1044,7 +998,7 @@ RETURN:
 		prof_malloc(ret, usize, cnt);
 	if (config_stats && ret != NULL) {
 		assert(usize == isalloc(ret));
-		ALLOCATED_ADD(usize, 0);
+		thread_allocated_tsd_get()->allocated += usize;
 	}
 	return (ret);
 }
@@ -1173,8 +1127,11 @@ RETURN:
 	if (config_prof && opt_prof)
 		prof_realloc(ret, usize, cnt, old_size, old_ctx);
 	if (config_stats && ret != NULL) {
+		thread_allocated_t *ta;
 		assert(usize == isalloc(ret));
-		ALLOCATED_ADD(usize, old_size);
+		ta = thread_allocated_tsd_get();
+		ta->allocated += usize;
+		ta->deallocated += old_size;
 	}
 	return (ret);
 }
@@ -1197,7 +1154,7 @@ je_free(void *ptr)
 			usize = isalloc(ptr);
 		}
 		if (config_stats)
-			ALLOCATED_ADD(0, usize);
+			thread_allocated_tsd_get()->deallocated += usize;
 		idalloc(ptr);
 	}
 }
@@ -1412,7 +1369,7 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
 	*ptr = p;
 	if (config_stats) {
 		assert(usize == isalloc(p));
-		ALLOCATED_ADD(usize, 0);
+		thread_allocated_tsd_get()->allocated += usize;
 	}
 	return (ALLOCM_SUCCESS);
 OOM:
@@ -1502,8 +1459,12 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
 	}
 
 	*ptr = q;
-	if (config_stats)
-		ALLOCATED_ADD(usize, old_size);
+	if (config_stats) {
+		thread_allocated_t *ta;
+		ta = thread_allocated_tsd_get();
+		ta->allocated += usize;
+		ta->deallocated += old_size;
+	}
 	return (ALLOCM_SUCCESS);
 ERR:
 	if (no_move)
@@ -1556,7 +1517,7 @@ je_dallocm(void *ptr, int flags)
 		prof_free(ptr, usize);
 	}
 	if (config_stats)
-		ALLOCATED_ADD(0, usize);
+		thread_allocated_tsd_get()->deallocated += usize;
 	idalloc(ptr);
 
 	return (ALLOCM_SUCCESS);
diff --git a/src/prof.c b/src/prof.c
index 9c32737..ba0b64e 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -14,6 +14,8 @@
 /******************************************************************************/
 /* Data. */
 
+malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL)
+
 bool		opt_prof = false;
 bool		opt_prof_active = true;
 size_t		opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
@@ -26,12 +28,6 @@ char		opt_prof_prefix[PATH_MAX + 1];
 uint64_t	prof_interval;
 bool		prof_promote;
 
-#ifdef JEMALLOC_TLS
-__thread prof_tdata_t	*prof_tdata_tls
-    JEMALLOC_ATTR(tls_model("initial-exec"));
-#endif
-pthread_key_t	prof_tdata_tsd;
-
 /*
  * Global hash of (prof_bt_t *)-->(prof_ctx_t *).  This is the master data
  * structure that knows about all backtraces currently captured.
@@ -50,7 +46,7 @@ static uint64_t		prof_dump_useq;
  * all profile dumps.  The buffer is implicitly protected by bt2ctx_mtx, since
  * it must be locked anyway during dumping.
  */
-static char		prof_dump_buf[PROF_DUMP_BUF_SIZE];
+static char		prof_dump_buf[PROF_DUMP_BUFSIZE];
 static unsigned		prof_dump_buf_end;
 static int		prof_dump_fd;
 
@@ -91,7 +87,6 @@ static void	prof_fdump(void);
 static void	prof_bt_hash(const void *key, unsigned minbits, size_t *hash1,
     size_t *hash2);
 static bool	prof_bt_keycomp(const void *k1, const void *k2);
-static void	prof_tdata_cleanup(void *arg);
 
 /******************************************************************************/
 
@@ -439,7 +434,7 @@ prof_lookup(prof_bt_t *bt)
 
 	cassert(config_prof);
 
-	prof_tdata = PROF_TCACHE_GET();
+	prof_tdata = *prof_tdata_tsd_get();
 	if (prof_tdata == NULL) {
 		prof_tdata = prof_tdata_init();
 		if (prof_tdata == NULL)
@@ -599,16 +594,16 @@ prof_write(bool propagate_err, const char *s)
 	slen = strlen(s);
 	while (i < slen) {
 		/* Flush the buffer if it is full. */
-		if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE)
+		if (prof_dump_buf_end == PROF_DUMP_BUFSIZE)
 			if (prof_flush(propagate_err) && propagate_err)
 				return (true);
 
-		if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) {
+		if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) {
 			/* Finish writing. */
 			n = slen - i;
 		} else {
 			/* Write as much of s as will fit. */
-			n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end;
+			n = PROF_DUMP_BUFSIZE - prof_dump_buf_end;
 		}
 		memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
 		prof_dump_buf_end += n;
@@ -624,10 +619,12 @@ prof_printf(bool propagate_err, const char *format, ...)
 {
 	bool ret;
 	va_list ap;
+	char buf[PROF_PRINTF_BUFSIZE];
 
 	va_start(ap, format);
-	ret = prof_write(propagate_err, malloc_vtprintf(format, ap));
+	malloc_snprintf(buf, sizeof(buf), format, ap);
 	va_end(ap);
+	ret = prof_write(propagate_err, buf);
 
 	return (ret);
 }
@@ -795,11 +792,13 @@ static bool
 prof_dump_maps(bool propagate_err)
 {
 	int mfd;
+	char filename[PATH_MAX + 1];
 
 	cassert(config_prof);
 
-	mfd = open(malloc_tprintf("/proc/%d/maps", (int)getpid()),
-	    O_RDONLY);
+	malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps",
+	    (int)getpid());
+	mfd = open(filename, O_RDONLY);
 	if (mfd != -1) {
 		ssize_t nread;
 
@@ -809,13 +808,13 @@ prof_dump_maps(bool propagate_err)
 		nread = 0;
 		do {
 			prof_dump_buf_end += nread;
-			if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) {
+			if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) {
 				/* Make space in prof_dump_buf before read(). */
 				if (prof_flush(propagate_err) && propagate_err)
 					return (true);
 			}
 			nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
-			    PROF_DUMP_BUF_SIZE - prof_dump_buf_end);
+			    PROF_DUMP_BUFSIZE - prof_dump_buf_end);
 		} while (nread > 0);
 		close(mfd);
 	} else
@@ -1098,16 +1097,16 @@ prof_tdata_init(void)
 	prof_tdata->threshold = 0;
 	prof_tdata->accum = 0;
 
-	PROF_TCACHE_SET(prof_tdata);
+	prof_tdata_tsd_set(&prof_tdata);
 
 	return (prof_tdata);
 }
 
-static void
+void
 prof_tdata_cleanup(void *arg)
 {
 	prof_thr_cnt_t *cnt;
-	prof_tdata_t *prof_tdata = (prof_tdata_t *)arg;
+	prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg;
 
 	cassert(config_prof);
 
@@ -1127,7 +1126,8 @@ prof_tdata_cleanup(void *arg)
 	idalloc(prof_tdata->vec);
 
 	idalloc(prof_tdata);
-	PROF_TCACHE_SET(NULL);
+	prof_tdata = NULL;
+	prof_tdata_tsd_set(&prof_tdata);
 }
 
 void
@@ -1182,8 +1182,7 @@ prof_boot2(void)
 			return (true);
 		if (malloc_mutex_init(&bt2ctx_mtx))
 			return (true);
-		if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup)
-		    != 0) {
+		if (prof_tdata_tsd_boot()) {
 			malloc_write(
 			    "<jemalloc>: Error in pthread_key_create()\n");
 			abort();
diff --git a/src/tcache.c b/src/tcache.c
index f90308c..3442406 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -4,30 +4,16 @@
 /******************************************************************************/
 /* Data. */
 
+malloc_tsd_data(, tcache, tcache_t *, NULL)
+
 bool	opt_tcache = true;
 ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
 
 tcache_bin_info_t	*tcache_bin_info;
 static unsigned		stack_nelms; /* Total stack elms per tcache. */
 
-/* Map of thread-specific caches. */
-#ifdef JEMALLOC_TLS
-__thread tcache_t	*tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
-#endif
-
-/*
- * Same contents as tcache, but initialized such that the TSD destructor is
- * called when a thread exits, so that the cache can be cleaned up.
- */
-pthread_key_t		tcache_tsd;
-
-size_t				nhbins;
-size_t				tcache_maxclass;
-
-/******************************************************************************/
-/* Function prototypes for non-inline static functions. */
-
-static void	tcache_thread_cleanup(void *arg);
+size_t			nhbins;
+size_t			tcache_maxclass;
 
 /******************************************************************************/
 
@@ -196,6 +182,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
 		tbin->low_water = tbin->ncached;
 }
 
+void
+tcache_arena_associate(tcache_t *tcache, arena_t *arena)
+{
+
+	if (config_stats) {
+		/* Link into list of extant tcaches. */
+		malloc_mutex_lock(&arena->lock);
+		ql_elm_new(tcache, link);
+		ql_tail_insert(&arena->tcache_ql, tcache, link);
+		malloc_mutex_unlock(&arena->lock);
+	}
+	tcache->arena = arena;
+}
+
+void
+tcache_arena_dissociate(tcache_t *tcache)
+{
+
+	if (config_stats) {
+		/* Unlink from list of extant tcaches. */
+		malloc_mutex_lock(&tcache->arena->lock);
+		ql_remove(&tcache->arena->tcache_ql, tcache, link);
+		malloc_mutex_unlock(&tcache->arena->lock);
+		tcache_stats_merge(tcache, tcache->arena);
+	}
+}
+
 tcache_t *
 tcache_create(arena_t *arena)
 {
@@ -228,15 +241,8 @@ tcache_create(arena_t *arena)
 	if (tcache == NULL)
 		return (NULL);
 
-	if (config_stats) {
-		/* Link into list of extant tcaches. */
-		malloc_mutex_lock(&arena->lock);
-		ql_elm_new(tcache, link);
-		ql_tail_insert(&arena->tcache_ql, tcache, link);
-		malloc_mutex_unlock(&arena->lock);
-	}
+	tcache_arena_associate(tcache, arena);
 
-	tcache->arena = arena;
 	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
 	for (i = 0; i < nhbins; i++) {
 		tcache->tbins[i].lg_fill_div = 1;
@@ -245,7 +251,7 @@ tcache_create(arena_t *arena)
 		stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
 	}
 
-	TCACHE_SET(tcache);
+	tcache_tsd_set(&tcache);
 
 	return (tcache);
 }
@@ -256,13 +262,7 @@ tcache_destroy(tcache_t *tcache)
 	unsigned i;
 	size_t tcache_size;
 
-	if (config_stats) {
-		/* Unlink from list of extant tcaches. */
-		malloc_mutex_lock(&tcache->arena->lock);
-		ql_remove(&tcache->arena->tcache_ql, tcache, link);
-		malloc_mutex_unlock(&tcache->arena->lock);
-		tcache_stats_merge(tcache, tcache->arena);
-	}
+	tcache_arena_dissociate(tcache);
 
 	for (i = 0; i < NBINS; i++) {
 		tcache_bin_t *tbin = &tcache->tbins[i];
@@ -323,10 +323,10 @@ tcache_destroy(tcache_t *tcache)
 		idalloc(tcache);
 }
 
-static void
+void
 tcache_thread_cleanup(void *arg)
 {
-	tcache_t *tcache = (tcache_t *)arg;
+	tcache_t *tcache = *(tcache_t **)arg;
 
 	if (tcache == (void *)(uintptr_t)1) {
 		/*
@@ -341,11 +341,13 @@ tcache_thread_cleanup(void *arg)
 		 * destructor was called.  Reset tcache to 1 in order to
 		 * receive another callback.
 		 */
-		TCACHE_SET((uintptr_t)1);
+		tcache = (tcache_t *)(uintptr_t)1;
+		tcache_tsd_set(&tcache);
 	} else if (tcache != NULL) {
 		assert(tcache != (void *)(uintptr_t)1);
 		tcache_destroy(tcache);
-		TCACHE_SET((uintptr_t)1);
+		tcache = (tcache_t *)(uintptr_t)1;
+		tcache_tsd_set(&tcache);
 	}
 }
 
@@ -374,7 +376,7 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena)
 }
 
 bool
-tcache_boot(void)
+tcache_boot0(void)
 {
 
 	if (opt_tcache) {
@@ -385,8 +387,8 @@ tcache_boot(void)
 		 * SMALL_MAXCLASS and arena_maxclass are known.
 		 * XXX Can this be done earlier?
 		 */
-		if (opt_lg_tcache_max < 0 || (1U <<
-		    opt_lg_tcache_max) < SMALL_MAXCLASS)
+		if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) <
+		    SMALL_MAXCLASS)
 			tcache_maxclass = SMALL_MAXCLASS;
 		else if ((1U << opt_lg_tcache_max) > arena_maxclass)
 			tcache_maxclass = arena_maxclass;
@@ -416,13 +418,18 @@ tcache_boot(void)
 			tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
 			stack_nelms += tcache_bin_info[i].ncached_max;
 		}
+	}
 
-		if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
-		    0) {
-			malloc_write(
-			    "<jemalloc>: Error in pthread_key_create()\n");
-			abort();
-		}
+	return (false);
+}
+
+bool
+tcache_boot1(void)
+{
+
+	if (opt_tcache) {
+		if (tcache_tsd_boot())
+			return (true);
 	}
 
 	return (false);
diff --git a/src/tsd.c b/src/tsd.c
new file mode 100644
index 0000000..669ea8f
--- /dev/null
+++ b/src/tsd.c
@@ -0,0 +1,72 @@
+#define	JEMALLOC_TSD_C_
+#include "jemalloc/internal/jemalloc_internal.h"
+
+/******************************************************************************/
+/* Data. */
+
+static unsigned ncleanups;
+static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX];
+
+/******************************************************************************/
+
+void *
+malloc_tsd_malloc(size_t size)
+{
+
+	/* Avoid choose_arena() in order to dodge bootstrapping issues. */
+	return arena_malloc_prechosen(arenas[0], size, false);
+}
+
+void
+malloc_tsd_dalloc(void *wrapper)
+{
+
+	idalloc(wrapper);
+}
+
+void
+malloc_tsd_no_cleanup(void *arg)
+{
+
+	not_reached();
+}
+
+#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP
+void
+_malloc_thread_cleanup(void)
+{
+	bool pending[ncleanups], again;
+	unsigned i;
+
+	for (i = 0; i < ncleanups; i++)
+		pending[i] = true;
+
+	do {
+		again = false;
+		for (i = 0; i < ncleanups; i++) {
+			if (pending[i]) {
+				pending[i] = cleanups[i].f(cleanups[i].arg);
+				if (pending[i])
+					again = true;
+			}
+		}
+	} while (again);
+}
+#endif
+
+void
+malloc_tsd_cleanup_register(bool (*f)(void *), void *arg)
+{
+
+	assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX);
+	cleanups[ncleanups].f = f;
+	cleanups[ncleanups].arg = arg;
+	ncleanups++;
+}
+
+void
+malloc_tsd_boot(void)
+{
+
+	ncleanups = 0;
+}
diff --git a/src/util.c b/src/util.c
index 47e7b66..96c87f7 100644
--- a/src/util.c
+++ b/src/util.c
@@ -222,6 +222,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 	case 'z':							\
 		val = va_arg(ap, size_t);				\
 		break;							\
+	case 'p': /* Synthetic; used for %p. */				\
+		val = va_arg(ap, uintptr_t);				\
+		break;							\
 	default: not_reached();						\
 	}								\
 } while (0)
@@ -410,7 +413,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
 				uintmax_t val;
 				char buf[X2S_BUFSIZE];
 
-				GET_ARG_NUMERIC(val, len);
+				GET_ARG_NUMERIC(val, 'p');
 				s = x2s(val, true, false, buf, &slen);
 				APPEND_PADDED_S(s, slen, width, left_justify);
 				f++;
@@ -466,34 +469,11 @@ malloc_snprintf(char *str, size_t size, const char *format, ...)
 	return (ret);
 }
 
-const char *
-malloc_vtprintf(const char *format, va_list ap)
-{
-	static __thread char buf[MALLOC_PRINTF_BUFSIZE];
-
-	malloc_vsnprintf(buf, sizeof(buf), format, ap);
-
-	return (buf);
-}
-
-JEMALLOC_ATTR(format(printf, 1, 2))
-const char *
-malloc_tprintf(const char *format, ...)
-{
-	const char *ret;
-	va_list ap;
-
-	va_start(ap, format);
-	ret = malloc_vtprintf(format, ap);
-	va_end(ap);
-
-	return (ret);
-}
-
 void
 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
     const char *format, va_list ap)
 {
+	char buf[MALLOC_PRINTF_BUFSIZE];
 
 	if (write_cb == NULL) {
 		/*
@@ -505,7 +485,8 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
 		cbopaque = NULL;
 	}
 
-	write_cb(cbopaque, malloc_vtprintf(format, ap));
+	malloc_vsnprintf(buf, sizeof(buf), format, ap);
+	write_cb(cbopaque, buf);
 }
 
 /*