Merge branch 'dev'5.0.0

9 files changed, 537 insertions, 268 deletions

diff --git a/test/include/test/SFMT-alti.h b/test/include/test/SFMT-alti.h
index 0005df6..a1885db 100644
--- a/test/include/test/SFMT-alti.h
+++ b/test/include/test/SFMT-alti.h
@@ -33,8 +33,8 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-/** 
- * @file SFMT-alti.h 
+/**
+ * @file SFMT-alti.h
  *
  * @brief SIMD oriented Fast Mersenne Twister(SFMT)
  * pseudorandom number generator
@@ -95,7 +95,7 @@ vector unsigned int vec_recursion(vector unsigned int a,
  * This function fills the internal state array with pseudorandom
  * integers.
  */
-JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+static inline void gen_rand_all(sfmt_t *ctx) {
     int i;
     vector unsigned int r, r1, r2;
 
@@ -119,10 +119,10 @@ JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
  * This function fills the user-specified array with pseudorandom
  * integers.
  *
- * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param array an 128-bit array to be filled by pseudorandom numbers.
  * @param size number of 128-bit pesudorandom numbers to be generated.
  */
-JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+static inline void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
     int i, j;
     vector unsigned int r, r1, r2;
 
@@ -173,7 +173,7 @@ JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
  * @param array an 128-bit array to be swaped.
  * @param size size of 128-bit array.
  */
-JEMALLOC_INLINE void swap(w128_t *array, int size) {
+static inline void swap(w128_t *array, int size) {
     int i;
     const vector unsigned char perm = ALTI_SWAP;
 
diff --git a/test/include/test/SFMT-sse2.h b/test/include/test/SFMT-sse2.h
index 0314a16..169ad55 100644
--- a/test/include/test/SFMT-sse2.h
+++ b/test/include/test/SFMT-sse2.h
@@ -33,7 +33,7 @@
  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
-/** 
+/**
  * @file  SFMT-sse2.h
  * @brief SIMD oriented Fast Mersenne Twister(SFMT) for Intel SSE2
  *
@@ -60,10 +60,10 @@
  * @param mask 128-bit mask
  * @return output
  */
-JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b, 
+JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b,
 				   __m128i c, __m128i d, __m128i mask) {
     __m128i v, x, y, z;
-    
+
     x = _mm_load_si128(a);
     y = _mm_srli_epi32(*b, SR1);
     z = _mm_srli_si128(c, SR2);
@@ -81,7 +81,7 @@ JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b,
  * This function fills the internal state array with pseudorandom
  * integers.
  */
-JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
+static inline void gen_rand_all(sfmt_t *ctx) {
     int i;
     __m128i r, r1, r2, mask;
     mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
@@ -108,10 +108,10 @@ JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) {
  * This function fills the user-specified array with pseudorandom
  * integers.
  *
- * @param array an 128-bit array to be filled by pseudorandom numbers.  
+ * @param array an 128-bit array to be filled by pseudorandom numbers.
  * @param size number of 128-bit pesudorandom numbers to be generated.
  */
-JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
+static inline void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) {
     int i, j;
     __m128i r, r1, r2, mask;
     mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1);
diff --git a/test/include/test/SFMT.h b/test/include/test/SFMT.h
index 09c1607..863fc55 100644
--- a/test/include/test/SFMT.h
+++ b/test/include/test/SFMT.h
@@ -81,91 +81,66 @@ const char *get_idstring(void);
 int get_min_array_size32(void);
 int get_min_array_size64(void);
 
-#ifndef JEMALLOC_ENABLE_INLINE
-double to_real1(uint32_t v);
-double genrand_real1(sfmt_t *ctx);
-double to_real2(uint32_t v);
-double genrand_real2(sfmt_t *ctx);
-double to_real3(uint32_t v);
-double genrand_real3(sfmt_t *ctx);
-double to_res53(uint64_t v);
-double to_res53_mix(uint32_t x, uint32_t y);
-double genrand_res53(sfmt_t *ctx);
-double genrand_res53_mix(sfmt_t *ctx);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(SFMT_C_))
 /* These real versions are due to Isaku Wada */
 /** generates a random number on [0,1]-real-interval */
-JEMALLOC_INLINE double to_real1(uint32_t v)
-{
+static inline double to_real1(uint32_t v) {
     return v * (1.0/4294967295.0); 
     /* divided by 2^32-1 */ 
 }
 
 /** generates a random number on [0,1]-real-interval */
-JEMALLOC_INLINE double genrand_real1(sfmt_t *ctx)
-{
+static inline double genrand_real1(sfmt_t *ctx) {
     return to_real1(gen_rand32(ctx));
 }
 
 /** generates a random number on [0,1)-real-interval */
-JEMALLOC_INLINE double to_real2(uint32_t v)
-{
+static inline double to_real2(uint32_t v) {
     return v * (1.0/4294967296.0); 
     /* divided by 2^32 */
 }
 
 /** generates a random number on [0,1)-real-interval */
-JEMALLOC_INLINE double genrand_real2(sfmt_t *ctx)
-{
+static inline double genrand_real2(sfmt_t *ctx) {
     return to_real2(gen_rand32(ctx));
 }
 
 /** generates a random number on (0,1)-real-interval */
-JEMALLOC_INLINE double to_real3(uint32_t v)
-{
+static inline double to_real3(uint32_t v) {
     return (((double)v) + 0.5)*(1.0/4294967296.0); 
     /* divided by 2^32 */
 }
 
 /** generates a random number on (0,1)-real-interval */
-JEMALLOC_INLINE double genrand_real3(sfmt_t *ctx)
-{
+static inline double genrand_real3(sfmt_t *ctx) {
     return to_real3(gen_rand32(ctx));
 }
 /** These real versions are due to Isaku Wada */
 
 /** generates a random number on [0,1) with 53-bit resolution*/
-JEMALLOC_INLINE double to_res53(uint64_t v) 
-{ 
+static inline double to_res53(uint64_t v) {
     return v * (1.0/18446744073709551616.0L);
 }
 
 /** generates a random number on [0,1) with 53-bit resolution from two
  * 32 bit integers */
-JEMALLOC_INLINE double to_res53_mix(uint32_t x, uint32_t y) 
-{ 
+static inline double to_res53_mix(uint32_t x, uint32_t y) {
     return to_res53(x | ((uint64_t)y << 32));
 }
 
 /** generates a random number on [0,1) with 53-bit resolution
  */
-JEMALLOC_INLINE double genrand_res53(sfmt_t *ctx) 
-{ 
+static inline double genrand_res53(sfmt_t *ctx) {
     return to_res53(gen_rand64(ctx));
-} 
+}
 
 /** generates a random number on [0,1) with 53-bit resolution
     using 32bit integer.
  */
-JEMALLOC_INLINE double genrand_res53_mix(sfmt_t *ctx) 
-{ 
+static inline double genrand_res53_mix(sfmt_t *ctx) {
     uint32_t x, y;
 
     x = gen_rand32(ctx);
     y = gen_rand32(ctx);
     return to_res53_mix(x, y);
-} 
-#endif
+}
 #endif
diff --git a/test/include/test/btalloc.h b/test/include/test/btalloc.h
index c3f9d4d..5877ea7 100644
--- a/test/include/test/btalloc.h
+++ b/test/include/test/btalloc.h
@@ -1,20 +1,19 @@
 /* btalloc() provides a mechanism for allocating via permuted backtraces. */
 void	*btalloc(size_t size, unsigned bits);
 
-#define	btalloc_n_proto(n)						\
+#define btalloc_n_proto(n)						\
 void	*btalloc_##n(size_t size, unsigned bits);
 btalloc_n_proto(0)
 btalloc_n_proto(1)
 
-#define	btalloc_n_gen(n)						\
+#define btalloc_n_gen(n)						\
 void *									\
-btalloc_##n(size_t size, unsigned bits)					\
-{									\
+btalloc_##n(size_t size, unsigned bits) {				\
 	void *p;							\
 									\
-	if (bits == 0)							\
+	if (bits == 0) {						\
 		p = mallocx(size, 0);					\
-	else {								\
+	} else {							\
 		switch (bits & 0x1U) {					\
 		case 0:							\
 			p = (btalloc_0(size, bits >> 1));		\
@@ -27,5 +26,5 @@ btalloc_##n(size_t size, unsigned bits)					\
 	}								\
 	/* Intentionally sabotage tail call optimization. */		\
 	assert_ptr_not_null(p, "Unexpected mallocx() failure");		\
-	return (p);							\
+	return p;							\
 }
diff --git a/test/include/test/extent_hooks.h b/test/include/test/extent_hooks.h
new file mode 100644
index 0000000..ea01285
--- /dev/null
+++ b/test/include/test/extent_hooks.h
@@ -0,0 +1,287 @@
+/*
+ * Boilerplate code used for testing extent hooks via interception and
+ * passthrough.
+ */
+
+static void	*extent_alloc_hook(extent_hooks_t *extent_hooks, void *new_addr,
+    size_t size, size_t alignment, bool *zero, bool *commit,
+    unsigned arena_ind);
+static bool	extent_dalloc_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, bool committed, unsigned arena_ind);
+static void	extent_destroy_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, bool committed, unsigned arena_ind);
+static bool	extent_commit_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_decommit_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_purge_lazy_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_purge_forced_hook(extent_hooks_t *extent_hooks,
+    void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind);
+static bool	extent_split_hook(extent_hooks_t *extent_hooks, void *addr,
+    size_t size, size_t size_a, size_t size_b, bool committed,
+    unsigned arena_ind);
+static bool	extent_merge_hook(extent_hooks_t *extent_hooks, void *addr_a,
+    size_t size_a, void *addr_b, size_t size_b, bool committed,
+    unsigned arena_ind);
+
+static extent_hooks_t *default_hooks;
+static extent_hooks_t hooks = {
+	extent_alloc_hook,
+	extent_dalloc_hook,
+	extent_destroy_hook,
+	extent_commit_hook,
+	extent_decommit_hook,
+	extent_purge_lazy_hook,
+	extent_purge_forced_hook,
+	extent_split_hook,
+	extent_merge_hook
+};
+
+/* Control whether hook functions pass calls through to default hooks. */
+static bool try_alloc = true;
+static bool try_dalloc = true;
+static bool try_destroy = true;
+static bool try_commit = true;
+static bool try_decommit = true;
+static bool try_purge_lazy = true;
+static bool try_purge_forced = true;
+static bool try_split = true;
+static bool try_merge = true;
+
+/* Set to false prior to operations, then introspect after operations. */
+static bool called_alloc;
+static bool called_dalloc;
+static bool called_destroy;
+static bool called_commit;
+static bool called_decommit;
+static bool called_purge_lazy;
+static bool called_purge_forced;
+static bool called_split;
+static bool called_merge;
+
+/* Set to false prior to operations, then introspect after operations. */
+static bool did_alloc;
+static bool did_dalloc;
+static bool did_destroy;
+static bool did_commit;
+static bool did_decommit;
+static bool did_purge_lazy;
+static bool did_purge_forced;
+static bool did_split;
+static bool did_merge;
+
+#if 0
+#  define TRACE_HOOK(fmt, ...) malloc_printf(fmt, __VA_ARGS__)
+#else
+#  define TRACE_HOOK(fmt, ...)
+#endif
+
+static void *
+extent_alloc_hook(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
+    size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
+	void *ret;
+
+	TRACE_HOOK("%s(extent_hooks=%p, new_addr=%p, size=%zu, alignment=%zu, "
+	    "*zero=%s, *commit=%s, arena_ind=%u)\n", __func__, extent_hooks,
+	    new_addr, size, alignment, *zero ?  "true" : "false", *commit ?
+	    "true" : "false", arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->alloc, extent_alloc_hook,
+	    "Wrong hook function");
+	called_alloc = true;
+	if (!try_alloc) {
+		return NULL;
+	}
+	ret = default_hooks->alloc(default_hooks, new_addr, size, alignment,
+	    zero, commit, 0);
+	did_alloc = (ret != NULL);
+	return ret;
+}
+
+static bool
+extent_dalloc_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    bool committed, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
+	    "arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
+	    "true" : "false", arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->dalloc, extent_dalloc_hook,
+	    "Wrong hook function");
+	called_dalloc = true;
+	if (!try_dalloc) {
+		return true;
+	}
+	err = default_hooks->dalloc(default_hooks, addr, size, committed, 0);
+	did_dalloc = !err;
+	return err;
+}
+
+static void
+extent_destroy_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    bool committed, unsigned arena_ind) {
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, committed=%s, "
+	    "arena_ind=%u)\n", __func__, extent_hooks, addr, size, committed ?
+	    "true" : "false", arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->destroy, extent_destroy_hook,
+	    "Wrong hook function");
+	called_destroy = true;
+	if (!try_destroy) {
+		return;
+	}
+	default_hooks->destroy(default_hooks, addr, size, committed, 0);
+	did_destroy = true;
+}
+
+static bool
+extent_commit_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t offset, size_t length, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
+	    "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
+	    offset, length, arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->commit, extent_commit_hook,
+	    "Wrong hook function");
+	called_commit = true;
+	if (!try_commit) {
+		return true;
+	}
+	err = default_hooks->commit(default_hooks, addr, size, offset, length,
+	    0);
+	did_commit = !err;
+	return err;
+}
+
+static bool
+extent_decommit_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t offset, size_t length, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
+	    "length=%zu, arena_ind=%u)\n", __func__, extent_hooks, addr, size,
+	    offset, length, arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->decommit, extent_decommit_hook,
+	    "Wrong hook function");
+	called_decommit = true;
+	if (!try_decommit) {
+		return true;
+	}
+	err = default_hooks->decommit(default_hooks, addr, size, offset, length,
+	    0);
+	did_decommit = !err;
+	return err;
+}
+
+static bool
+extent_purge_lazy_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t offset, size_t length, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
+	    "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
+	    offset, length, arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->purge_lazy, extent_purge_lazy_hook,
+	    "Wrong hook function");
+	called_purge_lazy = true;
+	if (!try_purge_lazy) {
+		return true;
+	}
+	err = default_hooks->purge_lazy == NULL ||
+	    default_hooks->purge_lazy(default_hooks, addr, size, offset, length,
+	    0);
+	did_purge_lazy = !err;
+	return err;
+}
+
+static bool
+extent_purge_forced_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t offset, size_t length, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, offset=%zu, "
+	    "length=%zu arena_ind=%u)\n", __func__, extent_hooks, addr, size,
+	    offset, length, arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->purge_forced, extent_purge_forced_hook,
+	    "Wrong hook function");
+	called_purge_forced = true;
+	if (!try_purge_forced) {
+		return true;
+	}
+	err = default_hooks->purge_forced == NULL ||
+	    default_hooks->purge_forced(default_hooks, addr, size, offset,
+	    length, 0);
+	did_purge_forced = !err;
+	return err;
+}
+
+static bool
+extent_split_hook(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr=%p, size=%zu, size_a=%zu, "
+	    "size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
+	    addr, size, size_a, size_b, committed ? "true" : "false",
+	    arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->split, extent_split_hook,
+	    "Wrong hook function");
+	called_split = true;
+	if (!try_split) {
+		return true;
+	}
+	err = (default_hooks->split == NULL ||
+	    default_hooks->split(default_hooks, addr, size, size_a, size_b,
+	    committed, 0));
+	did_split = !err;
+	return err;
+}
+
+static bool
+extent_merge_hook(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
+    void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
+	bool err;
+
+	TRACE_HOOK("%s(extent_hooks=%p, addr_a=%p, size_a=%zu, addr_b=%p "
+	    "size_b=%zu, committed=%s, arena_ind=%u)\n", __func__, extent_hooks,
+	    addr_a, size_a, addr_b, size_b, committed ? "true" : "false",
+	    arena_ind);
+	assert_ptr_eq(extent_hooks, &hooks,
+	    "extent_hooks should be same as pointer used to set hooks");
+	assert_ptr_eq(extent_hooks->merge, extent_merge_hook,
+	    "Wrong hook function");
+	called_merge = true;
+	if (!try_merge) {
+		return true;
+	}
+	err = (default_hooks->merge == NULL ||
+	    default_hooks->merge(default_hooks, addr_a, size_a, addr_b, size_b,
+	    committed, 0));
+	did_merge = !err;
+	return err;
+}
+
+static void
+extent_hooks_prep(void) {
+	size_t sz;
+
+	sz = sizeof(default_hooks);
+	assert_d_eq(mallctl("arena.0.extent_hooks", (void *)&default_hooks, &sz,
+	    NULL, 0), 0, "Unexpected mallctl() error");
+}
diff --git a/test/include/test/jemalloc_test.h.in b/test/include/test/jemalloc_test.h.in
index 1f36e46..67caa86 100644
--- a/test/include/test/jemalloc_test.h.in
+++ b/test/include/test/jemalloc_test.h.in
@@ -1,3 +1,7 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include <limits.h>
 #ifndef SIZE_T_MAX
 #  define SIZE_T_MAX	SIZE_MAX
@@ -39,7 +43,8 @@
 #ifdef JEMALLOC_UNIT_TEST
 #  define JEMALLOC_JET
 #  define JEMALLOC_MANGLE
-#  include "jemalloc/internal/jemalloc_internal.h"
+#  include "jemalloc/internal/jemalloc_preamble.h"
+#  include "jemalloc/internal/jemalloc_internal_includes.h"
 
 /******************************************************************************/
 /*
@@ -47,7 +52,8 @@
  * expose the minimum necessary internal utility code (to avoid re-implementing
  * essentially identical code within the test infrastructure).
  */
-#elif defined(JEMALLOC_INTEGRATION_TEST)
+#elif defined(JEMALLOC_INTEGRATION_TEST) || \
+    defined(JEMALLOC_INTEGRATION_CPP_TEST)
 #  define JEMALLOC_MANGLE
 #  include "jemalloc/jemalloc@install_suffix@.h"
 #  include "jemalloc/internal/jemalloc_internal_defs.h"
@@ -63,19 +69,17 @@ static const bool config_debug =
 
 #  define JEMALLOC_N(n) @private_namespace@##n
 #  include "jemalloc/internal/private_namespace.h"
+#  include "jemalloc/internal/hooks.h"
 
-#  define JEMALLOC_H_TYPES
-#  define JEMALLOC_H_STRUCTS
-#  define JEMALLOC_H_EXTERNS
-#  define JEMALLOC_H_INLINES
+/* Hermetic headers. */
+#  include "jemalloc/internal/assert.h"
+#  include "jemalloc/internal/malloc_io.h"
 #  include "jemalloc/internal/nstime.h"
 #  include "jemalloc/internal/util.h"
+
+/* Non-hermetic headers. */
 #  include "jemalloc/internal/qr.h"
 #  include "jemalloc/internal/ql.h"
-#  undef JEMALLOC_H_TYPES
-#  undef JEMALLOC_H_STRUCTS
-#  undef JEMALLOC_H_EXTERNS
-#  undef JEMALLOC_H_INLINES
 
 /******************************************************************************/
 /*
@@ -90,7 +94,8 @@ static const bool config_debug =
 #  include "jemalloc/jemalloc_protos_jet.h"
 
 #  define JEMALLOC_JET
-#  include "jemalloc/internal/jemalloc_internal.h"
+#  include "jemalloc/internal/jemalloc_preamble.h"
+#  include "jemalloc/internal/jemalloc_internal_includes.h"
 #  include "jemalloc/internal/public_unnamespace.h"
 #  undef JEMALLOC_JET
 
@@ -122,7 +127,7 @@ static const bool config_debug =
 #include "test/test.h"
 #include "test/timer.h"
 #include "test/thd.h"
-#define	MEXP 19937
+#define MEXP 19937
 #include "test/SFMT.h"
 
 /******************************************************************************/
@@ -135,7 +140,7 @@ static const bool config_debug =
 #undef not_implemented
 #undef assert_not_implemented
 
-#define	assert(e) do {							\
+#define assert(e) do {							\
 	if (!(e)) {							\
 		malloc_printf(						\
 		    "<jemalloc>: %s:%d: Failed assertion: \"%s\"\n",	\
@@ -144,20 +149,25 @@ static const bool config_debug =
 	}								\
 } while (0)
 
-#define	not_reached() do {						\
+#define not_reached() do {						\
 	malloc_printf(							\
 	    "<jemalloc>: %s:%d: Unreachable code reached\n",		\
 	    __FILE__, __LINE__);					\
 	abort();							\
 } while (0)
 
-#define	not_implemented() do {						\
+#define not_implemented() do {						\
 	malloc_printf("<jemalloc>: %s:%d: Not implemented\n",		\
 	    __FILE__, __LINE__);					\
 	abort();							\
 } while (0)
 
-#define	assert_not_implemented(e) do {					\
-	if (!(e))							\
+#define assert_not_implemented(e) do {					\
+	if (!(e)) {							\
 		not_implemented();					\
+	}								\
 } while (0)
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/test/include/test/math.h b/test/include/test/math.h
index b057b29..efba086 100644
--- a/test/include/test/math.h
+++ b/test/include/test/math.h
@@ -1,12 +1,3 @@
-#ifndef JEMALLOC_ENABLE_INLINE
-double	ln_gamma(double x);
-double	i_gamma(double x, double p, double ln_gamma_p);
-double	pt_norm(double p);
-double	pt_chi2(double p, double df, double ln_gamma_df_2);
-double	pt_gamma(double p, double shape, double scale, double ln_gamma_shape);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(MATH_C_))
 /*
  * Compute the natural log of Gamma(x), accurate to 10 decimal places.
  *
@@ -15,9 +6,8 @@ double	pt_gamma(double p, double shape, double scale, double ln_gamma_shape);
  *   Pike, M.C., I.D. Hill (1966) Algorithm 291: Logarithm of Gamma function
  *   [S14].  Communications of the ACM 9(9):684.
  */
-JEMALLOC_INLINE double
-ln_gamma(double x)
-{
+static inline double
+ln_gamma(double x) {
 	double f, z;
 
 	assert(x > 0.0);
@@ -31,14 +21,15 @@ ln_gamma(double x)
 		}
 		x = z;
 		f = -log(f);
-	} else
+	} else {
 		f = 0.0;
+	}
 
 	z = 1.0 / (x * x);
 
-	return (f + (x-0.5) * log(x) - x + 0.918938533204673 +
+	return f + (x-0.5) * log(x) - x + 0.918938533204673 +
 	    (((-0.000595238095238 * z + 0.000793650793651) * z -
-	    0.002777777777778) * z + 0.083333333333333) / x);
+	    0.002777777777778) * z + 0.083333333333333) / x;
 }
 
 /*
@@ -50,9 +41,8 @@ ln_gamma(double x)
  *   Bhattacharjee, G.P. (1970) Algorithm AS 32: The incomplete Gamma integral.
  *   Applied Statistics 19:285-287.
  */
-JEMALLOC_INLINE double
-i_gamma(double x, double p, double ln_gamma_p)
-{
+static inline double
+i_gamma(double x, double p, double ln_gamma_p) {
 	double acu, factor, oflo, gin, term, rn, a, b, an, dif;
 	double pn[6];
 	unsigned i;
@@ -60,8 +50,9 @@ i_gamma(double x, double p, double ln_gamma_p)
 	assert(p > 0.0);
 	assert(x >= 0.0);
 
-	if (x == 0.0)
-		return (0.0);
+	if (x == 0.0) {
+		return 0.0;
+	}
 
 	acu = 1.0e-10;
 	oflo = 1.0e30;
@@ -80,7 +71,7 @@ i_gamma(double x, double p, double ln_gamma_p)
 			gin += term;
 			if (term <= acu) {
 				gin *= factor / p;
-				return (gin);
+				return gin;
 			}
 		}
 	} else {
@@ -99,23 +90,26 @@ i_gamma(double x, double p, double ln_gamma_p)
 			b += 2.0;
 			term += 1.0;
 			an = a * term;
-			for (i = 0; i < 2; i++)
+			for (i = 0; i < 2; i++) {
 				pn[i+4] = b * pn[i+2] - an * pn[i];
+			}
 			if (pn[5] != 0.0) {
 				rn = pn[4] / pn[5];
 				dif = fabs(gin - rn);
 				if (dif <= acu && dif <= acu * rn) {
 					gin = 1.0 - factor * gin;
-					return (gin);
+					return gin;
 				}
 				gin = rn;
 			}
-			for (i = 0; i < 4; i++)
+			for (i = 0; i < 4; i++) {
 				pn[i] = pn[i+2];
+			}
 
 			if (fabs(pn[4]) >= oflo) {
-				for (i = 0; i < 4; i++)
+				for (i = 0; i < 4; i++) {
 					pn[i] /= oflo;
+				}
 			}
 		}
 	}
@@ -131,9 +125,8 @@ i_gamma(double x, double p, double ln_gamma_p)
  *   Wichura, M.J. (1988) Algorithm AS 241: The percentage points of the normal
  *   distribution.  Applied Statistics 37(3):477-484.
  */
-JEMALLOC_INLINE double
-pt_norm(double p)
-{
+static inline double
+pt_norm(double p) {
 	double q, r, ret;
 
 	assert(p > 0.0 && p < 1.0);
@@ -142,7 +135,7 @@ pt_norm(double p)
 	if (fabs(q) <= 0.425) {
 		/* p close to 1/2. */
 		r = 0.180625 - q * q;
-		return (q * (((((((2.5090809287301226727e3 * r +
+		return q * (((((((2.5090809287301226727e3 * r +
 		    3.3430575583588128105e4) * r + 6.7265770927008700853e4) * r
 		    + 4.5921953931549871457e4) * r + 1.3731693765509461125e4) *
 		    r + 1.9715909503065514427e3) * r + 1.3314166789178437745e2)
@@ -151,12 +144,13 @@ pt_norm(double p)
 		    2.8729085735721942674e4) * r + 3.9307895800092710610e4) * r
 		    + 2.1213794301586595867e4) * r + 5.3941960214247511077e3) *
 		    r + 6.8718700749205790830e2) * r + 4.2313330701600911252e1)
-		    * r + 1.0));
+		    * r + 1.0);
 	} else {
-		if (q < 0.0)
+		if (q < 0.0) {
 			r = p;
-		else
+		} else {
 			r = 1.0 - p;
+		}
 		assert(r > 0.0);
 
 		r = sqrt(-log(r));
@@ -198,9 +192,10 @@ pt_norm(double p)
 			    5.99832206555887937690e-1)
 			    * r + 1.0));
 		}
-		if (q < 0.0)
+		if (q < 0.0) {
 			ret = -ret;
-		return (ret);
+		}
+		return ret;
 	}
 }
 
@@ -218,9 +213,8 @@ pt_norm(double p)
  *   Shea, B.L. (1991) Algorithm AS R85: A remark on AS 91: The percentage
  *   points of the Chi^2 distribution.  Applied Statistics 40(1):233-235.
  */
-JEMALLOC_INLINE double
-pt_chi2(double p, double df, double ln_gamma_df_2)
-{
+static inline double
+pt_chi2(double p, double df, double ln_gamma_df_2) {
 	double e, aa, xx, c, ch, a, q, p1, p2, t, x, b, s1, s2, s3, s4, s5, s6;
 	unsigned i;
 
@@ -236,8 +230,9 @@ pt_chi2(double p, double df, double ln_gamma_df_2)
 	if (df < -1.24 * log(p)) {
 		/* Starting approximation for small Chi^2. */
 		ch = pow(p * xx * exp(ln_gamma_df_2 + xx * aa), 1.0 / xx);
-		if (ch - e < 0.0)
-			return (ch);
+		if (ch - e < 0.0) {
+			return ch;
+		}
 	} else {
 		if (df > 0.32) {
 			x = pt_norm(p);
@@ -263,8 +258,9 @@ pt_chi2(double p, double df, double ln_gamma_df_2)
 				    * (13.32 + 3.0 * ch)) / p2;
 				ch -= (1.0 - exp(a + ln_gamma_df_2 + 0.5 * ch +
 				    c * aa) * p2 / p1) / t;
-				if (fabs(q / ch - 1.0) - 0.01 <= 0.0)
+				if (fabs(q / ch - 1.0) - 0.01 <= 0.0) {
 					break;
+				}
 			}
 		}
 	}
@@ -273,8 +269,9 @@ pt_chi2(double p, double df, double ln_gamma_df_2)
 		/* Calculation of seven-term Taylor series. */
 		q = ch;
 		p1 = 0.5 * ch;
-		if (p1 < 0.0)
-			return (-1.0);
+		if (p1 < 0.0) {
+			return -1.0;
+		}
 		p2 = p - i_gamma(p1, xx, ln_gamma_df_2);
 		t = p2 * exp(xx * aa + ln_gamma_df_2 + p1 - c * log(ch));
 		b = t / ch;
@@ -290,11 +287,12 @@ pt_chi2(double p, double df, double ln_gamma_df_2)
 		s6 = (120.0 + c * (346.0 + 127.0 * c)) / 5040.0;
 		ch += t * (1.0 + 0.5 * t * s1 - b * c * (s1 - b * (s2 - b * (s3
 		    - b * (s4 - b * (s5 - b * s6))))));
-		if (fabs(q / ch - 1.0) <= e)
+		if (fabs(q / ch - 1.0) <= e) {
 			break;
+		}
 	}
 
-	return (ch);
+	return ch;
 }
 
 /*
@@ -302,10 +300,7 @@ pt_chi2(double p, double df, double ln_gamma_df_2)
  * compute the upper limit on the definite integral from [0..z] that satisfies
  * p.
  */
-JEMALLOC_INLINE double
-pt_gamma(double p, double shape, double scale, double ln_gamma_shape)
-{
-
-	return (pt_chi2(p, shape * 2.0, ln_gamma_shape) * 0.5 * scale);
+static inline double
+pt_gamma(double p, double shape, double scale, double ln_gamma_shape) {
+	return pt_chi2(p, shape * 2.0, ln_gamma_shape) * 0.5 * scale;
 }
-#endif
diff --git a/test/include/test/mq.h b/test/include/test/mq.h
index 7c4df49..af2c078 100644
--- a/test/include/test/mq.h
+++ b/test/include/test/mq.h
@@ -26,9 +26,9 @@ void	mq_nanosleep(unsigned ns);
  * does not perform any cleanup of messages, since it knows nothing of their
  * payloads.
  */
-#define	mq_msg(a_mq_msg_type)	ql_elm(a_mq_msg_type)
+#define mq_msg(a_mq_msg_type)	ql_elm(a_mq_msg_type)
 
-#define	mq_gen(a_attr, a_prefix, a_mq_type, a_mq_msg_type, a_field)	\
+#define mq_gen(a_attr, a_prefix, a_mq_type, a_mq_msg_type, a_field)	\
 typedef struct {							\
 	mtx_t			lock;					\
 	ql_head(a_mq_msg_type)	msgs;					\
@@ -37,31 +37,28 @@ typedef struct {							\
 a_attr bool								\
 a_prefix##init(a_mq_type *mq) {						\
 									\
-	if (mtx_init(&mq->lock))					\
-		return (true);						\
+	if (mtx_init(&mq->lock)) {					\
+		return true;						\
+	}								\
 	ql_new(&mq->msgs);						\
 	mq->count = 0;							\
-	return (false);							\
+	return false;							\
 }									\
 a_attr void								\
-a_prefix##fini(a_mq_type *mq)						\
-{									\
-									\
+a_prefix##fini(a_mq_type *mq) {						\
 	mtx_fini(&mq->lock);						\
 }									\
 a_attr unsigned								\
-a_prefix##count(a_mq_type *mq)						\
-{									\
+a_prefix##count(a_mq_type *mq) {					\
 	unsigned count;							\
 									\
 	mtx_lock(&mq->lock);						\
 	count = mq->count;						\
 	mtx_unlock(&mq->lock);						\
-	return (count);							\
+	return count;							\
 }									\
 a_attr a_mq_msg_type *							\
-a_prefix##tryget(a_mq_type *mq)						\
-{									\
+a_prefix##tryget(a_mq_type *mq) {					\
 	a_mq_msg_type *msg;						\
 									\
 	mtx_lock(&mq->lock);						\
@@ -71,35 +68,36 @@ a_prefix##tryget(a_mq_type *mq)						\
 		mq->count--;						\
 	}								\
 	mtx_unlock(&mq->lock);						\
-	return (msg);							\
+	return msg;							\
 }									\
 a_attr a_mq_msg_type *							\
-a_prefix##get(a_mq_type *mq)						\
-{									\
+a_prefix##get(a_mq_type *mq) {						\
 	a_mq_msg_type *msg;						\
 	unsigned ns;							\
 									\
 	msg = a_prefix##tryget(mq);					\
-	if (msg != NULL)						\
-		return (msg);						\
+	if (msg != NULL) {						\
+		return msg;						\
+	}								\
 									\
 	ns = 1;								\
 	while (true) {							\
 		mq_nanosleep(ns);					\
 		msg = a_prefix##tryget(mq);				\
-		if (msg != NULL)					\
-			return (msg);					\
+		if (msg != NULL) {					\
+			return msg;					\
+		}							\
 		if (ns < 1000*1000*1000) {				\
 			/* Double sleep time, up to max 1 second. */	\
 			ns <<= 1;					\
-			if (ns > 1000*1000*1000)			\
+			if (ns > 1000*1000*1000) {			\
 				ns = 1000*1000*1000;			\
+			}						\
 		}							\
 	}								\
 }									\
 a_attr void								\
-a_prefix##put(a_mq_type *mq, a_mq_msg_type *msg)			\
-{									\
+a_prefix##put(a_mq_type *mq, a_mq_msg_type *msg) {			\
 									\
 	mtx_lock(&mq->lock);						\
 	ql_elm_new(msg, a_field);					\
diff --git a/test/include/test/test.h b/test/include/test/test.h
index c8112eb..fd0e526 100644
--- a/test/include/test/test.h
+++ b/test/include/test/test.h
@@ -1,6 +1,6 @@
-#define	ASSERT_BUFSIZE	256
+#define ASSERT_BUFSIZE	256
 
-#define	assert_cmp(t, a, b, cmp, neg_cmp, pri, ...) do {		\
+#define assert_cmp(t, a, b, cmp, neg_cmp, pri, ...) do {		\
 	t a_ = (a);							\
 	t b_ = (b);							\
 	if (!(a_ cmp b_)) {						\
@@ -8,8 +8,8 @@
 		char message[ASSERT_BUFSIZE];				\
 		malloc_snprintf(prefix, sizeof(prefix),			\
 		    "%s:%s:%d: Failed assertion: "			\
-		    "(%s) "#cmp" (%s) --> "				\
-		    "%"pri" "#neg_cmp" %"pri": ",			\
+		    "(%s) " #cmp " (%s) --> "				\
+		    "%" pri " " #neg_cmp " %" pri ": ",			\
 		    __func__, __FILE__, __LINE__,			\
 		    #a, #b, a_, b_);					\
 		malloc_snprintf(message, sizeof(message), __VA_ARGS__);	\
@@ -17,200 +17,200 @@
 	}								\
 } while (0)
 
-#define	assert_ptr_eq(a, b, ...)	assert_cmp(void *, a, b, ==,	\
+#define assert_ptr_eq(a, b, ...)	assert_cmp(void *, a, b, ==,	\
     !=, "p", __VA_ARGS__)
-#define	assert_ptr_ne(a, b, ...)	assert_cmp(void *, a, b, !=,	\
+#define assert_ptr_ne(a, b, ...)	assert_cmp(void *, a, b, !=,	\
     ==, "p", __VA_ARGS__)
-#define	assert_ptr_null(a, ...)		assert_cmp(void *, a, NULL, ==,	\
+#define assert_ptr_null(a, ...)		assert_cmp(void *, a, NULL, ==,	\
     !=, "p", __VA_ARGS__)
-#define	assert_ptr_not_null(a, ...)	assert_cmp(void *, a, NULL, !=,	\
+#define assert_ptr_not_null(a, ...)	assert_cmp(void *, a, NULL, !=,	\
     ==, "p", __VA_ARGS__)
 
-#define	assert_c_eq(a, b, ...)	assert_cmp(char, a, b, ==, !=, "c", __VA_ARGS__)
-#define	assert_c_ne(a, b, ...)	assert_cmp(char, a, b, !=, ==, "c", __VA_ARGS__)
-#define	assert_c_lt(a, b, ...)	assert_cmp(char, a, b, <, >=, "c", __VA_ARGS__)
-#define	assert_c_le(a, b, ...)	assert_cmp(char, a, b, <=, >, "c", __VA_ARGS__)
-#define	assert_c_ge(a, b, ...)	assert_cmp(char, a, b, >=, <, "c", __VA_ARGS__)
-#define	assert_c_gt(a, b, ...)	assert_cmp(char, a, b, >, <=, "c", __VA_ARGS__)
-
-#define	assert_x_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "#x", __VA_ARGS__)
-#define	assert_x_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "#x", __VA_ARGS__)
-#define	assert_x_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "#x", __VA_ARGS__)
-#define	assert_x_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "#x", __VA_ARGS__)
-#define	assert_x_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "#x", __VA_ARGS__)
-#define	assert_x_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "#x", __VA_ARGS__)
-
-#define	assert_d_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "d", __VA_ARGS__)
-#define	assert_d_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "d", __VA_ARGS__)
-#define	assert_d_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "d", __VA_ARGS__)
-#define	assert_d_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "d", __VA_ARGS__)
-#define	assert_d_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "d", __VA_ARGS__)
-#define	assert_d_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "d", __VA_ARGS__)
-
-#define	assert_u_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "u", __VA_ARGS__)
-#define	assert_u_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "u", __VA_ARGS__)
-#define	assert_u_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "u", __VA_ARGS__)
-#define	assert_u_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "u", __VA_ARGS__)
-#define	assert_u_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "u", __VA_ARGS__)
-#define	assert_u_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "u", __VA_ARGS__)
-
-#define	assert_ld_eq(a, b, ...)	assert_cmp(long, a, b, ==,	\
+#define assert_c_eq(a, b, ...)	assert_cmp(char, a, b, ==, !=, "c", __VA_ARGS__)
+#define assert_c_ne(a, b, ...)	assert_cmp(char, a, b, !=, ==, "c", __VA_ARGS__)
+#define assert_c_lt(a, b, ...)	assert_cmp(char, a, b, <, >=, "c", __VA_ARGS__)
+#define assert_c_le(a, b, ...)	assert_cmp(char, a, b, <=, >, "c", __VA_ARGS__)
+#define assert_c_ge(a, b, ...)	assert_cmp(char, a, b, >=, <, "c", __VA_ARGS__)
+#define assert_c_gt(a, b, ...)	assert_cmp(char, a, b, >, <=, "c", __VA_ARGS__)
+
+#define assert_x_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "#x", __VA_ARGS__)
+#define assert_x_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "#x", __VA_ARGS__)
+#define assert_x_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "#x", __VA_ARGS__)
+#define assert_x_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "#x", __VA_ARGS__)
+#define assert_x_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "#x", __VA_ARGS__)
+#define assert_x_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "#x", __VA_ARGS__)
+
+#define assert_d_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "d", __VA_ARGS__)
+#define assert_d_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "d", __VA_ARGS__)
+#define assert_d_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "d", __VA_ARGS__)
+#define assert_d_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "d", __VA_ARGS__)
+#define assert_d_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "d", __VA_ARGS__)
+#define assert_d_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "d", __VA_ARGS__)
+
+#define assert_u_eq(a, b, ...)	assert_cmp(int, a, b, ==, !=, "u", __VA_ARGS__)
+#define assert_u_ne(a, b, ...)	assert_cmp(int, a, b, !=, ==, "u", __VA_ARGS__)
+#define assert_u_lt(a, b, ...)	assert_cmp(int, a, b, <, >=, "u", __VA_ARGS__)
+#define assert_u_le(a, b, ...)	assert_cmp(int, a, b, <=, >, "u", __VA_ARGS__)
+#define assert_u_ge(a, b, ...)	assert_cmp(int, a, b, >=, <, "u", __VA_ARGS__)
+#define assert_u_gt(a, b, ...)	assert_cmp(int, a, b, >, <=, "u", __VA_ARGS__)
+
+#define assert_ld_eq(a, b, ...)	assert_cmp(long, a, b, ==,	\
     !=, "ld", __VA_ARGS__)
-#define	assert_ld_ne(a, b, ...)	assert_cmp(long, a, b, !=,	\
+#define assert_ld_ne(a, b, ...)	assert_cmp(long, a, b, !=,	\
     ==, "ld", __VA_ARGS__)
-#define	assert_ld_lt(a, b, ...)	assert_cmp(long, a, b, <,	\
+#define assert_ld_lt(a, b, ...)	assert_cmp(long, a, b, <,	\
     >=, "ld", __VA_ARGS__)
-#define	assert_ld_le(a, b, ...)	assert_cmp(long, a, b, <=,	\
+#define assert_ld_le(a, b, ...)	assert_cmp(long, a, b, <=,	\
     >, "ld", __VA_ARGS__)
-#define	assert_ld_ge(a, b, ...)	assert_cmp(long, a, b, >=,	\
+#define assert_ld_ge(a, b, ...)	assert_cmp(long, a, b, >=,	\
     <, "ld", __VA_ARGS__)
-#define	assert_ld_gt(a, b, ...)	assert_cmp(long, a, b, >,	\
+#define assert_ld_gt(a, b, ...)	assert_cmp(long, a, b, >,	\
     <=, "ld", __VA_ARGS__)
 
-#define	assert_lu_eq(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_eq(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, ==, !=, "lu", __VA_ARGS__)
-#define	assert_lu_ne(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_ne(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, !=, ==, "lu", __VA_ARGS__)
-#define	assert_lu_lt(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_lt(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, <, >=, "lu", __VA_ARGS__)
-#define	assert_lu_le(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_le(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, <=, >, "lu", __VA_ARGS__)
-#define	assert_lu_ge(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_ge(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, >=, <, "lu", __VA_ARGS__)
-#define	assert_lu_gt(a, b, ...)	assert_cmp(unsigned long,	\
+#define assert_lu_gt(a, b, ...)	assert_cmp(unsigned long,	\
     a, b, >, <=, "lu", __VA_ARGS__)
 
-#define	assert_qd_eq(a, b, ...)	assert_cmp(long long, a, b, ==,	\
+#define assert_qd_eq(a, b, ...)	assert_cmp(long long, a, b, ==,	\
     !=, "qd", __VA_ARGS__)
-#define	assert_qd_ne(a, b, ...)	assert_cmp(long long, a, b, !=,	\
+#define assert_qd_ne(a, b, ...)	assert_cmp(long long, a, b, !=,	\
     ==, "qd", __VA_ARGS__)
-#define	assert_qd_lt(a, b, ...)	assert_cmp(long long, a, b, <,	\
+#define assert_qd_lt(a, b, ...)	assert_cmp(long long, a, b, <,	\
     >=, "qd", __VA_ARGS__)
-#define	assert_qd_le(a, b, ...)	assert_cmp(long long, a, b, <=,	\
+#define assert_qd_le(a, b, ...)	assert_cmp(long long, a, b, <=,	\
     >, "qd", __VA_ARGS__)
-#define	assert_qd_ge(a, b, ...)	assert_cmp(long long, a, b, >=,	\
+#define assert_qd_ge(a, b, ...)	assert_cmp(long long, a, b, >=,	\
     <, "qd", __VA_ARGS__)
-#define	assert_qd_gt(a, b, ...)	assert_cmp(long long, a, b, >,	\
+#define assert_qd_gt(a, b, ...)	assert_cmp(long long, a, b, >,	\
     <=, "qd", __VA_ARGS__)
 
-#define	assert_qu_eq(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_eq(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, ==, !=, "qu", __VA_ARGS__)
-#define	assert_qu_ne(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_ne(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, !=, ==, "qu", __VA_ARGS__)
-#define	assert_qu_lt(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_lt(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, <, >=, "qu", __VA_ARGS__)
-#define	assert_qu_le(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_le(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, <=, >, "qu", __VA_ARGS__)
-#define	assert_qu_ge(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_ge(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, >=, <, "qu", __VA_ARGS__)
-#define	assert_qu_gt(a, b, ...)	assert_cmp(unsigned long long,	\
+#define assert_qu_gt(a, b, ...)	assert_cmp(unsigned long long,	\
     a, b, >, <=, "qu", __VA_ARGS__)
 
-#define	assert_jd_eq(a, b, ...)	assert_cmp(intmax_t, a, b, ==,	\
+#define assert_jd_eq(a, b, ...)	assert_cmp(intmax_t, a, b, ==,	\
     !=, "jd", __VA_ARGS__)
-#define	assert_jd_ne(a, b, ...)	assert_cmp(intmax_t, a, b, !=,	\
+#define assert_jd_ne(a, b, ...)	assert_cmp(intmax_t, a, b, !=,	\
     ==, "jd", __VA_ARGS__)
-#define	assert_jd_lt(a, b, ...)	assert_cmp(intmax_t, a, b, <,	\
+#define assert_jd_lt(a, b, ...)	assert_cmp(intmax_t, a, b, <,	\
     >=, "jd", __VA_ARGS__)
-#define	assert_jd_le(a, b, ...)	assert_cmp(intmax_t, a, b, <=,	\
+#define assert_jd_le(a, b, ...)	assert_cmp(intmax_t, a, b, <=,	\
     >, "jd", __VA_ARGS__)
-#define	assert_jd_ge(a, b, ...)	assert_cmp(intmax_t, a, b, >=,	\
+#define assert_jd_ge(a, b, ...)	assert_cmp(intmax_t, a, b, >=,	\
     <, "jd", __VA_ARGS__)
-#define	assert_jd_gt(a, b, ...)	assert_cmp(intmax_t, a, b, >,	\
+#define assert_jd_gt(a, b, ...)	assert_cmp(intmax_t, a, b, >,	\
     <=, "jd", __VA_ARGS__)
 
-#define	assert_ju_eq(a, b, ...)	assert_cmp(uintmax_t, a, b, ==,	\
+#define assert_ju_eq(a, b, ...)	assert_cmp(uintmax_t, a, b, ==,	\
     !=, "ju", __VA_ARGS__)
-#define	assert_ju_ne(a, b, ...)	assert_cmp(uintmax_t, a, b, !=,	\
+#define assert_ju_ne(a, b, ...)	assert_cmp(uintmax_t, a, b, !=,	\
     ==, "ju", __VA_ARGS__)
-#define	assert_ju_lt(a, b, ...)	assert_cmp(uintmax_t, a, b, <,	\
+#define assert_ju_lt(a, b, ...)	assert_cmp(uintmax_t, a, b, <,	\
     >=, "ju", __VA_ARGS__)
-#define	assert_ju_le(a, b, ...)	assert_cmp(uintmax_t, a, b, <=,	\
+#define assert_ju_le(a, b, ...)	assert_cmp(uintmax_t, a, b, <=,	\
     >, "ju", __VA_ARGS__)
-#define	assert_ju_ge(a, b, ...)	assert_cmp(uintmax_t, a, b, >=,	\
+#define assert_ju_ge(a, b, ...)	assert_cmp(uintmax_t, a, b, >=,	\
     <, "ju", __VA_ARGS__)
-#define	assert_ju_gt(a, b, ...)	assert_cmp(uintmax_t, a, b, >,	\
+#define assert_ju_gt(a, b, ...)	assert_cmp(uintmax_t, a, b, >,	\
     <=, "ju", __VA_ARGS__)
 
-#define	assert_zd_eq(a, b, ...)	assert_cmp(ssize_t, a, b, ==,	\
+#define assert_zd_eq(a, b, ...)	assert_cmp(ssize_t, a, b, ==,	\
     !=, "zd", __VA_ARGS__)
-#define	assert_zd_ne(a, b, ...)	assert_cmp(ssize_t, a, b, !=,	\
+#define assert_zd_ne(a, b, ...)	assert_cmp(ssize_t, a, b, !=,	\
     ==, "zd", __VA_ARGS__)
-#define	assert_zd_lt(a, b, ...)	assert_cmp(ssize_t, a, b, <,	\
+#define assert_zd_lt(a, b, ...)	assert_cmp(ssize_t, a, b, <,	\
     >=, "zd", __VA_ARGS__)
-#define	assert_zd_le(a, b, ...)	assert_cmp(ssize_t, a, b, <=,	\
+#define assert_zd_le(a, b, ...)	assert_cmp(ssize_t, a, b, <=,	\
     >, "zd", __VA_ARGS__)
-#define	assert_zd_ge(a, b, ...)	assert_cmp(ssize_t, a, b, >=,	\
+#define assert_zd_ge(a, b, ...)	assert_cmp(ssize_t, a, b, >=,	\
     <, "zd", __VA_ARGS__)
-#define	assert_zd_gt(a, b, ...)	assert_cmp(ssize_t, a, b, >,	\
+#define assert_zd_gt(a, b, ...)	assert_cmp(ssize_t, a, b, >,	\
     <=, "zd", __VA_ARGS__)
 
-#define	assert_zu_eq(a, b, ...)	assert_cmp(size_t, a, b, ==,	\
+#define assert_zu_eq(a, b, ...)	assert_cmp(size_t, a, b, ==,	\
     !=, "zu", __VA_ARGS__)
-#define	assert_zu_ne(a, b, ...)	assert_cmp(size_t, a, b, !=,	\
+#define assert_zu_ne(a, b, ...)	assert_cmp(size_t, a, b, !=,	\
     ==, "zu", __VA_ARGS__)
-#define	assert_zu_lt(a, b, ...)	assert_cmp(size_t, a, b, <,	\
+#define assert_zu_lt(a, b, ...)	assert_cmp(size_t, a, b, <,	\
     >=, "zu", __VA_ARGS__)
-#define	assert_zu_le(a, b, ...)	assert_cmp(size_t, a, b, <=,	\
+#define assert_zu_le(a, b, ...)	assert_cmp(size_t, a, b, <=,	\
     >, "zu", __VA_ARGS__)
-#define	assert_zu_ge(a, b, ...)	assert_cmp(size_t, a, b, >=,	\
+#define assert_zu_ge(a, b, ...)	assert_cmp(size_t, a, b, >=,	\
     <, "zu", __VA_ARGS__)
-#define	assert_zu_gt(a, b, ...)	assert_cmp(size_t, a, b, >,	\
+#define assert_zu_gt(a, b, ...)	assert_cmp(size_t, a, b, >,	\
     <=, "zu", __VA_ARGS__)
 
-#define	assert_d32_eq(a, b, ...)	assert_cmp(int32_t, a, b, ==,	\
+#define assert_d32_eq(a, b, ...)	assert_cmp(int32_t, a, b, ==,	\
     !=, FMTd32, __VA_ARGS__)
-#define	assert_d32_ne(a, b, ...)	assert_cmp(int32_t, a, b, !=,	\
+#define assert_d32_ne(a, b, ...)	assert_cmp(int32_t, a, b, !=,	\
     ==, FMTd32, __VA_ARGS__)
-#define	assert_d32_lt(a, b, ...)	assert_cmp(int32_t, a, b, <,	\
+#define assert_d32_lt(a, b, ...)	assert_cmp(int32_t, a, b, <,	\
     >=, FMTd32, __VA_ARGS__)
-#define	assert_d32_le(a, b, ...)	assert_cmp(int32_t, a, b, <=,	\
+#define assert_d32_le(a, b, ...)	assert_cmp(int32_t, a, b, <=,	\
     >, FMTd32, __VA_ARGS__)
-#define	assert_d32_ge(a, b, ...)	assert_cmp(int32_t, a, b, >=,	\
+#define assert_d32_ge(a, b, ...)	assert_cmp(int32_t, a, b, >=,	\
     <, FMTd32, __VA_ARGS__)
-#define	assert_d32_gt(a, b, ...)	assert_cmp(int32_t, a, b, >,	\
+#define assert_d32_gt(a, b, ...)	assert_cmp(int32_t, a, b, >,	\
     <=, FMTd32, __VA_ARGS__)
 
-#define	assert_u32_eq(a, b, ...)	assert_cmp(uint32_t, a, b, ==,	\
+#define assert_u32_eq(a, b, ...)	assert_cmp(uint32_t, a, b, ==,	\
     !=, FMTu32, __VA_ARGS__)
-#define	assert_u32_ne(a, b, ...)	assert_cmp(uint32_t, a, b, !=,	\
+#define assert_u32_ne(a, b, ...)	assert_cmp(uint32_t, a, b, !=,	\
     ==, FMTu32, __VA_ARGS__)
-#define	assert_u32_lt(a, b, ...)	assert_cmp(uint32_t, a, b, <,	\
+#define assert_u32_lt(a, b, ...)	assert_cmp(uint32_t, a, b, <,	\
     >=, FMTu32, __VA_ARGS__)
-#define	assert_u32_le(a, b, ...)	assert_cmp(uint32_t, a, b, <=,	\
+#define assert_u32_le(a, b, ...)	assert_cmp(uint32_t, a, b, <=,	\
     >, FMTu32, __VA_ARGS__)
-#define	assert_u32_ge(a, b, ...)	assert_cmp(uint32_t, a, b, >=,	\
+#define assert_u32_ge(a, b, ...)	assert_cmp(uint32_t, a, b, >=,	\
     <, FMTu32, __VA_ARGS__)
-#define	assert_u32_gt(a, b, ...)	assert_cmp(uint32_t, a, b, >,	\
+#define assert_u32_gt(a, b, ...)	assert_cmp(uint32_t, a, b, >,	\
     <=, FMTu32, __VA_ARGS__)
 
-#define	assert_d64_eq(a, b, ...)	assert_cmp(int64_t, a, b, ==,	\
+#define assert_d64_eq(a, b, ...)	assert_cmp(int64_t, a, b, ==,	\
     !=, FMTd64, __VA_ARGS__)
-#define	assert_d64_ne(a, b, ...)	assert_cmp(int64_t, a, b, !=,	\
+#define assert_d64_ne(a, b, ...)	assert_cmp(int64_t, a, b, !=,	\
     ==, FMTd64, __VA_ARGS__)
-#define	assert_d64_lt(a, b, ...)	assert_cmp(int64_t, a, b, <,	\
+#define assert_d64_lt(a, b, ...)	assert_cmp(int64_t, a, b, <,	\
     >=, FMTd64, __VA_ARGS__)
-#define	assert_d64_le(a, b, ...)	assert_cmp(int64_t, a, b, <=,	\
+#define assert_d64_le(a, b, ...)	assert_cmp(int64_t, a, b, <=,	\
     >, FMTd64, __VA_ARGS__)
-#define	assert_d64_ge(a, b, ...)	assert_cmp(int64_t, a, b, >=,	\
+#define assert_d64_ge(a, b, ...)	assert_cmp(int64_t, a, b, >=,	\
     <, FMTd64, __VA_ARGS__)
-#define	assert_d64_gt(a, b, ...)	assert_cmp(int64_t, a, b, >,	\
+#define assert_d64_gt(a, b, ...)	assert_cmp(int64_t, a, b, >,	\
     <=, FMTd64, __VA_ARGS__)
 
-#define	assert_u64_eq(a, b, ...)	assert_cmp(uint64_t, a, b, ==,	\
+#define assert_u64_eq(a, b, ...)	assert_cmp(uint64_t, a, b, ==,	\
     !=, FMTu64, __VA_ARGS__)
-#define	assert_u64_ne(a, b, ...)	assert_cmp(uint64_t, a, b, !=,	\
+#define assert_u64_ne(a, b, ...)	assert_cmp(uint64_t, a, b, !=,	\
     ==, FMTu64, __VA_ARGS__)
-#define	assert_u64_lt(a, b, ...)	assert_cmp(uint64_t, a, b, <,	\
+#define assert_u64_lt(a, b, ...)	assert_cmp(uint64_t, a, b, <,	\
     >=, FMTu64, __VA_ARGS__)
-#define	assert_u64_le(a, b, ...)	assert_cmp(uint64_t, a, b, <=,	\
+#define assert_u64_le(a, b, ...)	assert_cmp(uint64_t, a, b, <=,	\
     >, FMTu64, __VA_ARGS__)
-#define	assert_u64_ge(a, b, ...)	assert_cmp(uint64_t, a, b, >=,	\
+#define assert_u64_ge(a, b, ...)	assert_cmp(uint64_t, a, b, >=,	\
     <, FMTu64, __VA_ARGS__)
-#define	assert_u64_gt(a, b, ...)	assert_cmp(uint64_t, a, b, >,	\
+#define assert_u64_gt(a, b, ...)	assert_cmp(uint64_t, a, b, >,	\
     <=, FMTu64, __VA_ARGS__)
 
-#define	assert_b_eq(a, b, ...) do {					\
+#define assert_b_eq(a, b, ...) do {					\
 	bool a_ = (a);							\
 	bool b_ = (b);							\
 	if (!(a_ == b_)) {						\
@@ -226,7 +226,7 @@
 		p_test_fail(prefix, message);				\
 	}								\
 } while (0)
-#define	assert_b_ne(a, b, ...) do {					\
+#define assert_b_ne(a, b, ...) do {					\
 	bool a_ = (a);							\
 	bool b_ = (b);							\
 	if (!(a_ != b_)) {						\
@@ -242,10 +242,10 @@
 		p_test_fail(prefix, message);				\
 	}								\
 } while (0)
-#define	assert_true(a, ...)	assert_b_eq(a, true, __VA_ARGS__)
-#define	assert_false(a, ...)	assert_b_eq(a, false, __VA_ARGS__)
+#define assert_true(a, ...)	assert_b_eq(a, true, __VA_ARGS__)
+#define assert_false(a, ...)	assert_b_eq(a, false, __VA_ARGS__)
 
-#define	assert_str_eq(a, b, ...) do {				\
+#define assert_str_eq(a, b, ...) do {				\
 	if (strcmp((a), (b))) {						\
 		char prefix[ASSERT_BUFSIZE];				\
 		char message[ASSERT_BUFSIZE];				\
@@ -258,7 +258,7 @@
 		p_test_fail(prefix, message);				\
 	}								\
 } while (0)
-#define	assert_str_ne(a, b, ...) do {				\
+#define assert_str_ne(a, b, ...) do {				\
 	if (!strcmp((a), (b))) {					\
 		char prefix[ASSERT_BUFSIZE];				\
 		char message[ASSERT_BUFSIZE];				\
@@ -272,7 +272,7 @@
 	}								\
 } while (0)
 
-#define	assert_not_reached(...) do {					\
+#define assert_not_reached(...) do {					\
 	char prefix[ASSERT_BUFSIZE];					\
 	char message[ASSERT_BUFSIZE];					\
 	malloc_snprintf(prefix, sizeof(prefix),				\
@@ -296,25 +296,27 @@ typedef enum {
 
 typedef void (test_t)(void);
 
-#define	TEST_BEGIN(f)							\
+#define TEST_BEGIN(f)							\
 static void								\
-f(void)									\
-{									\
+f(void) {								\
 	p_test_init(#f);
 
-#define	TEST_END							\
+#define TEST_END							\
 	goto label_test_end;						\
 label_test_end:								\
 	p_test_fini();							\
 }
 
-#define	test(...)							\
+#define test(...)							\
 	p_test(__VA_ARGS__, NULL)
 
-#define	test_no_malloc_init(...)					\
+#define test_no_reentrancy(...)							\
+	p_test_no_reentrancy(__VA_ARGS__, NULL)
+
+#define test_no_malloc_init(...)					\
 	p_test_no_malloc_init(__VA_ARGS__, NULL)
 
-#define	test_skip_if(e) do {						\
+#define test_skip_if(e) do {						\
 	if (e) {							\
 		test_skip("%s:%s:%d: Test skipped: (%s)",		\
 		    __func__, __FILE__, __LINE__, #e);			\
@@ -322,11 +324,14 @@ label_test_end:								\
 	}								\
 } while (0)
 
+bool test_is_reentrant();
+
 void	test_skip(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
 void	test_fail(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2);
 
 /* For private use by macros. */
 test_status_t	p_test(test_t *t, ...);
+test_status_t	p_test_no_reentrancy(test_t *t, ...);
 test_status_t	p_test_no_malloc_init(test_t *t, ...);
 void	p_test_init(const char *name);
 void	p_test_fini(void);