Stop depending on JEMALLOC_N() for function interception during testing.

Instead, always define function pointers for interceptable functions,
but mark them const unless testing, so that the compiler can optimize
out the pointer dereferences.

7 files changed, 106 insertions, 114 deletions

diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h
index 7d56e44..410709c 100644
--- a/include/jemalloc/internal/arena_externs.h
+++ b/include/jemalloc/internal/arena_externs.h
@@ -59,12 +59,10 @@ void	arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
     tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes);
 void	arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info,
     bool zero);
-#ifdef JEMALLOC_JET
+
 typedef void (arena_dalloc_junk_small_t)(void *, const arena_bin_info_t *);
-extern arena_dalloc_junk_small_t *arena_dalloc_junk_small;
-#else
-void	arena_dalloc_junk_small(void *ptr, const arena_bin_info_t *bin_info);
-#endif
+extern arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small;
+
 void	*arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size,
     szind_t ind, bool zero);
 void	*arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h
index a1712cf..4571895 100644
--- a/include/jemalloc/internal/jemalloc_internal_macros.h
+++ b/include/jemalloc/internal/jemalloc_internal_macros.h
@@ -30,4 +30,11 @@
 #  define restrict
 #endif
 
+/* Various function pointers are statick and immutable except during testing. */
+#ifdef JEMALLOC_JET
+#  define JET_MUTABLE
+#else
+#  define JET_MUTABLE const
+#endif
+
 #endif /* JEMALLOC_INTERNAL_MACROS_H */
diff --git a/include/jemalloc/internal/large_externs.h b/include/jemalloc/internal/large_externs.h
index 2a208c8..3f36282 100644
--- a/include/jemalloc/internal/large_externs.h
+++ b/include/jemalloc/internal/large_externs.h
@@ -1,28 +1,26 @@
 #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H
 #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H
 
-void	*large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
-void	*large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
-    size_t alignment, bool zero);
-bool	large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
+void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero);
+void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
+    bool zero);
+bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
     size_t usize_max, bool zero);
-void	*large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
-    size_t usize, size_t alignment, bool zero, tcache_t *tcache);
-#ifdef JEMALLOC_JET
+void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize,
+    size_t alignment, bool zero, tcache_t *tcache);
+
 typedef void (large_dalloc_junk_t)(void *, size_t);
-extern large_dalloc_junk_t *large_dalloc_junk;
+extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk;
+
 typedef void (large_dalloc_maybe_junk_t)(void *, size_t);
-extern large_dalloc_maybe_junk_t *large_dalloc_maybe_junk;
-#else
-void	large_dalloc_junk(void *ptr, size_t size);
-void	large_dalloc_maybe_junk(void *ptr, size_t size);
-#endif
-void	large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent);
-void	large_dalloc_finish(tsdn_t *tsdn, extent_t *extent);
-void	large_dalloc(tsdn_t *tsdn, extent_t *extent);
-size_t	large_salloc(tsdn_t *tsdn, const extent_t *extent);
-prof_tctx_t	*large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
-void	large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
-void	large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
+extern large_dalloc_maybe_junk_t *JET_MUTABLE large_dalloc_maybe_junk;
+
+void large_dalloc_prep_junked_locked(tsdn_t *tsdn, extent_t *extent);
+void large_dalloc_finish(tsdn_t *tsdn, extent_t *extent);
+void large_dalloc(tsdn_t *tsdn, extent_t *extent);
+size_t large_salloc(tsdn_t *tsdn, const extent_t *extent);
+prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent);
+void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx);
+void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent);
 
 #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */
diff --git a/include/jemalloc/internal/nstime.h b/include/jemalloc/internal/nstime.h
index cfccca0..ad7efb8 100644
--- a/include/jemalloc/internal/nstime.h
+++ b/include/jemalloc/internal/nstime.h
@@ -10,27 +10,23 @@ typedef struct {
 } nstime_t;
 
 void nstime_init(nstime_t *time, uint64_t ns);
-void	nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
-uint64_t	nstime_ns(const nstime_t *time);
-uint64_t	nstime_sec(const nstime_t *time);
-uint64_t	nstime_msec(const nstime_t *time);
-uint64_t	nstime_nsec(const nstime_t *time);
-void	nstime_copy(nstime_t *time, const nstime_t *source);
-int	nstime_compare(const nstime_t *a, const nstime_t *b);
-void	nstime_add(nstime_t *time, const nstime_t *addend);
-void	nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
-void	nstime_imultiply(nstime_t *time, uint64_t multiplier);
-void	nstime_idivide(nstime_t *time, uint64_t divisor);
-uint64_t	nstime_divide(const nstime_t *time, const nstime_t *divisor);
-#ifdef JEMALLOC_JET
+void nstime_init2(nstime_t *time, uint64_t sec, uint64_t nsec);
+uint64_t nstime_ns(const nstime_t *time);
+uint64_t nstime_sec(const nstime_t *time);
+uint64_t nstime_msec(const nstime_t *time);
+uint64_t nstime_nsec(const nstime_t *time);
+void nstime_copy(nstime_t *time, const nstime_t *source);
+int nstime_compare(const nstime_t *a, const nstime_t *b);
+void nstime_add(nstime_t *time, const nstime_t *addend);
+void nstime_subtract(nstime_t *time, const nstime_t *subtrahend);
+void nstime_imultiply(nstime_t *time, uint64_t multiplier);
+void nstime_idivide(nstime_t *time, uint64_t divisor);
+uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
+
 typedef bool (nstime_monotonic_t)(void);
-extern nstime_monotonic_t *nstime_monotonic;
-typedef bool (nstime_update_t)(nstime_t *);
-extern nstime_update_t *nstime_update;
-#else
-bool	nstime_monotonic(void);
-bool	nstime_update(nstime_t *time);
-#endif
+extern nstime_monotonic_t *JET_MUTABLE nstime_monotonic;
 
+typedef bool (nstime_update_t)(nstime_t *);
+extern nstime_update_t *JET_MUTABLE nstime_update;
 
 #endif /* JEMALLOC_INTERNAL_NSTIME_H */
diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h
index cbd9795..2891b8b 100644
--- a/include/jemalloc/internal/prof_externs.h
+++ b/include/jemalloc/internal/prof_externs.h
@@ -40,48 +40,51 @@ extern uint64_t	prof_interval;
  */
 extern size_t	lg_prof_sample;
 
-void	prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
-void	prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
+void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated);
+void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
     prof_tctx_t *tctx);
-void	prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
-void	bt_init(prof_bt_t *bt, void **vec);
-void	prof_backtrace(prof_bt_t *bt);
-prof_tctx_t	*prof_lookup(tsd_t *tsd, prof_bt_t *bt);
+void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx);
+void bt_init(prof_bt_t *bt, void **vec);
+void prof_backtrace(prof_bt_t *bt);
+prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt);
 #ifdef JEMALLOC_JET
-size_t	prof_tdata_count(void);
-size_t	prof_bt_count(void);
+size_t prof_tdata_count(void);
+size_t prof_bt_count(void);
+#endif
 typedef int (prof_dump_open_t)(bool, const char *);
-extern prof_dump_open_t *prof_dump_open;
+extern prof_dump_open_t *JET_MUTABLE prof_dump_open;
+
 typedef bool (prof_dump_header_t)(tsdn_t *, bool, const prof_cnt_t *);
-extern prof_dump_header_t *prof_dump_header;
-void	prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes,
-    uint64_t *accumobjs, uint64_t *accumbytes);
+extern prof_dump_header_t *JET_MUTABLE prof_dump_header;
+#ifdef JEMALLOC_JET
+void prof_cnt_all(uint64_t *curobjs, uint64_t *curbytes, uint64_t *accumobjs,
+    uint64_t *accumbytes);
 #endif
 bool prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum);
-void	prof_idump(tsdn_t *tsdn);
-bool	prof_mdump(tsd_t *tsd, const char *filename);
-void	prof_gdump(tsdn_t *tsdn);
-prof_tdata_t	*prof_tdata_init(tsd_t *tsd);
-prof_tdata_t	*prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
-void	prof_reset(tsd_t *tsd, size_t lg_sample);
-void	prof_tdata_cleanup(tsd_t *tsd);
-bool	prof_active_get(tsdn_t *tsdn);
-bool	prof_active_set(tsdn_t *tsdn, bool active);
-const char	*prof_thread_name_get(tsd_t *tsd);
-int	prof_thread_name_set(tsd_t *tsd, const char *thread_name);
-bool	prof_thread_active_get(tsd_t *tsd);
-bool	prof_thread_active_set(tsd_t *tsd, bool active);
-bool	prof_thread_active_init_get(tsdn_t *tsdn);
-bool	prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
-bool	prof_gdump_get(tsdn_t *tsdn);
-bool	prof_gdump_set(tsdn_t *tsdn, bool active);
-void	prof_boot0(void);
-void	prof_boot1(void);
-bool	prof_boot2(tsd_t *tsd);
-void	prof_prefork0(tsdn_t *tsdn);
-void	prof_prefork1(tsdn_t *tsdn);
-void	prof_postfork_parent(tsdn_t *tsdn);
-void	prof_postfork_child(tsdn_t *tsdn);
-void	prof_sample_threshold_update(prof_tdata_t *tdata);
+void prof_idump(tsdn_t *tsdn);
+bool prof_mdump(tsd_t *tsd, const char *filename);
+void prof_gdump(tsdn_t *tsdn);
+prof_tdata_t *prof_tdata_init(tsd_t *tsd);
+prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
+void prof_reset(tsd_t *tsd, size_t lg_sample);
+void prof_tdata_cleanup(tsd_t *tsd);
+bool prof_active_get(tsdn_t *tsdn);
+bool prof_active_set(tsdn_t *tsdn, bool active);
+const char *prof_thread_name_get(tsd_t *tsd);
+int prof_thread_name_set(tsd_t *tsd, const char *thread_name);
+bool prof_thread_active_get(tsd_t *tsd);
+bool prof_thread_active_set(tsd_t *tsd, bool active);
+bool prof_thread_active_init_get(tsdn_t *tsdn);
+bool prof_thread_active_init_set(tsdn_t *tsdn, bool active_init);
+bool prof_gdump_get(tsdn_t *tsdn);
+bool prof_gdump_set(tsdn_t *tsdn, bool active);
+void prof_boot0(void);
+void prof_boot1(void);
+bool prof_boot2(tsd_t *tsd);
+void prof_prefork0(tsdn_t *tsdn);
+void prof_prefork1(tsdn_t *tsdn);
+void prof_postfork_parent(tsdn_t *tsdn);
+void prof_postfork_child(tsdn_t *tsdn);
+void prof_sample_threshold_update(prof_tdata_t *tdata);
 
 #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */
diff --git a/include/jemalloc/internal/rtree_externs.h b/include/jemalloc/internal/rtree_externs.h
index 5145c12..5742f58 100644
--- a/include/jemalloc/internal/rtree_externs.h
+++ b/include/jemalloc/internal/rtree_externs.h
@@ -24,15 +24,19 @@ static const rtree_level_t rtree_levels[] = {
 };
 
 bool rtree_new(rtree_t *rtree, bool zeroed);
-#ifdef JEMALLOC_JET
+
 typedef rtree_node_elm_t *(rtree_node_alloc_t)(tsdn_t *, rtree_t *, size_t);
-extern rtree_node_alloc_t *rtree_node_alloc;
+extern rtree_node_alloc_t *JET_MUTABLE rtree_node_alloc;
+
 typedef rtree_leaf_elm_t *(rtree_leaf_alloc_t)(tsdn_t *, rtree_t *, size_t);
-extern rtree_leaf_alloc_t *rtree_leaf_alloc;
+extern rtree_leaf_alloc_t *JET_MUTABLE rtree_leaf_alloc;
+
 typedef void (rtree_node_dalloc_t)(tsdn_t *, rtree_t *, rtree_node_elm_t *);
-extern rtree_node_dalloc_t *rtree_node_dalloc;
+extern rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc;
+
 typedef void (rtree_leaf_dalloc_t)(tsdn_t *, rtree_t *, rtree_leaf_elm_t *);
-extern rtree_leaf_dalloc_t *rtree_leaf_dalloc;
+extern rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc;
+#ifdef JEMALLOC_JET
 void rtree_delete(tsdn_t *tsdn, rtree_t *rtree);
 #endif
 rtree_leaf_elm_t *rtree_leaf_elm_lookup_hard(tsdn_t *tsdn, rtree_t *rtree,
diff --git a/include/jemalloc/internal/witness_externs.h b/include/jemalloc/internal/witness_externs.h
index 5d91fde..99df4c5 100644
--- a/include/jemalloc/internal/witness_externs.h
+++ b/include/jemalloc/internal/witness_externs.h
@@ -1,39 +1,25 @@
 #ifndef JEMALLOC_INTERNAL_WITNESS_EXTERNS_H
 #define JEMALLOC_INTERNAL_WITNESS_EXTERNS_H
 
-void	witness_init(witness_t *witness, const char *name, witness_rank_t rank,
+void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
     witness_comp_t *comp, void *opaque);
-#ifdef JEMALLOC_JET
+
 typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
-extern witness_lock_error_t *witness_lock_error;
-#else
-void	witness_lock_error(const witness_list_t *witnesses,
-    const witness_t *witness);
-#endif
-#ifdef JEMALLOC_JET
+extern witness_lock_error_t *JET_MUTABLE witness_lock_error;
+
 typedef void (witness_owner_error_t)(const witness_t *);
-extern witness_owner_error_t *witness_owner_error;
-#else
-void	witness_owner_error(const witness_t *witness);
-#endif
-#ifdef JEMALLOC_JET
+extern witness_owner_error_t *JET_MUTABLE witness_owner_error;
+
 typedef void (witness_not_owner_error_t)(const witness_t *);
-extern witness_not_owner_error_t *witness_not_owner_error;
-#else
-void	witness_not_owner_error(const witness_t *witness);
-#endif
-#ifdef JEMALLOC_JET
+extern witness_not_owner_error_t *JET_MUTABLE witness_not_owner_error;
+
 typedef void (witness_depth_error_t)(const witness_list_t *,
     witness_rank_t rank_inclusive, unsigned depth);
-extern witness_depth_error_t *witness_depth_error;
-#else
-void	witness_depth_error(const witness_list_t *witnesses,
-    witness_rank_t rank_inclusive, unsigned depth);
-#endif
+extern witness_depth_error_t *JET_MUTABLE witness_depth_error;
 
-void	witnesses_cleanup(tsd_t *tsd);
-void	witness_prefork(tsd_t *tsd);
-void	witness_postfork_parent(tsd_t *tsd);
-void	witness_postfork_child(tsd_t *tsd);
+void witnesses_cleanup(tsd_t *tsd);
+void witness_prefork(tsd_t *tsd);
+void witness_postfork_parent(tsd_t *tsd);
+void witness_postfork_child(tsd_t *tsd);
 
 #endif /* JEMALLOC_INTERNAL_WITNESS_EXTERNS_H */