summaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorDavid Goldblatt <davidgoldblatt@fb.com>2017-04-21 16:37:34 (GMT)
committerDavid Goldblatt <davidtgoldblatt@gmail.com>2017-04-24 17:33:21 (GMT)
commit4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98 (patch)
treeecb8c4f306175db5b74f0dde83a28993d71f9888 /include
parent7d86c92c61c60b771cdf146f6187c1550a089ad1 (diff)
downloadjemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.zip
jemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.tar.gz
jemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.tar.bz2
Get rid of most of the various inline macros.
Diffstat (limited to 'include')
-rw-r--r--include/jemalloc/internal/arena_inlines_a.h25
-rw-r--r--include/jemalloc/internal/arena_inlines_b.h32
-rw-r--r--include/jemalloc/internal/base_inlines.h8
-rw-r--r--include/jemalloc/internal/bitmap_inlines.h25
-rw-r--r--include/jemalloc/internal/extent_inlines.h160
-rw-r--r--include/jemalloc/internal/hash_inlines.h32
-rw-r--r--include/jemalloc/internal/jemalloc_internal_inlines_a.h60
-rw-r--r--include/jemalloc/internal/jemalloc_internal_inlines_b.h14
-rw-r--r--include/jemalloc/internal/jemalloc_internal_inlines_c.h31
-rw-r--r--include/jemalloc/internal/jemalloc_internal_macros.h34
-rw-r--r--include/jemalloc/internal/mutex_inlines.h30
-rw-r--r--include/jemalloc/internal/prng_inlines.h18
-rw-r--r--include/jemalloc/internal/prof_inlines_a.h12
-rw-r--r--include/jemalloc/internal/prof_inlines_b.h24
-rw-r--r--include/jemalloc/internal/rtree_inlines.h74
-rw-r--r--include/jemalloc/internal/tcache_inlines.h23
-rw-r--r--include/jemalloc/internal/ticker_inlines.h20
-rw-r--r--include/jemalloc/internal/tsd_inlines.h26
-rw-r--r--include/jemalloc/internal/witness_inlines.h30
19 files changed, 134 insertions, 544 deletions
diff --git a/include/jemalloc/internal/arena_inlines_a.h b/include/jemalloc/internal/arena_inlines_a.h
index 2bd5ce7..da58770 100644
--- a/include/jemalloc/internal/arena_inlines_a.h
+++ b/include/jemalloc/internal/arena_inlines_a.h
@@ -1,38 +1,27 @@
#ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H
#define JEMALLOC_INTERNAL_ARENA_INLINES_A_H
-#ifndef JEMALLOC_ENABLE_INLINE
-unsigned arena_ind_get(const arena_t *arena);
-void arena_internal_add(arena_t *arena, size_t size);
-void arena_internal_sub(arena_t *arena, size_t size);
-size_t arena_internal_get(arena_t *arena);
-bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes);
-void percpu_arena_update(tsd_t *tsd, unsigned cpu);
-#endif /* JEMALLOC_ENABLE_INLINE */
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
-
-JEMALLOC_INLINE unsigned
+static inline unsigned
arena_ind_get(const arena_t *arena) {
return base_ind_get(arena->base);
}
-JEMALLOC_INLINE void
+static inline void
arena_internal_add(arena_t *arena, size_t size) {
atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
-JEMALLOC_INLINE void
+static inline void
arena_internal_sub(arena_t *arena, size_t size) {
atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED);
}
-JEMALLOC_INLINE size_t
+static inline size_t
arena_internal_get(arena_t *arena) {
return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED);
}
-JEMALLOC_INLINE bool
+static inline bool
arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) {
cassert(config_prof);
@@ -43,7 +32,7 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) {
return prof_accum_add(tsdn, &arena->prof_accum, accumbytes);
}
-JEMALLOC_INLINE void
+static inline void
percpu_arena_update(tsd_t *tsd, unsigned cpu) {
assert(have_percpu_arena);
arena_t *oldarena = tsd_arena_get(tsd);
@@ -65,6 +54,4 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) {
}
}
-#endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */
-
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */
diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index 4264f4b..526103b 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -3,30 +3,7 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
-prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr,
- alloc_ctx_t *ctx);
-void arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
- alloc_ctx_t *ctx, prof_tctx_t *tctx);
-void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
-void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks);
-void arena_decay_tick(tsdn_t *tsdn, arena_t *arena);
-void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
- bool zero, tcache_t *tcache, bool slow_path);
-arena_t *arena_aalloc(tsdn_t *tsdn, const void *ptr);
-size_t arena_salloc(tsdn_t *tsdn, const void *ptr);
-size_t arena_vsalloc(tsdn_t *tsdn, const void *ptr);
-void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr);
-void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
- alloc_ctx_t *alloc_ctx, bool slow_path);
-void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size);
-void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
- alloc_ctx_t *alloc_ctx, bool slow_path);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
-JEMALLOC_INLINE szind_t
+static inline szind_t
arena_bin_index(arena_t *arena, arena_bin_t *bin) {
szind_t binind = (szind_t)(bin - arena->bins);
assert(binind < NBINS);
@@ -71,7 +48,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
}
}
-JEMALLOC_INLINE void
+static inline void
arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) {
cassert(config_prof);
assert(ptr != NULL);
@@ -182,7 +159,7 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
return index2size(szind);
}
-JEMALLOC_INLINE void
+static inline void
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
@@ -264,7 +241,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
}
}
-JEMALLOC_INLINE void
+static inline void
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= LARGE_MAXCLASS);
@@ -376,5 +353,4 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
}
}
-#endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */
#endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */
diff --git a/include/jemalloc/internal/base_inlines.h b/include/jemalloc/internal/base_inlines.h
index aa8306a..931560b 100644
--- a/include/jemalloc/internal/base_inlines.h
+++ b/include/jemalloc/internal/base_inlines.h
@@ -1,15 +1,9 @@
#ifndef JEMALLOC_INTERNAL_BASE_INLINES_H
#define JEMALLOC_INTERNAL_BASE_INLINES_H
-#ifndef JEMALLOC_ENABLE_INLINE
-unsigned base_ind_get(const base_t *base);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BASE_C_))
-JEMALLOC_INLINE unsigned
+static inline unsigned
base_ind_get(const base_t *base) {
return base->ind;
}
-#endif
#endif /* JEMALLOC_INTERNAL_BASE_INLINES_H */
diff --git a/include/jemalloc/internal/bitmap_inlines.h b/include/jemalloc/internal/bitmap_inlines.h
index c236201..84425b3 100644
--- a/include/jemalloc/internal/bitmap_inlines.h
+++ b/include/jemalloc/internal/bitmap_inlines.h
@@ -3,18 +3,7 @@
#include "jemalloc/internal/bit_util.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo);
-bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
-void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
-size_t bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo,
- size_t min_bit);
-size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo);
-void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_))
-JEMALLOC_INLINE bool
+static inline bool
bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
#ifdef BITMAP_USE_TREE
size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1;
@@ -33,7 +22,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) {
#endif
}
-JEMALLOC_INLINE bool
+static inline bool
bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t g;
@@ -44,7 +33,7 @@ bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
return !(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK)));
}
-JEMALLOC_INLINE void
+static inline void
bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t *gp;
@@ -80,7 +69,7 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
}
/* ffu: find first unset >= bit. */
-JEMALLOC_INLINE size_t
+static inline size_t
bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
assert(min_bit < binfo->nbits);
@@ -139,7 +128,7 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) {
}
/* sfu: set first unset. */
-JEMALLOC_INLINE size_t
+static inline size_t
bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
size_t bit;
bitmap_t g;
@@ -169,7 +158,7 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
return bit;
}
-JEMALLOC_INLINE void
+static inline void
bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
size_t goff;
bitmap_t *gp;
@@ -208,6 +197,4 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) {
#endif /* BITMAP_USE_TREE */
}
-#endif
-
#endif /* JEMALLOC_INTERNAL_BITMAP_INLINES_H */
diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h
index fbe51e4..22d45ce 100644
--- a/include/jemalloc/internal/extent_inlines.h
+++ b/include/jemalloc/internal/extent_inlines.h
@@ -3,64 +3,7 @@
#include "jemalloc/internal/ql.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-arena_t *extent_arena_get(const extent_t *extent);
-szind_t extent_szind_get_maybe_invalid(const extent_t *extent);
-szind_t extent_szind_get(const extent_t *extent);
-size_t extent_usize_get(const extent_t *extent);
-size_t extent_sn_get(const extent_t *extent);
-extent_state_t extent_state_get(const extent_t *extent);
-bool extent_zeroed_get(const extent_t *extent);
-bool extent_committed_get(const extent_t *extent);
-bool extent_slab_get(const extent_t *extent);
-unsigned extent_nfree_get(const extent_t *extent);
-void *extent_base_get(const extent_t *extent);
-void *extent_addr_get(const extent_t *extent);
-size_t extent_size_get(const extent_t *extent);
-size_t extent_esn_get(const extent_t *extent);
-size_t extent_bsize_get(const extent_t *extent);
-void *extent_before_get(const extent_t *extent);
-void *extent_last_get(const extent_t *extent);
-void *extent_past_get(const extent_t *extent);
-arena_slab_data_t *extent_slab_data_get(extent_t *extent);
-const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent);
-prof_tctx_t *extent_prof_tctx_get(const extent_t *extent);
-void extent_arena_set(extent_t *extent, arena_t *arena);
-void extent_addr_set(extent_t *extent, void *addr);
-void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment);
-void extent_size_set(extent_t *extent, size_t size);
-void extent_esn_set(extent_t *extent, size_t esn);
-void extent_bsize_set(extent_t *extent, size_t bsize);
-void extent_szind_set(extent_t *extent, szind_t szind);
-void extent_nfree_set(extent_t *extent, unsigned nfree);
-void extent_nfree_inc(extent_t *extent);
-void extent_nfree_dec(extent_t *extent);
-void extent_sn_set(extent_t *extent, size_t sn);
-void extent_state_set(extent_t *extent, extent_state_t state);
-void extent_zeroed_set(extent_t *extent, bool zeroed);
-void extent_committed_set(extent_t *extent, bool committed);
-void extent_slab_set(extent_t *extent, bool slab);
-void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx);
-void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
- bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
- bool committed);
-void extent_binit(extent_t *extent, void *addr, size_t size, size_t sn);
-void extent_list_init(extent_list_t *list);
-extent_t *extent_list_first(const extent_list_t *list);
-extent_t *extent_list_last(const extent_list_t *list);
-void extent_list_append(extent_list_t *list, extent_t *extent);
-void extent_list_replace(extent_list_t *list, extent_t *to_remove,
- extent_t *to_insert);
-void extent_list_remove(extent_list_t *list, extent_t *extent);
-int extent_sn_comp(const extent_t *a, const extent_t *b);
-int extent_esn_comp(const extent_t *a, const extent_t *b);
-int extent_ad_comp(const extent_t *a, const extent_t *b);
-int extent_snad_comp(const extent_t *a, const extent_t *b);
-int extent_esnead_comp(const extent_t *a, const extent_t *b);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_))
-JEMALLOC_INLINE arena_t *
+static inline arena_t *
extent_arena_get(const extent_t *extent) {
unsigned arena_ind = (unsigned)((extent->e_bits &
EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT);
@@ -75,7 +18,7 @@ extent_arena_get(const extent_t *extent) {
return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE);
}
-JEMALLOC_INLINE szind_t
+static inline szind_t
extent_szind_get_maybe_invalid(const extent_t *extent) {
szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >>
EXTENT_BITS_SZIND_SHIFT);
@@ -83,120 +26,120 @@ extent_szind_get_maybe_invalid(const extent_t *extent) {
return szind;
}
-JEMALLOC_INLINE szind_t
+static inline szind_t
extent_szind_get(const extent_t *extent) {
szind_t szind = extent_szind_get_maybe_invalid(extent);
assert(szind < NSIZES); /* Never call when "invalid". */
return szind;
}
-JEMALLOC_INLINE size_t
+static inline size_t
extent_usize_get(const extent_t *extent) {
return index2size(extent_szind_get(extent));
}
-JEMALLOC_INLINE size_t
+static inline size_t
extent_sn_get(const extent_t *extent) {
return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >>
EXTENT_BITS_SN_SHIFT);
}
-JEMALLOC_INLINE extent_state_t
+static inline extent_state_t
extent_state_get(const extent_t *extent) {
return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >>
EXTENT_BITS_STATE_SHIFT);
}
-JEMALLOC_INLINE bool
+static inline bool
extent_zeroed_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >>
EXTENT_BITS_ZEROED_SHIFT);
}
-JEMALLOC_INLINE bool
+static inline bool
extent_committed_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >>
EXTENT_BITS_COMMITTED_SHIFT);
}
-JEMALLOC_INLINE bool
+static inline bool
extent_slab_get(const extent_t *extent) {
return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >>
EXTENT_BITS_SLAB_SHIFT);
}
-JEMALLOC_INLINE unsigned
+static inline unsigned
extent_nfree_get(const extent_t *extent) {
assert(extent_slab_get(extent));
return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >>
EXTENT_BITS_NFREE_SHIFT);
}
-JEMALLOC_INLINE void *
+static inline void *
extent_base_get(const extent_t *extent) {
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent_slab_get(extent));
return PAGE_ADDR2BASE(extent->e_addr);
}
-JEMALLOC_INLINE void *
+static inline void *
extent_addr_get(const extent_t *extent) {
assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) ||
!extent_slab_get(extent));
return extent->e_addr;
}
-JEMALLOC_INLINE size_t
+static inline size_t
extent_size_get(const extent_t *extent) {
return (extent->e_size_esn & EXTENT_SIZE_MASK);
}
-JEMALLOC_INLINE size_t
+static inline size_t
extent_esn_get(const extent_t *extent) {
return (extent->e_size_esn & EXTENT_ESN_MASK);
}
-JEMALLOC_INLINE size_t
+static inline size_t
extent_bsize_get(const extent_t *extent) {
return extent->e_bsize;
}
-JEMALLOC_INLINE void *
+static inline void *
extent_before_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) - PAGE);
}
-JEMALLOC_INLINE void *
+static inline void *
extent_last_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent) - PAGE);
}
-JEMALLOC_INLINE void *
+static inline void *
extent_past_get(const extent_t *extent) {
return (void *)((uintptr_t)extent_base_get(extent) +
extent_size_get(extent));
}
-JEMALLOC_INLINE arena_slab_data_t *
+static inline arena_slab_data_t *
extent_slab_data_get(extent_t *extent) {
assert(extent_slab_get(extent));
return &extent->e_slab_data;
}
-JEMALLOC_INLINE const arena_slab_data_t *
+static inline const arena_slab_data_t *
extent_slab_data_get_const(const extent_t *extent) {
assert(extent_slab_get(extent));
return &extent->e_slab_data;
}
-JEMALLOC_INLINE prof_tctx_t *
+static inline prof_tctx_t *
extent_prof_tctx_get(const extent_t *extent) {
return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx,
ATOMIC_ACQUIRE);
}
-JEMALLOC_INLINE void
+static inline void
extent_arena_set(extent_t *extent, arena_t *arena) {
unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U <<
MALLOCX_ARENA_BITS) - 1);
@@ -204,12 +147,12 @@ extent_arena_set(extent_t *extent, arena_t *arena) {
((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_addr_set(extent_t *extent, void *addr) {
extent->e_addr = addr;
}
-JEMALLOC_INLINE void
+static inline void
extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) {
assert(extent_base_get(extent) == extent_addr_get(extent));
@@ -228,85 +171,85 @@ extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) {
}
}
-JEMALLOC_INLINE void
+static inline void
extent_size_set(extent_t *extent, size_t size) {
assert((size & ~EXTENT_SIZE_MASK) == 0);
extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK);
}
-JEMALLOC_INLINE void
+static inline void
extent_esn_set(extent_t *extent, size_t esn) {
extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn &
EXTENT_ESN_MASK);
}
-JEMALLOC_INLINE void
+static inline void
extent_bsize_set(extent_t *extent, size_t bsize) {
extent->e_bsize = bsize;
}
-JEMALLOC_INLINE void
+static inline void
extent_szind_set(extent_t *extent, szind_t szind) {
assert(szind <= NSIZES); /* NSIZES means "invalid". */
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) |
((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_nfree_set(extent_t *extent, unsigned nfree) {
assert(extent_slab_get(extent));
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) |
((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_nfree_inc(extent_t *extent) {
assert(extent_slab_get(extent));
extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_nfree_dec(extent_t *extent) {
assert(extent_slab_get(extent));
extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_sn_set(extent_t *extent, size_t sn) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) |
((uint64_t)sn << EXTENT_BITS_SN_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_state_set(extent_t *extent, extent_state_t state) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) |
((uint64_t)state << EXTENT_BITS_STATE_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_zeroed_set(extent_t *extent, bool zeroed) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) |
((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_committed_set(extent_t *extent, bool committed) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) |
((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_slab_set(extent_t *extent, bool slab) {
extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) |
((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT);
}
-JEMALLOC_INLINE void
+static inline void
extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) {
atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE);
}
-JEMALLOC_INLINE void
+static inline void
extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed,
bool committed) {
@@ -327,7 +270,7 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size,
}
}
-JEMALLOC_INLINE void
+static inline void
extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
extent_arena_set(extent, NULL);
extent_addr_set(extent, addr);
@@ -340,39 +283,39 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) {
extent_committed_set(extent, true);
}
-JEMALLOC_INLINE void
+static inline void
extent_list_init(extent_list_t *list) {
ql_new(list);
}
-JEMALLOC_INLINE extent_t *
+static inline extent_t *
extent_list_first(const extent_list_t *list) {
return ql_first(list);
}
-JEMALLOC_INLINE extent_t *
+static inline extent_t *
extent_list_last(const extent_list_t *list) {
return ql_last(list, ql_link);
}
-JEMALLOC_INLINE void
+static inline void
extent_list_append(extent_list_t *list, extent_t *extent) {
ql_tail_insert(list, extent, ql_link);
}
-JEMALLOC_INLINE void
+static inline void
extent_list_replace(extent_list_t *list, extent_t *to_remove,
extent_t *to_insert) {
ql_after_insert(to_remove, to_insert, ql_link);
ql_remove(list, to_remove, ql_link);
}
-JEMALLOC_INLINE void
+static inline void
extent_list_remove(extent_list_t *list, extent_t *extent) {
ql_remove(list, extent, ql_link);
}
-JEMALLOC_INLINE int
+static inline int
extent_sn_comp(const extent_t *a, const extent_t *b) {
size_t a_sn = extent_sn_get(a);
size_t b_sn = extent_sn_get(b);
@@ -380,7 +323,7 @@ extent_sn_comp(const extent_t *a, const extent_t *b) {
return (a_sn > b_sn) - (a_sn < b_sn);
}
-JEMALLOC_INLINE int
+static inline int
extent_esn_comp(const extent_t *a, const extent_t *b) {
size_t a_esn = extent_esn_get(a);
size_t b_esn = extent_esn_get(b);
@@ -388,7 +331,7 @@ extent_esn_comp(const extent_t *a, const extent_t *b) {
return (a_esn > b_esn) - (a_esn < b_esn);
}
-JEMALLOC_INLINE int
+static inline int
extent_ad_comp(const extent_t *a, const extent_t *b) {
uintptr_t a_addr = (uintptr_t)extent_addr_get(a);
uintptr_t b_addr = (uintptr_t)extent_addr_get(b);
@@ -396,7 +339,7 @@ extent_ad_comp(const extent_t *a, const extent_t *b) {
return (a_addr > b_addr) - (a_addr < b_addr);
}
-JEMALLOC_INLINE int
+static inline int
extent_ead_comp(const extent_t *a, const extent_t *b) {
uintptr_t a_eaddr = (uintptr_t)a;
uintptr_t b_eaddr = (uintptr_t)b;
@@ -404,7 +347,7 @@ extent_ead_comp(const extent_t *a, const extent_t *b) {
return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr);
}
-JEMALLOC_INLINE int
+static inline int
extent_snad_comp(const extent_t *a, const extent_t *b) {
int ret;
@@ -417,7 +360,7 @@ extent_snad_comp(const extent_t *a, const extent_t *b) {
return ret;
}
-JEMALLOC_INLINE int
+static inline int
extent_esnead_comp(const extent_t *a, const extent_t *b) {
int ret;
@@ -429,6 +372,5 @@ extent_esnead_comp(const extent_t *a, const extent_t *b) {
ret = extent_ead_comp(a, b);
return ret;
}
-#endif
#endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */
diff --git a/include/jemalloc/internal/hash_inlines.h b/include/jemalloc/internal/hash_inlines.h
index 321c17c..2cd7e3e 100644
--- a/include/jemalloc/internal/hash_inlines.h
+++ b/include/jemalloc/internal/hash_inlines.h
@@ -9,30 +9,19 @@
* details.
*/
-#ifndef JEMALLOC_ENABLE_INLINE
-uint32_t hash_x86_32(const void *key, int len, uint32_t seed);
-void hash_x86_128(const void *key, const int len, uint32_t seed,
- uint64_t r_out[2]);
-void hash_x64_128(const void *key, const int len, const uint32_t seed,
- uint64_t r_out[2]);
-void hash(const void *key, size_t len, const uint32_t seed,
- size_t r_hash[2]);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_))
/******************************************************************************/
/* Internal implementation. */
-JEMALLOC_INLINE uint32_t
+static inline uint32_t
hash_rotl_32(uint32_t x, int8_t r) {
return ((x << r) | (x >> (32 - r)));
}
-JEMALLOC_INLINE uint64_t
+static inline uint64_t
hash_rotl_64(uint64_t x, int8_t r) {
return ((x << r) | (x >> (64 - r)));
}
-JEMALLOC_INLINE uint32_t
+static inline uint32_t
hash_get_block_32(const uint32_t *p, int i) {
/* Handle unaligned read. */
if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) {
@@ -45,7 +34,7 @@ hash_get_block_32(const uint32_t *p, int i) {
return p[i];
}
-JEMALLOC_INLINE uint64_t
+static inline uint64_t
hash_get_block_64(const uint64_t *p, int i) {
/* Handle unaligned read. */
if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) {
@@ -58,7 +47,7 @@ hash_get_block_64(const uint64_t *p, int i) {
return p[i];
}
-JEMALLOC_INLINE uint32_t
+static inline uint32_t
hash_fmix_32(uint32_t h) {
h ^= h >> 16;
h *= 0x85ebca6b;
@@ -69,7 +58,7 @@ hash_fmix_32(uint32_t h) {
return h;
}
-JEMALLOC_INLINE uint64_t
+static inline uint64_t
hash_fmix_64(uint64_t k) {
k ^= k >> 33;
k *= KQU(0xff51afd7ed558ccd);
@@ -80,7 +69,7 @@ hash_fmix_64(uint64_t k) {
return k;
}
-JEMALLOC_INLINE uint32_t
+static inline uint32_t
hash_x86_32(const void *key, int len, uint32_t seed) {
const uint8_t *data = (const uint8_t *) key;
const int nblocks = len / 4;
@@ -130,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) {
return h1;
}
-UNUSED JEMALLOC_INLINE void
+UNUSED static inline void
hash_x86_128(const void *key, const int len, uint32_t seed,
uint64_t r_out[2]) {
const uint8_t * data = (const uint8_t *) key;
@@ -231,7 +220,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed,
r_out[1] = (((uint64_t) h4) << 32) | h3;
}
-UNUSED JEMALLOC_INLINE void
+UNUSED static inline void
hash_x64_128(const void *key, const int len, const uint32_t seed,
uint64_t r_out[2]) {
const uint8_t *data = (const uint8_t *) key;
@@ -310,7 +299,7 @@ hash_x64_128(const void *key, const int len, const uint32_t seed,
/******************************************************************************/
/* API. */
-JEMALLOC_INLINE void
+static inline void
hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) {
assert(len <= INT_MAX); /* Unfortunate implementation limitation. */
@@ -325,6 +314,5 @@ hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) {
}
#endif
}
-#endif
#endif /* JEMALLOC_INTERNAL_HASH_INLINES_H */
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h
index c28bd7c..9cb933c 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h
@@ -5,42 +5,6 @@
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-pszind_t psz2ind(size_t psz);
-size_t pind2sz_compute(pszind_t pind);
-size_t pind2sz_lookup(pszind_t pind);
-size_t pind2sz(pszind_t pind);
-size_t psz2u(size_t psz);
-szind_t size2index_compute(size_t size);
-szind_t size2index_lookup(size_t size);
-szind_t size2index(size_t size);
-size_t index2size_compute(szind_t index);
-size_t index2size_lookup(szind_t index);
-size_t index2size(szind_t index);
-size_t s2u_compute(size_t size);
-size_t s2u_lookup(size_t size);
-size_t s2u(size_t size);
-size_t sa2u(size_t size, size_t alignment);
-arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
-arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
-arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
-bool arena_is_auto(arena_t *arena);
-arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
- bool refresh_if_missing);
-arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
-ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind);
-bool tcache_available(tsd_t *tsd);
-tcache_bin_t *tcache_small_bin_get(tcache_t *tcache, szind_t binind);
-tcache_bin_t *tcache_large_bin_get(tcache_t *tcache, szind_t binind);
-tcache_t *tcache_get(tsd_t *tsd);
-malloc_cpuid_t malloc_getcpu(void);
-unsigned percpu_arena_choose(void);
-unsigned percpu_arena_ind_limit(void);
-void pre_reentrancy(tsd_t *tsd);
-void post_reentrancy(tsd_t *tsd);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE pszind_t
psz2ind(size_t psz) {
if (unlikely(psz > LARGE_MAXCLASS)) {
@@ -64,7 +28,7 @@ psz2ind(size_t psz) {
}
}
-JEMALLOC_INLINE size_t
+static inline size_t
pind2sz_compute(pszind_t pind) {
if (unlikely(pind == NPSIZES)) {
return LARGE_MAXCLASS + PAGE;
@@ -86,20 +50,20 @@ pind2sz_compute(pszind_t pind) {
}
}
-JEMALLOC_INLINE size_t
+static inline size_t
pind2sz_lookup(pszind_t pind) {
size_t ret = (size_t)pind2sz_tab[pind];
assert(ret == pind2sz_compute(pind));
return ret;
}
-JEMALLOC_INLINE size_t
+static inline size_t
pind2sz(pszind_t pind) {
assert(pind < NPSIZES+1);
return pind2sz_lookup(pind);
}
-JEMALLOC_INLINE size_t
+static inline size_t
psz2u(size_t psz) {
if (unlikely(psz > LARGE_MAXCLASS)) {
return LARGE_MAXCLASS + PAGE;
@@ -115,7 +79,7 @@ psz2u(size_t psz) {
}
}
-JEMALLOC_INLINE szind_t
+static inline szind_t
size2index_compute(size_t size) {
if (unlikely(size > LARGE_MAXCLASS)) {
return NSIZES;
@@ -164,7 +128,7 @@ size2index(size_t size) {
return size2index_compute(size);
}
-JEMALLOC_INLINE size_t
+static inline size_t
index2size_compute(szind_t index) {
#if (NTBINS > 0)
if (index < NTBINS) {
@@ -355,7 +319,7 @@ percpu_arena_ind_limit(void) {
}
}
-JEMALLOC_INLINE arena_tdata_t *
+static inline arena_tdata_t *
arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) {
arena_tdata_t *tdata;
arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd);
@@ -380,7 +344,7 @@ arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) {
return arena_tdata_get_hard(tsd, ind);
}
-JEMALLOC_INLINE arena_t *
+static inline arena_t *
arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) {
arena_t *ret;
@@ -396,7 +360,7 @@ arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) {
return ret;
}
-JEMALLOC_INLINE ticker_t *
+static inline ticker_t *
decay_ticker_get(tsd_t *tsd, unsigned ind) {
arena_tdata_t *tdata;
@@ -446,7 +410,7 @@ tcache_get(tsd_t *tsd) {
return tsd_tcachep_get(tsd);
}
-JEMALLOC_INLINE void
+static inline void
pre_reentrancy(tsd_t *tsd) {
bool fast = tsd_fast(tsd);
++*tsd_reentrancy_levelp_get(tsd);
@@ -457,7 +421,7 @@ pre_reentrancy(tsd_t *tsd) {
}
}
-JEMALLOC_INLINE void
+static inline void
post_reentrancy(tsd_t *tsd) {
int8_t *reentrancy_level = tsd_reentrancy_levelp_get(tsd);
assert(*reentrancy_level > 0);
@@ -466,6 +430,4 @@ post_reentrancy(tsd_t *tsd) {
}
}
-#endif
-
#endif /* JEMALLOC_INTERNAL_INLINES_A_H */
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
index 2fd371c..cfc5209 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h
@@ -1,13 +1,8 @@
#ifndef JEMALLOC_INTERNAL_INLINES_B_H
#define JEMALLOC_INTERNAL_INLINES_B_H
-#ifndef JEMALLOC_ENABLE_INLINE
-extent_t *iealloc(tsdn_t *tsdn, const void *ptr);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
/* Choose an arena based on a per-thread value. */
-JEMALLOC_INLINE arena_t *
+static inline arena_t *
arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
arena_t *ret;
@@ -60,17 +55,17 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) {
return ret;
}
-JEMALLOC_INLINE arena_t *
+static inline arena_t *
arena_choose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, false);
}
-JEMALLOC_INLINE arena_t *
+static inline arena_t *
arena_ichoose(tsd_t *tsd, arena_t *arena) {
return arena_choose_impl(tsd, arena, true);
}
-JEMALLOC_INLINE bool
+static inline bool
arena_is_auto(arena_t *arena) {
assert(narenas_auto > 0);
return (arena_ind_get(arena) < narenas_auto);
@@ -84,6 +79,5 @@ iealloc(tsdn_t *tsdn, const void *ptr) {
return rtree_extent_read(tsdn, &extents_rtree, rtree_ctx,
(uintptr_t)ptr, true);
}
-#endif
#endif /* JEMALLOC_INTERNAL_INLINES_B_H */
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
index 70ac666..415c503 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
@@ -3,36 +3,6 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-arena_t *iaalloc(tsdn_t *tsdn, const void *ptr);
-size_t isalloc(tsdn_t *tsdn, const void *ptr);
-void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
- tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path);
-void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero,
- bool slow_path);
-void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
- tcache_t *tcache, bool is_internal, arena_t *arena);
-void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
- tcache_t *tcache, arena_t *arena);
-void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero);
-size_t ivsalloc(tsdn_t *tsdn, const void *ptr);
-void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
- alloc_ctx_t *alloc_ctx, bool is_internal, bool slow_path);
-void idalloc(tsd_t *tsd, void *ptr);
-void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
- alloc_ctx_t *alloc_ctx, bool slow_path);
-void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
- size_t extra, size_t alignment, bool zero, tcache_t *tcache,
- arena_t *arena);
-void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
- size_t alignment, bool zero, tcache_t *tcache, arena_t *arena);
-void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size,
- size_t alignment, bool zero);
-bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_ALWAYS_INLINE arena_t *
iaalloc(tsdn_t *tsdn, const void *ptr) {
assert(ptr != NULL);
@@ -214,6 +184,5 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra,
return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero);
}
-#endif
#endif /* JEMALLOC_INTERNAL_INLINES_C_H */
diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h
index c5dd9b3..1b2802a 100644
--- a/include/jemalloc/internal/jemalloc_internal_macros.h
+++ b/include/jemalloc/internal/jemalloc_internal_macros.h
@@ -1,37 +1,13 @@
#ifndef JEMALLOC_INTERNAL_MACROS_H
#define JEMALLOC_INTERNAL_MACROS_H
-/*
- * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for
- * functions that are static inline functions if inlining is enabled, and
- * single-definition library-private functions if inlining is disabled.
- *
- * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in
- * which case the denoted functions are always static, regardless of whether
- * inlining is enabled.
- */
#if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE)
- /* Disable inlining to make debugging/profiling easier. */
-# define JEMALLOC_ALWAYS_INLINE
-# define JEMALLOC_ALWAYS_INLINE_C static
-# define JEMALLOC_INLINE
-# define JEMALLOC_INLINE_C static
+# define JEMALLOC_ALWAYS_INLINE static inline
#else
-# define JEMALLOC_ENABLE_INLINE
-# ifdef JEMALLOC_HAVE_ATTR
-# define JEMALLOC_ALWAYS_INLINE \
- static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline)
-# define JEMALLOC_ALWAYS_INLINE_C \
- static inline JEMALLOC_ATTR(always_inline)
-# else
-# define JEMALLOC_ALWAYS_INLINE static inline
-# define JEMALLOC_ALWAYS_INLINE_C static inline
-# endif
-# define JEMALLOC_INLINE static inline
-# define JEMALLOC_INLINE_C static inline
-# ifdef _MSC_VER
-# define inline _inline
-# endif
+# define JEMALLOC_ALWAYS_INLINE JEMALLOC_ATTR(always_inline) static inline
+#endif
+#ifdef _MSC_VER
+# define inline _inline
#endif
#ifdef JEMALLOC_CC_SILENCE
diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h
index 5ec439f..2856d84 100644
--- a/include/jemalloc/internal/mutex_inlines.h
+++ b/include/jemalloc/internal/mutex_inlines.h
@@ -5,31 +5,19 @@
void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
-#ifndef JEMALLOC_ENABLE_INLINE
-void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex);
-bool malloc_mutex_trylock(malloc_mutex_t *mutex);
-void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex);
-void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
-void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex);
-void malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
- malloc_mutex_t *mutex);
-void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_lock_final(malloc_mutex_t *mutex) {
MALLOC_MUTEX_LOCK(mutex);
}
/* Trylock: return false if the lock is successfully acquired. */
-JEMALLOC_INLINE bool
+static inline bool
malloc_mutex_trylock(malloc_mutex_t *mutex) {
return MALLOC_MUTEX_TRYLOCK(mutex);
}
/* Aggregate lock prof data. */
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
@@ -52,7 +40,7 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
sum->n_lock_ops += data->n_lock_ops;
}
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_not_owner(tsdn, &mutex->witness);
if (isthreaded) {
@@ -72,7 +60,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_lock(tsdn, &mutex->witness);
}
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_unlock(tsdn, &mutex->witness);
if (isthreaded) {
@@ -80,18 +68,18 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
}
}
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_owner(tsdn, &mutex->witness);
}
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
witness_assert_not_owner(tsdn, &mutex->witness);
}
/* Copy the prof data from mutex for processing. */
-JEMALLOC_INLINE void
+static inline void
malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
malloc_mutex_t *mutex) {
mutex_prof_data_t *source = &mutex->prof_data;
@@ -108,6 +96,4 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
}
-#endif
-
#endif /* JEMALLOC_INTERNAL_MUTEX_INLINES_H */
diff --git a/include/jemalloc/internal/prng_inlines.h b/include/jemalloc/internal/prng_inlines.h
index c39c63f..0275dfc 100644
--- a/include/jemalloc/internal/prng_inlines.h
+++ b/include/jemalloc/internal/prng_inlines.h
@@ -4,23 +4,6 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-uint32_t prng_state_next_u32(uint32_t state);
-uint64_t prng_state_next_u64(uint64_t state);
-size_t prng_state_next_zu(size_t state);
-
-uint32_t prng_lg_range_u32(atomic_u32_t *state, unsigned lg_range,
- bool atomic);
-uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range);
-size_t prng_lg_range_zu(atomic_zu_t *state, unsigned lg_range, bool atomic);
-
-uint32_t prng_range_u32(atomic_u32_t *state, uint32_t range,
- bool atomic);
-uint64_t prng_range_u64(uint64_t *state, uint64_t range);
-size_t prng_range_zu(atomic_zu_t *state, size_t range, bool atomic);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
JEMALLOC_ALWAYS_INLINE uint32_t
prng_state_next_u32(uint32_t state) {
return (state * PRNG_A_32) + PRNG_C_32;
@@ -156,6 +139,5 @@ prng_range_zu(atomic_zu_t *state, size_t range, bool atomic) {
return ret;
}
-#endif
#endif /* JEMALLOC_INTERNAL_PRNG_INLINES_H */
diff --git a/include/jemalloc/internal/prof_inlines_a.h b/include/jemalloc/internal/prof_inlines_a.h
index d0d2968..6203cbd 100644
--- a/include/jemalloc/internal/prof_inlines_a.h
+++ b/include/jemalloc/internal/prof_inlines_a.h
@@ -1,14 +1,7 @@
#ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H
#define JEMALLOC_INTERNAL_PROF_INLINES_A_H
-#ifndef JEMALLOC_ENABLE_INLINE
-bool prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum,
- uint64_t accumbytes);
-void prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
-JEMALLOC_INLINE bool
+static inline bool
prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) {
cassert(config_prof);
@@ -46,7 +39,7 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) {
return overflow;
}
-JEMALLOC_INLINE void
+static inline void
prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) {
cassert(config_prof);
@@ -73,6 +66,5 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) {
malloc_mutex_unlock(tsdn, &prof_accum->mtx);
#endif
}
-#endif
#endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */
diff --git a/include/jemalloc/internal/prof_inlines_b.h b/include/jemalloc/internal/prof_inlines_b.h
index 6a79c01..eba981b 100644
--- a/include/jemalloc/internal/prof_inlines_b.h
+++ b/include/jemalloc/internal/prof_inlines_b.h
@@ -1,29 +1,6 @@
#ifndef JEMALLOC_INTERNAL_PROF_INLINES_B_H
#define JEMALLOC_INTERNAL_PROF_INLINES_B_H
-#ifndef JEMALLOC_ENABLE_INLINE
-bool prof_active_get_unlocked(void);
-bool prof_gdump_get_unlocked(void);
-prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create);
-prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr,
- alloc_ctx_t *alloc_ctx);
-void prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize,
- alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx);
-void prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx);
-bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update,
- prof_tdata_t **tdata_out);
-prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active,
- bool update);
-void prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize,
- alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx);
-void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize,
- prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr,
- size_t old_usize, prof_tctx_t *old_tctx);
-void prof_free(tsd_t *tsd, const void *ptr, size_t usize,
- alloc_ctx_t *alloc_ctx);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_))
JEMALLOC_ALWAYS_INLINE bool
prof_active_get_unlocked(void) {
/*
@@ -231,6 +208,5 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) {
prof_free_sampled_object(tsd, usize, tctx);
}
}
-#endif
#endif /* JEMALLOC_INTERNAL_PROF_INLINES_B_H */
diff --git a/include/jemalloc/internal/rtree_inlines.h b/include/jemalloc/internal/rtree_inlines.h
index 030e578..f4f7c2c 100644
--- a/include/jemalloc/internal/rtree_inlines.h
+++ b/include/jemalloc/internal/rtree_inlines.h
@@ -3,59 +3,6 @@
#include "jemalloc/internal/spin.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-uintptr_t rtree_leafkey(uintptr_t key);
-uintptr_t rtree_subkey(uintptr_t key, unsigned level);
-# ifdef RTREE_LEAF_COMPACT
-uintptr_t rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, bool dependent);
-extent_t *rtree_leaf_elm_bits_extent_get(uintptr_t bits);
-szind_t rtree_leaf_elm_bits_szind_get(uintptr_t bits);
-bool rtree_leaf_elm_bits_slab_get(uintptr_t bits);
-bool rtree_leaf_elm_bits_locked_get(uintptr_t bits);
-# endif
-extent_t *rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, bool dependent);
-szind_t rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, bool dependent);
-bool rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, bool dependent);
-void rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, extent_t *extent);
-void rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, szind_t szind);
-void rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, bool acquired, bool slab);
-void rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
- bool acquired, extent_t *extent, szind_t szind, bool slab);
-void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm, szind_t szind, bool slab);
-rtree_leaf_elm_t *rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
-bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
- uintptr_t key, extent_t *extent, szind_t szind, bool slab);
-rtree_leaf_elm_t *rtree_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent);
-extent_t *rtree_extent_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent);
-szind_t rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent);
-bool rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, extent_t **r_extent,
- szind_t *r_szind);
-bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
- uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab);
-rtree_leaf_elm_t *rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing);
-void rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree,
- rtree_leaf_elm_t *elm);
-void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree,
- rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab);
-void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
- uintptr_t key);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_))
JEMALLOC_ALWAYS_INLINE uintptr_t
rtree_leafkey(uintptr_t key) {
unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3);
@@ -194,7 +141,7 @@ rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
#endif
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
bool acquired, extent_t *extent) {
if (config_debug && acquired) {
@@ -219,7 +166,7 @@ rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
#endif
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
bool acquired, szind_t szind) {
if (config_debug && acquired) {
@@ -241,7 +188,7 @@ rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
#endif
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
bool acquired, bool slab) {
if (config_debug && acquired) {
@@ -261,7 +208,7 @@ rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
#endif
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
bool acquired, extent_t *extent, szind_t szind, bool slab) {
if (config_debug && acquired) {
@@ -287,7 +234,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm,
#endif
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree,
rtree_leaf_elm_t *elm, szind_t szind, bool slab) {
assert(!slab || szind < NBINS);
@@ -384,7 +331,7 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
dependent, init_missing);
}
-JEMALLOC_INLINE bool
+static inline bool
rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key,
extent_t *extent, szind_t szind, bool slab) {
/* Use rtree_clear() to set the extent to NULL. */
@@ -471,7 +418,7 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
return false;
}
-JEMALLOC_INLINE rtree_leaf_elm_t *
+static inline rtree_leaf_elm_t *
rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, bool dependent, bool init_missing) {
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx,
@@ -511,7 +458,7 @@ rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
return elm;
}
-JEMALLOC_INLINE void
+static inline void
rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm) {
extent_t *extent = rtree_leaf_elm_extent_read(tsdn, rtree, elm, true,
true);
@@ -521,7 +468,7 @@ rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm) {
}
}
-JEMALLOC_INLINE void
+static inline void
rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key, szind_t szind, bool slab) {
assert(!slab || szind < NBINS);
@@ -530,7 +477,7 @@ rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab);
}
-JEMALLOC_INLINE void
+static inline void
rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
uintptr_t key) {
rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true);
@@ -538,6 +485,5 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx,
NULL);
rtree_leaf_elm_write(tsdn, rtree, elm, false, NULL, NSIZES, false);
}
-#endif
#endif /* JEMALLOC_INTERNAL_RTREE_INLINES_H */
diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h
index 67d35b5..8b42af6 100644
--- a/include/jemalloc/internal/tcache_inlines.h
+++ b/include/jemalloc/internal/tcache_inlines.h
@@ -4,30 +4,12 @@
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/util.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-void tcache_event(tsd_t *tsd, tcache_t *tcache);
-bool tcache_enabled_get(tsd_t *tsd);
-tcache_t *tcache_get(tsd_t *tsd);
-void tcache_enabled_set(tsd_t *tsd, bool enabled);
-void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success);
-void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
- size_t size, szind_t ind, bool zero, bool slow_path);
-void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache,
- size_t size, szind_t ind, bool zero, bool slow_path);
-void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr,
- szind_t binind, bool slow_path);
-void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr,
- szind_t binind, bool slow_path);
-tcache_t *tcaches_get(tsd_t *tsd, unsigned ind);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_))
-JEMALLOC_INLINE bool
+static inline bool
tcache_enabled_get(tsd_t *tsd) {
return tsd_tcache_enabled_get(tsd);
}
-JEMALLOC_INLINE void
+static inline void
tcache_enabled_set(tsd_t *tsd, bool enabled) {
bool was_enabled = tsd_tcache_enabled_get(tsd);
@@ -261,6 +243,5 @@ tcaches_get(tsd_t *tsd, unsigned ind) {
}
return elm->tcache;
}
-#endif
#endif /* JEMALLOC_INTERNAL_TCACHE_INLINES_H */
diff --git a/include/jemalloc/internal/ticker_inlines.h b/include/jemalloc/internal/ticker_inlines.h
index 9102ba6..cd5821f 100644
--- a/include/jemalloc/internal/ticker_inlines.h
+++ b/include/jemalloc/internal/ticker_inlines.h
@@ -1,32 +1,23 @@
#ifndef JEMALLOC_INTERNAL_TICKER_INLINES_H
#define JEMALLOC_INTERNAL_TICKER_INLINES_H
-#ifndef JEMALLOC_ENABLE_INLINE
-void ticker_init(ticker_t *ticker, int32_t nticks);
-void ticker_copy(ticker_t *ticker, const ticker_t *other);
-int32_t ticker_read(const ticker_t *ticker);
-bool ticker_ticks(ticker_t *ticker, int32_t nticks);
-bool ticker_tick(ticker_t *ticker);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TICKER_C_))
-JEMALLOC_INLINE void
+static inline void
ticker_init(ticker_t *ticker, int32_t nticks) {
ticker->tick = nticks;
ticker->nticks = nticks;
}
-JEMALLOC_INLINE void
+static inline void
ticker_copy(ticker_t *ticker, const ticker_t *other) {
*ticker = *other;
}
-JEMALLOC_INLINE int32_t
+static inline int32_t
ticker_read(const ticker_t *ticker) {
return ticker->tick;
}
-JEMALLOC_INLINE bool
+static inline bool
ticker_ticks(ticker_t *ticker, int32_t nticks) {
if (unlikely(ticker->tick < nticks)) {
ticker->tick = ticker->nticks;
@@ -36,10 +27,9 @@ ticker_ticks(ticker_t *ticker, int32_t nticks) {
return(false);
}
-JEMALLOC_INLINE bool
+static inline bool
ticker_tick(ticker_t *ticker) {
return ticker_ticks(ticker, 1);
}
-#endif
#endif /* JEMALLOC_INTERNAL_TICKER_INLINES_H */
diff --git a/include/jemalloc/internal/tsd_inlines.h b/include/jemalloc/internal/tsd_inlines.h
index 93469bc..f0f77e4 100644
--- a/include/jemalloc/internal/tsd_inlines.h
+++ b/include/jemalloc/internal/tsd_inlines.h
@@ -1,29 +1,6 @@
#ifndef JEMALLOC_INTERNAL_TSD_INLINES_H
#define JEMALLOC_INTERNAL_TSD_INLINES_H
-#ifndef JEMALLOC_ENABLE_INLINE
-malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
-
-tsd_t *tsd_fetch_impl(bool init);
-tsd_t *tsd_fetch(void);
-tsdn_t *tsd_tsdn(tsd_t *tsd);
-bool tsd_nominal(tsd_t *tsd);
-#define O(n, t, gs, i, c) \
-t *tsd_##n##p_get(tsd_t *tsd); \
-t tsd_##n##_get(tsd_t *tsd); \
-void tsd_##n##_set(tsd_t *tsd, t n);
-MALLOC_TSD
-#undef O
-tsdn_t *tsdn_fetch(void);
-bool tsdn_null(const tsdn_t *tsdn);
-tsd_t *tsdn_tsd(tsdn_t *tsdn);
-rtree_ctx_t *tsd_rtree_ctx(tsd_t *tsd);
-rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback);
-bool tsd_fast(tsd_t *tsd);
-bool tsd_assert_fast(tsd_t *tsd);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_))
malloc_tsd_externs(, tsd_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
@@ -97,7 +74,7 @@ tsd_tsdn(tsd_t *tsd) {
return (tsdn_t *)tsd;
}
-JEMALLOC_INLINE bool
+static inline bool
tsd_nominal(tsd_t *tsd) {
return (tsd->state <= tsd_state_nominal_max);
}
@@ -140,6 +117,5 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) {
}
return tsd_rtree_ctx(tsdn_tsd(tsdn));
}
-#endif
#endif /* JEMALLOC_INTERNAL_TSD_INLINES_H */
diff --git a/include/jemalloc/internal/witness_inlines.h b/include/jemalloc/internal/witness_inlines.h
index c5027f1..51d1af3 100644
--- a/include/jemalloc/internal/witness_inlines.h
+++ b/include/jemalloc/internal/witness_inlines.h
@@ -3,21 +3,8 @@
#include "jemalloc/internal/ql.h"
-#ifndef JEMALLOC_ENABLE_INLINE
-bool witness_owner(tsd_t *tsd, const witness_t *witness);
-void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
-void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
-void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
- unsigned depth);
-void witness_assert_depth(tsdn_t *tsdn, unsigned depth);
-void witness_assert_lockless(tsdn_t *tsdn);
-void witness_lock(tsdn_t *tsdn, witness_t *witness);
-void witness_unlock(tsdn_t *tsdn, witness_t *witness);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
/* Helper, not intended for direct use. */
-JEMALLOC_INLINE bool
+static inline bool
witness_owner(tsd_t *tsd, const witness_t *witness) {
witness_list_t *witnesses;
witness_t *w;
@@ -34,7 +21,7 @@ witness_owner(tsd_t *tsd, const witness_t *witness) {
return false;
}
-JEMALLOC_INLINE void
+static inline void
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) {
tsd_t *tsd;
@@ -56,7 +43,7 @@ witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) {
witness_owner_error(witness);
}
-JEMALLOC_INLINE void
+static inline void
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
@@ -82,7 +69,7 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) {
}
}
-JEMALLOC_INLINE void
+static inline void
witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
unsigned depth) {
tsd_t *tsd;
@@ -115,17 +102,17 @@ witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
}
}
-JEMALLOC_INLINE void
+static inline void
witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
}
-JEMALLOC_INLINE void
+static inline void
witness_assert_lockless(tsdn_t *tsdn) {
witness_assert_depth(tsdn, 0);
}
-JEMALLOC_INLINE void
+static inline void
witness_lock(tsdn_t *tsdn, witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
@@ -168,7 +155,7 @@ witness_lock(tsdn_t *tsdn, witness_t *witness) {
ql_tail_insert(witnesses, witness, link);
}
-JEMALLOC_INLINE void
+static inline void
witness_unlock(tsdn_t *tsdn, witness_t *witness) {
tsd_t *tsd;
witness_list_t *witnesses;
@@ -197,6 +184,5 @@ witness_unlock(tsdn_t *tsdn, witness_t *witness) {
witness_assert_owner(tsdn, witness);
}
}
-#endif
#endif /* JEMALLOC_INTERNAL_WITNESS_INLINES_H */