diff options
author | David Goldblatt <davidgoldblatt@fb.com> | 2017-04-21 16:37:34 (GMT) |
---|---|---|
committer | David Goldblatt <davidtgoldblatt@gmail.com> | 2017-04-24 17:33:21 (GMT) |
commit | 4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98 (patch) | |
tree | ecb8c4f306175db5b74f0dde83a28993d71f9888 /include | |
parent | 7d86c92c61c60b771cdf146f6187c1550a089ad1 (diff) | |
download | jemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.zip jemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.tar.gz jemalloc-4d2e4bf5ebb1e37a9348fdbf51af0b63304d7c98.tar.bz2 |
Get rid of most of the various inline macros.
Diffstat (limited to 'include')
19 files changed, 134 insertions, 544 deletions
diff --git a/include/jemalloc/internal/arena_inlines_a.h b/include/jemalloc/internal/arena_inlines_a.h index 2bd5ce7..da58770 100644 --- a/include/jemalloc/internal/arena_inlines_a.h +++ b/include/jemalloc/internal/arena_inlines_a.h @@ -1,38 +1,27 @@ #ifndef JEMALLOC_INTERNAL_ARENA_INLINES_A_H #define JEMALLOC_INTERNAL_ARENA_INLINES_A_H -#ifndef JEMALLOC_ENABLE_INLINE -unsigned arena_ind_get(const arena_t *arena); -void arena_internal_add(arena_t *arena, size_t size); -void arena_internal_sub(arena_t *arena, size_t size); -size_t arena_internal_get(arena_t *arena); -bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes); -void percpu_arena_update(tsd_t *tsd, unsigned cpu); -#endif /* JEMALLOC_ENABLE_INLINE */ - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) - -JEMALLOC_INLINE unsigned +static inline unsigned arena_ind_get(const arena_t *arena) { return base_ind_get(arena->base); } -JEMALLOC_INLINE void +static inline void arena_internal_add(arena_t *arena, size_t size) { atomic_fetch_add_zu(&arena->stats.internal, size, ATOMIC_RELAXED); } -JEMALLOC_INLINE void +static inline void arena_internal_sub(arena_t *arena, size_t size) { atomic_fetch_sub_zu(&arena->stats.internal, size, ATOMIC_RELAXED); } -JEMALLOC_INLINE size_t +static inline size_t arena_internal_get(arena_t *arena) { return atomic_load_zu(&arena->stats.internal, ATOMIC_RELAXED); } -JEMALLOC_INLINE bool +static inline bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) { cassert(config_prof); @@ -43,7 +32,7 @@ arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) { return prof_accum_add(tsdn, &arena->prof_accum, accumbytes); } -JEMALLOC_INLINE void +static inline void percpu_arena_update(tsd_t *tsd, unsigned cpu) { assert(have_percpu_arena); arena_t *oldarena = tsd_arena_get(tsd); @@ -65,6 +54,4 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) { } } -#endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */ - #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_A_H */ diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 4264f4b..526103b 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -3,30 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" -#ifndef JEMALLOC_ENABLE_INLINE -szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin); -prof_tctx_t *arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, - alloc_ctx_t *ctx); -void arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, - alloc_ctx_t *ctx, prof_tctx_t *tctx); -void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx); -void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks); -void arena_decay_tick(tsdn_t *tsdn, arena_t *arena); -void *arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, - bool zero, tcache_t *tcache, bool slow_path); -arena_t *arena_aalloc(tsdn_t *tsdn, const void *ptr); -size_t arena_salloc(tsdn_t *tsdn, const void *ptr); -size_t arena_vsalloc(tsdn_t *tsdn, const void *ptr); -void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr); -void arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, - alloc_ctx_t *alloc_ctx, bool slow_path); -void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size); -void arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - alloc_ctx_t *alloc_ctx, bool slow_path); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) -JEMALLOC_INLINE szind_t +static inline szind_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { szind_t binind = (szind_t)(bin - arena->bins); assert(binind < NBINS); @@ -71,7 +48,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, } } -JEMALLOC_INLINE void +static inline void arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -182,7 +159,7 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { return index2size(szind); } -JEMALLOC_INLINE void +static inline void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { assert(ptr != NULL); @@ -264,7 +241,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, } } -JEMALLOC_INLINE void +static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); assert(size <= LARGE_MAXCLASS); @@ -376,5 +353,4 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, } } -#endif /* (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) */ #endif /* JEMALLOC_INTERNAL_ARENA_INLINES_B_H */ diff --git a/include/jemalloc/internal/base_inlines.h b/include/jemalloc/internal/base_inlines.h index aa8306a..931560b 100644 --- a/include/jemalloc/internal/base_inlines.h +++ b/include/jemalloc/internal/base_inlines.h @@ -1,15 +1,9 @@ #ifndef JEMALLOC_INTERNAL_BASE_INLINES_H #define JEMALLOC_INTERNAL_BASE_INLINES_H -#ifndef JEMALLOC_ENABLE_INLINE -unsigned base_ind_get(const base_t *base); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BASE_C_)) -JEMALLOC_INLINE unsigned +static inline unsigned base_ind_get(const base_t *base) { return base->ind; } -#endif #endif /* JEMALLOC_INTERNAL_BASE_INLINES_H */ diff --git a/include/jemalloc/internal/bitmap_inlines.h b/include/jemalloc/internal/bitmap_inlines.h index c236201..84425b3 100644 --- a/include/jemalloc/internal/bitmap_inlines.h +++ b/include/jemalloc/internal/bitmap_inlines.h @@ -3,18 +3,7 @@ #include "jemalloc/internal/bit_util.h" -#ifndef JEMALLOC_ENABLE_INLINE -bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo); -bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -size_t bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, - size_t min_bit); -size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo); -void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_)) -JEMALLOC_INLINE bool +static inline bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) { #ifdef BITMAP_USE_TREE size_t rgoff = binfo->levels[binfo->nlevels].group_offset - 1; @@ -33,7 +22,7 @@ bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) { #endif } -JEMALLOC_INLINE bool +static inline bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { size_t goff; bitmap_t g; @@ -44,7 +33,7 @@ bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { return !(g & (ZU(1) << (bit & BITMAP_GROUP_NBITS_MASK))); } -JEMALLOC_INLINE void +static inline void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { size_t goff; bitmap_t *gp; @@ -80,7 +69,7 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { } /* ffu: find first unset >= bit. */ -JEMALLOC_INLINE size_t +static inline size_t bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) { assert(min_bit < binfo->nbits); @@ -139,7 +128,7 @@ bitmap_ffu(const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t min_bit) { } /* sfu: set first unset. */ -JEMALLOC_INLINE size_t +static inline size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) { size_t bit; bitmap_t g; @@ -169,7 +158,7 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) { return bit; } -JEMALLOC_INLINE void +static inline void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { size_t goff; bitmap_t *gp; @@ -208,6 +197,4 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) { #endif /* BITMAP_USE_TREE */ } -#endif - #endif /* JEMALLOC_INTERNAL_BITMAP_INLINES_H */ diff --git a/include/jemalloc/internal/extent_inlines.h b/include/jemalloc/internal/extent_inlines.h index fbe51e4..22d45ce 100644 --- a/include/jemalloc/internal/extent_inlines.h +++ b/include/jemalloc/internal/extent_inlines.h @@ -3,64 +3,7 @@ #include "jemalloc/internal/ql.h" -#ifndef JEMALLOC_ENABLE_INLINE -arena_t *extent_arena_get(const extent_t *extent); -szind_t extent_szind_get_maybe_invalid(const extent_t *extent); -szind_t extent_szind_get(const extent_t *extent); -size_t extent_usize_get(const extent_t *extent); -size_t extent_sn_get(const extent_t *extent); -extent_state_t extent_state_get(const extent_t *extent); -bool extent_zeroed_get(const extent_t *extent); -bool extent_committed_get(const extent_t *extent); -bool extent_slab_get(const extent_t *extent); -unsigned extent_nfree_get(const extent_t *extent); -void *extent_base_get(const extent_t *extent); -void *extent_addr_get(const extent_t *extent); -size_t extent_size_get(const extent_t *extent); -size_t extent_esn_get(const extent_t *extent); -size_t extent_bsize_get(const extent_t *extent); -void *extent_before_get(const extent_t *extent); -void *extent_last_get(const extent_t *extent); -void *extent_past_get(const extent_t *extent); -arena_slab_data_t *extent_slab_data_get(extent_t *extent); -const arena_slab_data_t *extent_slab_data_get_const(const extent_t *extent); -prof_tctx_t *extent_prof_tctx_get(const extent_t *extent); -void extent_arena_set(extent_t *extent, arena_t *arena); -void extent_addr_set(extent_t *extent, void *addr); -void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment); -void extent_size_set(extent_t *extent, size_t size); -void extent_esn_set(extent_t *extent, size_t esn); -void extent_bsize_set(extent_t *extent, size_t bsize); -void extent_szind_set(extent_t *extent, szind_t szind); -void extent_nfree_set(extent_t *extent, unsigned nfree); -void extent_nfree_inc(extent_t *extent); -void extent_nfree_dec(extent_t *extent); -void extent_sn_set(extent_t *extent, size_t sn); -void extent_state_set(extent_t *extent, extent_state_t state); -void extent_zeroed_set(extent_t *extent, bool zeroed); -void extent_committed_set(extent_t *extent, bool committed); -void extent_slab_set(extent_t *extent, bool slab); -void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx); -void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, - bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, - bool committed); -void extent_binit(extent_t *extent, void *addr, size_t size, size_t sn); -void extent_list_init(extent_list_t *list); -extent_t *extent_list_first(const extent_list_t *list); -extent_t *extent_list_last(const extent_list_t *list); -void extent_list_append(extent_list_t *list, extent_t *extent); -void extent_list_replace(extent_list_t *list, extent_t *to_remove, - extent_t *to_insert); -void extent_list_remove(extent_list_t *list, extent_t *extent); -int extent_sn_comp(const extent_t *a, const extent_t *b); -int extent_esn_comp(const extent_t *a, const extent_t *b); -int extent_ad_comp(const extent_t *a, const extent_t *b); -int extent_snad_comp(const extent_t *a, const extent_t *b); -int extent_esnead_comp(const extent_t *a, const extent_t *b); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) -JEMALLOC_INLINE arena_t * +static inline arena_t * extent_arena_get(const extent_t *extent) { unsigned arena_ind = (unsigned)((extent->e_bits & EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); @@ -75,7 +18,7 @@ extent_arena_get(const extent_t *extent) { return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); } -JEMALLOC_INLINE szind_t +static inline szind_t extent_szind_get_maybe_invalid(const extent_t *extent) { szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> EXTENT_BITS_SZIND_SHIFT); @@ -83,120 +26,120 @@ extent_szind_get_maybe_invalid(const extent_t *extent) { return szind; } -JEMALLOC_INLINE szind_t +static inline szind_t extent_szind_get(const extent_t *extent) { szind_t szind = extent_szind_get_maybe_invalid(extent); assert(szind < NSIZES); /* Never call when "invalid". */ return szind; } -JEMALLOC_INLINE size_t +static inline size_t extent_usize_get(const extent_t *extent) { return index2size(extent_szind_get(extent)); } -JEMALLOC_INLINE size_t +static inline size_t extent_sn_get(const extent_t *extent) { return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> EXTENT_BITS_SN_SHIFT); } -JEMALLOC_INLINE extent_state_t +static inline extent_state_t extent_state_get(const extent_t *extent) { return (extent_state_t)((extent->e_bits & EXTENT_BITS_STATE_MASK) >> EXTENT_BITS_STATE_SHIFT); } -JEMALLOC_INLINE bool +static inline bool extent_zeroed_get(const extent_t *extent) { return (bool)((extent->e_bits & EXTENT_BITS_ZEROED_MASK) >> EXTENT_BITS_ZEROED_SHIFT); } -JEMALLOC_INLINE bool +static inline bool extent_committed_get(const extent_t *extent) { return (bool)((extent->e_bits & EXTENT_BITS_COMMITTED_MASK) >> EXTENT_BITS_COMMITTED_SHIFT); } -JEMALLOC_INLINE bool +static inline bool extent_slab_get(const extent_t *extent) { return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >> EXTENT_BITS_SLAB_SHIFT); } -JEMALLOC_INLINE unsigned +static inline unsigned extent_nfree_get(const extent_t *extent) { assert(extent_slab_get(extent)); return (unsigned)((extent->e_bits & EXTENT_BITS_NFREE_MASK) >> EXTENT_BITS_NFREE_SHIFT); } -JEMALLOC_INLINE void * +static inline void * extent_base_get(const extent_t *extent) { assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)); return PAGE_ADDR2BASE(extent->e_addr); } -JEMALLOC_INLINE void * +static inline void * extent_addr_get(const extent_t *extent) { assert(extent->e_addr == PAGE_ADDR2BASE(extent->e_addr) || !extent_slab_get(extent)); return extent->e_addr; } -JEMALLOC_INLINE size_t +static inline size_t extent_size_get(const extent_t *extent) { return (extent->e_size_esn & EXTENT_SIZE_MASK); } -JEMALLOC_INLINE size_t +static inline size_t extent_esn_get(const extent_t *extent) { return (extent->e_size_esn & EXTENT_ESN_MASK); } -JEMALLOC_INLINE size_t +static inline size_t extent_bsize_get(const extent_t *extent) { return extent->e_bsize; } -JEMALLOC_INLINE void * +static inline void * extent_before_get(const extent_t *extent) { return (void *)((uintptr_t)extent_base_get(extent) - PAGE); } -JEMALLOC_INLINE void * +static inline void * extent_last_get(const extent_t *extent) { return (void *)((uintptr_t)extent_base_get(extent) + extent_size_get(extent) - PAGE); } -JEMALLOC_INLINE void * +static inline void * extent_past_get(const extent_t *extent) { return (void *)((uintptr_t)extent_base_get(extent) + extent_size_get(extent)); } -JEMALLOC_INLINE arena_slab_data_t * +static inline arena_slab_data_t * extent_slab_data_get(extent_t *extent) { assert(extent_slab_get(extent)); return &extent->e_slab_data; } -JEMALLOC_INLINE const arena_slab_data_t * +static inline const arena_slab_data_t * extent_slab_data_get_const(const extent_t *extent) { assert(extent_slab_get(extent)); return &extent->e_slab_data; } -JEMALLOC_INLINE prof_tctx_t * +static inline prof_tctx_t * extent_prof_tctx_get(const extent_t *extent) { return (prof_tctx_t *)atomic_load_p(&extent->e_prof_tctx, ATOMIC_ACQUIRE); } -JEMALLOC_INLINE void +static inline void extent_arena_set(extent_t *extent, arena_t *arena) { unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << MALLOCX_ARENA_BITS) - 1); @@ -204,12 +147,12 @@ extent_arena_set(extent_t *extent, arena_t *arena) { ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_addr_set(extent_t *extent, void *addr) { extent->e_addr = addr; } -JEMALLOC_INLINE void +static inline void extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { assert(extent_base_get(extent) == extent_addr_get(extent)); @@ -228,85 +171,85 @@ extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { } } -JEMALLOC_INLINE void +static inline void extent_size_set(extent_t *extent, size_t size) { assert((size & ~EXTENT_SIZE_MASK) == 0); extent->e_size_esn = size | (extent->e_size_esn & ~EXTENT_SIZE_MASK); } -JEMALLOC_INLINE void +static inline void extent_esn_set(extent_t *extent, size_t esn) { extent->e_size_esn = (extent->e_size_esn & ~EXTENT_ESN_MASK) | (esn & EXTENT_ESN_MASK); } -JEMALLOC_INLINE void +static inline void extent_bsize_set(extent_t *extent, size_t bsize) { extent->e_bsize = bsize; } -JEMALLOC_INLINE void +static inline void extent_szind_set(extent_t *extent, szind_t szind) { assert(szind <= NSIZES); /* NSIZES means "invalid". */ extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_nfree_set(extent_t *extent, unsigned nfree) { assert(extent_slab_get(extent)); extent->e_bits = (extent->e_bits & ~EXTENT_BITS_NFREE_MASK) | ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_nfree_inc(extent_t *extent) { assert(extent_slab_get(extent)); extent->e_bits += ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_nfree_dec(extent_t *extent) { assert(extent_slab_get(extent)); extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | ((uint64_t)sn << EXTENT_BITS_SN_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_state_set(extent_t *extent, extent_state_t state) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_STATE_MASK) | ((uint64_t)state << EXTENT_BITS_STATE_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_zeroed_set(extent_t *extent, bool zeroed) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_ZEROED_MASK) | ((uint64_t)zeroed << EXTENT_BITS_ZEROED_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_committed_set(extent_t *extent, bool committed) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_COMMITTED_MASK) | ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_slab_set(extent_t *extent, bool slab) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) | ((uint64_t)slab << EXTENT_BITS_SLAB_SHIFT); } -JEMALLOC_INLINE void +static inline void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); } -JEMALLOC_INLINE void +static inline void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, bool committed) { @@ -327,7 +270,7 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, } } -JEMALLOC_INLINE void +static inline void extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_arena_set(extent, NULL); extent_addr_set(extent, addr); @@ -340,39 +283,39 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_committed_set(extent, true); } -JEMALLOC_INLINE void +static inline void extent_list_init(extent_list_t *list) { ql_new(list); } -JEMALLOC_INLINE extent_t * +static inline extent_t * extent_list_first(const extent_list_t *list) { return ql_first(list); } -JEMALLOC_INLINE extent_t * +static inline extent_t * extent_list_last(const extent_list_t *list) { return ql_last(list, ql_link); } -JEMALLOC_INLINE void +static inline void extent_list_append(extent_list_t *list, extent_t *extent) { ql_tail_insert(list, extent, ql_link); } -JEMALLOC_INLINE void +static inline void extent_list_replace(extent_list_t *list, extent_t *to_remove, extent_t *to_insert) { ql_after_insert(to_remove, to_insert, ql_link); ql_remove(list, to_remove, ql_link); } -JEMALLOC_INLINE void +static inline void extent_list_remove(extent_list_t *list, extent_t *extent) { ql_remove(list, extent, ql_link); } -JEMALLOC_INLINE int +static inline int extent_sn_comp(const extent_t *a, const extent_t *b) { size_t a_sn = extent_sn_get(a); size_t b_sn = extent_sn_get(b); @@ -380,7 +323,7 @@ extent_sn_comp(const extent_t *a, const extent_t *b) { return (a_sn > b_sn) - (a_sn < b_sn); } -JEMALLOC_INLINE int +static inline int extent_esn_comp(const extent_t *a, const extent_t *b) { size_t a_esn = extent_esn_get(a); size_t b_esn = extent_esn_get(b); @@ -388,7 +331,7 @@ extent_esn_comp(const extent_t *a, const extent_t *b) { return (a_esn > b_esn) - (a_esn < b_esn); } -JEMALLOC_INLINE int +static inline int extent_ad_comp(const extent_t *a, const extent_t *b) { uintptr_t a_addr = (uintptr_t)extent_addr_get(a); uintptr_t b_addr = (uintptr_t)extent_addr_get(b); @@ -396,7 +339,7 @@ extent_ad_comp(const extent_t *a, const extent_t *b) { return (a_addr > b_addr) - (a_addr < b_addr); } -JEMALLOC_INLINE int +static inline int extent_ead_comp(const extent_t *a, const extent_t *b) { uintptr_t a_eaddr = (uintptr_t)a; uintptr_t b_eaddr = (uintptr_t)b; @@ -404,7 +347,7 @@ extent_ead_comp(const extent_t *a, const extent_t *b) { return (a_eaddr > b_eaddr) - (a_eaddr < b_eaddr); } -JEMALLOC_INLINE int +static inline int extent_snad_comp(const extent_t *a, const extent_t *b) { int ret; @@ -417,7 +360,7 @@ extent_snad_comp(const extent_t *a, const extent_t *b) { return ret; } -JEMALLOC_INLINE int +static inline int extent_esnead_comp(const extent_t *a, const extent_t *b) { int ret; @@ -429,6 +372,5 @@ extent_esnead_comp(const extent_t *a, const extent_t *b) { ret = extent_ead_comp(a, b); return ret; } -#endif #endif /* JEMALLOC_INTERNAL_EXTENT_INLINES_H */ diff --git a/include/jemalloc/internal/hash_inlines.h b/include/jemalloc/internal/hash_inlines.h index 321c17c..2cd7e3e 100644 --- a/include/jemalloc/internal/hash_inlines.h +++ b/include/jemalloc/internal/hash_inlines.h @@ -9,30 +9,19 @@ * details. */ -#ifndef JEMALLOC_ENABLE_INLINE -uint32_t hash_x86_32(const void *key, int len, uint32_t seed); -void hash_x86_128(const void *key, const int len, uint32_t seed, - uint64_t r_out[2]); -void hash_x64_128(const void *key, const int len, const uint32_t seed, - uint64_t r_out[2]); -void hash(const void *key, size_t len, const uint32_t seed, - size_t r_hash[2]); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_)) /******************************************************************************/ /* Internal implementation. */ -JEMALLOC_INLINE uint32_t +static inline uint32_t hash_rotl_32(uint32_t x, int8_t r) { return ((x << r) | (x >> (32 - r))); } -JEMALLOC_INLINE uint64_t +static inline uint64_t hash_rotl_64(uint64_t x, int8_t r) { return ((x << r) | (x >> (64 - r))); } -JEMALLOC_INLINE uint32_t +static inline uint32_t hash_get_block_32(const uint32_t *p, int i) { /* Handle unaligned read. */ if (unlikely((uintptr_t)p & (sizeof(uint32_t)-1)) != 0) { @@ -45,7 +34,7 @@ hash_get_block_32(const uint32_t *p, int i) { return p[i]; } -JEMALLOC_INLINE uint64_t +static inline uint64_t hash_get_block_64(const uint64_t *p, int i) { /* Handle unaligned read. */ if (unlikely((uintptr_t)p & (sizeof(uint64_t)-1)) != 0) { @@ -58,7 +47,7 @@ hash_get_block_64(const uint64_t *p, int i) { return p[i]; } -JEMALLOC_INLINE uint32_t +static inline uint32_t hash_fmix_32(uint32_t h) { h ^= h >> 16; h *= 0x85ebca6b; @@ -69,7 +58,7 @@ hash_fmix_32(uint32_t h) { return h; } -JEMALLOC_INLINE uint64_t +static inline uint64_t hash_fmix_64(uint64_t k) { k ^= k >> 33; k *= KQU(0xff51afd7ed558ccd); @@ -80,7 +69,7 @@ hash_fmix_64(uint64_t k) { return k; } -JEMALLOC_INLINE uint32_t +static inline uint32_t hash_x86_32(const void *key, int len, uint32_t seed) { const uint8_t *data = (const uint8_t *) key; const int nblocks = len / 4; @@ -130,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { return h1; } -UNUSED JEMALLOC_INLINE void +UNUSED static inline void hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { const uint8_t * data = (const uint8_t *) key; @@ -231,7 +220,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, r_out[1] = (((uint64_t) h4) << 32) | h3; } -UNUSED JEMALLOC_INLINE void +UNUSED static inline void hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *) key; @@ -310,7 +299,7 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, /******************************************************************************/ /* API. */ -JEMALLOC_INLINE void +static inline void hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) { assert(len <= INT_MAX); /* Unfortunate implementation limitation. */ @@ -325,6 +314,5 @@ hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) { } #endif } -#endif #endif /* JEMALLOC_INTERNAL_HASH_INLINES_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h index c28bd7c..9cb933c 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -5,42 +5,6 @@ #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#ifndef JEMALLOC_ENABLE_INLINE -pszind_t psz2ind(size_t psz); -size_t pind2sz_compute(pszind_t pind); -size_t pind2sz_lookup(pszind_t pind); -size_t pind2sz(pszind_t pind); -size_t psz2u(size_t psz); -szind_t size2index_compute(size_t size); -szind_t size2index_lookup(size_t size); -szind_t size2index(size_t size); -size_t index2size_compute(szind_t index); -size_t index2size_lookup(szind_t index); -size_t index2size(szind_t index); -size_t s2u_compute(size_t size); -size_t s2u_lookup(size_t size); -size_t s2u(size_t size); -size_t sa2u(size_t size, size_t alignment); -arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); -arena_t *arena_choose(tsd_t *tsd, arena_t *arena); -arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); -bool arena_is_auto(arena_t *arena); -arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, - bool refresh_if_missing); -arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); -ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); -bool tcache_available(tsd_t *tsd); -tcache_bin_t *tcache_small_bin_get(tcache_t *tcache, szind_t binind); -tcache_bin_t *tcache_large_bin_get(tcache_t *tcache, szind_t binind); -tcache_t *tcache_get(tsd_t *tsd); -malloc_cpuid_t malloc_getcpu(void); -unsigned percpu_arena_choose(void); -unsigned percpu_arena_ind_limit(void); -void pre_reentrancy(tsd_t *tsd); -void post_reentrancy(tsd_t *tsd); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_ALWAYS_INLINE pszind_t psz2ind(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) { @@ -64,7 +28,7 @@ psz2ind(size_t psz) { } } -JEMALLOC_INLINE size_t +static inline size_t pind2sz_compute(pszind_t pind) { if (unlikely(pind == NPSIZES)) { return LARGE_MAXCLASS + PAGE; @@ -86,20 +50,20 @@ pind2sz_compute(pszind_t pind) { } } -JEMALLOC_INLINE size_t +static inline size_t pind2sz_lookup(pszind_t pind) { size_t ret = (size_t)pind2sz_tab[pind]; assert(ret == pind2sz_compute(pind)); return ret; } -JEMALLOC_INLINE size_t +static inline size_t pind2sz(pszind_t pind) { assert(pind < NPSIZES+1); return pind2sz_lookup(pind); } -JEMALLOC_INLINE size_t +static inline size_t psz2u(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) { return LARGE_MAXCLASS + PAGE; @@ -115,7 +79,7 @@ psz2u(size_t psz) { } } -JEMALLOC_INLINE szind_t +static inline szind_t size2index_compute(size_t size) { if (unlikely(size > LARGE_MAXCLASS)) { return NSIZES; @@ -164,7 +128,7 @@ size2index(size_t size) { return size2index_compute(size); } -JEMALLOC_INLINE size_t +static inline size_t index2size_compute(szind_t index) { #if (NTBINS > 0) if (index < NTBINS) { @@ -355,7 +319,7 @@ percpu_arena_ind_limit(void) { } } -JEMALLOC_INLINE arena_tdata_t * +static inline arena_tdata_t * arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { arena_tdata_t *tdata; arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); @@ -380,7 +344,7 @@ arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { return arena_tdata_get_hard(tsd, ind); } -JEMALLOC_INLINE arena_t * +static inline arena_t * arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { arena_t *ret; @@ -396,7 +360,7 @@ arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { return ret; } -JEMALLOC_INLINE ticker_t * +static inline ticker_t * decay_ticker_get(tsd_t *tsd, unsigned ind) { arena_tdata_t *tdata; @@ -446,7 +410,7 @@ tcache_get(tsd_t *tsd) { return tsd_tcachep_get(tsd); } -JEMALLOC_INLINE void +static inline void pre_reentrancy(tsd_t *tsd) { bool fast = tsd_fast(tsd); ++*tsd_reentrancy_levelp_get(tsd); @@ -457,7 +421,7 @@ pre_reentrancy(tsd_t *tsd) { } } -JEMALLOC_INLINE void +static inline void post_reentrancy(tsd_t *tsd) { int8_t *reentrancy_level = tsd_reentrancy_levelp_get(tsd); assert(*reentrancy_level > 0); @@ -466,6 +430,4 @@ post_reentrancy(tsd_t *tsd) { } } -#endif - #endif /* JEMALLOC_INTERNAL_INLINES_A_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 2fd371c..cfc5209 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -1,13 +1,8 @@ #ifndef JEMALLOC_INTERNAL_INLINES_B_H #define JEMALLOC_INTERNAL_INLINES_B_H -#ifndef JEMALLOC_ENABLE_INLINE -extent_t *iealloc(tsdn_t *tsdn, const void *ptr); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) /* Choose an arena based on a per-thread value. */ -JEMALLOC_INLINE arena_t * +static inline arena_t * arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { arena_t *ret; @@ -60,17 +55,17 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { return ret; } -JEMALLOC_INLINE arena_t * +static inline arena_t * arena_choose(tsd_t *tsd, arena_t *arena) { return arena_choose_impl(tsd, arena, false); } -JEMALLOC_INLINE arena_t * +static inline arena_t * arena_ichoose(tsd_t *tsd, arena_t *arena) { return arena_choose_impl(tsd, arena, true); } -JEMALLOC_INLINE bool +static inline bool arena_is_auto(arena_t *arena) { assert(narenas_auto > 0); return (arena_ind_get(arena) < narenas_auto); @@ -84,6 +79,5 @@ iealloc(tsdn_t *tsdn, const void *ptr) { return rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); } -#endif #endif /* JEMALLOC_INTERNAL_INLINES_B_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 70ac666..415c503 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -3,36 +3,6 @@ #include "jemalloc/internal/jemalloc_internal_types.h" -#ifndef JEMALLOC_ENABLE_INLINE -arena_t *iaalloc(tsdn_t *tsdn, const void *ptr); -size_t isalloc(tsdn_t *tsdn, const void *ptr); -void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path); -void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, - bool slow_path); -void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena); -void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, arena_t *arena); -void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); -size_t ivsalloc(tsdn_t *tsdn, const void *ptr); -void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, - alloc_ctx_t *alloc_ctx, bool is_internal, bool slow_path); -void idalloc(tsd_t *tsd, void *ptr); -void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - alloc_ctx_t *alloc_ctx, bool slow_path); -void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena); -void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); -void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero); -bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_ALWAYS_INLINE arena_t * iaalloc(tsdn_t *tsdn, const void *ptr) { assert(ptr != NULL); @@ -214,6 +184,5 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); } -#endif #endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index c5dd9b3..1b2802a 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -1,37 +1,13 @@ #ifndef JEMALLOC_INTERNAL_MACROS_H #define JEMALLOC_INTERNAL_MACROS_H -/* - * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for - * functions that are static inline functions if inlining is enabled, and - * single-definition library-private functions if inlining is disabled. - * - * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in - * which case the denoted functions are always static, regardless of whether - * inlining is enabled. - */ #if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE) - /* Disable inlining to make debugging/profiling easier. */ -# define JEMALLOC_ALWAYS_INLINE -# define JEMALLOC_ALWAYS_INLINE_C static -# define JEMALLOC_INLINE -# define JEMALLOC_INLINE_C static +# define JEMALLOC_ALWAYS_INLINE static inline #else -# define JEMALLOC_ENABLE_INLINE -# ifdef JEMALLOC_HAVE_ATTR -# define JEMALLOC_ALWAYS_INLINE \ - static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline) -# define JEMALLOC_ALWAYS_INLINE_C \ - static inline JEMALLOC_ATTR(always_inline) -# else -# define JEMALLOC_ALWAYS_INLINE static inline -# define JEMALLOC_ALWAYS_INLINE_C static inline -# endif -# define JEMALLOC_INLINE static inline -# define JEMALLOC_INLINE_C static inline -# ifdef _MSC_VER -# define inline _inline -# endif +# define JEMALLOC_ALWAYS_INLINE JEMALLOC_ATTR(always_inline) static inline +#endif +#ifdef _MSC_VER +# define inline _inline #endif #ifdef JEMALLOC_CC_SILENCE diff --git a/include/jemalloc/internal/mutex_inlines.h b/include/jemalloc/internal/mutex_inlines.h index 5ec439f..2856d84 100644 --- a/include/jemalloc/internal/mutex_inlines.h +++ b/include/jemalloc/internal/mutex_inlines.h @@ -5,31 +5,19 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex); -#ifndef JEMALLOC_ENABLE_INLINE -void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex); -bool malloc_mutex_trylock(malloc_mutex_t *mutex); -void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex); -void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex); -void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex); -void malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, - malloc_mutex_t *mutex); -void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) -JEMALLOC_INLINE void +static inline void malloc_mutex_lock_final(malloc_mutex_t *mutex) { MALLOC_MUTEX_LOCK(mutex); } /* Trylock: return false if the lock is successfully acquired. */ -JEMALLOC_INLINE bool +static inline bool malloc_mutex_trylock(malloc_mutex_t *mutex) { return MALLOC_MUTEX_TRYLOCK(mutex); } /* Aggregate lock prof data. */ -JEMALLOC_INLINE void +static inline void malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { nstime_add(&sum->tot_wait_time, &data->tot_wait_time); if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { @@ -52,7 +40,7 @@ malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { sum->n_lock_ops += data->n_lock_ops; } -JEMALLOC_INLINE void +static inline void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn, &mutex->witness); if (isthreaded) { @@ -72,7 +60,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_lock(tsdn, &mutex->witness); } -JEMALLOC_INLINE void +static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_unlock(tsdn, &mutex->witness); if (isthreaded) { @@ -80,18 +68,18 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { } } -JEMALLOC_INLINE void +static inline void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_owner(tsdn, &mutex->witness); } -JEMALLOC_INLINE void +static inline void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn, &mutex->witness); } /* Copy the prof data from mutex for processing. */ -JEMALLOC_INLINE void +static inline void malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, malloc_mutex_t *mutex) { mutex_prof_data_t *source = &mutex->prof_data; @@ -108,6 +96,4 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); } -#endif - #endif /* JEMALLOC_INTERNAL_MUTEX_INLINES_H */ diff --git a/include/jemalloc/internal/prng_inlines.h b/include/jemalloc/internal/prng_inlines.h index c39c63f..0275dfc 100644 --- a/include/jemalloc/internal/prng_inlines.h +++ b/include/jemalloc/internal/prng_inlines.h @@ -4,23 +4,6 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" -#ifndef JEMALLOC_ENABLE_INLINE -uint32_t prng_state_next_u32(uint32_t state); -uint64_t prng_state_next_u64(uint64_t state); -size_t prng_state_next_zu(size_t state); - -uint32_t prng_lg_range_u32(atomic_u32_t *state, unsigned lg_range, - bool atomic); -uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range); -size_t prng_lg_range_zu(atomic_zu_t *state, unsigned lg_range, bool atomic); - -uint32_t prng_range_u32(atomic_u32_t *state, uint32_t range, - bool atomic); -uint64_t prng_range_u64(uint64_t *state, uint64_t range); -size_t prng_range_zu(atomic_zu_t *state, size_t range, bool atomic); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_)) JEMALLOC_ALWAYS_INLINE uint32_t prng_state_next_u32(uint32_t state) { return (state * PRNG_A_32) + PRNG_C_32; @@ -156,6 +139,5 @@ prng_range_zu(atomic_zu_t *state, size_t range, bool atomic) { return ret; } -#endif #endif /* JEMALLOC_INTERNAL_PRNG_INLINES_H */ diff --git a/include/jemalloc/internal/prof_inlines_a.h b/include/jemalloc/internal/prof_inlines_a.h index d0d2968..6203cbd 100644 --- a/include/jemalloc/internal/prof_inlines_a.h +++ b/include/jemalloc/internal/prof_inlines_a.h @@ -1,14 +1,7 @@ #ifndef JEMALLOC_INTERNAL_PROF_INLINES_A_H #define JEMALLOC_INTERNAL_PROF_INLINES_A_H -#ifndef JEMALLOC_ENABLE_INLINE -bool prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, - uint64_t accumbytes); -void prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) -JEMALLOC_INLINE bool +static inline bool prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { cassert(config_prof); @@ -46,7 +39,7 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { return overflow; } -JEMALLOC_INLINE void +static inline void prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { cassert(config_prof); @@ -73,6 +66,5 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { malloc_mutex_unlock(tsdn, &prof_accum->mtx); #endif } -#endif #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ diff --git a/include/jemalloc/internal/prof_inlines_b.h b/include/jemalloc/internal/prof_inlines_b.h index 6a79c01..eba981b 100644 --- a/include/jemalloc/internal/prof_inlines_b.h +++ b/include/jemalloc/internal/prof_inlines_b.h @@ -1,29 +1,6 @@ #ifndef JEMALLOC_INTERNAL_PROF_INLINES_B_H #define JEMALLOC_INTERNAL_PROF_INLINES_B_H -#ifndef JEMALLOC_ENABLE_INLINE -bool prof_active_get_unlocked(void); -bool prof_gdump_get_unlocked(void); -prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); -prof_tctx_t *prof_tctx_get(tsdn_t *tsdn, const void *ptr, - alloc_ctx_t *alloc_ctx); -void prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, - alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx); -void prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx); -bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, - prof_tdata_t **tdata_out); -prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool prof_active, - bool update); -void prof_malloc(tsdn_t *tsdn, const void *ptr, size_t usize, - alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx); -void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, - prof_tctx_t *tctx, bool prof_active, bool updated, const void *old_ptr, - size_t old_usize, prof_tctx_t *old_tctx); -void prof_free(tsd_t *tsd, const void *ptr, size_t usize, - alloc_ctx_t *alloc_ctx); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) JEMALLOC_ALWAYS_INLINE bool prof_active_get_unlocked(void) { /* @@ -231,6 +208,5 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { prof_free_sampled_object(tsd, usize, tctx); } } -#endif #endif /* JEMALLOC_INTERNAL_PROF_INLINES_B_H */ diff --git a/include/jemalloc/internal/rtree_inlines.h b/include/jemalloc/internal/rtree_inlines.h index 030e578..f4f7c2c 100644 --- a/include/jemalloc/internal/rtree_inlines.h +++ b/include/jemalloc/internal/rtree_inlines.h @@ -3,59 +3,6 @@ #include "jemalloc/internal/spin.h" -#ifndef JEMALLOC_ENABLE_INLINE -uintptr_t rtree_leafkey(uintptr_t key); -uintptr_t rtree_subkey(uintptr_t key, unsigned level); -# ifdef RTREE_LEAF_COMPACT -uintptr_t rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, bool dependent); -extent_t *rtree_leaf_elm_bits_extent_get(uintptr_t bits); -szind_t rtree_leaf_elm_bits_szind_get(uintptr_t bits); -bool rtree_leaf_elm_bits_slab_get(uintptr_t bits); -bool rtree_leaf_elm_bits_locked_get(uintptr_t bits); -# endif -extent_t *rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, bool dependent); -szind_t rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, bool dependent); -bool rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, bool dependent); -void rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, extent_t *extent); -void rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, szind_t szind); -void rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool acquired, bool slab); -void rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool acquired, extent_t *extent, szind_t szind, bool slab); -void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, szind_t szind, bool slab); -rtree_leaf_elm_t *rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); -bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, extent_t *extent, szind_t szind, bool slab); -rtree_leaf_elm_t *rtree_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); -extent_t *rtree_extent_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); -szind_t rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent); -bool rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, extent_t **r_extent, - szind_t *r_szind); -bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab); -rtree_leaf_elm_t *rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing); -void rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm); -void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, - rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab); -void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) JEMALLOC_ALWAYS_INLINE uintptr_t rtree_leafkey(uintptr_t key) { unsigned ptrbits = ZU(1) << (LG_SIZEOF_PTR+3); @@ -194,7 +141,7 @@ rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, #endif } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool acquired, extent_t *extent) { if (config_debug && acquired) { @@ -219,7 +166,7 @@ rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, #endif } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool acquired, szind_t szind) { if (config_debug && acquired) { @@ -241,7 +188,7 @@ rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, #endif } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool acquired, bool slab) { if (config_debug && acquired) { @@ -261,7 +208,7 @@ rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, #endif } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, bool acquired, extent_t *extent, szind_t szind, bool slab) { if (config_debug && acquired) { @@ -287,7 +234,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, #endif } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind, bool slab) { assert(!slab || szind < NBINS); @@ -384,7 +331,7 @@ rtree_leaf_elm_lookup(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, dependent, init_missing); } -JEMALLOC_INLINE bool +static inline bool rtree_write(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, extent_t *extent, szind_t szind, bool slab) { /* Use rtree_clear() to set the extent to NULL. */ @@ -471,7 +418,7 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return false; } -JEMALLOC_INLINE rtree_leaf_elm_t * +static inline rtree_leaf_elm_t * rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, bool init_missing) { rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, rtree, rtree_ctx, @@ -511,7 +458,7 @@ rtree_leaf_elm_acquire(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return elm; } -JEMALLOC_INLINE void +static inline void rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm) { extent_t *extent = rtree_leaf_elm_extent_read(tsdn, rtree, elm, true, true); @@ -521,7 +468,7 @@ rtree_leaf_elm_release(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm) { } } -JEMALLOC_INLINE void +static inline void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab) { assert(!slab || szind < NBINS); @@ -530,7 +477,7 @@ rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab); } -JEMALLOC_INLINE void +static inline void rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key) { rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); @@ -538,6 +485,5 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, NULL); rtree_leaf_elm_write(tsdn, rtree, elm, false, NULL, NSIZES, false); } -#endif #endif /* JEMALLOC_INTERNAL_RTREE_INLINES_H */ diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index 67d35b5..8b42af6 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -4,30 +4,12 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/util.h" -#ifndef JEMALLOC_ENABLE_INLINE -void tcache_event(tsd_t *tsd, tcache_t *tcache); -bool tcache_enabled_get(tsd_t *tsd); -tcache_t *tcache_get(tsd_t *tsd); -void tcache_enabled_set(tsd_t *tsd, bool enabled); -void *tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success); -void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - size_t size, szind_t ind, bool zero, bool slow_path); -void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - size_t size, szind_t ind, bool zero, bool slow_path); -void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, - szind_t binind, bool slow_path); -void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, - szind_t binind, bool slow_path); -tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) -JEMALLOC_INLINE bool +static inline bool tcache_enabled_get(tsd_t *tsd) { return tsd_tcache_enabled_get(tsd); } -JEMALLOC_INLINE void +static inline void tcache_enabled_set(tsd_t *tsd, bool enabled) { bool was_enabled = tsd_tcache_enabled_get(tsd); @@ -261,6 +243,5 @@ tcaches_get(tsd_t *tsd, unsigned ind) { } return elm->tcache; } -#endif #endif /* JEMALLOC_INTERNAL_TCACHE_INLINES_H */ diff --git a/include/jemalloc/internal/ticker_inlines.h b/include/jemalloc/internal/ticker_inlines.h index 9102ba6..cd5821f 100644 --- a/include/jemalloc/internal/ticker_inlines.h +++ b/include/jemalloc/internal/ticker_inlines.h @@ -1,32 +1,23 @@ #ifndef JEMALLOC_INTERNAL_TICKER_INLINES_H #define JEMALLOC_INTERNAL_TICKER_INLINES_H -#ifndef JEMALLOC_ENABLE_INLINE -void ticker_init(ticker_t *ticker, int32_t nticks); -void ticker_copy(ticker_t *ticker, const ticker_t *other); -int32_t ticker_read(const ticker_t *ticker); -bool ticker_ticks(ticker_t *ticker, int32_t nticks); -bool ticker_tick(ticker_t *ticker); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TICKER_C_)) -JEMALLOC_INLINE void +static inline void ticker_init(ticker_t *ticker, int32_t nticks) { ticker->tick = nticks; ticker->nticks = nticks; } -JEMALLOC_INLINE void +static inline void ticker_copy(ticker_t *ticker, const ticker_t *other) { *ticker = *other; } -JEMALLOC_INLINE int32_t +static inline int32_t ticker_read(const ticker_t *ticker) { return ticker->tick; } -JEMALLOC_INLINE bool +static inline bool ticker_ticks(ticker_t *ticker, int32_t nticks) { if (unlikely(ticker->tick < nticks)) { ticker->tick = ticker->nticks; @@ -36,10 +27,9 @@ ticker_ticks(ticker_t *ticker, int32_t nticks) { return(false); } -JEMALLOC_INLINE bool +static inline bool ticker_tick(ticker_t *ticker) { return ticker_ticks(ticker, 1); } -#endif #endif /* JEMALLOC_INTERNAL_TICKER_INLINES_H */ diff --git a/include/jemalloc/internal/tsd_inlines.h b/include/jemalloc/internal/tsd_inlines.h index 93469bc..f0f77e4 100644 --- a/include/jemalloc/internal/tsd_inlines.h +++ b/include/jemalloc/internal/tsd_inlines.h @@ -1,29 +1,6 @@ #ifndef JEMALLOC_INTERNAL_TSD_INLINES_H #define JEMALLOC_INTERNAL_TSD_INLINES_H -#ifndef JEMALLOC_ENABLE_INLINE -malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) - -tsd_t *tsd_fetch_impl(bool init); -tsd_t *tsd_fetch(void); -tsdn_t *tsd_tsdn(tsd_t *tsd); -bool tsd_nominal(tsd_t *tsd); -#define O(n, t, gs, i, c) \ -t *tsd_##n##p_get(tsd_t *tsd); \ -t tsd_##n##_get(tsd_t *tsd); \ -void tsd_##n##_set(tsd_t *tsd, t n); -MALLOC_TSD -#undef O -tsdn_t *tsdn_fetch(void); -bool tsdn_null(const tsdn_t *tsdn); -tsd_t *tsdn_tsd(tsdn_t *tsdn); -rtree_ctx_t *tsd_rtree_ctx(tsd_t *tsd); -rtree_ctx_t *tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback); -bool tsd_fast(tsd_t *tsd); -bool tsd_assert_fast(tsd_t *tsd); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) malloc_tsd_externs(, tsd_t) malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) @@ -97,7 +74,7 @@ tsd_tsdn(tsd_t *tsd) { return (tsdn_t *)tsd; } -JEMALLOC_INLINE bool +static inline bool tsd_nominal(tsd_t *tsd) { return (tsd->state <= tsd_state_nominal_max); } @@ -140,6 +117,5 @@ tsdn_rtree_ctx(tsdn_t *tsdn, rtree_ctx_t *fallback) { } return tsd_rtree_ctx(tsdn_tsd(tsdn)); } -#endif #endif /* JEMALLOC_INTERNAL_TSD_INLINES_H */ diff --git a/include/jemalloc/internal/witness_inlines.h b/include/jemalloc/internal/witness_inlines.h index c5027f1..51d1af3 100644 --- a/include/jemalloc/internal/witness_inlines.h +++ b/include/jemalloc/internal/witness_inlines.h @@ -3,21 +3,8 @@ #include "jemalloc/internal/ql.h" -#ifndef JEMALLOC_ENABLE_INLINE -bool witness_owner(tsd_t *tsd, const witness_t *witness); -void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, - unsigned depth); -void witness_assert_depth(tsdn_t *tsdn, unsigned depth); -void witness_assert_lockless(tsdn_t *tsdn); -void witness_lock(tsdn_t *tsdn, witness_t *witness); -void witness_unlock(tsdn_t *tsdn, witness_t *witness); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) /* Helper, not intended for direct use. */ -JEMALLOC_INLINE bool +static inline bool witness_owner(tsd_t *tsd, const witness_t *witness) { witness_list_t *witnesses; witness_t *w; @@ -34,7 +21,7 @@ witness_owner(tsd_t *tsd, const witness_t *witness) { return false; } -JEMALLOC_INLINE void +static inline void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) { tsd_t *tsd; @@ -56,7 +43,7 @@ witness_assert_owner(tsdn_t *tsdn, const witness_t *witness) { witness_owner_error(witness); } -JEMALLOC_INLINE void +static inline void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) { tsd_t *tsd; witness_list_t *witnesses; @@ -82,7 +69,7 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) { } } -JEMALLOC_INLINE void +static inline void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, unsigned depth) { tsd_t *tsd; @@ -115,17 +102,17 @@ witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, } } -JEMALLOC_INLINE void +static inline void witness_assert_depth(tsdn_t *tsdn, unsigned depth) { witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth); } -JEMALLOC_INLINE void +static inline void witness_assert_lockless(tsdn_t *tsdn) { witness_assert_depth(tsdn, 0); } -JEMALLOC_INLINE void +static inline void witness_lock(tsdn_t *tsdn, witness_t *witness) { tsd_t *tsd; witness_list_t *witnesses; @@ -168,7 +155,7 @@ witness_lock(tsdn_t *tsdn, witness_t *witness) { ql_tail_insert(witnesses, witness, link); } -JEMALLOC_INLINE void +static inline void witness_unlock(tsdn_t *tsdn, witness_t *witness) { tsd_t *tsd; witness_list_t *witnesses; @@ -197,6 +184,5 @@ witness_unlock(tsdn_t *tsdn, witness_t *witness) { witness_assert_owner(tsdn, witness); } } -#endif #endif /* JEMALLOC_INTERNAL_WITNESS_INLINES_H */ |