diff options
38 files changed, 1396 insertions, 1337 deletions
@@ -21,7 +21,7 @@ /Makefile -/include/jemalloc/internal/jemalloc_internal.h +/include/jemalloc/internal/jemalloc_preamble.h /include/jemalloc/internal/jemalloc_internal_defs.h /include/jemalloc/internal/private_namespace.h /include/jemalloc/internal/private_unnamespace.h diff --git a/configure.ac b/configure.ac index 894e72d..0086813 100644 --- a/configure.ac +++ b/configure.ac @@ -875,7 +875,7 @@ cfgoutputs_in="${cfgoutputs_in} doc/jemalloc.xml.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_typedefs.h.in" -cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_preamble.h.in" cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" @@ -887,7 +887,7 @@ cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_typedefs.h" -cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_preamble.h" cfgoutputs_out="${cfgoutputs_out} test/test.sh" cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" @@ -899,7 +899,7 @@ cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jemalloc/jemalloc_macros.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_typedefs.h:include/jemalloc/jemalloc_typedefs.h.in" -cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_preamble.h" cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in deleted file mode 100644 index 58240e3..0000000 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ /dev/null @@ -1,1303 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_H -#define JEMALLOC_INTERNAL_H - -#ifdef __cplusplus -# define CPP_PROLOGUE extern "C" { -# define CPP_EPILOGUE } -#else -# define CPP_PROLOGUE -# define CPP_EPILOGUE -#endif - -CPP_PROLOGUE - -#include "jemalloc_internal_defs.h" -#include "jemalloc/internal/jemalloc_internal_decls.h" - -#ifdef JEMALLOC_UTRACE -#include <sys/ktrace.h> -#endif - -#define JEMALLOC_NO_DEMANGLE -#ifdef JEMALLOC_JET -# define JEMALLOC_N(n) jet_##n -# include "jemalloc/internal/public_namespace.h" -# define JEMALLOC_NO_RENAME -# include "../jemalloc@install_suffix@.h" -# undef JEMALLOC_NO_RENAME -#else -# define JEMALLOC_N(n) @private_namespace@##n -# include "../jemalloc@install_suffix@.h" -#endif - -/* - * Note that the ordering matters here; the hook itself is name-mangled. We - * want the inclusion of hooks to happen early, so that we hook as much as - * possible. - */ -#include "jemalloc/internal/private_namespace.h" -#include "jemalloc/internal/hooks.h" - -static const bool config_debug = -#ifdef JEMALLOC_DEBUG - true -#else - false -#endif - ; -static const bool have_dss = -#ifdef JEMALLOC_DSS - true -#else - false -#endif - ; -static const bool config_fill = -#ifdef JEMALLOC_FILL - true -#else - false -#endif - ; -static const bool config_lazy_lock = -#ifdef JEMALLOC_LAZY_LOCK - true -#else - false -#endif - ; -static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; -static const bool config_prof = -#ifdef JEMALLOC_PROF - true -#else - false -#endif - ; -static const bool config_prof_libgcc = -#ifdef JEMALLOC_PROF_LIBGCC - true -#else - false -#endif - ; -static const bool config_prof_libunwind = -#ifdef JEMALLOC_PROF_LIBUNWIND - true -#else - false -#endif - ; -static const bool maps_coalesce = -#ifdef JEMALLOC_MAPS_COALESCE - true -#else - false -#endif - ; -static const bool config_munmap = -#ifdef JEMALLOC_MUNMAP - true -#else - false -#endif - ; -static const bool config_stats = -#ifdef JEMALLOC_STATS - true -#else - false -#endif - ; -static const bool config_tcache = -#ifdef JEMALLOC_TCACHE - true -#else - false -#endif - ; -static const bool config_tls = -#ifdef JEMALLOC_TLS - true -#else - false -#endif - ; -static const bool config_utrace = -#ifdef JEMALLOC_UTRACE - true -#else - false -#endif - ; -static const bool config_xmalloc = -#ifdef JEMALLOC_XMALLOC - true -#else - false -#endif - ; -static const bool config_ivsalloc = -#ifdef JEMALLOC_IVSALLOC - true -#else - false -#endif - ; -static const bool config_cache_oblivious = -#ifdef JEMALLOC_CACHE_OBLIVIOUS - true -#else - false -#endif - ; -static const bool have_thp = -#ifdef JEMALLOC_THP - true -#else - false -#endif - ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU -/* Currently percpu_arena depends on sched_getcpu. */ -#define JEMALLOC_PERCPU_ARENA -#endif -static const bool have_percpu_arena = -#ifdef JEMALLOC_PERCPU_ARENA - true -#else - false -#endif - ; - -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) -#include <libkern/OSAtomic.h> -#endif - -#ifdef JEMALLOC_ZONE -#include <mach/mach_error.h> -#include <mach/mach_init.h> -#include <mach/vm_map.h> -#endif - -/* - * jemalloc can conceptually be broken into components (arena, tcache, etc.), - * but there are circular dependencies that cannot be broken without - * substantial performance degradation. - * - * Historically, we dealt with this by each header into four sections (types, - * structs, externs, and inlines), and included each header file multiple times - * in this file, picking out the portion we want on each pass using the - * following #defines: - * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data - * types. - * JEMALLOC_H_STRUCTS : Data structures. - * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. - * JEMALLOC_H_INLINES : Inline functions. - * - * We're moving toward a world in which the dependencies are explicit; each file - * will #include the headers it depends on (rather than relying on them being - * implicitly available via this file including every header file in the - * project). - * - * We're now in an intermediate state: we've broken up the header files to avoid - * having to include each one multiple times, but have not yet moved the - * dependency information into the header files (i.e. we still rely on the - * ordering in this file to ensure all a header's dependencies are available in - * its translation unit). Each component is now broken up into multiple header - * files, corresponding to the sections above (e.g. instead of "tsd.h", we now - * have "tsd_types.h", "tsd_structs.h", "tsd_externs.h", "tsd_inlines.h"). - * - * Those files which have been converted to explicitly include their - * inter-component dependencies are now in the initial HERMETIC HEADERS - * section. These headers may still rely on this file for system headers and - * global jemalloc headers, however. - */ - -#include "jemalloc/internal/jemalloc_internal_macros.h" - -/******************************************************************************/ -/* HERMETIC HEADERS */ -/******************************************************************************/ - -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/malloc_io.h" -#include "jemalloc/internal/util.h" - -/******************************************************************************/ -/* TYPES */ -/******************************************************************************/ - -/* Page size index type. */ -typedef unsigned pszind_t; - -/* Size class index type. */ -typedef unsigned szind_t; - -/* Processor / core id type. */ -typedef int malloc_cpuid_t; - -/* - * Flags bits: - * - * a: arena - * t: tcache - * 0: unused - * z: zero - * n: alignment - * - * aaaaaaaa aaaatttt tttttttt 0znnnnnn - */ -#define MALLOCX_ARENA_BITS 12 -#define MALLOCX_TCACHE_BITS 12 -#define MALLOCX_LG_ALIGN_BITS 6 -#define MALLOCX_ARENA_SHIFT 20 -#define MALLOCX_TCACHE_SHIFT 8 -#define MALLOCX_ARENA_MASK \ - (((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT) -/* NB: Arena index bias decreases the maximum number of arenas by 1. */ -#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2) -#define MALLOCX_TCACHE_MASK \ - (((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT) -#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3) -#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1) -/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ -#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ - (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) -#define MALLOCX_ALIGN_GET(flags) \ - (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) -#define MALLOCX_ZERO_GET(flags) \ - ((bool)(flags & MALLOCX_ZERO)) - -#define MALLOCX_TCACHE_GET(flags) \ - (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2) -#define MALLOCX_ARENA_GET(flags) \ - (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1) - -/* Smallest size class to support. */ -#define TINY_MIN (1U << LG_TINY_MIN) - -/* - * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size - * classes). - */ -#ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __mips__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# ifdef __riscv__ -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# ifdef __SH4__ -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - -#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) -#define LONG_MASK (LONG - 1) - -/* Return the smallest long multiple that is >= a. */ -#define LONG_CEILING(a) \ - (((a) + LONG_MASK) & ~LONG_MASK) - -#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) -#define PTR_MASK (SIZEOF_PTR - 1) - -/* Return the smallest (void *) multiple that is >= a. */ -#define PTR_CEILING(a) \ - (((a) + PTR_MASK) & ~PTR_MASK) - -/* - * Maximum size of L1 cache line. This is used to avoid cache line aliasing. - * In addition, this controls the spacing of cacheline-spaced size classes. - * - * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can - * only handle raw constants. - */ -#define LG_CACHELINE 6 -#define CACHELINE 64 -#define CACHELINE_MASK (CACHELINE - 1) - -/* Return the smallest cacheline multiple that is >= s. */ -#define CACHELINE_CEILING(s) \ - (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) - -/* Return the nearest aligned address at or below a. */ -#define ALIGNMENT_ADDR2BASE(a, alignment) \ - ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) - -/* Return the offset between a and the nearest aligned address at or below a. */ -#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ - ((size_t)((uintptr_t)(a) & (alignment - 1))) - -/* Return the smallest alignment multiple that is >= s. */ -#define ALIGNMENT_CEILING(s, alignment) \ - (((s) + (alignment - 1)) & ((~(alignment)) + 1)) - -/* Declare a variable-length array. */ -#if __STDC_VERSION__ < 199901L -# ifdef _MSC_VER -# include <malloc.h> -# define alloca _alloca -# else -# ifdef JEMALLOC_HAS_ALLOCA_H -# include <alloca.h> -# else -# include <stdlib.h> -# endif -# endif -# define VARIABLE_ARRAY(type, name, count) \ - type *name = alloca(sizeof(type) * (count)) -#else -# define VARIABLE_ARRAY(type, name, count) type name[(count)] -#endif - -#include "jemalloc/internal/nstime_types.h" -#include "jemalloc/internal/spin_types.h" -#include "jemalloc/internal/prng_types.h" -#include "jemalloc/internal/ticker_types.h" -#include "jemalloc/internal/ckh_types.h" -#include "jemalloc/internal/size_classes.h" -#include "jemalloc/internal/smoothstep.h" -#include "jemalloc/internal/stats_types.h" -#include "jemalloc/internal/ctl_types.h" -#include "jemalloc/internal/witness_types.h" -#include "jemalloc/internal/mutex_types.h" -#include "jemalloc/internal/tsd_types.h" -#include "jemalloc/internal/extent_types.h" -#include "jemalloc/internal/extent_dss_types.h" -#include "jemalloc/internal/base_types.h" -#include "jemalloc/internal/arena_types.h" -#include "jemalloc/internal/bitmap_types.h" -#include "jemalloc/internal/rtree_types.h" -#include "jemalloc/internal/pages_types.h" -#include "jemalloc/internal/tcache_types.h" -#include "jemalloc/internal/prof_types.h" - - -/******************************************************************************/ -/* STRUCTS */ -/******************************************************************************/ - -#include "jemalloc/internal/nstime_structs.h" -#include "jemalloc/internal/spin_structs.h" -#include "jemalloc/internal/ticker_structs.h" -#include "jemalloc/internal/ckh_structs.h" -#include "jemalloc/internal/witness_structs.h" -#include "jemalloc/internal/mutex_structs.h" -#include "jemalloc/internal/stats_structs.h" -#include "jemalloc/internal/ctl_structs.h" -#include "jemalloc/internal/bitmap_structs.h" -#include "jemalloc/internal/arena_structs_a.h" -#include "jemalloc/internal/extent_structs.h" -#include "jemalloc/internal/extent_dss_structs.h" -#include "jemalloc/internal/base_structs.h" -#include "jemalloc/internal/prof_structs.h" -#include "jemalloc/internal/arena_structs_b.h" -#include "jemalloc/internal/rtree_structs.h" -#include "jemalloc/internal/tcache_structs.h" -#include "jemalloc/internal/tsd_structs.h" - - -/******************************************************************************/ -/* EXTERNS */ -/******************************************************************************/ - -extern bool opt_abort; -extern const char *opt_junk; -extern bool opt_junk_alloc; -extern bool opt_junk_free; -extern bool opt_utrace; -extern bool opt_xmalloc; -extern bool opt_zero; -extern unsigned opt_narenas; - -/* Number of CPUs. */ -extern unsigned ncpus; - -/* Number of arenas used for automatic multiplexing of threads and arenas. */ -extern unsigned narenas_auto; - -/* - * Arenas that are used to service external requests. Not all elements of the - * arenas array are necessarily used; arenas are created lazily as needed. - */ -extern atomic_p_t arenas[]; - -/* - * pind2sz_tab encodes the same information as could be computed by - * pind2sz_compute(). - */ -extern size_t const pind2sz_tab[NPSIZES+1]; -/* - * index2size_tab encodes the same information as could be computed (at - * unacceptable cost in some code paths) by index2size_compute(). - */ -extern size_t const index2size_tab[NSIZES]; -/* - * size2index_tab is a compact lookup table that rounds request sizes up to - * size classes. In order to reduce cache footprint, the table is compressed, - * and all accesses are via size2index(). - */ -extern uint8_t const size2index_tab[]; - -void *a0malloc(size_t size); -void a0dalloc(void *ptr); -void *bootstrap_malloc(size_t size); -void *bootstrap_calloc(size_t num, size_t size); -void bootstrap_free(void *ptr); -void arena_set(unsigned ind, arena_t *arena); -unsigned narenas_total_get(void); -arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); -arena_t *arena_choose_hard(tsd_t *tsd, bool internal); -void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); -void iarena_cleanup(tsd_t *tsd); -void arena_cleanup(tsd_t *tsd); -void arenas_tdata_cleanup(tsd_t *tsd); -void jemalloc_prefork(void); -void jemalloc_postfork_parent(void); -void jemalloc_postfork_child(void); -bool malloc_initialized(void); - -#include "jemalloc/internal/nstime_externs.h" -#include "jemalloc/internal/ckh_externs.h" -#include "jemalloc/internal/stats_externs.h" -#include "jemalloc/internal/ctl_externs.h" -#include "jemalloc/internal/witness_externs.h" -#include "jemalloc/internal/mutex_externs.h" -#include "jemalloc/internal/bitmap_externs.h" -#include "jemalloc/internal/extent_externs.h" -#include "jemalloc/internal/extent_dss_externs.h" -#include "jemalloc/internal/extent_mmap_externs.h" -#include "jemalloc/internal/base_externs.h" -#include "jemalloc/internal/arena_externs.h" -#include "jemalloc/internal/rtree_externs.h" -#include "jemalloc/internal/pages_externs.h" -#include "jemalloc/internal/large_externs.h" -#include "jemalloc/internal/tcache_externs.h" -#include "jemalloc/internal/prof_externs.h" -#include "jemalloc/internal/tsd_externs.h" - -/******************************************************************************/ -/* INLINES */ -/******************************************************************************/ - -#include "jemalloc/internal/spin_inlines.h" -#include "jemalloc/internal/prng_inlines.h" -#include "jemalloc/internal/ticker_inlines.h" -#include "jemalloc/internal/tsd_inlines.h" -#include "jemalloc/internal/witness_inlines.h" -#include "jemalloc/internal/mutex_inlines.h" - -#ifndef JEMALLOC_ENABLE_INLINE -pszind_t psz2ind(size_t psz); -size_t pind2sz_compute(pszind_t pind); -size_t pind2sz_lookup(pszind_t pind); -size_t pind2sz(pszind_t pind); -size_t psz2u(size_t psz); -szind_t size2index_compute(size_t size); -szind_t size2index_lookup(size_t size); -szind_t size2index(size_t size); -size_t index2size_compute(szind_t index); -size_t index2size_lookup(szind_t index); -size_t index2size(szind_t index); -size_t s2u_compute(size_t size); -size_t s2u_lookup(size_t size); -size_t s2u(size_t size); -size_t sa2u(size_t size, size_t alignment); -arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); -arena_t *arena_choose(tsd_t *tsd, arena_t *arena); -arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); -arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, - bool refresh_if_missing); -arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); -ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); -bool tcache_available(tsd_t *tsd); -tcache_bin_t *tcache_small_bin_get(tcache_t *tcache, szind_t binind); -tcache_bin_t *tcache_large_bin_get(tcache_t *tcache, szind_t binind); -tcache_t *tcache_get(tsd_t *tsd); -malloc_cpuid_t malloc_getcpu(void); -unsigned percpu_arena_choose(void); -unsigned percpu_arena_ind_limit(void); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_ALWAYS_INLINE pszind_t -psz2ind(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return NPSIZES; - } - { - pszind_t x = lg_floor((psz<<1)-1); - pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - - (LG_SIZE_CLASS_GROUP + LG_PAGE); - pszind_t grp = shift << LG_SIZE_CLASS_GROUP; - - pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - - size_t delta_inverse_mask = ZD(-1) << lg_delta; - pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - - pszind_t ind = grp + mod; - return ind; - } -} - -JEMALLOC_INLINE size_t -pind2sz_compute(pszind_t pind) { - if (unlikely(pind == NPSIZES)) { - return LARGE_MAXCLASS + PAGE; - } - { - size_t grp = pind >> LG_SIZE_CLASS_GROUP; - size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; - - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; - - size_t sz = grp_size + mod_size; - return sz; - } -} - -JEMALLOC_INLINE size_t -pind2sz_lookup(pszind_t pind) { - size_t ret = (size_t)pind2sz_tab[pind]; - assert(ret == pind2sz_compute(pind)); - return ret; -} - -JEMALLOC_INLINE size_t -pind2sz(pszind_t pind) { - assert(pind < NPSIZES+1); - return pind2sz_lookup(pind); -} - -JEMALLOC_INLINE size_t -psz2u(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return LARGE_MAXCLASS + PAGE; - } - { - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (psz + delta_mask) & ~delta_mask; - return usize; - } -} - -JEMALLOC_INLINE szind_t -size2index_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { - return NSIZES; - } -#if (NTBINS != 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; - szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); - return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); - } -#endif - { - szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : - x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); - szind_t grp = shift << LG_SIZE_CLASS_GROUP; - - szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; - - size_t delta_inverse_mask = ZD(-1) << lg_delta; - szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - - szind_t index = NTBINS + grp + mod; - return index; - } -} - -JEMALLOC_ALWAYS_INLINE szind_t -size2index_lookup(size_t size) { - assert(size <= LOOKUP_MAXCLASS); - { - szind_t ret = (size2index_tab[(size-1) >> LG_TINY_MIN]); - assert(ret == size2index_compute(size)); - return ret; - } -} - -JEMALLOC_ALWAYS_INLINE szind_t -size2index(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { - return size2index_lookup(size); - } - return size2index_compute(size); -} - -JEMALLOC_INLINE size_t -index2size_compute(szind_t index) { -#if (NTBINS > 0) - if (index < NTBINS) { - return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); - } -#endif - { - size_t reduced_index = index - NTBINS; - size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; - size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - - 1); - - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; - - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_QUANTUM-1); - size_t mod_size = (mod+1) << lg_delta; - - size_t usize = grp_size + mod_size; - return usize; - } -} - -JEMALLOC_ALWAYS_INLINE size_t -index2size_lookup(szind_t index) { - size_t ret = (size_t)index2size_tab[index]; - assert(ret == index2size_compute(index)); - return ret; -} - -JEMALLOC_ALWAYS_INLINE size_t -index2size(szind_t index) { - assert(index < NSIZES); - return index2size_lookup(index); -} - -JEMALLOC_ALWAYS_INLINE size_t -s2u_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { - return 0; - } -#if (NTBINS > 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; - size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); - return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : - (ZU(1) << lg_ceil)); - } -#endif - { - size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (size + delta_mask) & ~delta_mask; - return usize; - } -} - -JEMALLOC_ALWAYS_INLINE size_t -s2u_lookup(size_t size) { - size_t ret = index2size_lookup(size2index_lookup(size)); - - assert(ret == s2u_compute(size)); - return ret; -} - -/* - * Compute usable size that would result from allocating an object with the - * specified size. - */ -JEMALLOC_ALWAYS_INLINE size_t -s2u(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { - return s2u_lookup(size); - } - return s2u_compute(size); -} - -/* - * Compute usable size that would result from allocating an object with the - * specified size and alignment. - */ -JEMALLOC_ALWAYS_INLINE size_t -sa2u(size_t size, size_t alignment) { - size_t usize; - - assert(alignment != 0 && ((alignment - 1) & alignment) == 0); - - /* Try for a small size class. */ - if (size <= SMALL_MAXCLASS && alignment < PAGE) { - /* - * Round size up to the nearest multiple of alignment. - * - * This done, we can take advantage of the fact that for each - * small size class, every object is aligned at the smallest - * power of two that is non-zero in the base two representation - * of the size. For example: - * - * Size | Base 2 | Minimum alignment - * -----+----------+------------------ - * 96 | 1100000 | 32 - * 144 | 10100000 | 32 - * 192 | 11000000 | 64 - */ - usize = s2u(ALIGNMENT_CEILING(size, alignment)); - if (usize < LARGE_MINCLASS) { - return usize; - } - } - - /* Large size class. Beware of overflow. */ - - if (unlikely(alignment > LARGE_MAXCLASS)) { - return 0; - } - - /* Make sure result is a large size class. */ - if (size <= LARGE_MINCLASS) { - usize = LARGE_MINCLASS; - } else { - usize = s2u(size); - if (usize < size) { - /* size_t overflow. */ - return 0; - } - } - - /* - * Calculate the multi-page mapping that large_palloc() would need in - * order to guarantee the alignment. - */ - if (usize + large_pad + PAGE_CEILING(alignment) - PAGE < usize) { - /* size_t overflow. */ - return 0; - } - return usize; -} - -JEMALLOC_ALWAYS_INLINE malloc_cpuid_t -malloc_getcpu(void) { - assert(have_percpu_arena); -#if defined(JEMALLOC_HAVE_SCHED_GETCPU) - return (malloc_cpuid_t)sched_getcpu(); -#else - not_reached(); - return -1; -#endif -} - -/* Return the chosen arena index based on current cpu. */ -JEMALLOC_ALWAYS_INLINE unsigned -percpu_arena_choose(void) { - unsigned arena_ind; - assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled)); - - malloc_cpuid_t cpuid = malloc_getcpu(); - assert(cpuid >= 0); - if ((percpu_arena_mode == percpu_arena) || - ((unsigned)cpuid < ncpus / 2)) { - arena_ind = cpuid; - } else { - assert(percpu_arena_mode == per_phycpu_arena); - /* Hyper threads on the same physical CPU share arena. */ - arena_ind = cpuid - ncpus / 2; - } - - return arena_ind; -} - -/* Return the limit of percpu auto arena range, i.e. arenas[0...ind_limit). */ -JEMALLOC_ALWAYS_INLINE unsigned -percpu_arena_ind_limit(void) { - assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled)); - if (percpu_arena_mode == per_phycpu_arena && ncpus > 1) { - if (ncpus % 2) { - /* This likely means a misconfig. */ - return ncpus / 2 + 1; - } - return ncpus / 2; - } else { - return ncpus; - } -} - -JEMALLOC_INLINE arena_tdata_t * -arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { - arena_tdata_t *tdata; - arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); - - if (unlikely(arenas_tdata == NULL)) { - /* arenas_tdata hasn't been initialized yet. */ - return arena_tdata_get_hard(tsd, ind); - } - if (unlikely(ind >= tsd_narenas_tdata_get(tsd))) { - /* - * ind is invalid, cache is old (too small), or tdata to be - * initialized. - */ - return (refresh_if_missing ? arena_tdata_get_hard(tsd, ind) : - NULL); - } - - tdata = &arenas_tdata[ind]; - if (likely(tdata != NULL) || !refresh_if_missing) { - return tdata; - } - return arena_tdata_get_hard(tsd, ind); -} - -JEMALLOC_INLINE arena_t * -arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { - arena_t *ret; - - assert(ind <= MALLOCX_ARENA_MAX); - - ret = (arena_t *)atomic_load_p(&arenas[ind], ATOMIC_ACQUIRE); - if (unlikely(ret == NULL)) { - if (init_if_missing) { - ret = arena_init(tsdn, ind, - (extent_hooks_t *)&extent_hooks_default); - } - } - return ret; -} - -JEMALLOC_INLINE ticker_t * -decay_ticker_get(tsd_t *tsd, unsigned ind) { - arena_tdata_t *tdata; - - tdata = arena_tdata_get(tsd, ind, true); - if (unlikely(tdata == NULL)) { - return NULL; - } - return &tdata->decay_ticker; -} - -JEMALLOC_ALWAYS_INLINE tcache_bin_t * -tcache_small_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind < NBINS); - return &tcache->tbins_small[binind]; -} - -JEMALLOC_ALWAYS_INLINE tcache_bin_t * -tcache_large_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind >= NBINS &&binind < nhbins); - return &tcache->tbins_large[binind - NBINS]; -} - -JEMALLOC_ALWAYS_INLINE bool -tcache_available(tsd_t *tsd) { - cassert(config_tcache); - - /* - * Thread specific auto tcache might be unavailable if: 1) during tcache - * initialization, or 2) disabled through thread.tcache.enabled mallctl - * or config options. This check covers all cases. - */ - if (likely(tsd_tcache_enabled_get(tsd) == true)) { - /* Associated arena == null implies tcache init in progress. */ - if (tsd_tcachep_get(tsd)->arena != NULL) { - assert(tcache_small_bin_get(tsd_tcachep_get(tsd), - 0)->avail != NULL); - } - return true; - } - - return false; -} - -JEMALLOC_ALWAYS_INLINE tcache_t * -tcache_get(tsd_t *tsd) { - if (!config_tcache) { - return NULL; - } - if (!tcache_available(tsd)) { - return NULL; - } - - return tsd_tcachep_get(tsd); -} -#endif - -#include "jemalloc/internal/rtree_inlines.h" -#include "jemalloc/internal/base_inlines.h" -#include "jemalloc/internal/bitmap_inlines.h" -/* - * Include portions of arena code interleaved with tcache code in order to - * resolve circular dependencies. - */ -#include "jemalloc/internal/prof_inlines_a.h" -#include "jemalloc/internal/arena_inlines_a.h" -#include "jemalloc/internal/extent_inlines.h" - -#ifndef JEMALLOC_ENABLE_INLINE -extent_t *iealloc(tsdn_t *tsdn, const void *ptr); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -/* Choose an arena based on a per-thread value. */ -JEMALLOC_INLINE arena_t * -arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { - arena_t *ret; - - if (arena != NULL) { - return arena; - } - - /* During reentrancy, arena 0 is the safest bet. */ - if (*tsd_reentrancy_levelp_get(tsd) > 1) { - return arena_get(tsd_tsdn(tsd), 0, true); - } - - ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); - if (unlikely(ret == NULL)) { - ret = arena_choose_hard(tsd, internal); - assert(ret); - if (config_tcache && tcache_available(tsd)) { - tcache_t *tcache = tcache_get(tsd); - if (tcache->arena != NULL) { - /* See comments in tcache_data_init().*/ - assert(tcache->arena == - arena_get(tsd_tsdn(tsd), 0, false)); - if (tcache->arena != ret) { - tcache_arena_reassociate(tsd_tsdn(tsd), - tcache, ret); - } - } else { - tcache_arena_associate(tsd_tsdn(tsd), tcache, - ret); - } - } - } - - /* - * Note that for percpu arena, if the current arena is outside of the - * auto percpu arena range, (i.e. thread is assigned to a manually - * managed arena), then percpu arena is skipped. - */ - if (have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled) && - (arena_ind_get(ret) < percpu_arena_ind_limit()) && - (ret->last_thd != tsd_tsdn(tsd))) { - unsigned ind = percpu_arena_choose(); - if (arena_ind_get(ret) != ind) { - percpu_arena_update(tsd, ind); - ret = tsd_arena_get(tsd); - } - ret->last_thd = tsd_tsdn(tsd); - } - - return ret; -} - -JEMALLOC_INLINE arena_t * -arena_choose(tsd_t *tsd, arena_t *arena) { - return arena_choose_impl(tsd, arena, false); -} - -JEMALLOC_INLINE arena_t * -arena_ichoose(tsd_t *tsd, arena_t *arena) { - return arena_choose_impl(tsd, arena, true); -} - -JEMALLOC_ALWAYS_INLINE extent_t * -iealloc(tsdn_t *tsdn, const void *ptr) { - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - - return rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)ptr, true); -} -#endif - -#include "jemalloc/internal/tcache_inlines.h" -#include "jemalloc/internal/arena_inlines_b.h" -#include "jemalloc/internal/hash_inlines.h" - -#ifndef JEMALLOC_ENABLE_INLINE -arena_t *iaalloc(tsdn_t *tsdn, const void *ptr); -size_t isalloc(tsdn_t *tsdn, const void *ptr); -void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path); -void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, - bool slow_path); -void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena); -void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, arena_t *arena); -void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); -size_t ivsalloc(tsdn_t *tsdn, const void *ptr); -void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, - dalloc_ctx_t *dalloc_ctx, bool is_internal, bool slow_path); -void idalloc(tsd_t *tsd, void *ptr); -void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, - bool slow_path); -void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena); -void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); -void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero); -bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_ALWAYS_INLINE arena_t * -iaalloc(tsdn_t *tsdn, const void *ptr) { - assert(ptr != NULL); - - return arena_aalloc(tsdn, ptr); -} - -JEMALLOC_ALWAYS_INLINE size_t -isalloc(tsdn_t *tsdn, const void *ptr) { - assert(ptr != NULL); - - return arena_salloc(tsdn, ptr); -} - -JEMALLOC_ALWAYS_INLINE void * -iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, - bool is_internal, arena_t *arena, bool slow_path) { - void *ret; - - assert(size != 0); - assert(!is_internal || tcache == NULL); - assert(!is_internal || arena == NULL || arena_ind_get(arena) < - narenas_auto); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); - if (config_stats && is_internal && likely(ret != NULL)) { - arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); - } - return ret; -} - -JEMALLOC_ALWAYS_INLINE void * -ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { - return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false, - NULL, slow_path); -} - -JEMALLOC_ALWAYS_INLINE void * -ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, bool is_internal, arena_t *arena) { - void *ret; - - assert(usize != 0); - assert(usize == sa2u(usize, alignment)); - assert(!is_internal || tcache == NULL); - assert(!is_internal || arena == NULL || arena_ind_get(arena) < - narenas_auto); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); - assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); - if (config_stats && is_internal && likely(ret != NULL)) { - arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); - } - return ret; -} - -JEMALLOC_ALWAYS_INLINE void * -ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, - tcache_t *tcache, arena_t *arena) { - return ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena); -} - -JEMALLOC_ALWAYS_INLINE void * -ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { - return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, - tcache_get(tsd), false, NULL); -} - -JEMALLOC_ALWAYS_INLINE size_t -ivsalloc(tsdn_t *tsdn, const void *ptr) { - return arena_vsalloc(tsdn, ptr); -} - -JEMALLOC_ALWAYS_INLINE void -idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, dalloc_ctx_t *dalloc_ctx, - bool is_internal, bool slow_path) { - assert(ptr != NULL); - assert(!is_internal || tcache == NULL); - assert(!is_internal || arena_ind_get(iaalloc(tsdn, ptr)) < - narenas_auto); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - if (config_stats && is_internal) { - arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); - } - if (!is_internal && *tsd_reentrancy_levelp_get(tsdn_tsd(tsdn)) != 0) { - tcache = NULL; - } - arena_dalloc(tsdn, ptr, tcache, dalloc_ctx, slow_path); -} - -JEMALLOC_ALWAYS_INLINE void -idalloc(tsd_t *tsd, void *ptr) { - idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), NULL, false, true); -} - -JEMALLOC_ALWAYS_INLINE void -isdalloct(tsdn_t *tsdn, void *ptr, size_t size, - tcache_t *tcache, bool slow_path) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - arena_sdalloc(tsdn, ptr, size, tcache, slow_path); -} - -JEMALLOC_ALWAYS_INLINE void * -iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena) { - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - void *p; - size_t usize, copysize; - - usize = sa2u(size + extra, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { - return NULL; - } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); - if (p == NULL) { - if (extra == 0) { - return NULL; - } - /* Try again, without extra this time. */ - usize = sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { - return NULL; - } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); - if (p == NULL) { - return NULL; - } - } - /* - * Copy at most size bytes (not size+extra), since the caller has no - * expectation that the extra bytes will be reliably preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - memcpy(p, ptr, copysize); - isdalloct(tsdn, ptr, oldsize, tcache, true); - return p; -} - -JEMALLOC_ALWAYS_INLINE void * -iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena) { - assert(ptr != NULL); - assert(size != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) - != 0) { - /* - * Existing object alignment is inadequate; allocate new space - * and copy. - */ - return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, - zero, tcache, arena); - } - - return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache); -} - -JEMALLOC_ALWAYS_INLINE void * -iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero) { - return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL); -} - -JEMALLOC_ALWAYS_INLINE bool -ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) { - assert(ptr != NULL); - assert(size != 0); - witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); - - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) - != 0) { - /* Existing object alignment is inadequate. */ - return true; - } - - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); -} -#endif - -#include "jemalloc/internal/prof_inlines_b.h" - -CPP_EPILOGUE - -#endif /* JEMALLOC_INTERNAL_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h new file mode 100644 index 0000000..7ac39be --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -0,0 +1,61 @@ +#ifndef JEMALLOC_INTERNAL_EXTERNS_H +#define JEMALLOC_INTERNAL_EXTERNS_H + +extern bool opt_abort; +extern const char *opt_junk; +extern bool opt_junk_alloc; +extern bool opt_junk_free; +extern bool opt_utrace; +extern bool opt_xmalloc; +extern bool opt_zero; +extern unsigned opt_narenas; + +/* Number of CPUs. */ +extern unsigned ncpus; + +/* Number of arenas used for automatic multiplexing of threads and arenas. */ +extern unsigned narenas_auto; + +/* + * Arenas that are used to service external requests. Not all elements of the + * arenas array are necessarily used; arenas are created lazily as needed. + */ +extern atomic_p_t arenas[]; + +/* + * pind2sz_tab encodes the same information as could be computed by + * pind2sz_compute(). + */ +extern size_t const pind2sz_tab[NPSIZES+1]; +/* + * index2size_tab encodes the same information as could be computed (at + * unacceptable cost in some code paths) by index2size_compute(). + */ +extern size_t const index2size_tab[NSIZES]; +/* + * size2index_tab is a compact lookup table that rounds request sizes up to + * size classes. In order to reduce cache footprint, the table is compressed, + * and all accesses are via size2index(). + */ +extern uint8_t const size2index_tab[]; + +void *a0malloc(size_t size); +void a0dalloc(void *ptr); +void *bootstrap_malloc(size_t size); +void *bootstrap_calloc(size_t num, size_t size); +void bootstrap_free(void *ptr); +void arena_set(unsigned ind, arena_t *arena); +unsigned narenas_total_get(void); +arena_t *arena_init(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); +arena_tdata_t *arena_tdata_get_hard(tsd_t *tsd, unsigned ind); +arena_t *arena_choose_hard(tsd_t *tsd, bool internal); +void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); +void iarena_cleanup(tsd_t *tsd); +void arena_cleanup(tsd_t *tsd); +void arenas_tdata_cleanup(tsd_t *tsd); +void jemalloc_prefork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); +bool malloc_initialized(void); + +#endif /* JEMALLOC_INTERNAL_EXTERNS_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h new file mode 100644 index 0000000..4503513 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_includes.h @@ -0,0 +1,155 @@ +#ifndef JEMALLOC_INTERNAL_INCLUDES_H +#define JEMALLOC_INTERNAL_INCLUDES_H + +/* + * jemalloc can conceptually be broken into components (arena, tcache, etc.), + * but there are circular dependencies that cannot be broken without + * substantial performance degradation. + * + * Historically, we dealt with this by each header into four sections (types, + * structs, externs, and inlines), and included each header file multiple times + * in this file, picking out the portion we want on each pass using the + * following #defines: + * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data + * types. + * JEMALLOC_H_STRUCTS : Data structures. + * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. + * JEMALLOC_H_INLINES : Inline functions. + * + * We're moving toward a world in which the dependencies are explicit; each file + * will #include the headers it depends on (rather than relying on them being + * implicitly available via this file including every header file in the + * project). + * + * We're now in an intermediate state: we've broken up the header files to avoid + * having to include each one multiple times, but have not yet moved the + * dependency information into the header files (i.e. we still rely on the + * ordering in this file to ensure all a header's dependencies are available in + * its translation unit). Each component is now broken up into multiple header + * files, corresponding to the sections above (e.g. instead of "tsd.h", we now + * have "tsd_types.h", "tsd_structs.h", "tsd_externs.h", "tsd_inlines.h"). + * + * Those files which have been converted to explicitly include their + * inter-component dependencies are now in the initial HERMETIC HEADERS + * section. All headers may still rely on jemalloc_preamble.h (which, by fiat, + * must be included first in every translation unit) for system headers and + * global jemalloc definitions, however. + */ + +CPP_PROLOGUE + +/******************************************************************************/ +/* HERMETIC HEADERS */ +/******************************************************************************/ + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/bit_util.h" +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/util.h" + +/******************************************************************************/ +/* TYPES */ +/******************************************************************************/ + +#include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/nstime_types.h" +#include "jemalloc/internal/spin_types.h" +#include "jemalloc/internal/prng_types.h" +#include "jemalloc/internal/ticker_types.h" +#include "jemalloc/internal/ckh_types.h" +#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/smoothstep.h" +#include "jemalloc/internal/stats_types.h" +#include "jemalloc/internal/ctl_types.h" +#include "jemalloc/internal/witness_types.h" +#include "jemalloc/internal/mutex_types.h" +#include "jemalloc/internal/tsd_types.h" +#include "jemalloc/internal/extent_types.h" +#include "jemalloc/internal/extent_dss_types.h" +#include "jemalloc/internal/base_types.h" +#include "jemalloc/internal/arena_types.h" +#include "jemalloc/internal/bitmap_types.h" +#include "jemalloc/internal/rtree_types.h" +#include "jemalloc/internal/pages_types.h" +#include "jemalloc/internal/tcache_types.h" +#include "jemalloc/internal/prof_types.h" + +/******************************************************************************/ +/* STRUCTS */ +/******************************************************************************/ + +#include "jemalloc/internal/nstime_structs.h" +#include "jemalloc/internal/spin_structs.h" +#include "jemalloc/internal/ticker_structs.h" +#include "jemalloc/internal/ckh_structs.h" +#include "jemalloc/internal/witness_structs.h" +#include "jemalloc/internal/mutex_structs.h" +#include "jemalloc/internal/stats_structs.h" +#include "jemalloc/internal/ctl_structs.h" +#include "jemalloc/internal/bitmap_structs.h" +#include "jemalloc/internal/arena_structs_a.h" +#include "jemalloc/internal/extent_structs.h" +#include "jemalloc/internal/extent_dss_structs.h" +#include "jemalloc/internal/base_structs.h" +#include "jemalloc/internal/prof_structs.h" +#include "jemalloc/internal/arena_structs_b.h" +#include "jemalloc/internal/rtree_structs.h" +#include "jemalloc/internal/tcache_structs.h" +#include "jemalloc/internal/tsd_structs.h" + +/******************************************************************************/ +/* EXTERNS */ +/******************************************************************************/ + +#include "jemalloc/internal/jemalloc_internal_externs.h" +#include "jemalloc/internal/nstime_externs.h" +#include "jemalloc/internal/ckh_externs.h" +#include "jemalloc/internal/stats_externs.h" +#include "jemalloc/internal/ctl_externs.h" +#include "jemalloc/internal/witness_externs.h" +#include "jemalloc/internal/mutex_externs.h" +#include "jemalloc/internal/bitmap_externs.h" +#include "jemalloc/internal/extent_externs.h" +#include "jemalloc/internal/extent_dss_externs.h" +#include "jemalloc/internal/extent_mmap_externs.h" +#include "jemalloc/internal/base_externs.h" +#include "jemalloc/internal/arena_externs.h" +#include "jemalloc/internal/rtree_externs.h" +#include "jemalloc/internal/pages_externs.h" +#include "jemalloc/internal/large_externs.h" +#include "jemalloc/internal/tcache_externs.h" +#include "jemalloc/internal/prof_externs.h" +#include "jemalloc/internal/tsd_externs.h" + +/******************************************************************************/ +/* INLINES */ +/******************************************************************************/ + +#include "jemalloc/internal/spin_inlines.h" +#include "jemalloc/internal/prng_inlines.h" +#include "jemalloc/internal/ticker_inlines.h" +#include "jemalloc/internal/tsd_inlines.h" +#include "jemalloc/internal/witness_inlines.h" +#include "jemalloc/internal/mutex_inlines.h" +#include "jemalloc/internal/jemalloc_internal_inlines_a.h" +#include "jemalloc/internal/rtree_inlines.h" +#include "jemalloc/internal/base_inlines.h" +#include "jemalloc/internal/bitmap_inlines.h" +/* + * Include portions of arena code interleaved with tcache code in order to + * resolve circular dependencies. + */ +#include "jemalloc/internal/prof_inlines_a.h" +#include "jemalloc/internal/arena_inlines_a.h" +#include "jemalloc/internal/extent_inlines.h" +#include "jemalloc/internal/jemalloc_internal_inlines_b.h" +#include "jemalloc/internal/tcache_inlines.h" +#include "jemalloc/internal/arena_inlines_b.h" +#include "jemalloc/internal/hash_inlines.h" +#include "jemalloc/internal/jemalloc_internal_inlines_c.h" +#include "jemalloc/internal/prof_inlines_b.h" + +CPP_EPILOGUE + +#endif /* JEMALLOC_INTERNAL_INCLUDES_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h new file mode 100644 index 0000000..256329a --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -0,0 +1,448 @@ +#ifndef JEMALLOC_INTERNAL_INLINES_A_H +#define JEMALLOC_INTERNAL_INLINES_A_H + +#ifndef JEMALLOC_ENABLE_INLINE +pszind_t psz2ind(size_t psz); +size_t pind2sz_compute(pszind_t pind); +size_t pind2sz_lookup(pszind_t pind); +size_t pind2sz(pszind_t pind); +size_t psz2u(size_t psz); +szind_t size2index_compute(size_t size); +szind_t size2index_lookup(size_t size); +szind_t size2index(size_t size); +size_t index2size_compute(szind_t index); +size_t index2size_lookup(szind_t index); +size_t index2size(szind_t index); +size_t s2u_compute(size_t size); +size_t s2u_lookup(size_t size); +size_t s2u(size_t size); +size_t sa2u(size_t size, size_t alignment); +arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal); +arena_t *arena_choose(tsd_t *tsd, arena_t *arena); +arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena); +arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind, + bool refresh_if_missing); +arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing); +ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); +bool tcache_available(tsd_t *tsd); +tcache_bin_t *tcache_small_bin_get(tcache_t *tcache, szind_t binind); +tcache_bin_t *tcache_large_bin_get(tcache_t *tcache, szind_t binind); +tcache_t *tcache_get(tsd_t *tsd); +malloc_cpuid_t malloc_getcpu(void); +unsigned percpu_arena_choose(void); +unsigned percpu_arena_ind_limit(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_ALWAYS_INLINE pszind_t +psz2ind(size_t psz) { + if (unlikely(psz > LARGE_MAXCLASS)) { + return NPSIZES; + } + { + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - + (LG_SIZE_CLASS_GROUP + LG_PAGE); + pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + + pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZD(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + pszind_t ind = grp + mod; + return ind; + } +} + +JEMALLOC_INLINE size_t +pind2sz_compute(pszind_t pind) { + if (unlikely(pind == NPSIZES)) { + return LARGE_MAXCLASS + PAGE; + } + { + size_t grp = pind >> LG_SIZE_CLASS_GROUP; + size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t sz = grp_size + mod_size; + return sz; + } +} + +JEMALLOC_INLINE size_t +pind2sz_lookup(pszind_t pind) { + size_t ret = (size_t)pind2sz_tab[pind]; + assert(ret == pind2sz_compute(pind)); + return ret; +} + +JEMALLOC_INLINE size_t +pind2sz(pszind_t pind) { + assert(pind < NPSIZES+1); + return pind2sz_lookup(pind); +} + +JEMALLOC_INLINE size_t +psz2u(size_t psz) { + if (unlikely(psz > LARGE_MAXCLASS)) { + return LARGE_MAXCLASS + PAGE; + } + { + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return usize; + } +} + +JEMALLOC_INLINE szind_t +size2index_compute(size_t size) { + if (unlikely(size > LARGE_MAXCLASS)) { + return NSIZES; + } +#if (NTBINS != 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); + return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); + } +#endif + { + szind_t x = lg_floor((size<<1)-1); + szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : + x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); + szind_t grp = shift << LG_SIZE_CLASS_GROUP; + + szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZD(-1) << lg_delta; + szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + szind_t index = NTBINS + grp + mod; + return index; + } +} + +JEMALLOC_ALWAYS_INLINE szind_t +size2index_lookup(size_t size) { + assert(size <= LOOKUP_MAXCLASS); + { + szind_t ret = (size2index_tab[(size-1) >> LG_TINY_MIN]); + assert(ret == size2index_compute(size)); + return ret; + } +} + +JEMALLOC_ALWAYS_INLINE szind_t +size2index(size_t size) { + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) { + return size2index_lookup(size); + } + return size2index_compute(size); +} + +JEMALLOC_INLINE size_t +index2size_compute(szind_t index) { +#if (NTBINS > 0) + if (index < NTBINS) { + return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); + } +#endif + { + size_t reduced_index = index - NTBINS; + size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; + size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_QUANTUM + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_QUANTUM-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t usize = grp_size + mod_size; + return usize; + } +} + +JEMALLOC_ALWAYS_INLINE size_t +index2size_lookup(szind_t index) { + size_t ret = (size_t)index2size_tab[index]; + assert(ret == index2size_compute(index)); + return ret; +} + +JEMALLOC_ALWAYS_INLINE size_t +index2size(szind_t index) { + assert(index < NSIZES); + return index2size_lookup(index); +} + +JEMALLOC_ALWAYS_INLINE size_t +s2u_compute(size_t size) { + if (unlikely(size > LARGE_MAXCLASS)) { + return 0; + } +#if (NTBINS > 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); + return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : + (ZU(1) << lg_ceil)); + } +#endif + { + size_t x = lg_floor((size<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (size + delta_mask) & ~delta_mask; + return usize; + } +} + +JEMALLOC_ALWAYS_INLINE size_t +s2u_lookup(size_t size) { + size_t ret = index2size_lookup(size2index_lookup(size)); + + assert(ret == s2u_compute(size)); + return ret; +} + +/* + * Compute usable size that would result from allocating an object with the + * specified size. + */ +JEMALLOC_ALWAYS_INLINE size_t +s2u(size_t size) { + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) { + return s2u_lookup(size); + } + return s2u_compute(size); +} + +/* + * Compute usable size that would result from allocating an object with the + * specified size and alignment. + */ +JEMALLOC_ALWAYS_INLINE size_t +sa2u(size_t size, size_t alignment) { + size_t usize; + + assert(alignment != 0 && ((alignment - 1) & alignment) == 0); + + /* Try for a small size class. */ + if (size <= SMALL_MAXCLASS && alignment < PAGE) { + /* + * Round size up to the nearest multiple of alignment. + * + * This done, we can take advantage of the fact that for each + * small size class, every object is aligned at the smallest + * power of two that is non-zero in the base two representation + * of the size. For example: + * + * Size | Base 2 | Minimum alignment + * -----+----------+------------------ + * 96 | 1100000 | 32 + * 144 | 10100000 | 32 + * 192 | 11000000 | 64 + */ + usize = s2u(ALIGNMENT_CEILING(size, alignment)); + if (usize < LARGE_MINCLASS) { + return usize; + } + } + + /* Large size class. Beware of overflow. */ + + if (unlikely(alignment > LARGE_MAXCLASS)) { + return 0; + } + + /* Make sure result is a large size class. */ + if (size <= LARGE_MINCLASS) { + usize = LARGE_MINCLASS; + } else { + usize = s2u(size); + if (usize < size) { + /* size_t overflow. */ + return 0; + } + } + + /* + * Calculate the multi-page mapping that large_palloc() would need in + * order to guarantee the alignment. + */ + if (usize + large_pad + PAGE_CEILING(alignment) - PAGE < usize) { + /* size_t overflow. */ + return 0; + } + return usize; +} + +JEMALLOC_ALWAYS_INLINE malloc_cpuid_t +malloc_getcpu(void) { + assert(have_percpu_arena); +#if defined(JEMALLOC_HAVE_SCHED_GETCPU) + return (malloc_cpuid_t)sched_getcpu(); +#else + not_reached(); + return -1; +#endif +} + +/* Return the chosen arena index based on current cpu. */ +JEMALLOC_ALWAYS_INLINE unsigned +percpu_arena_choose(void) { + unsigned arena_ind; + assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled)); + + malloc_cpuid_t cpuid = malloc_getcpu(); + assert(cpuid >= 0); + if ((percpu_arena_mode == percpu_arena) || + ((unsigned)cpuid < ncpus / 2)) { + arena_ind = cpuid; + } else { + assert(percpu_arena_mode == per_phycpu_arena); + /* Hyper threads on the same physical CPU share arena. */ + arena_ind = cpuid - ncpus / 2; + } + + return arena_ind; +} + +/* Return the limit of percpu auto arena range, i.e. arenas[0...ind_limit). */ +JEMALLOC_ALWAYS_INLINE unsigned +percpu_arena_ind_limit(void) { + assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled)); + if (percpu_arena_mode == per_phycpu_arena && ncpus > 1) { + if (ncpus % 2) { + /* This likely means a misconfig. */ + return ncpus / 2 + 1; + } + return ncpus / 2; + } else { + return ncpus; + } +} + +JEMALLOC_INLINE arena_tdata_t * +arena_tdata_get(tsd_t *tsd, unsigned ind, bool refresh_if_missing) { + arena_tdata_t *tdata; + arena_tdata_t *arenas_tdata = tsd_arenas_tdata_get(tsd); + + if (unlikely(arenas_tdata == NULL)) { + /* arenas_tdata hasn't been initialized yet. */ + return arena_tdata_get_hard(tsd, ind); + } + if (unlikely(ind >= tsd_narenas_tdata_get(tsd))) { + /* + * ind is invalid, cache is old (too small), or tdata to be + * initialized. + */ + return (refresh_if_missing ? arena_tdata_get_hard(tsd, ind) : + NULL); + } + + tdata = &arenas_tdata[ind]; + if (likely(tdata != NULL) || !refresh_if_missing) { + return tdata; + } + return arena_tdata_get_hard(tsd, ind); +} + +JEMALLOC_INLINE arena_t * +arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing) { + arena_t *ret; + + assert(ind <= MALLOCX_ARENA_MAX); + + ret = (arena_t *)atomic_load_p(&arenas[ind], ATOMIC_ACQUIRE); + if (unlikely(ret == NULL)) { + if (init_if_missing) { + ret = arena_init(tsdn, ind, + (extent_hooks_t *)&extent_hooks_default); + } + } + return ret; +} + +JEMALLOC_INLINE ticker_t * +decay_ticker_get(tsd_t *tsd, unsigned ind) { + arena_tdata_t *tdata; + + tdata = arena_tdata_get(tsd, ind, true); + if (unlikely(tdata == NULL)) { + return NULL; + } + return &tdata->decay_ticker; +} + +JEMALLOC_ALWAYS_INLINE tcache_bin_t * +tcache_small_bin_get(tcache_t *tcache, szind_t binind) { + assert(binind < NBINS); + return &tcache->tbins_small[binind]; +} + +JEMALLOC_ALWAYS_INLINE tcache_bin_t * +tcache_large_bin_get(tcache_t *tcache, szind_t binind) { + assert(binind >= NBINS &&binind < nhbins); + return &tcache->tbins_large[binind - NBINS]; +} + +JEMALLOC_ALWAYS_INLINE bool +tcache_available(tsd_t *tsd) { + cassert(config_tcache); + + /* + * Thread specific auto tcache might be unavailable if: 1) during tcache + * initialization, or 2) disabled through thread.tcache.enabled mallctl + * or config options. This check covers all cases. + */ + if (likely(tsd_tcache_enabled_get(tsd) == true)) { + /* Associated arena == null implies tcache init in progress. */ + if (tsd_tcachep_get(tsd)->arena != NULL) { + assert(tcache_small_bin_get(tsd_tcachep_get(tsd), + 0)->avail != NULL); + } + return true; + } + + return false; +} + +JEMALLOC_ALWAYS_INLINE tcache_t * +tcache_get(tsd_t *tsd) { + if (!config_tcache) { + return NULL; + } + if (!tcache_available(tsd)) { + return NULL; + } + + return tsd_tcachep_get(tsd); +} +#endif + +#endif /* JEMALLOC_INTERNAL_INLINES_A_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h new file mode 100644 index 0000000..52afb42 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -0,0 +1,83 @@ +#ifndef JEMALLOC_INTERNAL_INLINES_B_H +#define JEMALLOC_INTERNAL_INLINES_B_H + +#ifndef JEMALLOC_ENABLE_INLINE +extent_t *iealloc(tsdn_t *tsdn, const void *ptr); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +/* Choose an arena based on a per-thread value. */ +JEMALLOC_INLINE arena_t * +arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { + arena_t *ret; + + if (arena != NULL) { + return arena; + } + + /* During reentrancy, arena 0 is the safest bet. */ + if (*tsd_reentrancy_levelp_get(tsd) > 1) { + return arena_get(tsd_tsdn(tsd), 0, true); + } + + ret = internal ? tsd_iarena_get(tsd) : tsd_arena_get(tsd); + if (unlikely(ret == NULL)) { + ret = arena_choose_hard(tsd, internal); + assert(ret); + if (config_tcache && tcache_available(tsd)) { + tcache_t *tcache = tcache_get(tsd); + if (tcache->arena != NULL) { + /* See comments in tcache_data_init().*/ + assert(tcache->arena == + arena_get(tsd_tsdn(tsd), 0, false)); + if (tcache->arena != ret) { + tcache_arena_reassociate(tsd_tsdn(tsd), + tcache, ret); + } + } else { + tcache_arena_associate(tsd_tsdn(tsd), tcache, + ret); + } + } + } + + /* + * Note that for percpu arena, if the current arena is outside of the + * auto percpu arena range, (i.e. thread is assigned to a manually + * managed arena), then percpu arena is skipped. + */ + if (have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled) && + (arena_ind_get(ret) < percpu_arena_ind_limit()) && + (ret->last_thd != tsd_tsdn(tsd))) { + unsigned ind = percpu_arena_choose(); + if (arena_ind_get(ret) != ind) { + percpu_arena_update(tsd, ind); + ret = tsd_arena_get(tsd); + } + ret->last_thd = tsd_tsdn(tsd); + } + + return ret; +} + +JEMALLOC_INLINE arena_t * +arena_choose(tsd_t *tsd, arena_t *arena) { + return arena_choose_impl(tsd, arena, false); +} + +JEMALLOC_INLINE arena_t * +arena_ichoose(tsd_t *tsd, arena_t *arena) { + return arena_choose_impl(tsd, arena, true); +} + +JEMALLOC_ALWAYS_INLINE extent_t * +iealloc(tsdn_t *tsdn, const void *ptr) { + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + return rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, + (uintptr_t)ptr, true); +} +#endif + +#endif /* JEMALLOC_INTERNAL_INLINES_B_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h new file mode 100644 index 0000000..05debd2 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -0,0 +1,220 @@ +#ifndef JEMALLOC_INTERNAL_INLINES_C_H +#define JEMALLOC_INTERNAL_INLINES_C_H + +#ifndef JEMALLOC_ENABLE_INLINE +arena_t *iaalloc(tsdn_t *tsdn, const void *ptr); +size_t isalloc(tsdn_t *tsdn, const void *ptr); +void *iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, + tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path); +void *ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, + bool slow_path); +void *ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, bool is_internal, arena_t *arena); +void *ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, arena_t *arena); +void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); +size_t ivsalloc(tsdn_t *tsdn, const void *ptr); +void idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, + dalloc_ctx_t *dalloc_ctx, bool is_internal, bool slow_path); +void idalloc(tsd_t *tsd, void *ptr); +void isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, + bool slow_path); +void *iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, tcache_t *tcache, + arena_t *arena); +void *iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); +void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero); +bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_ALWAYS_INLINE arena_t * +iaalloc(tsdn_t *tsdn, const void *ptr) { + assert(ptr != NULL); + + return arena_aalloc(tsdn, ptr); +} + +JEMALLOC_ALWAYS_INLINE size_t +isalloc(tsdn_t *tsdn, const void *ptr) { + assert(ptr != NULL); + + return arena_salloc(tsdn, ptr); +} + +JEMALLOC_ALWAYS_INLINE void * +iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, + bool is_internal, arena_t *arena, bool slow_path) { + void *ret; + + assert(size != 0); + assert(!is_internal || tcache == NULL); + assert(!is_internal || arena == NULL || arena_ind_get(arena) < + narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); + if (config_stats && is_internal && likely(ret != NULL)) { + arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); + } + return ret; +} + +JEMALLOC_ALWAYS_INLINE void * +ialloc(tsd_t *tsd, size_t size, szind_t ind, bool zero, bool slow_path) { + return iallocztm(tsd_tsdn(tsd), size, ind, zero, tcache_get(tsd), false, + NULL, slow_path); +} + +JEMALLOC_ALWAYS_INLINE void * +ipallocztm(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, bool is_internal, arena_t *arena) { + void *ret; + + assert(usize != 0); + assert(usize == sa2u(usize, alignment)); + assert(!is_internal || tcache == NULL); + assert(!is_internal || arena == NULL || arena_ind_get(arena) < + narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + ret = arena_palloc(tsdn, arena, usize, alignment, zero, tcache); + assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); + if (config_stats && is_internal && likely(ret != NULL)) { + arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret)); + } + return ret; +} + +JEMALLOC_ALWAYS_INLINE void * +ipalloct(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, arena_t *arena) { + return ipallocztm(tsdn, usize, alignment, zero, tcache, false, arena); +} + +JEMALLOC_ALWAYS_INLINE void * +ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { + return ipallocztm(tsd_tsdn(tsd), usize, alignment, zero, + tcache_get(tsd), false, NULL); +} + +JEMALLOC_ALWAYS_INLINE size_t +ivsalloc(tsdn_t *tsdn, const void *ptr) { + return arena_vsalloc(tsdn, ptr); +} + +JEMALLOC_ALWAYS_INLINE void +idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, dalloc_ctx_t *dalloc_ctx, + bool is_internal, bool slow_path) { + assert(ptr != NULL); + assert(!is_internal || tcache == NULL); + assert(!is_internal || arena_ind_get(iaalloc(tsdn, ptr)) < + narenas_auto); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + if (config_stats && is_internal) { + arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); + } + if (!is_internal && *tsd_reentrancy_levelp_get(tsdn_tsd(tsdn)) != 0) { + tcache = NULL; + } + arena_dalloc(tsdn, ptr, tcache, dalloc_ctx, slow_path); +} + +JEMALLOC_ALWAYS_INLINE void +idalloc(tsd_t *tsd, void *ptr) { + idalloctm(tsd_tsdn(tsd), ptr, tcache_get(tsd), NULL, false, true); +} + +JEMALLOC_ALWAYS_INLINE void +isdalloct(tsdn_t *tsdn, void *ptr, size_t size, + tcache_t *tcache, bool slow_path) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + arena_sdalloc(tsdn, ptr, size, tcache, slow_path); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, tcache_t *tcache, + arena_t *arena) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + void *p; + size_t usize, copysize; + + usize = sa2u(size + extra, alignment); + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + return NULL; + } + p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); + if (p == NULL) { + if (extra == 0) { + return NULL; + } + /* Try again, without extra this time. */ + usize = sa2u(size, alignment); + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + return NULL; + } + p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); + if (p == NULL) { + return NULL; + } + } + /* + * Copy at most size bytes (not size+extra), since the caller has no + * expectation that the extra bytes will be reliably preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + memcpy(p, ptr, copysize); + isdalloct(tsdn, ptr, oldsize, tcache, true); + return p; +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, + bool zero, tcache_t *tcache, arena_t *arena) { + assert(ptr != NULL); + assert(size != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) + != 0) { + /* + * Existing object alignment is inadequate; allocate new space + * and copy. + */ + return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, + zero, tcache, arena); + } + + return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, + tcache); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, + bool zero) { + return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, + tcache_get(tsd), NULL); +} + +JEMALLOC_ALWAYS_INLINE bool +ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero) { + assert(ptr != NULL); + assert(size != 0); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); + + if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) + != 0) { + /* Existing object alignment is inadequate. */ + return true; + } + + return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); +} +#endif + +#endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_types.h b/include/jemalloc/internal/jemalloc_internal_types.h new file mode 100644 index 0000000..663ed8b --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_types.h @@ -0,0 +1,178 @@ +#ifndef JEMALLOC_INTERNAL_TYPES_H +#define JEMALLOC_INTERNAL_TYPES_H + +/* Page size index type. */ +typedef unsigned pszind_t; + +/* Size class index type. */ +typedef unsigned szind_t; + +/* Processor / core id type. */ +typedef int malloc_cpuid_t; + +/* + * Flags bits: + * + * a: arena + * t: tcache + * 0: unused + * z: zero + * n: alignment + * + * aaaaaaaa aaaatttt tttttttt 0znnnnnn + */ +#define MALLOCX_ARENA_BITS 12 +#define MALLOCX_TCACHE_BITS 12 +#define MALLOCX_LG_ALIGN_BITS 6 +#define MALLOCX_ARENA_SHIFT 20 +#define MALLOCX_TCACHE_SHIFT 8 +#define MALLOCX_ARENA_MASK \ + (((1 << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT) +/* NB: Arena index bias decreases the maximum number of arenas by 1. */ +#define MALLOCX_ARENA_MAX ((1 << MALLOCX_ARENA_BITS) - 2) +#define MALLOCX_TCACHE_MASK \ + (((1 << MALLOCX_TCACHE_BITS) - 1) << MALLOCX_TCACHE_SHIFT) +#define MALLOCX_TCACHE_MAX ((1 << MALLOCX_TCACHE_BITS) - 3) +#define MALLOCX_LG_ALIGN_MASK ((1 << MALLOCX_LG_ALIGN_BITS) - 1) +/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ +#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ + (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) +#define MALLOCX_ALIGN_GET(flags) \ + (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) +#define MALLOCX_ZERO_GET(flags) \ + ((bool)(flags & MALLOCX_ZERO)) + +#define MALLOCX_TCACHE_GET(flags) \ + (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> MALLOCX_TCACHE_SHIFT)) - 2) +#define MALLOCX_ARENA_GET(flags) \ + (((unsigned)(((unsigned)flags) >> MALLOCX_ARENA_SHIFT)) - 1) + +/* Smallest size class to support. */ +#define TINY_MIN (1U << LG_TINY_MIN) + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# ifdef __riscv__ +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# ifdef __SH4__ +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" +# endif +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + +#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) +#define LONG_MASK (LONG - 1) + +/* Return the smallest long multiple that is >= a. */ +#define LONG_CEILING(a) \ + (((a) + LONG_MASK) & ~LONG_MASK) + +#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) +#define PTR_MASK (SIZEOF_PTR - 1) + +/* Return the smallest (void *) multiple that is >= a. */ +#define PTR_CEILING(a) \ + (((a) + PTR_MASK) & ~PTR_MASK) + +/* + * Maximum size of L1 cache line. This is used to avoid cache line aliasing. + * In addition, this controls the spacing of cacheline-spaced size classes. + * + * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can + * only handle raw constants. + */ +#define LG_CACHELINE 6 +#define CACHELINE 64 +#define CACHELINE_MASK (CACHELINE - 1) + +/* Return the smallest cacheline multiple that is >= s. */ +#define CACHELINE_CEILING(s) \ + (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) + +/* Return the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2BASE(a, alignment) \ + ((void *)((uintptr_t)(a) & ((~(alignment)) + 1))) + +/* Return the offset between a and the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ + ((size_t)((uintptr_t)(a) & (alignment - 1))) + +/* Return the smallest alignment multiple that is >= s. */ +#define ALIGNMENT_CEILING(s, alignment) \ + (((s) + (alignment - 1)) & ((~(alignment)) + 1)) + +/* Declare a variable-length array. */ +#if __STDC_VERSION__ < 199901L +# ifdef _MSC_VER +# include <malloc.h> +# define alloca _alloca +# else +# ifdef JEMALLOC_HAS_ALLOCA_H +# include <alloca.h> +# else +# include <stdlib.h> +# endif +# endif +# define VARIABLE_ARRAY(type, name, count) \ + type *name = alloca(sizeof(type) * (count)) +#else +# define VARIABLE_ARRAY(type, name, count) type name[(count)] +#endif + +#endif /* JEMALLOC_INTERNAL_TYPES_H */ diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in new file mode 100644 index 0000000..6e38fe6 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_preamble.h.in @@ -0,0 +1,187 @@ +#ifndef JEMALLOC_PREAMBLE_H +#define JEMALLOC_PREAMBLE_H + +#ifdef __cplusplus +# define CPP_PROLOGUE extern "C" { +# define CPP_EPILOGUE } +#else +# define CPP_PROLOGUE +# define CPP_EPILOGUE +#endif + +CPP_PROLOGUE + +#include "jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" + +#ifdef JEMALLOC_UTRACE +#include <sys/ktrace.h> +#endif + +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "../jemalloc@install_suffix@.h" +# undef JEMALLOC_NO_RENAME +#else +# define JEMALLOC_N(n) @private_namespace@##n +# include "../jemalloc@install_suffix@.h" +#endif + +#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#include <libkern/OSAtomic.h> +#endif + +#ifdef JEMALLOC_ZONE +#include <mach/mach_error.h> +#include <mach/mach_init.h> +#include <mach/vm_map.h> +#endif + +#include "jemalloc/internal/jemalloc_internal_macros.h" + +/* + * Note that the ordering matters here; the hook itself is name-mangled. We + * want the inclusion of hooks to happen early, so that we hook as much as + * possible. + */ +#include "jemalloc/internal/private_namespace.h" +#include "jemalloc/internal/hooks.h" + +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool have_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool maps_coalesce = +#ifdef JEMALLOC_MAPS_COALESCE + true +#else + false +#endif + ; +static const bool config_munmap = +#ifdef JEMALLOC_MUNMAP + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_ivsalloc = +#ifdef JEMALLOC_IVSALLOC + true +#else + false +#endif + ; +static const bool config_cache_oblivious = +#ifdef JEMALLOC_CACHE_OBLIVIOUS + true +#else + false +#endif + ; +static const bool have_thp = +#ifdef JEMALLOC_THP + true +#else + false +#endif + ; +#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* Currently percpu_arena depends on sched_getcpu. */ +#define JEMALLOC_PERCPU_ARENA +#endif +static const bool have_percpu_arena = +#ifdef JEMALLOC_PERCPU_ARENA + true +#else + false +#endif + ; + +CPP_EPILOGUE + +#endif /* JEMALLOC_PREAMBLE_H */ diff --git a/src/arena.c b/src/arena.c index 198c6e4..5d313e3 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1,5 +1,6 @@ #define JEMALLOC_ARENA_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ @@ -1,5 +1,6 @@ #define JEMALLOC_BASE_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ diff --git a/src/bitmap.c b/src/bitmap.c index a629aca..2eb50f1 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1,5 +1,6 @@ #define JEMALLOC_BITMAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ @@ -35,7 +35,8 @@ * ******************************************************************************/ #define JEMALLOC_CKH_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -1,5 +1,6 @@ #define JEMALLOC_CTL_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ diff --git a/src/extent.c b/src/extent.c index b6c3f4b..2344e9c 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1,8 +1,10 @@ #define JEMALLOC_EXTENT_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/ph.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + /******************************************************************************/ /* Data. */ diff --git a/src/extent_dss.c b/src/extent_dss.c index 9991909..c609f14 100644 --- a/src/extent_dss.c +++ b/src/extent_dss.c @@ -1,5 +1,7 @@ #define JEMALLOC_EXTENT_DSS_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + /******************************************************************************/ /* Data. */ diff --git a/src/extent_mmap.c b/src/extent_mmap.c index 7265159..5717573 100644 --- a/src/extent_mmap.c +++ b/src/extent_mmap.c @@ -1,5 +1,6 @@ #define JEMALLOC_EXTENT_MMAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ @@ -1,2 +1,3 @@ #define JEMALLOC_HASH_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" diff --git a/src/hooks.c b/src/hooks.c index c32471e..6266ecd 100644 --- a/src/hooks.c +++ b/src/hooks.c @@ -1,4 +1,4 @@ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" /* * The hooks are a little bit screwy -- they're not genuinely exported in the diff --git a/src/jemalloc.c b/src/jemalloc.c index bc65932..27f9711 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1,5 +1,6 @@ #define JEMALLOC_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ diff --git a/src/jemalloc_cpp.cpp b/src/jemalloc_cpp.cpp index 9692b5b..b6d7c9a 100644 --- a/src/jemalloc_cpp.cpp +++ b/src/jemalloc_cpp.cpp @@ -2,7 +2,8 @@ #include <new> #define JEMALLOC_CPP_CPP_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" // All operators in this file are exported. diff --git a/src/large.c b/src/large.c index 3f96c52..18987c1 100644 --- a/src/large.c +++ b/src/large.c @@ -1,5 +1,6 @@ #define JEMALLOC_LARGE_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ diff --git a/src/malloc_io.c b/src/malloc_io.c index fd6ff0f..98ef7a6 100644 --- a/src/malloc_io.c +++ b/src/malloc_io.c @@ -1,5 +1,6 @@ #define JEMALLOC_MALLOC_IO_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" #ifdef assert # undef assert diff --git a/src/mutex.c b/src/mutex.c index 8c59310..26af523 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -1,5 +1,6 @@ #define JEMALLOC_MUTEX_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) #include <dlfcn.h> diff --git a/src/nstime.c b/src/nstime.c index 70b2f9d..ee8d78e 100644 --- a/src/nstime.c +++ b/src/nstime.c @@ -1,4 +1,5 @@ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" #define BILLION UINT64_C(1000000000) #define MILLION UINT64_C(1000000) diff --git a/src/pages.c b/src/pages.c index fa986ba..53ca653 100644 --- a/src/pages.c +++ b/src/pages.c @@ -1,5 +1,6 @@ #define JEMALLOC_PAGES_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT #include <sys/sysctl.h> @@ -1,2 +1,3 @@ #define JEMALLOC_PRNG_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" @@ -1,5 +1,7 @@ #define JEMALLOC_PROF_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + /******************************************************************************/ #ifdef JEMALLOC_PROF_LIBUNWIND diff --git a/src/rtree.c b/src/rtree.c index de4990b..051428f 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -1,5 +1,6 @@ #define JEMALLOC_RTREE_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /* * Only the most significant bits of keys passed to rtree_{read,write}() are @@ -1,2 +1,3 @@ #define JEMALLOC_SPIN_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" diff --git a/src/stats.c b/src/stats.c index aa7ca50..110d62f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1,5 +1,6 @@ #define JEMALLOC_STATS_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" const char *global_mutex_names[num_global_prof_mutexes] = { #define OP(mtx) #mtx, diff --git a/src/tcache.c b/src/tcache.c index 09776e1..99749fb 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -1,5 +1,6 @@ #define JEMALLOC_TCACHE_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ diff --git a/src/ticker.c b/src/ticker.c index b0149e1..d7b8cd2 100644 --- a/src/ticker.c +++ b/src/ticker.c @@ -1,2 +1,3 @@ #define JEMALLOC_TICKER_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" @@ -1,5 +1,6 @@ #define JEMALLOC_TSD_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* Data. */ diff --git a/src/witness.c b/src/witness.c index cbffaea..26b16e7 100644 --- a/src/witness.c +++ b/src/witness.c @@ -1,5 +1,6 @@ #define JEMALLOC_WITNESS_C_ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" void witness_init(witness_t *witness, const char *name, witness_rank_t rank, @@ -1,4 +1,5 @@ -#include "jemalloc/internal/jemalloc_internal.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" #ifndef JEMALLOC_ZONE # error "This source file is for zones on Darwin (OS X)." #endif diff --git a/test/include/test/jemalloc_test.h.in b/test/include/test/jemalloc_test.h.in index e3882b2..02eaac2 100644 --- a/test/include/test/jemalloc_test.h.in +++ b/test/include/test/jemalloc_test.h.in @@ -43,8 +43,8 @@ extern "C" { #ifdef JEMALLOC_UNIT_TEST # define JEMALLOC_JET # define JEMALLOC_MANGLE -# include "jemalloc/internal/jemalloc_internal.h" - +# include "jemalloc/internal/jemalloc_preamble.h" +# include "jemalloc/internal/jemalloc_internal_includes.h" /******************************************************************************/ /* @@ -96,7 +96,8 @@ static const bool config_debug = # include "jemalloc/jemalloc_protos_jet.h" # define JEMALLOC_JET -# include "jemalloc/internal/jemalloc_internal.h" +# include "jemalloc/internal/jemalloc_preamble.h" +# include "jemalloc/internal/jemalloc_internal_includes.h" # include "jemalloc/internal/public_unnamespace.h" # undef JEMALLOC_JET |