diff options
| -rw-r--r-- | ChangeLog | 3 | ||||
| -rw-r--r-- | Makefile.in | 7 | ||||
| -rw-r--r-- | doc/jemalloc.xml.in | 201 | ||||
| -rw-r--r-- | include/jemalloc/internal/arena.h | 19 | ||||
| -rw-r--r-- | include/jemalloc/internal/chunk.h | 44 | ||||
| -rw-r--r-- | include/jemalloc/internal/chunk_mmap.h | 2 | ||||
| -rw-r--r-- | include/jemalloc/internal/extent.h | 28 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal.h.in | 4 | ||||
| -rw-r--r-- | include/jemalloc/internal/pages.h | 26 | ||||
| -rw-r--r-- | include/jemalloc/internal/private_symbols.txt | 12 | ||||
| -rw-r--r-- | include/jemalloc/jemalloc_typedefs.h.in | 54 | ||||
| -rw-r--r-- | src/arena.c | 184 | ||||
| -rw-r--r-- | src/base.c | 4 | ||||
| -rw-r--r-- | src/chunk.c | 346 | ||||
| -rw-r--r-- | src/chunk_dss.c | 8 | ||||
| -rw-r--r-- | src/chunk_mmap.c | 131 | ||||
| -rw-r--r-- | src/ctl.c | 75 | ||||
| -rw-r--r-- | src/huge.c | 44 | ||||
| -rw-r--r-- | src/pages.c | 167 | ||||
| -rw-r--r-- | test/integration/chunk.c | 216 |
20 files changed, 1022 insertions, 553 deletions
@@ -37,8 +37,7 @@ brevity. Much more detail can be found in the git revision history: "opt.prof_thread_active_init", "prof.thread_active_init", and "thread.prof.active" mallctls. - Add support for per arena application-specified chunk allocators, configured - via the "arena<i>.chunk.alloc", "arena<i>.chunk.dalloc", and - "arena.<i>.chunk.purge" mallctls. + via the "arena.<i>.chunk_hooks" mallctl. - Refactor huge allocation to be managed by arenas, so that arenas now function as general purpose independent allocators. This is important in the context of user-specified chunk allocators, aside from the scalability diff --git a/Makefile.in b/Makefile.in index 25c2d5a..5084b1a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -82,9 +82,10 @@ C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \ $(srcroot)src/chunk.c $(srcroot)src/chunk_dss.c \ $(srcroot)src/chunk_mmap.c $(srcroot)src/ckh.c $(srcroot)src/ctl.c \ $(srcroot)src/extent.c $(srcroot)src/hash.c $(srcroot)src/huge.c \ - $(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \ - $(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \ - $(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c + $(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/pages.c \ + $(srcroot)src/prof.c $(srcroot)src/quarantine.c $(srcroot)src/rtree.c \ + $(srcroot)src/stats.c $(srcroot)src/tcache.c $(srcroot)src/util.c \ + $(srcroot)src/tsd.c ifeq ($(enable_valgrind), 1) C_SRCS += $(srcroot)src/valgrind.c endif diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index dbbe837..4cb74a0 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1518,18 +1518,48 @@ malloc_conf = "xmalloc:true";]]></programlisting> for additional information.</para></listitem> </varlistentry> - <varlistentry id="arena.i.chunk.alloc"> + <varlistentry id="arena.i.chunk_hooks"> <term> - <mallctl>arena.<i>.chunk.alloc</mallctl> - (<type>chunk_alloc_t *</type>) + <mallctl>arena.<i>.chunk_hooks</mallctl> + (<type>chunk_hooks_t</type>) <literal>rw</literal> </term> - <listitem><para>Get or set the chunk allocation function for arena - <i>. If setting, the chunk deallocation function should - also be set via <link linkend="arena.i.chunk.dalloc"> - <mallctl>arena.<i>.chunk.dalloc</mallctl></link> to a companion - function that knows how to deallocate the chunks. - <funcsynopsis><funcprototype> + <listitem><para>Get or set the chunk management hook functions for arena + <i>. The functions must be capable of operating on all extant + chunks associated with arena <i>, usually by passing unknown + chunks to the replaced functions. In practice, it is feasible to + control allocation for arenas created via <link + linkend="arenas.extend"><mallctl>arenas.extend</mallctl></link> such + that all chunks originate from an application-supplied chunk allocator + (by setting custom chunk hook functions just after arena creation), but + the automatically created arenas may have already created chunks prior + to the application having an opportunity to take over chunk + allocation.</para> + + <para><programlisting language="C"><![CDATA[ +typedef struct { + chunk_alloc_t *alloc; + chunk_dalloc_t *dalloc; + chunk_commit_t *commit; + chunk_decommit_t *decommit; + chunk_purge_t *purge; + chunk_split_t *split; + chunk_merge_t *merge; +} chunk_hooks_t;]]></programlisting> + The <type>chunk_hooks_t</type> structure comprises function pointers + which are described individually below. jemalloc uses these + functions to manage chunk lifetime, which starts off with allocation of + mapped committed memory, in the simplest case followed by deallocation. + However, there are performance and platform reasons to retain chunks for + later reuse. Cleanup attempts cascade from deallocation to decommit to + purging, which gives the chunk management functions opportunities to + reject the most permanent cleanup operations in favor of less permanent + (and often less costly) operations. The chunk splitting and merging + operations can also be opted out of, but this is mainly intended to + support platforms on which virtual memory mappings provided by the + operating system kernel do not automatically coalesce and split.</para> + + <para><funcsynopsis><funcprototype> <funcdef>typedef void *<function>(chunk_alloc_t)</function></funcdef> <paramdef>void *<parameter>chunk</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef> @@ -1539,9 +1569,9 @@ malloc_conf = "xmalloc:true";]]></programlisting> </funcprototype></funcsynopsis> A chunk allocation function conforms to the <type>chunk_alloc_t</type> type and upon success returns a pointer to <parameter>size</parameter> - bytes of memory on behalf of arena <parameter>arena_ind</parameter> such - that the chunk's base address is a multiple of - <parameter>alignment</parameter>, as well as setting + bytes of mapped committed memory on behalf of arena + <parameter>arena_ind</parameter> such that the chunk's base address is a + multiple of <parameter>alignment</parameter>, as well as setting <parameter>*zero</parameter> to indicate whether the chunk is zeroed. Upon error the function returns <constant>NULL</constant> and leaves <parameter>*zero</parameter> unmodified. The @@ -1550,34 +1580,16 @@ malloc_conf = "xmalloc:true";]]></programlisting> of two at least as large as the chunk size. Zeroing is mandatory if <parameter>*zero</parameter> is true upon function entry. If <parameter>chunk</parameter> is not <constant>NULL</constant>, the - returned pointer must be <parameter>chunk</parameter> or - <constant>NULL</constant> if it could not be allocated.</para> - - <para>Note that replacing the default chunk allocation function makes - the arena's <link + returned pointer must be <parameter>chunk</parameter> on success or + <constant>NULL</constant> on error. Committed memory may be committed + in absolute terms as on a system that does not overcommit, or in + implicit terms as on a system that overcommits and satisfies physical + memory needs on demand via soft page faults. Note that replacing the + default chunk allocation function makes the arena's <link linkend="arena.i.dss"><mallctl>arena.<i>.dss</mallctl></link> - setting irrelevant.</para></listitem> - </varlistentry> + setting irrelevant.</para> - <varlistentry id="arena.i.chunk.dalloc"> - <term> - <mallctl>arena.<i>.chunk.dalloc</mallctl> - (<type>chunk_dalloc_t *</type>) - <literal>rw</literal> - </term> - <listitem><para>Get or set the chunk deallocation function for arena - <i>. If setting, the chunk deallocation function must - be capable of deallocating all extant chunks associated with arena - <i>, usually by passing unknown chunks to the deallocation - function that was replaced. In practice, it is feasible to control - allocation for arenas created via <link - linkend="arenas.extend"><mallctl>arenas.extend</mallctl></link> such - that all chunks originate from an application-supplied chunk allocator - (by setting custom chunk allocation/deallocation/purge functions just - after arena creation), but the automatically created arenas may have - already created chunks prior to the application having an opportunity to - take over chunk allocation. - <funcsynopsis><funcprototype> + <para><funcsynopsis><funcprototype> <funcdef>typedef bool <function>(chunk_dalloc_t)</function></funcdef> <paramdef>void *<parameter>chunk</parameter></paramdef> <paramdef>size_t <parameter>size</parameter></paramdef> @@ -1587,46 +1599,99 @@ malloc_conf = "xmalloc:true";]]></programlisting> <type>chunk_dalloc_t</type> type and deallocates a <parameter>chunk</parameter> of given <parameter>size</parameter> on behalf of arena <parameter>arena_ind</parameter>, returning false upon - success.</para></listitem> - </varlistentry> + success. If the function returns true, this indicates opt-out from + deallocation; the virtual memory mapping associated with the chunk + remains mapped, committed, and available for future use, in which case + it will be automatically retained for later reuse.</para> - <varlistentry id="arena.i.chunk.purge"> - <term> - <mallctl>arena.<i>.chunk.purge</mallctl> - (<type>chunk_purge_t *</type>) - <literal>rw</literal> - </term> - <listitem><para>Get or set the chunk purge function for arena <i>. - A chunk purge function optionally discards physical pages associated - with pages in the chunk's virtual memory range but leaves the virtual - memory mapping intact, and indicates via its return value whether pages - in the virtual memory range will be zero-filled the next time they are - accessed. If setting, the chunk purge function must be capable of - purging all extant chunks associated with arena <i>, usually by - passing unknown chunks to the purge function that was replaced. In - practice, it is feasible to control allocation for arenas created via - <link linkend="arenas.extend"><mallctl>arenas.extend</mallctl></link> - such that all chunks originate from an application-supplied chunk - allocator (by setting custom chunk allocation/deallocation/purge - functions just after arena creation), but the automatically created - arenas may have already created chunks prior to the application having - an opportunity to take over chunk allocation. - <funcsynopsis><funcprototype> + <para><funcsynopsis><funcprototype> + <funcdef>typedef bool <function>(chunk_commit_t)</function></funcdef> + <paramdef>void *<parameter>chunk</parameter></paramdef> + <paramdef>size_t <parameter>size</parameter></paramdef> + <paramdef>unsigned <parameter>arena_ind</parameter></paramdef> + </funcprototype></funcsynopsis> + A chunk commit function conforms to the <type>chunk_commit_t</type> type + and commits zeroed physical memory to back a + <parameter>chunk</parameter> of given <parameter>size</parameter> on + behalf of arena <parameter>arena_ind</parameter>, returning false upon + success. Committed memory may be committed in absolute terms as on a + system that does not overcommit, or in implicit terms as on a system + that overcommits and satisfies physical memory needs on demand via soft + page faults. If the function returns true, this indicates insufficient + physical memory to satisfy the request.</para> + + <para><funcsynopsis><funcprototype> + <funcdef>typedef bool <function>(chunk_decommit_t)</function></funcdef> + <paramdef>void *<parameter>chunk</parameter></paramdef> + <paramdef>size_t <parameter>size</parameter></paramdef> + <paramdef>unsigned <parameter>arena_ind</parameter></paramdef> + </funcprototype></funcsynopsis> + A chunk decommit function conforms to the <type>chunk_decommit_t</type> + type and decommits any physical memory that is backing a + <parameter>chunk</parameter> of given <parameter>size</parameter> on + behalf of arena <parameter>arena_ind</parameter>, returning false upon + success, in which case the chunk will be committed via the chunk commit + function before being reused. If the function returns true, this + indicates opt-out from decommit; the memory remains committed and + available for future use, in which case it will be automatically + retained for later reuse.</para> + + <para><funcsynopsis><funcprototype> <funcdef>typedef bool <function>(chunk_purge_t)</function></funcdef> <paramdef>void *<parameter>chunk</parameter></paramdef> + <paramdef>size_t<parameter>size</parameter></paramdef> <paramdef>size_t <parameter>offset</parameter></paramdef> <paramdef>size_t <parameter>length</parameter></paramdef> <paramdef>unsigned <parameter>arena_ind</parameter></paramdef> </funcprototype></funcsynopsis> A chunk purge function conforms to the <type>chunk_purge_t</type> type - and purges pages within <parameter>chunk</parameter> at - <parameter>offset</parameter> bytes, extending for - <parameter>length</parameter> on behalf of arena + and optionally discards physical pages within the virtual memory mapping + associated with <parameter>chunk</parameter> of given + <parameter>size</parameter> at <parameter>offset</parameter> bytes, + extending for <parameter>length</parameter> on behalf of arena <parameter>arena_ind</parameter>, returning false if pages within the purged virtual memory range will be zero-filled the next time they are - accessed. Note that the memory range being purged may span multiple - contiguous chunks, e.g. when purging memory that backed a huge - allocation.</para></listitem> + accessed.</para> + + <para><funcsynopsis><funcprototype> + <funcdef>typedef bool <function>(chunk_split_t)</function></funcdef> + <paramdef>void *<parameter>chunk</parameter></paramdef> + <paramdef>size_t <parameter>size</parameter></paramdef> + <paramdef>size_t <parameter>size_a</parameter></paramdef> + <paramdef>size_t <parameter>size_b</parameter></paramdef> + <paramdef>bool <parameter>committed</parameter></paramdef> + <paramdef>unsigned <parameter>arena_ind</parameter></paramdef> + </funcprototype></funcsynopsis> + A chunk split function conforms to the <type>chunk_split_t</type> type + and optionally splits <parameter>chunk</parameter> of given + <parameter>size</parameter> into two adjacent chunks, the first of + <parameter>size_a</parameter> bytes, and the second of + <parameter>size_b</parameter> bytes, operating on + <parameter>committed</parameter>/decommitted memory as indicated, on + behalf of arena <parameter>arena_ind</parameter>, returning false upon + success. If the function returns true, this indicates that the chunk + remains unsplit and therefore should continue to be operated on as a + whole.</para> + + <para><funcsynopsis><funcprototype> + <funcdef>typedef bool <function>(chunk_merge_t)</function></funcdef> + <paramdef>void *<parameter>chunk_a</parameter></paramdef> + <paramdef>size_t <parameter>size_a</parameter></paramdef> + <paramdef>void *<parameter>chunk_b</parameter></paramdef> + <paramdef>size_t <parameter>size_b</parameter></paramdef> + <paramdef>bool <parameter>committed</parameter></paramdef> + <paramdef>unsigned <parameter>arena_ind</parameter></paramdef> + </funcprototype></funcsynopsis> + A chunk merge function conforms to the <type>chunk_merge_t</type> type + and optionally merges adjacent chunks, <parameter>chunk_a</parameter> of + given <parameter>size_a</parameter> and <parameter>chunk_b</parameter> + of given <parameter>size_b</parameter> into one contiguous chunk, + operating on <parameter>committed</parameter>/decommitted memory as + indicated, on behalf of arena <parameter>arena_ind</parameter>, + returning false upon success. If the function returns true, this + indicates that the chunks remain distinct mappings and therefore should + continue to be operated on independently.</para> + </listitem> </varlistentry> <varlistentry id="arenas.narenas"> diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 8811f2e..29f73e7 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -379,23 +379,18 @@ struct arena_s { * orderings are needed, which is why there are two trees with the same * contents. */ - extent_tree_t chunks_szad_cache; - extent_tree_t chunks_ad_cache; - extent_tree_t chunks_szad_mmap; - extent_tree_t chunks_ad_mmap; - extent_tree_t chunks_szad_dss; - extent_tree_t chunks_ad_dss; + extent_tree_t chunks_szad_cached; + extent_tree_t chunks_ad_cached; + extent_tree_t chunks_szad_retained; + extent_tree_t chunks_ad_retained; + malloc_mutex_t chunks_mtx; /* Cache of nodes that were allocated via base_alloc(). */ ql_head(extent_node_t) node_cache; malloc_mutex_t node_cache_mtx; - /* - * User-configurable chunk allocation/deallocation/purge functions. - */ - chunk_alloc_t *chunk_alloc; - chunk_dalloc_t *chunk_dalloc; - chunk_purge_t *chunk_purge; + /* User-configurable chunk hook functions. */ + chunk_hooks_t chunk_hooks; /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 91aefad..8e51134 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -19,6 +19,16 @@ #define CHUNK_CEILING(s) \ (((s) + chunksize_mask) & ~chunksize_mask) +#define CHUNK_HOOKS_INITIALIZER { \ + NULL, \ + NULL, \ + NULL, \ + NULL, \ + NULL, \ + NULL, \ + NULL \ +} + #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS @@ -36,30 +46,30 @@ extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunk_npages; +extern const chunk_hooks_t chunk_hooks_default; + +chunk_hooks_t chunk_hooks_get(arena_t *arena); +chunk_hooks_t chunk_hooks_set(arena_t *arena, + const chunk_hooks_t *chunk_hooks); + bool chunk_register(const void *chunk, const extent_node_t *node); void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); -void *chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size, - size_t alignment, bool *zero, bool dalloc_node); -void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, - bool *zero, unsigned arena_ind); -void *chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc, +void *chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, + void *new_addr, size_t size, size_t alignment, bool *zero, + bool dalloc_node); +void *chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero); -void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, - bool zeroed); -void chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size); -void chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size, - bool zeroed); -bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); -void chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, +void chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, + void *chunk, size_t size); +void chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks, + void *chunk, size_t size, bool zeroed); +void chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size); bool chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length); -bool chunk_purge_default(void *chunk, size_t offset, size_t length, - unsigned arena_ind); -bool chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge, - void *chunk, size_t offset, size_t length); +bool chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, + void *chunk, size_t size, size_t offset, size_t length); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/chunk_mmap.h b/include/jemalloc/internal/chunk_mmap.h index c5d5c6c..e81dc3a 100644 --- a/include/jemalloc/internal/chunk_mmap.h +++ b/include/jemalloc/internal/chunk_mmap.h @@ -9,8 +9,6 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -bool pages_purge(void *addr, size_t length); - void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero); bool chunk_dalloc_mmap(void *chunk, size_t size); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 3751adc..b2ac2b6 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -19,6 +19,13 @@ struct extent_node_s { size_t en_size; /* + * True if physical memory is committed to the extent, whether + * explicitly or implicitly as on a system that overcommits and + * satisfies physical mamory needs on demand via soft page faults. + */ + bool en_committed; + + /* * The zeroed flag is used by chunk recycling code to track whether * memory is zero-filled. */ @@ -66,17 +73,19 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) arena_t *extent_node_arena_get(const extent_node_t *node); void *extent_node_addr_get(const extent_node_t *node); size_t extent_node_size_get(const extent_node_t *node); +bool extent_node_committed_get(const extent_node_t *node); bool extent_node_zeroed_get(const extent_node_t *node); bool extent_node_achunk_get(const extent_node_t *node); prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node); void extent_node_arena_set(extent_node_t *node, arena_t *arena); void extent_node_addr_set(extent_node_t *node, void *addr); void extent_node_size_set(extent_node_t *node, size_t size); +void extent_node_committed_set(extent_node_t *node, bool committed); void extent_node_zeroed_set(extent_node_t *node, bool zeroed); void extent_node_achunk_set(extent_node_t *node, bool achunk); void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, - size_t size, bool zeroed); + size_t size, bool committed, bool zeroed); void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_insert(extent_node_t *node, arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty); @@ -106,6 +115,13 @@ extent_node_size_get(const extent_node_t *node) } JEMALLOC_INLINE bool +extent_node_committed_get(const extent_node_t *node) +{ + + return (node->en_committed); +} + +JEMALLOC_INLINE bool extent_node_zeroed_get(const extent_node_t *node) { @@ -148,6 +164,13 @@ extent_node_size_set(extent_node_t *node, size_t size) } JEMALLOC_INLINE void +extent_node_committed_set(extent_node_t *node, bool committed) +{ + + node->en_committed = committed; +} + +JEMALLOC_INLINE void extent_node_zeroed_set(extent_node_t *node, bool zeroed) { @@ -170,12 +193,13 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx) JEMALLOC_INLINE void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, - bool zeroed) + bool committed, bool zeroed) { extent_node_arena_set(node, arena); extent_node_addr_set(node, addr); extent_node_size_set(node, size); + extent_node_committed_set(node, committed); extent_node_zeroed_set(node, zeroed); extent_node_achunk_set(node, false); if (config_prof) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 496997d..7a137b6 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -367,6 +367,7 @@ typedef unsigned index_t; #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/pages.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/tcache.h" @@ -398,6 +399,7 @@ typedef unsigned index_t; #undef JEMALLOC_ARENA_STRUCTS_B #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/pages.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/tcache.h" @@ -477,6 +479,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/arena.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/pages.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" #include "jemalloc/internal/tcache.h" @@ -503,6 +506,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/pages.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h new file mode 100644 index 0000000..da7eb96 --- /dev/null +++ b/include/jemalloc/internal/pages.h @@ -0,0 +1,26 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void *pages_map(void *addr, size_t size); +void pages_unmap(void *addr, size_t size); +void *pages_trim(void *addr, size_t alloc_size, size_t leadsize, + size_t size); +bool pages_commit(void *addr, size_t size); +bool pages_decommit(void *addr, size_t size); +bool pages_purge(void *addr, size_t size); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index aaf6978..0e6216f 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -132,14 +132,12 @@ bt_init buferror chunk_alloc_cache chunk_alloc_base -chunk_alloc_default chunk_alloc_dss chunk_alloc_mmap chunk_alloc_wrapper chunk_boot chunk_dalloc_arena chunk_dalloc_cache -chunk_dalloc_default chunk_dalloc_mmap chunk_dalloc_wrapper chunk_deregister @@ -149,6 +147,9 @@ chunk_dss_postfork_parent chunk_dss_prec_get chunk_dss_prec_set chunk_dss_prefork +chunk_hooks_default +chunk_hooks_get +chunk_hooks_set chunk_in_dss chunk_lookup chunk_npages @@ -156,9 +157,7 @@ chunk_postfork_child chunk_postfork_parent chunk_prefork chunk_purge_arena -chunk_purge_default chunk_purge_wrapper -chunk_record chunk_register chunks_rtree chunksize @@ -347,7 +346,12 @@ opt_utrace opt_xmalloc opt_zero p2rz +pages_commit +pages_decommit +pages_map pages_purge +pages_trim +pages_unmap pow2_ceil prof_active_get prof_active_get_unlocked diff --git a/include/jemalloc/jemalloc_typedefs.h.in b/include/jemalloc/jemalloc_typedefs.h.in index d4b4690..26eb9ad 100644 --- a/include/jemalloc/jemalloc_typedefs.h.in +++ b/include/jemalloc/jemalloc_typedefs.h.in @@ -1,3 +1,55 @@ +/* + * void * + * chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, + * unsigned arena_ind); + */ typedef void *(chunk_alloc_t)(void *, size_t, size_t, bool *, unsigned); + +/* + * bool + * chunk_dalloc(void *chunk, size_t size, unsigned arena_ind); + */ typedef bool (chunk_dalloc_t)(void *, size_t, unsigned); -typedef bool (chunk_purge_t)(void *, size_t, size_t, unsigned); + +/* + * bool + * chunk_commit(void *chunk, size_t size, unsigned arena_ind); + */ +typedef bool (chunk_commit_t)(void *, size_t, unsigned); + +/* + * bool + * chunk_decommit(void *chunk, size_t size, unsigned arena_ind); + */ +typedef bool (chunk_decommit_t)(void *, size_t, unsigned); + +/* + * bool + * chunk_purge(void *chunk, size_t size, size_t offset, size_t length, + * unsigned arena_ind); + */ +typedef bool (chunk_purge_t)(void *, size_t, size_t, size_t, unsigned); + +/* + * bool + * chunk_split(void *chunk, size_t size, size_t size_a, size_t size_b, + * bool committed, unsigned arena_ind); + */ +typedef bool (chunk_split_t)(void *, size_t, size_t, size_t, bool, unsigned); + +/* + * bool + * chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b, + * bool committed, unsigned arena_ind); + */ +typedef bool (chunk_merge_t)(void *, size_t, void *, size_t, bool, unsigned); + +typedef struct { + chunk_alloc_t *alloc; + chunk_dalloc_t *dalloc; + chunk_commit_t *commit; + chunk_decommit_t *decommit; + chunk_purge_t *purge; + chunk_split_t *split; + chunk_merge_t *merge; +} chunk_hooks_t; diff --git a/src/arena.c b/src/arena.c index e2f49bd..ceeef81 100644 --- a/src/arena.c +++ b/src/arena.c @@ -516,23 +516,23 @@ static bool arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero) { - extent_node_init(&chunk->node, arena, chunk, chunksize, zero); + extent_node_init(&chunk->node, arena, chunk, chunksize, true, zero); extent_node_achunk_set(&chunk->node, true); return (chunk_register(chunk, &chunk->node)); } static arena_chunk_t * -arena_chunk_alloc_internal_hard(arena_t *arena, bool *zero) +arena_chunk_alloc_internal_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, + bool *zero) { arena_chunk_t *chunk; - chunk_alloc_t *chunk_alloc = arena->chunk_alloc; - chunk_dalloc_t *chunk_dalloc = arena->chunk_dalloc; malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_alloc, NULL, + + chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_hooks, NULL, chunksize, chunksize, zero); if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) { - chunk_dalloc_wrapper(arena, chunk_dalloc, (void *)chunk, + chunk_dalloc_wrapper(arena, chunk_hooks, (void *)chunk, chunksize); chunk = NULL; } @@ -545,19 +545,18 @@ static arena_chunk_t * arena_chunk_alloc_internal(arena_t *arena, bool *zero) { arena_chunk_t *chunk; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - if (likely(arena->chunk_alloc == chunk_alloc_default)) { - chunk = chunk_alloc_cache(arena, NULL, chunksize, chunksize, - zero, true); - if (chunk != NULL && arena_chunk_register(arena, chunk, - *zero)) { - chunk_dalloc_cache(arena, chunk, chunksize); - return (NULL); - } - } else - chunk = NULL; - if (chunk == NULL) - chunk = arena_chunk_alloc_internal_hard(arena, zero); + chunk = chunk_alloc_cache(arena, &chunk_hooks, NULL, chunksize, + chunksize, zero, true); + if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) { + chunk_dalloc_cache(arena, &chunk_hooks, chunk, chunksize); + return (NULL); + } + if (chunk == NULL) { + chunk = arena_chunk_alloc_internal_hard(arena, &chunk_hooks, + zero); + } if (config_stats && chunk != NULL) { arena->stats.mapped += chunksize; @@ -657,7 +656,7 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; - chunk_dalloc_t *chunk_dalloc; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; arena->spare = chunk; if (arena_mapbits_dirty_get(spare, map_bias) != 0) { @@ -667,15 +666,8 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) chunk_deregister(spare, &spare->node); - chunk_dalloc = arena->chunk_dalloc; - if (likely(chunk_dalloc == chunk_dalloc_default)) - chunk_dalloc_cache(arena, (void *)spare, chunksize); - else { - malloc_mutex_unlock(&arena->lock); - chunk_dalloc_wrapper(arena, chunk_dalloc, (void *)spare, - chunksize); - malloc_mutex_lock(&arena->lock); - } + chunk_dalloc_cache(arena, &chunk_hooks, (void *)spare, + chunksize); if (config_stats) { arena->stats.mapped -= chunksize; @@ -781,12 +773,12 @@ arena_node_dalloc(arena_t *arena, extent_node_t *node) } static void * -arena_chunk_alloc_huge_hard(arena_t *arena, chunk_alloc_t *chunk_alloc, +arena_chunk_alloc_huge_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, size_t usize, size_t alignment, bool *zero, size_t csize) { void *ret; - ret = chunk_alloc_wrapper(arena, chunk_alloc, NULL, csize, alignment, + ret = chunk_alloc_wrapper(arena, chunk_hooks, NULL, csize, alignment, zero); if (ret == NULL) { /* Revert optimistic stats updates. */ @@ -807,7 +799,7 @@ arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, bool *zero) { void *ret; - chunk_alloc_t *chunk_alloc; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; size_t csize = CHUNK_CEILING(usize); malloc_mutex_lock(&arena->lock); @@ -819,15 +811,11 @@ arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, } arena->nactive += (usize >> LG_PAGE); - chunk_alloc = arena->chunk_alloc; - if (likely(chunk_alloc == chunk_alloc_default)) { - ret = chunk_alloc_cache(arena, NULL, csize, alignment, zero, - true); - } else - ret = NULL; + ret = chunk_alloc_cache(arena, &chunk_hooks, NULL, csize, alignment, + zero, true); malloc_mutex_unlock(&arena->lock); if (ret == NULL) { - ret = arena_chunk_alloc_huge_hard(arena, chunk_alloc, usize, + ret = arena_chunk_alloc_huge_hard(arena, &chunk_hooks, usize, alignment, zero, csize); } @@ -839,12 +827,11 @@ arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) { - chunk_dalloc_t *chunk_dalloc; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; size_t csize; csize = CHUNK_CEILING(usize); malloc_mutex_lock(&arena->lock); - chunk_dalloc = arena->chunk_dalloc; if (config_stats) { arena_huge_dalloc_stats_update(arena, usize); arena->stats.mapped -= usize; @@ -852,13 +839,8 @@ arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) } arena->nactive -= (usize >> LG_PAGE); - if (likely(chunk_dalloc == chunk_dalloc_default)) { - chunk_dalloc_cache(arena, chunk, csize); - malloc_mutex_unlock(&arena->lock); - } else { - malloc_mutex_unlock(&arena->lock); - chunk_dalloc_wrapper(arena, chunk_dalloc, chunk, csize); - } + chunk_dalloc_cache(arena, &chunk_hooks, chunk, csize); + malloc_mutex_unlock(&arena->lock); } void @@ -904,30 +886,23 @@ arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, arena->nactive -= udiff >> LG_PAGE; if (cdiff != 0) { - chunk_dalloc_t *chunk_dalloc = arena->chunk_dalloc; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(usize)); - if (likely(chunk_dalloc == chunk_dalloc_default)) { - chunk_dalloc_cache(arena, nchunk, cdiff); - malloc_mutex_unlock(&arena->lock); - } else { - malloc_mutex_unlock(&arena->lock); - chunk_dalloc_wrapper(arena, chunk_dalloc, nchunk, - cdiff); - } - } else - malloc_mutex_unlock(&arena->lock); + chunk_dalloc_cache(arena, &chunk_hooks, nchunk, cdiff); + } + malloc_mutex_unlock(&arena->lock); } -bool -arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_alloc_t *chunk_alloc, - size_t oldsize, size_t usize, bool *zero, void *nchunk, size_t udiff, - size_t cdiff) +static bool +arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_hooks_t *chunk_hooks, + void *chunk, size_t oldsize, size_t usize, bool *zero, void *nchunk, + size_t udiff, size_t cdiff) { bool err; - err = (chunk_alloc_wrapper(arena, chunk_alloc, nchunk, cdiff, chunksize, + err = (chunk_alloc_wrapper(arena, chunk_hooks, nchunk, cdiff, chunksize, zero) == NULL); if (err) { /* Revert optimistic stats updates. */ @@ -939,6 +914,10 @@ arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_alloc_t *chunk_alloc, } arena->nactive -= (udiff >> LG_PAGE); malloc_mutex_unlock(&arena->lock); + } else if (chunk_hooks->merge(chunk, CHUNK_CEILING(oldsize), nchunk, + cdiff, true, arena->ind)) { + chunk_dalloc_arena(arena, chunk_hooks, nchunk, cdiff, *zero); + err = true; } return (err); } @@ -948,11 +927,13 @@ arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero) { bool err; - chunk_alloc_t *chunk_alloc; + chunk_hooks_t chunk_hooks; void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); size_t udiff = usize - oldsize; size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); + chunk_hooks = chunk_hooks_get(arena); + malloc_mutex_lock(&arena->lock); /* Optimistically update stats. */ @@ -962,16 +943,17 @@ arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, } arena->nactive += (udiff >> LG_PAGE); - chunk_alloc = arena->chunk_alloc; - if (likely(chunk_alloc == chunk_alloc_default)) { - err = (chunk_alloc_cache(arena, nchunk, cdiff, chunksize, zero, - true) == NULL); - } else - err = true; + err = (chunk_alloc_cache(arena, &arena->chunk_hooks, nchunk, cdiff, + chunksize, zero, true) == NULL); malloc_mutex_unlock(&arena->lock); if (err) { - err = arena_chunk_ralloc_huge_expand_hard(arena, chunk_alloc, - oldsize, usize, zero, nchunk, udiff, cdiff); + err = arena_chunk_ralloc_huge_expand_hard(arena, &chunk_hooks, + chunk, oldsize, usize, zero, nchunk, udiff, + cdiff); + } else if (chunk_hooks.merge(chunk, CHUNK_CEILING(oldsize), nchunk, + cdiff, true, arena->ind)) { + chunk_dalloc_arena(arena, &chunk_hooks, nchunk, cdiff, *zero); + err = true; } if (config_stats && !err) @@ -1198,8 +1180,8 @@ arena_compute_npurge(arena_t *arena, bool all) } static size_t -arena_stash_dirty(arena_t *arena, bool all, size_t npurge, - arena_runs_dirty_link_t *purge_runs_sentinel, +arena_stash_dirty(arena_t *arena, chunk_hooks_t *chunk_hooks, bool all, + size_t npurge, arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { arena_runs_dirty_link_t *rdelm, *rdelm_next; @@ -1224,7 +1206,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, * dalloc_node=false argument to chunk_alloc_cache(). */ zero = false; - chunk = chunk_alloc_cache(arena, + chunk = chunk_alloc_cache(arena, chunk_hooks, extent_node_addr_get(chunkselm), extent_node_size_get(chunkselm), chunksize, &zero, false); @@ -1278,12 +1260,11 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, } static size_t -arena_purge_stashed(arena_t *arena, +arena_purge_stashed(arena_t *arena, chunk_hooks_t *chunk_hooks, arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { size_t npurged, nmadvise; - chunk_purge_t *chunk_purge; arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; @@ -1291,7 +1272,6 @@ arena_purge_stashed(arena_t *arena, nmadvise = 0; npurged = 0; - chunk_purge = arena->chunk_purge; malloc_mutex_unlock(&arena->lock); for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); @@ -1299,13 +1279,16 @@ arena_purge_stashed(arena_t *arena, size_t npages; if (rdelm == &chunkselm->rd) { + /* + * Don't actually purge the chunk here because 1) + * chunkselm is embedded in the chunk and must remain + * valid, and 2) we deallocate the chunk in + * arena_unstash_purged(), where it is destroyed, + * decommitted, or purged, depending on chunk + * deallocation policy. + */ size_t size = extent_node_size_get(chunkselm); - bool unzeroed; - npages = size >> LG_PAGE; - unzeroed = chunk_purge_wrapper(arena, chunk_purge, - extent_node_addr_get(chunkselm), 0, size); - extent_node_zeroed_set(chunkselm, !unzeroed); chunkselm = qr_next(chunkselm, cc_link); } else { size_t pageind, run_size, flag_unzeroed, i; @@ -1319,8 +1302,9 @@ arena_purge_stashed(arena_t *arena, npages = run_size >> LG_PAGE; assert(pageind + npages <= chunk_npages); - unzeroed = chunk_purge_wrapper(arena, chunk_purge, - chunk, pageind << LG_PAGE, run_size); + unzeroed = chunk_purge_wrapper(arena, + chunk_hooks, chunk, chunksize, pageind << LG_PAGE, + run_size); flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; /* @@ -1355,14 +1339,14 @@ arena_purge_stashed(arena_t *arena, } static void -arena_unstash_purged(arena_t *arena, +arena_unstash_purged(arena_t *arena, chunk_hooks_t *chunk_hooks, arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { arena_runs_dirty_link_t *rdelm, *rdelm_next; extent_node_t *chunkselm; - /* Deallocate runs. */ + /* Deallocate chunks/runs. */ for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); rdelm != purge_runs_sentinel; rdelm = rdelm_next) { @@ -1376,7 +1360,8 @@ arena_unstash_purged(arena_t *arena, extent_node_dirty_remove(chunkselm); arena_node_dalloc(arena, chunkselm); chunkselm = chunkselm_next; - chunk_dalloc_arena(arena, addr, size, zeroed); + chunk_dalloc_arena(arena, chunk_hooks, addr, size, + zeroed); } else { arena_chunk_map_misc_t *miscelm = arena_rd_to_miscelm(rdelm); @@ -1390,6 +1375,7 @@ arena_unstash_purged(arena_t *arena, static void arena_purge(arena_t *arena, bool all) { + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; size_t npurge, npurgeable, npurged; arena_runs_dirty_link_t purge_runs_sentinel; extent_node_t purge_chunks_sentinel; @@ -1413,13 +1399,13 @@ arena_purge(arena_t *arena, bool all) qr_new(&purge_runs_sentinel, rd_link); extent_node_dirty_linkage_init(&purge_chunks_sentinel); - npurgeable = arena_stash_dirty(arena, all, npurge, &purge_runs_sentinel, - &purge_chunks_sentinel); + npurgeable = arena_stash_dirty(arena, &chunk_hooks, all, npurge, + &purge_runs_sentinel, &purge_chunks_sentinel); assert(npurgeable >= npurge); - npurged = arena_purge_stashed(arena, &purge_runs_sentinel, + npurged = arena_purge_stashed(arena, &chunk_hooks, &purge_runs_sentinel, &purge_chunks_sentinel); assert(npurged == npurgeable); - arena_unstash_purged(arena, &purge_runs_sentinel, + arena_unstash_purged(arena, &chunk_hooks, &purge_runs_sentinel, &purge_chunks_sentinel); arena->purging = false; @@ -2874,21 +2860,17 @@ arena_new(unsigned ind) if (malloc_mutex_init(&arena->huge_mtx)) return (NULL); - extent_tree_szad_new(&arena->chunks_szad_cache); - extent_tree_ad_new(&arena->chunks_ad_cache); - extent_tree_szad_new(&arena->chunks_szad_mmap); - extent_tree_ad_new(&arena->chunks_ad_mmap); - extent_tree_szad_new(&arena->chunks_szad_dss); - extent_tree_ad_new(&arena->chunks_ad_dss); + extent_tree_szad_new(&arena->chunks_szad_cached); + extent_tree_ad_new(&arena->chunks_ad_cached); + extent_tree_szad_new(&arena->chunks_szad_retained); + extent_tree_ad_new(&arena->chunks_ad_retained); if (malloc_mutex_init(&arena->chunks_mtx)) return (NULL); ql_new(&arena->node_cache); if (malloc_mutex_init(&arena->node_cache_mtx)) return (NULL); - arena->chunk_alloc = chunk_alloc_default; - arena->chunk_dalloc = chunk_dalloc_default; - arena->chunk_purge = chunk_purge_default; + arena->chunk_hooks = chunk_hooks_default; /* Initialize bins. */ for (i = 0; i < NBINS; i++) { @@ -66,7 +66,7 @@ base_chunk_alloc(size_t minsize) base_resident += PAGE_CEILING(nsize); } } - extent_node_init(node, NULL, addr, csize, true); + extent_node_init(node, NULL, addr, csize, true, true); return (node); } @@ -90,7 +90,7 @@ base_alloc(size_t size) csize = CACHELINE_CEILING(size); usize = s2u(csize); - extent_node_init(&key, NULL, NULL, usize, false); + extent_node_init(&key, NULL, NULL, usize, true, false); malloc_mutex_lock(&base_mtx); node = extent_tree_szad_nsearch(&base_avail_szad, &key); if (node != NULL) { diff --git a/src/chunk.c b/src/chunk.c index 7a4ede8..cdd5311 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -18,7 +18,103 @@ size_t chunksize; size_t chunksize_mask; /* (chunksize - 1). */ size_t chunk_npages; +static void *chunk_alloc_default(void *new_addr, size_t size, + size_t alignment, bool *zero, unsigned arena_ind); +static bool chunk_dalloc_default(void *chunk, size_t size, + unsigned arena_ind); +static bool chunk_commit_default(void *chunk, size_t size, + unsigned arena_ind); +static bool chunk_decommit_default(void *chunk, size_t size, + unsigned arena_ind); +static bool chunk_purge_default(void *chunk, size_t size, size_t offset, + size_t length, unsigned arena_ind); +static bool chunk_split_default(void *chunk, size_t size, size_t size_a, + size_t size_b, bool committed, unsigned arena_ind); +static bool chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, + size_t size_b, bool committed, unsigned arena_ind); + +const chunk_hooks_t chunk_hooks_default = { + chunk_alloc_default, + chunk_dalloc_default, + chunk_commit_default, + chunk_decommit_default, + chunk_purge_default, + chunk_split_default, + chunk_merge_default +}; + /******************************************************************************/ +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + +static void chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks, + extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, + void *chunk, size_t size, bool committed, bool zeroed); + +/******************************************************************************/ + +static chunk_hooks_t +chunk_hooks_get_locked(arena_t *arena) +{ + + return (arena->chunk_hooks); +} + +chunk_hooks_t +chunk_hooks_get(arena_t *arena) +{ + chunk_hooks_t chunk_hooks; + + malloc_mutex_lock(&arena->chunks_mtx); + chunk_hooks = chunk_hooks_get_locked(arena); + malloc_mutex_unlock(&arena->chunks_mtx); + + return (chunk_hooks); +} + +chunk_hooks_t +chunk_hooks_set(arena_t *arena, const chunk_hooks_t *chunk_hooks) +{ + chunk_hooks_t old_chunk_hooks; + + malloc_mutex_lock(&arena->chunks_mtx); + old_chunk_hooks = arena->chunk_hooks; + arena->chunk_hooks = *chunk_hooks; + malloc_mutex_unlock(&arena->chunks_mtx); + + return (old_chunk_hooks); +} + +static void +chunk_hooks_assure_initialized_impl(arena_t *arena, chunk_hooks_t *chunk_hooks, + bool locked) +{ + static const chunk_hooks_t uninitialized_hooks = + CHUNK_HOOKS_INITIALIZER; + + if (memcmp(chunk_hooks, &uninitialized_hooks, sizeof(chunk_hooks_t)) == + 0) { + *chunk_hooks = locked ? chunk_hooks_get_locked(arena) : + chunk_hooks_get(arena); + } +} + +static void +chunk_hooks_assure_initialized_locked(arena_t *arena, + chunk_hooks_t *chunk_hooks) +{ + + chunk_hooks_assure_initialized_impl(arena, chunk_hooks, true); +} + +static void +chunk_hooks_assure_initialized(arena_t *arena, chunk_hooks_t *chunk_hooks) +{ + + chunk_hooks_assure_initialized_impl(arena, chunk_hooks, false); +} bool chunk_register(const void *chunk, const extent_node_t *node) @@ -74,21 +170,26 @@ chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad, assert(size == CHUNK_CEILING(size)); - extent_node_init(&key, arena, NULL, size, false); + extent_node_init(&key, arena, NULL, size, false, false); return (extent_tree_szad_nsearch(chunks_szad, &key)); } static void * -chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool cache, void *new_addr, size_t size, - size_t alignment, bool *zero, bool dalloc_node) +chunk_recycle(arena_t *arena, chunk_hooks_t *chunk_hooks, + extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, + void *new_addr, size_t size, size_t alignment, bool *zero, bool dalloc_node) { void *ret; extent_node_t *node; size_t alloc_size, leadsize, trailsize; - bool zeroed; + bool committed, zeroed; assert(new_addr == NULL || alignment == chunksize); + /* + * Cached chunks use the node linkage embedded in their headers, in + * which case dalloc_node is true, and new_addr is non-NULL because + * we're operating on a specific chunk. + */ assert(dalloc_node || new_addr != NULL); alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize)); @@ -96,9 +197,11 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, if (alloc_size < size) return (NULL); malloc_mutex_lock(&arena->chunks_mtx); + chunk_hooks_assure_initialized_locked(arena, chunk_hooks); if (new_addr != NULL) { extent_node_t key; - extent_node_init(&key, arena, new_addr, alloc_size, false); + extent_node_init(&key, arena, new_addr, alloc_size, false, + false); node = extent_tree_ad_search(chunks_ad, &key); } else { node = chunk_first_best_fit(arena, chunks_szad, chunks_ad, @@ -115,9 +218,17 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, assert(extent_node_size_get(node) >= leadsize + size); trailsize = extent_node_size_get(node) - leadsize - size; ret = (void *)((uintptr_t)extent_node_addr_get(node) + leadsize); + committed = extent_node_committed_get(node); zeroed = extent_node_zeroed_get(node); if (zeroed) *zero = true; + /* Split the lead. */ + if (leadsize != 0 && + chunk_hooks->split(extent_node_addr_get(node), + extent_node_size_get(node), leadsize, size, false, arena->ind)) { + malloc_mutex_unlock(&arena->chunks_mtx); + return (NULL); + } /* Remove node from the tree. */ extent_tree_szad_remove(chunks_szad, node); extent_tree_ad_remove(chunks_ad, node); @@ -131,23 +242,40 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, node = NULL; } if (trailsize != 0) { + /* Split the trail. */ + if (chunk_hooks->split(ret, size + trailsize, size, + trailsize, false, arena->ind)) { + if (dalloc_node && node != NULL) + arena_node_dalloc(arena, node); + malloc_mutex_unlock(&arena->chunks_mtx); + chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad, + cache, ret, size + trailsize, committed, zeroed); + return (NULL); + } /* Insert the trailing space as a smaller chunk. */ if (node == NULL) { node = arena_node_alloc(arena); if (node == NULL) { malloc_mutex_unlock(&arena->chunks_mtx); - chunk_record(arena, chunks_szad, chunks_ad, - cache, ret, size, zeroed); + chunk_record(arena, chunk_hooks, chunks_szad, + chunks_ad, cache, ret, size + trailsize, + committed, zeroed); return (NULL); } } extent_node_init(node, arena, (void *)((uintptr_t)(ret) + size), - trailsize, zeroed); + trailsize, committed, zeroed); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); arena_chunk_cache_maybe_insert(arena, node, cache); node = NULL; } + if (!committed && chunk_hooks->commit(ret, size, arena->ind)) { + malloc_mutex_unlock(&arena->chunks_mtx); + chunk_record(arena, chunk_hooks, chunks_szad, chunks_ad, cache, + ret, size, committed, zeroed); + return (NULL); + } malloc_mutex_unlock(&arena->chunks_mtx); assert(dalloc_node || node != NULL); @@ -168,20 +296,6 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, return (ret); } -static void * -chunk_alloc_core_dss(arena_t *arena, void *new_addr, size_t size, - size_t alignment, bool *zero) -{ - void *ret; - - if ((ret = chunk_recycle(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, false, new_addr, size, alignment, zero, - true)) != NULL) - return (ret); - ret = chunk_alloc_dss(arena, new_addr, size, alignment, zero); - return (ret); -} - /* * If the caller specifies (!*zero), it is still possible to receive zeroed * memory, in which case *zero is toggled to true. arena_chunk_alloc() takes @@ -193,33 +307,33 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, dss_prec_t dss_prec) { void *ret; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; assert(size != 0); assert((size & chunksize_mask) == 0); assert(alignment != 0); assert((alignment & chunksize_mask) == 0); + /* Retained. */ + if ((ret = chunk_recycle(arena, &chunk_hooks, + &arena->chunks_szad_retained, &arena->chunks_ad_retained, false, + new_addr, size, alignment, zero, true)) != NULL) + return (ret); + /* "primary" dss. */ if (have_dss && dss_prec == dss_prec_primary && (ret = - chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != - NULL) - return (ret); - /* mmap. */ - if (!config_munmap && (ret = chunk_recycle(arena, - &arena->chunks_szad_mmap, &arena->chunks_ad_mmap, false, new_addr, - size, alignment, zero, true)) != NULL) + chunk_alloc_dss(arena, new_addr, size, alignment, zero)) != NULL) return (ret); /* - * Requesting an address is not implemented for chunk_alloc_mmap(), so - * only call it if (new_addr == NULL). + * mmap. Requesting an address is not implemented for + * chunk_alloc_mmap(), so only call it if (new_addr == NULL). */ if (new_addr == NULL && (ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) return (ret); /* "secondary" dss. */ if (have_dss && dss_prec == dss_prec_secondary && (ret = - chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != - NULL) + chunk_alloc_dss(arena, new_addr, size, alignment, zero)) != NULL) return (ret); /* All strategies for allocation failed. */ @@ -248,8 +362,8 @@ chunk_alloc_base(size_t size) } void * -chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size, size_t alignment, - bool *zero, bool dalloc_node) +chunk_alloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, + size_t size, size_t alignment, bool *zero, bool dalloc_node) { void *ret; @@ -258,8 +372,8 @@ chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size, size_t alignment, assert(alignment != 0); assert((alignment & chunksize_mask) == 0); - ret = chunk_recycle(arena, &arena->chunks_szad_cache, - &arena->chunks_ad_cache, true, new_addr, size, alignment, zero, + ret = chunk_recycle(arena, chunk_hooks, &arena->chunks_szad_cached, + &arena->chunks_ad_cached, true, new_addr, size, alignment, zero, dalloc_node); if (ret == NULL) return (NULL); @@ -285,11 +399,13 @@ chunk_arena_get(unsigned arena_ind) } static void * -chunk_alloc_arena(arena_t *arena, void *new_addr, size_t size, size_t alignment, - bool *zero) +chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, + unsigned arena_ind) { void *ret; + arena_t *arena; + arena = chunk_arena_get(arena_ind); ret = chunk_alloc_core(arena, new_addr, size, alignment, zero, arena->dss_prec); if (ret == NULL) @@ -300,55 +416,45 @@ chunk_alloc_arena(arena_t *arena, void *new_addr, size_t size, size_t alignment, return (ret); } -/* - * Default arena chunk allocation routine in the absence of user override. This - * function isn't actually used by jemalloc, but it does the right thing if the - * application passes calls through to it during chunk allocation. - */ void * -chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, - unsigned arena_ind) -{ - arena_t *arena; - - arena = chunk_arena_get(arena_ind); - return (chunk_alloc_arena(arena, new_addr, size, alignment, zero)); -} - -void * -chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc, void *new_addr, +chunk_alloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment, bool *zero) { void *ret; - ret = chunk_alloc(new_addr, size, alignment, zero, arena->ind); + chunk_hooks_assure_initialized(arena, chunk_hooks); + ret = chunk_hooks->alloc(new_addr, size, alignment, zero, arena->ind); if (ret == NULL) return (NULL); - if (config_valgrind && chunk_alloc != chunk_alloc_default) + if (config_valgrind && chunk_hooks->alloc != chunk_alloc_default) JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, chunksize); return (ret); } -void -chunk_record(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed) +static void +chunk_record(arena_t *arena, chunk_hooks_t *chunk_hooks, + extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, + void *chunk, size_t size, bool committed, bool zeroed) { bool unzeroed; extent_node_t *node, *prev; extent_node_t key; - assert(maps_coalesce || size == chunksize); assert(!cache || !zeroed); unzeroed = cache || !zeroed; JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); malloc_mutex_lock(&arena->chunks_mtx); + chunk_hooks_assure_initialized_locked(arena, chunk_hooks); extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, - false); + false, false); node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ if (node != NULL && extent_node_addr_get(node) == - extent_node_addr_get(&key)) { + extent_node_addr_get(&key) && extent_node_committed_get(node) == + committed && !chunk_hooks->merge(chunk, size, + extent_node_addr_get(node), extent_node_size_get(node), false, + arena->ind)) { /* * Coalesce chunk with the following address range. This does * not change the position within chunks_ad, so only @@ -373,12 +479,13 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, * a virtual memory leak. */ if (cache) { - chunk_purge_wrapper(arena, arena->chunk_purge, - chunk, 0, size); + chunk_purge_wrapper(arena, chunk_hooks, chunk, + size, 0, size); } goto label_return; } - extent_node_init(node, arena, chunk, size, !unzeroed); + extent_node_init(node, arena, chunk, size, committed, + !unzeroed); extent_tree_ad_insert(chunks_ad, node); extent_tree_szad_insert(chunks_szad, node); arena_chunk_cache_maybe_insert(arena, node, cache); @@ -387,7 +494,10 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, /* Try to coalesce backward. */ prev = extent_tree_ad_prev(chunks_ad, node); if (prev != NULL && (void *)((uintptr_t)extent_node_addr_get(prev) + - extent_node_size_get(prev)) == chunk) { + extent_node_size_get(prev)) == chunk && + extent_node_committed_get(prev) == committed && + !chunk_hooks->merge(extent_node_addr_get(prev), + extent_node_size_get(prev), chunk, size, false, arena->ind)) { /* * Coalesce chunk with the previous address range. This does * not change the position within chunks_ad, so only @@ -414,7 +524,8 @@ label_return: } void -chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size) +chunk_dalloc_cache(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, + size_t size) { assert(chunk != NULL); @@ -422,57 +533,68 @@ chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size) assert(size != 0); assert((size & chunksize_mask) == 0); - if (!maps_coalesce && size != chunksize) { - chunk_dalloc_arena(arena, chunk, size, false); - return; - } - - chunk_record(arena, &arena->chunks_szad_cache, &arena->chunks_ad_cache, - true, chunk, size, false); + chunk_record(arena, chunk_hooks, &arena->chunks_szad_cached, + &arena->chunks_ad_cached, true, chunk, size, true, false); arena_maybe_purge(arena); } void -chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size, bool zeroed) +chunk_dalloc_arena(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, + size_t size, bool zeroed) { + bool committed; assert(chunk != NULL); assert(CHUNK_ADDR2BASE(chunk) == chunk); assert(size != 0); assert((size & chunksize_mask) == 0); - if (have_dss && chunk_in_dss(chunk)) { - chunk_record(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, false, chunk, size, zeroed); - } else if (chunk_dalloc_mmap(chunk, size)) { - chunk_record(arena, &arena->chunks_szad_mmap, - &arena->chunks_ad_mmap, false, chunk, size, zeroed); - } + chunk_hooks_assure_initialized(arena, chunk_hooks); + /* Try to deallocate. */ + if (!chunk_hooks->dalloc(chunk, size, arena->ind)) + return; + /* Try to decommit; purge if that fails. */ + committed = chunk_hooks->decommit(chunk, size, arena->ind); + zeroed = !committed || chunk_hooks->purge(chunk, size, 0, size, + arena->ind); + chunk_record(arena, chunk_hooks, &arena->chunks_szad_retained, + &arena->chunks_ad_retained, false, chunk, size, committed, zeroed); } -/* - * Default arena chunk deallocation routine in the absence of user override. - * This function isn't actually used by jemalloc, but it does the right thing if - * the application passes calls through to it during chunk deallocation. - */ -bool +static bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) { - chunk_dalloc_arena(chunk_arena_get(arena_ind), chunk, size, false); - return (false); + if (!have_dss || !chunk_in_dss(chunk)) + return (chunk_dalloc_mmap(chunk, size)); + return (true); } void -chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, void *chunk, +chunk_dalloc_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, size_t size) { - chunk_dalloc(chunk, size, arena->ind); - if (config_valgrind && chunk_dalloc != chunk_dalloc_default) + chunk_hooks_assure_initialized(arena, chunk_hooks); + chunk_hooks->dalloc(chunk, size, arena->ind); + if (config_valgrind && chunk_hooks->dalloc != chunk_dalloc_default) JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); } +static bool +chunk_commit_default(void *chunk, size_t size, unsigned arena_ind) +{ + + return (pages_commit(chunk, size)); +} + +static bool +chunk_decommit_default(void *chunk, size_t size, unsigned arena_ind) +{ + + return (pages_decommit(chunk, size)); +} + bool chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length) { @@ -487,8 +609,8 @@ chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length) length)); } -bool -chunk_purge_default(void *chunk, size_t offset, size_t length, +static bool +chunk_purge_default(void *chunk, size_t size, size_t offset, size_t length, unsigned arena_ind) { @@ -497,11 +619,35 @@ chunk_purge_default(void *chunk, size_t offset, size_t length, } bool -chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge, void *chunk, - size_t offset, size_t length) +chunk_purge_wrapper(arena_t *arena, chunk_hooks_t *chunk_hooks, void *chunk, + size_t size, size_t offset, size_t length) +{ + + chunk_hooks_assure_initialized(arena, chunk_hooks); + return (chunk_hooks->purge(chunk, size, offset, length, arena->ind)); +} + +static bool +chunk_split_default(void *chunk, size_t size, size_t size_a, size_t size_b, + bool committed, unsigned arena_ind) { - return (chunk_purge(chunk, offset, length, arena->ind)); + if (!maps_coalesce) + return (true); + return (false); +} + +static bool +chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b, + bool committed, unsigned arena_ind) +{ + + if (!maps_coalesce) + return (true); + if (have_dss && chunk_in_dss(chunk_a) != chunk_in_dss(chunk_b)) + return (true); + + return (false); } static rtree_node_elm_t * diff --git a/src/chunk_dss.c b/src/chunk_dss.c index 6fbe31b..2c115e0 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -134,10 +134,10 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); if (cpad_size != 0) { - chunk_record(arena, - &arena->chunks_szad_dss, - &arena->chunks_ad_dss, false, cpad, - cpad_size, false); + chunk_hooks_t chunk_hooks = + CHUNK_HOOKS_INITIALIZER; + chunk_dalloc_wrapper(arena, + &chunk_hooks, cpad, cpad_size); } if (*zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index 30ac10b..f243615 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -2,137 +2,6 @@ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *pages_map(void *addr, size_t size); -static void pages_unmap(void *addr, size_t size); -static void *chunk_alloc_mmap_slow(size_t size, size_t alignment, - bool *zero); - -/******************************************************************************/ - -static void * -pages_map(void *addr, size_t size) -{ - void *ret; - - assert(size != 0); - -#ifdef _WIN32 - /* - * If VirtualAlloc can't allocate at the given address when one is - * given, it fails and returns NULL. - */ - ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, - PAGE_READWRITE); -#else - /* - * We don't use MAP_FIXED here, because it can cause the *replacement* - * of existing mappings, and we only want to create new mappings. - */ - ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, - -1, 0); - assert(ret != NULL); - - if (ret == MAP_FAILED) - ret = NULL; - else if (addr != NULL && ret != addr) { - /* - * We succeeded in mapping memory, but not in the right place. - */ - pages_unmap(ret, size); - ret = NULL; - } -#endif - assert(ret == NULL || (addr == NULL && ret != addr) - || (addr != NULL && ret == addr)); - return (ret); -} - -static void -pages_unmap(void *addr, size_t size) -{ - -#ifdef _WIN32 - if (VirtualFree(addr, 0, MEM_RELEASE) == 0) -#else - if (munmap(addr, size) == -1) -#endif - { - char buf[BUFERROR_BUF]; - - buferror(get_errno(), buf, sizeof(buf)); - malloc_printf("<jemalloc>: Error in " -#ifdef _WIN32 - "VirtualFree" -#else - "munmap" -#endif - "(): %s\n", buf); - if (opt_abort) - abort(); - } -} - -static void * -pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size) -{ - void *ret = (void *)((uintptr_t)addr + leadsize); - - assert(alloc_size >= leadsize + size); -#ifdef _WIN32 - { - void *new_addr; - - pages_unmap(addr, alloc_size); - new_addr = pages_map(ret, size); - if (new_addr == ret) - return (ret); - if (new_addr) - pages_unmap(new_addr, size); - return (NULL); - } -#else - { - size_t trailsize = alloc_size - leadsize - size; - - if (leadsize != 0) - pages_unmap(addr, leadsize); - if (trailsize != 0) - pages_unmap((void *)((uintptr_t)ret + size), trailsize); - return (ret); - } -#endif -} - -bool -pages_purge(void *addr, size_t length) -{ - bool unzeroed; - -#ifdef _WIN32 - VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE); - unzeroed = true; -#elif defined(JEMALLOC_HAVE_MADVISE) -# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED -# define JEMALLOC_MADV_PURGE MADV_DONTNEED -# define JEMALLOC_MADV_ZEROS true -# elif defined(JEMALLOC_PURGE_MADVISE_FREE) -# define JEMALLOC_MADV_PURGE MADV_FREE -# define JEMALLOC_MADV_ZEROS false -# else -# error "No madvise(2) flag defined for purging unused dirty pages." -# endif - int err = madvise(addr, length, JEMALLOC_MADV_PURGE); - unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); -# undef JEMALLOC_MADV_PURGE -# undef JEMALLOC_MADV_ZEROS -#else - /* Last resort no-op. */ - unzeroed = true; -#endif - return (unzeroed); -} static void * chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero) @@ -118,9 +118,7 @@ CTL_PROTO(arena_i_purge) static void arena_purge(unsigned arena_ind); CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_lg_dirty_mult) -CTL_PROTO(arena_i_chunk_alloc) -CTL_PROTO(arena_i_chunk_dalloc) -CTL_PROTO(arena_i_chunk_purge) +CTL_PROTO(arena_i_chunk_hooks) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) @@ -288,17 +286,11 @@ static const ctl_named_node_t tcache_node[] = { {NAME("destroy"), CTL(tcache_destroy)} }; -static const ctl_named_node_t chunk_node[] = { - {NAME("alloc"), CTL(arena_i_chunk_alloc)}, - {NAME("dalloc"), CTL(arena_i_chunk_dalloc)}, - {NAME("purge"), CTL(arena_i_chunk_purge)} -}; - static const ctl_named_node_t arena_i_node[] = { {NAME("purge"), CTL(arena_i_purge)}, {NAME("dss"), CTL(arena_i_dss)}, {NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)}, - {NAME("chunk"), CHILD(named, chunk)}, + {NAME("chunk_hooks"), CTL(arena_i_chunk_hooks)} }; static const ctl_named_node_t super_arena_i_node[] = { {NAME(""), CHILD(named, arena_i)} @@ -1064,8 +1056,8 @@ ctl_postfork_child(void) memcpy(oldp, (void *)&(v), copylen); \ ret = EINVAL; \ goto label_return; \ - } else \ - *(t *)oldp = (v); \ + } \ + *(t *)oldp = (v); \ } \ } while (0) @@ -1682,37 +1674,36 @@ label_return: return (ret); } -#define CHUNK_FUNC(n) \ -static int \ -arena_i_chunk_##n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) \ -{ \ - \ - int ret; \ - unsigned arena_ind = mib[1]; \ - arena_t *arena; \ - \ - malloc_mutex_lock(&ctl_mtx); \ - if (arena_ind < narenas_total_get() && (arena = \ - arena_get(tsd_fetch(), arena_ind, false, true)) != NULL) { \ - malloc_mutex_lock(&arena->lock); \ - READ(arena->chunk_##n, chunk_##n##_t *); \ - WRITE(arena->chunk_##n, chunk_##n##_t *); \ - } else { \ - ret = EFAULT; \ - goto label_outer_return; \ - } \ - ret = 0; \ -label_return: \ - malloc_mutex_unlock(&arena->lock); \ -label_outer_return: \ - malloc_mutex_unlock(&ctl_mtx); \ - return (ret); \ +static int +arena_i_chunk_hooks_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + unsigned arena_ind = mib[1]; + arena_t *arena; + + malloc_mutex_lock(&ctl_mtx); + if (arena_ind < narenas_total_get() && (arena = + arena_get(tsd_fetch(), arena_ind, false, true)) != NULL) { + if (newp != NULL) { + chunk_hooks_t old_chunk_hooks, new_chunk_hooks; + WRITE(new_chunk_hooks, chunk_hooks_t); + old_chunk_hooks = chunk_hooks_set(arena, + &new_chunk_hooks); + READ(old_chunk_hooks, chunk_hooks_t); + } else { + chunk_hooks_t old_chunk_hooks = chunk_hooks_get(arena); + READ(old_chunk_hooks, chunk_hooks_t); + } + } else { + ret = EFAULT; + goto label_return; + } + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); } -CHUNK_FUNC(alloc) -CHUNK_FUNC(dalloc) -CHUNK_FUNC(purge) -#undef CHUNK_FUNC static const ctl_named_node_t * arena_i_index(const size_t *mib, size_t miblen, size_t i) @@ -79,7 +79,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, return (NULL); } - extent_node_init(node, arena, ret, size, is_zeroed); + extent_node_init(node, arena, ret, size, true, is_zeroed); if (huge_node_set(ret, node)) { arena_chunk_dalloc_huge(arena, ret, size); @@ -132,7 +132,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, size_t usize_next; extent_node_t *node; arena_t *arena; - chunk_purge_t *chunk_purge; + chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; bool zeroed; /* Increase usize to incorporate extra. */ @@ -145,15 +145,11 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, node = huge_node_get(ptr); arena = extent_node_arena_get(node); - malloc_mutex_lock(&arena->lock); - chunk_purge = arena->chunk_purge; - malloc_mutex_unlock(&arena->lock); - /* Fill if necessary (shrinking). */ if (oldsize > usize) { size_t sdiff = oldsize - usize; - zeroed = !chunk_purge_wrapper(arena, chunk_purge, ptr, usize, - sdiff); + zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr, + CHUNK_CEILING(usize), usize, sdiff); if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); zeroed = false; @@ -185,26 +181,31 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, } } -static void +static bool huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { extent_node_t *node; arena_t *arena; - chunk_purge_t *chunk_purge; + chunk_hooks_t chunk_hooks; + size_t cdiff; bool zeroed; node = huge_node_get(ptr); arena = extent_node_arena_get(node); + chunk_hooks = chunk_hooks_get(arena); - malloc_mutex_lock(&arena->lock); - chunk_purge = arena->chunk_purge; - malloc_mutex_unlock(&arena->lock); + /* Split excess chunks. */ + cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); + if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize), + CHUNK_CEILING(usize), cdiff, true, arena->ind)) + return (true); if (oldsize > usize) { size_t sdiff = oldsize - usize; - zeroed = !chunk_purge_wrapper(arena, chunk_purge, + zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, CHUNK_ADDR2BASE((uintptr_t)ptr + usize), - CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); + CHUNK_CEILING(usize), CHUNK_ADDR2OFFSET((uintptr_t)ptr + + usize), sdiff); if (config_fill && unlikely(opt_junk_free)) { huge_dalloc_junk((void *)((uintptr_t)ptr + usize), sdiff); @@ -222,6 +223,8 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) /* Zap the excess chunks. */ arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); + + return (false); } static bool @@ -304,14 +307,9 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, return (false); } - if (!maps_coalesce) - return (true); - - /* Shrink the allocation in-place. */ - if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) { - huge_ralloc_no_move_shrink(ptr, oldsize, usize); - return (false); - } + /* Attempt to shrink the allocation in-place. */ + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) + return (huge_ralloc_no_move_shrink(ptr, oldsize, usize)); /* Attempt to expand the allocation in-place. */ if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) { diff --git a/src/pages.c b/src/pages.c new file mode 100644 index 0000000..6f775dc --- /dev/null +++ b/src/pages.c @@ -0,0 +1,167 @@ +#define JEMALLOC_PAGES_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ + +void * +pages_map(void *addr, size_t size) +{ + void *ret; + + assert(size != 0); + +#ifdef _WIN32 + /* + * If VirtualAlloc can't allocate at the given address when one is + * given, it fails and returns NULL. + */ + ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE); +#else + /* + * We don't use MAP_FIXED here, because it can cause the *replacement* + * of existing mappings, and we only want to create new mappings. + */ + ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, + -1, 0); + assert(ret != NULL); + + if (ret == MAP_FAILED) + ret = NULL; + else if (addr != NULL && ret != addr) { + /* + * We succeeded in mapping memory, but not in the right place. + */ + pages_unmap(ret, size); + ret = NULL; + } +#endif + assert(ret == NULL || (addr == NULL && ret != addr) + || (addr != NULL && ret == addr)); + return (ret); +} + +void +pages_unmap(void *addr, size_t size) +{ + +#ifdef _WIN32 + if (VirtualFree(addr, 0, MEM_RELEASE) == 0) +#else + if (munmap(addr, size) == -1) +#endif + { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf("<jemalloc>: Error in " +#ifdef _WIN32 + "VirtualFree" +#else + "munmap" +#endif + "(): %s\n", buf); + if (opt_abort) + abort(); + } +} + +void * +pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size) +{ + void *ret = (void *)((uintptr_t)addr + leadsize); + + assert(alloc_size >= leadsize + size); +#ifdef _WIN32 + { + void *new_addr; + + pages_unmap(addr, alloc_size); + new_addr = pages_map(ret, size); + if (new_addr == ret) + return (ret); + if (new_addr) + pages_unmap(new_addr, size); + return (NULL); + } +#else + { + size_t trailsize = alloc_size - leadsize - size; + + if (leadsize != 0) + pages_unmap(addr, leadsize); + if (trailsize != 0) + pages_unmap((void *)((uintptr_t)ret + size), trailsize); + return (ret); + } +#endif +} + +static bool +pages_commit_impl(void *addr, size_t size, bool commit) +{ + +#ifndef _WIN32 + if (config_debug) { + int prot = commit ? (PROT_READ | PROT_WRITE) : PROT_NONE; + void *result = mmap(addr, size, prot, MAP_PRIVATE | MAP_ANON | + MAP_FIXED, -1, 0); + if (result == MAP_FAILED) + return (true); + if (result != addr) { + /* + * We succeeded in mapping memory, but not in the right + * place. + */ + pages_unmap(result, size); + return (true); + } + return (false); + } +#endif + return (true); +} + +bool +pages_commit(void *addr, size_t size) +{ + + return (pages_commit_impl(addr, size, true)); +} + +bool +pages_decommit(void *addr, size_t size) +{ + + return (pages_commit_impl(addr, size, false)); +} + +bool +pages_purge(void *addr, size_t size) +{ + bool unzeroed; + +#ifdef _WIN32 + VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); + unzeroed = true; +#elif defined(JEMALLOC_HAVE_MADVISE) +# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED +# define JEMALLOC_MADV_PURGE MADV_DONTNEED +# define JEMALLOC_MADV_ZEROS true +# elif defined(JEMALLOC_PURGE_MADVISE_FREE) +# define JEMALLOC_MADV_PURGE MADV_FREE +# define JEMALLOC_MADV_ZEROS false +# else +# error "No madvise(2) flag defined for purging unused dirty pages." +# endif + int err = madvise(addr, size, JEMALLOC_MADV_PURGE); + unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); +# undef JEMALLOC_MADV_PURGE +# undef JEMALLOC_MADV_ZEROS +#else + /* Last resort no-op. */ + unzeroed = true; +#endif + return (unzeroed); +} + diff --git a/test/integration/chunk.c b/test/integration/chunk.c index c94b2d4..62d00ba 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -1,59 +1,140 @@ #include "test/jemalloc_test.h" -chunk_alloc_t *old_alloc; -chunk_dalloc_t *old_dalloc; -chunk_purge_t *old_purge; -bool purged; +static chunk_hooks_t orig_hooks; +static chunk_hooks_t old_hooks; + +static bool do_dalloc = true; +static bool do_decommit; + +static bool did_alloc; +static bool did_dalloc; +static bool did_commit; +static bool did_decommit; +static bool did_purge; +static bool did_split; +static bool did_merge; + +#if 0 +# define TRACE_HOOK(fmt, ...) malloc_printf(fmt, __VA_ARGS__) +#else +# define TRACE_HOOK(fmt, ...) +#endif void * chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind) { - return (old_alloc(new_addr, size, alignment, zero, arena_ind)); + TRACE_HOOK("%s(new_addr=%p, size=%zu, alignment=%zu, *zero=%s, " + "arena_ind=%u)\n", __func__, new_addr, size, alignment, *zero ? + "true" : "false", arena_ind); + did_alloc = true; + return (old_hooks.alloc(new_addr, size, alignment, zero, arena_ind)); } bool chunk_dalloc(void *chunk, size_t size, unsigned arena_ind) { - return (old_dalloc(chunk, size, arena_ind)); + TRACE_HOOK("%s(chunk=%p, size=%zu, arena_ind=%u)\n", __func__, chunk, + size, arena_ind); + did_dalloc = true; + if (!do_dalloc) + return (true); + return (old_hooks.dalloc(chunk, size, arena_ind)); } bool -chunk_purge(void *chunk, size_t offset, size_t length, unsigned arena_ind) +chunk_commit(void *chunk, size_t size, unsigned arena_ind) { - purged = true; - return (old_purge(chunk, offset, length, arena_ind)); + TRACE_HOOK("%s(chunk=%p, size=%zu, arena_ind=%u)\n", __func__, chunk, + size, arena_ind); + did_commit = true; + memset(chunk, 0, size); + return (false); } -TEST_BEGIN(test_chunk) +bool +chunk_decommit(void *chunk, size_t size, unsigned arena_ind) { - void *p; - chunk_alloc_t *new_alloc; - chunk_dalloc_t *new_dalloc; - chunk_purge_t *new_purge; - size_t old_size, new_size, huge0, huge1, huge2, sz; - new_alloc = chunk_alloc; - new_dalloc = chunk_dalloc; - new_purge = chunk_purge; - old_size = sizeof(chunk_alloc_t *); - new_size = sizeof(chunk_alloc_t *); + TRACE_HOOK("%s(chunk=%p, size=%zu, arena_ind=%u)\n", __func__, chunk, + size, arena_ind); + did_decommit = true; + return (!do_decommit); +} - assert_d_eq(mallctl("arena.0.chunk.alloc", &old_alloc, &old_size, - &new_alloc, new_size), 0, "Unexpected alloc error"); - assert_ptr_ne(old_alloc, new_alloc, "Unexpected alloc error"); +bool +chunk_purge(void *chunk, size_t size, size_t offset, size_t length, + unsigned arena_ind) +{ + + TRACE_HOOK("%s(chunk=%p, size=%zu, offset=%zu, length=%zu " + "arena_ind=%u)\n", __func__, chunk, size, offset, length, + arena_ind); + did_purge = true; + return (old_hooks.purge(chunk, size, offset, length, arena_ind)); +} - assert_d_eq(mallctl("arena.0.chunk.dalloc", &old_dalloc, &old_size, - &new_dalloc, new_size), 0, "Unexpected dalloc error"); - assert_ptr_ne(old_dalloc, new_dalloc, "Unexpected dalloc error"); +bool +chunk_split(void *chunk, size_t size, size_t size_a, size_t size_b, + bool committed, unsigned arena_ind) +{ - assert_d_eq(mallctl("arena.0.chunk.purge", &old_purge, &old_size, - &new_purge, new_size), 0, "Unexpected purge error"); - assert_ptr_ne(old_purge, new_purge, "Unexpected purge error"); + TRACE_HOOK("%s(chunk=%p, size=%zu, size_a=%zu, size_b=%zu, " + "committed=%s, arena_ind=%u)\n", __func__, chunk, size, size_a, + size_b, committed ? "true" : "false", arena_ind); + did_split = true; + return (old_hooks.split(chunk, size, size_a, size_b, committed, + arena_ind)); +} +bool +chunk_merge(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b, + bool committed, unsigned arena_ind) +{ + + TRACE_HOOK("%s(chunk_a=%p, size_a=%zu, chunk_b=%p size_b=%zu, " + "committed=%s, arena_ind=%u)\n", __func__, chunk_a, size_a, chunk_b, + size_b, committed ? "true" : "false", arena_ind); + did_merge = true; + return (old_hooks.merge(chunk_a, size_a, chunk_b, size_b, + committed, arena_ind)); +} + +TEST_BEGIN(test_chunk) +{ + void *p; + size_t old_size, new_size, huge0, huge1, huge2, sz; + chunk_hooks_t new_hooks = { + chunk_alloc, + chunk_dalloc, + chunk_commit, + chunk_decommit, + chunk_purge, + chunk_split, + chunk_merge + }; + + /* Install custom chunk hooks. */ + old_size = sizeof(chunk_hooks_t); + new_size = sizeof(chunk_hooks_t); + assert_d_eq(mallctl("arena.0.chunk_hooks", &old_hooks, &old_size, + &new_hooks, new_size), 0, "Unexpected chunk_hooks error"); + orig_hooks = old_hooks; + assert_ptr_ne(old_hooks.alloc, chunk_alloc, "Unexpected alloc error"); + assert_ptr_ne(old_hooks.dalloc, chunk_dalloc, + "Unexpected dalloc error"); + assert_ptr_ne(old_hooks.commit, chunk_commit, + "Unexpected commit error"); + assert_ptr_ne(old_hooks.decommit, chunk_decommit, + "Unexpected decommit error"); + assert_ptr_ne(old_hooks.purge, chunk_purge, "Unexpected purge error"); + assert_ptr_ne(old_hooks.split, chunk_split, "Unexpected split error"); + assert_ptr_ne(old_hooks.merge, chunk_merge, "Unexpected merge error"); + + /* Get huge size classes. */ sz = sizeof(size_t); assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0, "Unexpected arenas.hchunk.0.size failure"); @@ -61,6 +142,49 @@ TEST_BEGIN(test_chunk) "Unexpected arenas.hchunk.1.size failure"); assert_d_eq(mallctl("arenas.hchunk.2.size", &huge2, &sz, NULL, 0), 0, "Unexpected arenas.hchunk.2.size failure"); + + /* Test dalloc/decommit/purge cascade. */ + do_dalloc = false; + do_decommit = false; + p = mallocx(huge0 * 2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + did_dalloc = false; + did_decommit = false; + did_purge = false; + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() failure"); + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected arena.0.purge error"); + assert_true(did_dalloc, "Expected dalloc"); + assert_true(did_decommit, "Expected decommit"); + assert_true(did_purge, "Expected purge"); + dallocx(p, 0); + do_dalloc = true; + + /* Test decommit/commit and observe split/merge. */ + do_dalloc = false; + do_decommit = true; + p = mallocx(huge0 * 2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + did_decommit = false; + did_commit = false; + did_split = false; + did_merge = false; + assert_zu_eq(xallocx(p, huge0, 0, 0), huge0, + "Unexpected xallocx() failure"); + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected arena.0.purge error"); + assert_true(did_decommit, "Expected decommit"); + assert_true(did_split, "Expected split"); + assert_zu_eq(xallocx(p, huge0 * 2, 0, 0), huge0 * 2, + "Unexpected xallocx() failure"); + assert_true(did_commit, "Expected commit"); + assert_true(did_commit, "Expected merge"); + dallocx(p, 0); + do_dalloc = true; + do_decommit = false; + + /* Test purge for partial-chunk huge allocations. */ if (huge0 * 2 > huge2) { /* * There are at least four size classes per doubling, so a @@ -69,23 +193,37 @@ TEST_BEGIN(test_chunk) */ p = mallocx(huge2, 0); assert_ptr_not_null(p, "Unexpected mallocx() error"); - purged = false; + did_purge = false; assert_zu_eq(xallocx(p, huge1, 0, 0), huge1, "Unexpected xallocx() failure"); - assert_true(purged, "Unexpected purge"); + assert_true(did_purge, "Unexpected purge"); dallocx(p, 0); } + /* Make sure non-huge allocation succeeds. */ p = mallocx(42, 0); assert_ptr_not_null(p, "Unexpected mallocx() error"); - free(p); - - assert_d_eq(mallctl("arena.0.chunk.alloc", NULL, NULL, &old_alloc, - old_size), 0, "Unexpected alloc error"); - assert_d_eq(mallctl("arena.0.chunk.dalloc", NULL, NULL, &old_dalloc, - old_size), 0, "Unexpected dalloc error"); - assert_d_eq(mallctl("arena.0.chunk.purge", NULL, NULL, &old_purge, - old_size), 0, "Unexpected purge error"); + dallocx(p, 0); + + /* Restore chunk hooks. */ + assert_d_eq(mallctl("arena.0.chunk_hooks", NULL, NULL, &old_hooks, + new_size), 0, "Unexpected chunk_hooks error"); + assert_d_eq(mallctl("arena.0.chunk_hooks", &old_hooks, &old_size, + NULL, 0), 0, "Unexpected chunk_hooks error"); + assert_ptr_eq(old_hooks.alloc, orig_hooks.alloc, + "Unexpected alloc error"); + assert_ptr_eq(old_hooks.dalloc, orig_hooks.dalloc, + "Unexpected dalloc error"); + assert_ptr_eq(old_hooks.commit, orig_hooks.commit, + "Unexpected commit error"); + assert_ptr_eq(old_hooks.decommit, orig_hooks.decommit, + "Unexpected decommit error"); + assert_ptr_eq(old_hooks.purge, orig_hooks.purge, + "Unexpected purge error"); + assert_ptr_eq(old_hooks.split, orig_hooks.split, + "Unexpected split error"); + assert_ptr_eq(old_hooks.merge, orig_hooks.merge, + "Unexpected merge error"); } TEST_END |