summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--.gitignore2
-rw-r--r--ChangeLog26
-rw-r--r--Makefile.in6
-rw-r--r--configure.ac3
-rw-r--r--doc/jemalloc.xml.in119
-rw-r--r--include/jemalloc/internal/arena.h15
-rw-r--r--include/jemalloc/internal/chunk.h8
-rw-r--r--include/jemalloc/internal/chunk_dss.h14
-rw-r--r--include/jemalloc/internal/chunk_mmap.h2
-rw-r--r--include/jemalloc/internal/ctl.h5
-rw-r--r--include/jemalloc/internal/extent.h3
-rw-r--r--include/jemalloc/internal/huge.h2
-rw-r--r--include/jemalloc/internal/jemalloc_internal.h.in123
-rw-r--r--include/jemalloc/internal/private_namespace.h40
-rw-r--r--include/jemalloc/internal/prof.h5
-rw-r--r--include/jemalloc/internal/rtree.h3
-rw-r--r--include/jemalloc/jemalloc.h.in5
-rw-r--r--include/jemalloc/jemalloc_defs.h.in9
-rw-r--r--src/arena.c184
-rw-r--r--src/base.c3
-rw-r--r--src/chunk.c179
-rw-r--r--src/chunk_dss.c37
-rw-r--r--src/chunk_mmap.c12
-rw-r--r--src/ctl.c365
-rw-r--r--src/huge.c7
-rw-r--r--src/jemalloc.c212
-rw-r--r--src/mutex.c2
-rw-r--r--src/prof.c42
-rw-r--r--src/rtree.c21
-rw-r--r--src/stats.c10
-rw-r--r--src/tcache.c4
-rw-r--r--src/util.c5
-rw-r--r--test/ALLOCM_ARENA.c66
-rw-r--r--test/ALLOCM_ARENA.exp2
-rw-r--r--test/thread_arena.c8
35 files changed, 1216 insertions, 333 deletions
diff --git a/.gitignore b/.gitignore
index e6e8bb0..6607a5f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,7 +18,7 @@
/src/*.[od]
/test/*.[od]
/test/*.out
-/test/[a-z]*
+/test/[a-zA-Z_]*
!test/*.c
!test/*.exp
/VERSION
diff --git a/ChangeLog b/ChangeLog
index 231dd6d..0ae4f11 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -6,6 +6,32 @@ found in the git revision history:
http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git
git://canonware.com/jemalloc.git
+* 3.1.0 (October 16, 2012)
+
+ New features:
+ - Auto-detect whether running inside Valgrind, thus removing the need to
+ manually specify MALLOC_CONF=valgrind:true.
+ - Add the "arenas.extend" mallctl, which allows applications to create
+ manually managed arenas.
+ - Add the ALLOCM_ARENA() flag for {,r,d}allocm().
+ - Add the "opt.dss", "arena.<i>.dss", and "stats.arenas.<i>.dss" mallctls,
+ which provide control over dss/mmap precedence.
+ - Add the "arena.<i>.purge" mallctl, which obsoletes "arenas.purge".
+ - Define LG_QUANTUM for hppa.
+
+ Incompatible changes:
+ - Disable tcache by default if running inside Valgrind, in order to avoid
+ making unallocated objects appear reachable to Valgrind.
+ - Drop const from malloc_usable_size() argument on Linux.
+
+ Bug fixes:
+ - Fix heap profiling crash if sampled object is freed via realloc(p, 0).
+ - Remove const from __*_hook variable declarations, so that glibc can modify
+ them during process forking.
+ - Fix mlockall(2)/madvise(2) interaction.
+ - Fix fork(2)-related deadlocks.
+ - Fix error return value for "thread.tcache.enabled" mallctl.
+
* 3.0.0 (May 11, 2012)
Although this version adds some major new features, the primary focus is on
diff --git a/Makefile.in b/Makefile.in
index 6675b59..3644818 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -101,9 +101,9 @@ DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html)
DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3)
DOCS := $(DOCS_HTML) $(DOCS_MAN3)
CTESTS := $(srcroot)test/aligned_alloc.c $(srcroot)test/allocated.c \
- $(srcroot)test/bitmap.c $(srcroot)test/mremap.c \
- $(srcroot)test/posix_memalign.c $(srcroot)test/thread_arena.c \
- $(srcroot)test/thread_tcache_enabled.c
+ $(srcroot)test/ALLOCM_ARENA.c $(srcroot)test/bitmap.c \
+ $(srcroot)test/mremap.c $(srcroot)test/posix_memalign.c \
+ $(srcroot)test/thread_arena.c $(srcroot)test/thread_tcache_enabled.c
ifeq ($(enable_experimental), 1)
CTESTS += $(srcroot)test/allocm.c $(srcroot)test/rallocm.c
endif
diff --git a/configure.ac b/configure.ac
index a72019e..1c52439 100644
--- a/configure.ac
+++ b/configure.ac
@@ -237,6 +237,7 @@ dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the
dnl definitions need to be seen before any headers are included, which is a pain
dnl to make happen otherwise.
default_munmap="1"
+JEMALLOC_USABLE_SIZE_CONST="const"
case "${host}" in
*-*-darwin*)
CFLAGS="$CFLAGS"
@@ -262,6 +263,7 @@ case "${host}" in
abi="elf"
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ])
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ])
+ JEMALLOC_USABLE_SIZE_CONST=""
default_munmap="0"
;;
*-*-netbsd*)
@@ -323,6 +325,7 @@ case "${host}" in
abi="elf"
;;
esac
+AC_DEFINE_UNQUOTED([JEMALLOC_USABLE_SIZE_CONST], [$JEMALLOC_USABLE_SIZE_CONST])
AC_SUBST([abi])
AC_SUBST([RPATH])
AC_SUBST([LD_PRELOAD_VAR])
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 877c500..441c1a4 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -368,6 +368,15 @@ for (i = 0; i < nbins; i++) {
object. This constraint can apply to both growth and
shrinkage.</para></listitem>
</varlistentry>
+ <varlistentry>
+ <term><constant>ALLOCM_ARENA(<parameter>a</parameter>)
+ </constant></term>
+
+ <listitem><para>Use the arena specified by the index
+ <parameter>a</parameter>. This macro does not validate that
+ <parameter>a</parameter> specifies an arena in the valid
+ range.</para></listitem>
+ </varlistentry>
</variablelist>
</para>
@@ -785,15 +794,29 @@ for (i = 0; i < nbins; i++) {
chunk size is 4 MiB (2^22).</para></listitem>
</varlistentry>
+ <varlistentry id="opt.dss">
+ <term>
+ <mallctl>opt.dss</mallctl>
+ (<type>const char *</type>)
+ <literal>r-</literal>
+ </term>
+ <listitem><para>dss (<citerefentry><refentrytitle>sbrk</refentrytitle>
+ <manvolnum>2</manvolnum></citerefentry>) allocation precedence as
+ related to <citerefentry><refentrytitle>mmap</refentrytitle>
+ <manvolnum>2</manvolnum></citerefentry> allocation. The following
+ settings are supported: &ldquo;disabled&rdquo;, &ldquo;primary&rdquo;,
+ and &ldquo;secondary&rdquo; (default).</para></listitem>
+ </varlistentry>
+
<varlistentry id="opt.narenas">
<term>
<mallctl>opt.narenas</mallctl>
(<type>size_t</type>)
<literal>r-</literal>
</term>
- <listitem><para>Maximum number of arenas to use. The default maximum
- number of arenas is four times the number of CPUs, or one if there is a
- single CPU.</para></listitem>
+ <listitem><para>Maximum number of arenas to use for automatic
+ multiplexing of threads and arenas. The default is four times the
+ number of CPUs, or one if there is a single CPU.</para></listitem>
</varlistentry>
<varlistentry id="opt.lg_dirty_mult">
@@ -846,7 +869,9 @@ for (i = 0; i < nbins; i++) {
<literal>0x5a</literal>. This is intended for debugging and will
impact performance negatively. This option is disabled by default
unless <option>--enable-debug</option> is specified during
- configuration, in which case it is enabled by default.</para></listitem>
+ configuration, in which case it is enabled by default unless running
+ inside <ulink
+ url="http://valgrind.org/">Valgrind</ulink>.</para></listitem>
</varlistentry>
<varlistentry id="opt.quarantine">
@@ -865,8 +890,9 @@ for (i = 0; i < nbins; i++) {
enabled. This feature is of particular use in combination with <ulink
url="http://valgrind.org/">Valgrind</ulink>, which can detect attempts
to access quarantined objects. This is intended for debugging and will
- impact performance negatively. The default quarantine size is
- 0.</para></listitem>
+ impact performance negatively. The default quarantine size is 0 unless
+ running inside Valgrind, in which case the default is 16
+ MiB.</para></listitem>
</varlistentry>
<varlistentry id="opt.redzone">
@@ -885,7 +911,7 @@ for (i = 0; i < nbins; i++) {
which needs redzones in order to do effective buffer overflow/underflow
detection. This option is intended for debugging and will impact
performance negatively. This option is disabled by
- default.</para></listitem>
+ default unless running inside Valgrind.</para></listitem>
</varlistentry>
<varlistentry id="opt.zero">
@@ -926,15 +952,9 @@ for (i = 0; i < nbins; i++) {
[<option>--enable-valgrind</option>]
</term>
<listitem><para><ulink url="http://valgrind.org/">Valgrind</ulink>
- support enabled/disabled. If enabled, several other options are
- automatically modified during options processing to work well with
- Valgrind: <link linkend="opt.junk"><mallctl>opt.junk</mallctl></link>
- and <link linkend="opt.zero"><mallctl>opt.zero</mallctl></link> are set
- to false, <link
- linkend="opt.quarantine"><mallctl>opt.quarantine</mallctl></link> is
- set to 16 MiB, and <link
- linkend="opt.redzone"><mallctl>opt.redzone</mallctl></link> is set to
- true. This option is disabled by default.</para></listitem>
+ support enabled/disabled. This option is vestigal because jemalloc
+ auto-detects whether it is running inside Valgrind. This option is
+ disabled by default, unless running inside Valgrind.</para></listitem>
</varlistentry>
<varlistentry id="opt.xmalloc">
@@ -972,7 +992,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<link
linkend="opt.lg_tcache_max"><mallctl>opt.lg_tcache_max</mallctl></link>
option for related tuning information. This option is enabled by
- default.</para></listitem>
+ default unless running inside <ulink
+ url="http://valgrind.org/">Valgrind</ulink>.</para></listitem>
</varlistentry>
<varlistentry id="opt.lg_tcache_max">
@@ -1151,11 +1172,8 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<literal>rw</literal>
</term>
<listitem><para>Get or set the arena associated with the calling
- thread. The arena index must be less than the maximum number of arenas
- (see the <link
- linkend="arenas.narenas"><mallctl>arenas.narenas</mallctl></link>
- mallctl). If the specified arena was not initialized beforehand (see
- the <link
+ thread. If the specified arena was not initialized beforehand (see the
+ <link
linkend="arenas.initialized"><mallctl>arenas.initialized</mallctl></link>
mallctl), it will be automatically initialized as a side effect of
calling this interface.</para></listitem>
@@ -1247,13 +1265,40 @@ malloc_conf = "xmalloc:true";]]></programlisting>
the developer may find manual flushing useful.</para></listitem>
</varlistentry>
+ <varlistentry id="arena.i.purge">
+ <term>
+ <mallctl>arena.&lt;i&gt;.purge</mallctl>
+ (<type>unsigned</type>)
+ <literal>--</literal>
+ </term>
+ <listitem><para>Purge unused dirty pages for arena &lt;i&gt;, or for
+ all arenas if &lt;i&gt; equals <link
+ linkend="arenas.narenas"><mallctl>arenas.narenas</mallctl></link>.
+ </para></listitem>
+ </varlistentry>
+
+ <varlistentry id="arena.i.dss">
+ <term>
+ <mallctl>arena.&lt;i&gt;.dss</mallctl>
+ (<type>const char *</type>)
+ <literal>rw</literal>
+ </term>
+ <listitem><para>Set the precedence of dss allocation as related to mmap
+ allocation for arena &lt;i&gt;, or for all arenas if &lt;i&gt; equals
+ <link
+ linkend="arenas.narenas"><mallctl>arenas.narenas</mallctl></link>. See
+ <link linkend="opt.dss"><mallctl>opt.dss</mallctl></link> for supported
+ settings.
+ </para></listitem>
+ </varlistentry>
+
<varlistentry id="arenas.narenas">
<term>
<mallctl>arenas.narenas</mallctl>
(<type>unsigned</type>)
<literal>r-</literal>
</term>
- <listitem><para>Maximum number of arenas.</para></listitem>
+ <listitem><para>Current limit on number of arenas.</para></listitem>
</varlistentry>
<varlistentry id="arenas.initialized">
@@ -1372,6 +1417,16 @@ malloc_conf = "xmalloc:true";]]></programlisting>
for all arenas if none is specified.</para></listitem>
</varlistentry>
+ <varlistentry>
+ <term>
+ <mallctl>arenas.extend</mallctl>
+ (<type>unsigned</type>)
+ <literal>r-</literal>
+ </term>
+ <listitem><para>Extend the array of arenas by appending a new arena,
+ and returning the new arena index.</para></listitem>
+ </varlistentry>
+
<varlistentry id="prof.active">
<term>
<mallctl>prof.active</mallctl>
@@ -1542,6 +1597,20 @@ malloc_conf = "xmalloc:true";]]></programlisting>
<varlistentry>
<term>
+ <mallctl>stats.arenas.&lt;i&gt;.dss</mallctl>
+ (<type>const char *</type>)
+ <literal>r-</literal>
+ </term>
+ <listitem><para>dss (<citerefentry><refentrytitle>sbrk</refentrytitle>
+ <manvolnum>2</manvolnum></citerefentry>) allocation precedence as
+ related to <citerefentry><refentrytitle>mmap</refentrytitle>
+ <manvolnum>2</manvolnum></citerefentry> allocation. See <link
+ linkend="opt.dss"><mallctl>opt.dss</mallctl></link> for details.
+ </para></listitem>
+ </varlistentry>
+
+ <varlistentry>
+ <term>
<mallctl>stats.arenas.&lt;i&gt;.nthreads</mallctl>
(<type>unsigned</type>)
<literal>r-</literal>
@@ -1865,9 +1934,7 @@ malloc_conf = "xmalloc:true";]]></programlisting>
it detects, because the performance impact for storing such information
would be prohibitive. However, jemalloc does integrate with the most
excellent <ulink url="http://valgrind.org/">Valgrind</ulink> tool if the
- <option>--enable-valgrind</option> configuration option is enabled and the
- <link linkend="opt.valgrind"><mallctl>opt.valgrind</mallctl></link> option
- is enabled.</para>
+ <option>--enable-valgrind</option> configuration option is enabled.</para>
</refsect1>
<refsect1 id="diagnostic_messages">
<title>DIAGNOSTIC MESSAGES</title>
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index 0b0f640..49213e3 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -331,6 +331,8 @@ struct arena_s {
uint64_t prof_accumbytes;
+ dss_prec_t dss_prec;
+
/* List of dirty-page-containing chunks this arena manages. */
ql_head(arena_chunk_t) chunks_dirty;
@@ -422,13 +424,16 @@ void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk,
void *ptr);
void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
- arena_stats_t *astats, malloc_bin_stats_t *bstats,
- malloc_large_stats_t *lstats);
void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero);
-void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero, bool try_tcache);
+void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+ size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
+ bool try_tcache_dalloc);
+dss_prec_t arena_dss_prec_get(arena_t *arena);
+void arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec);
+void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
+ size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
+ malloc_large_stats_t *lstats);
bool arena_new(arena_t *arena, unsigned ind);
void arena_boot(void);
void arena_prefork(arena_t *arena);
diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h
index 8fb1fe6..87d8700 100644
--- a/include/jemalloc/internal/chunk.h
+++ b/include/jemalloc/internal/chunk.h
@@ -28,6 +28,7 @@
#ifdef JEMALLOC_H_EXTERNS
extern size_t opt_lg_chunk;
+extern const char *opt_dss;
/* Protects stats_chunks; currently not used for any other purpose. */
extern malloc_mutex_t chunks_mtx;
@@ -42,9 +43,14 @@ extern size_t chunk_npages;
extern size_t map_bias; /* Number of arena chunk header pages. */
extern size_t arena_maxclass; /* Max size class for arenas. */
-void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero);
+void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
+ dss_prec_t dss_prec);
+void chunk_unmap(void *chunk, size_t size);
void chunk_dealloc(void *chunk, size_t size, bool unmap);
bool chunk_boot(void);
+void chunk_prefork(void);
+void chunk_postfork_parent(void);
+void chunk_postfork_child(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h
index 6e2643b..6585f07 100644
--- a/include/jemalloc/internal/chunk_dss.h
+++ b/include/jemalloc/internal/chunk_dss.h
@@ -1,14 +1,28 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
+typedef enum {
+ dss_prec_disabled = 0,
+ dss_prec_primary = 1,
+ dss_prec_secondary = 2,
+
+ dss_prec_limit = 3
+} dss_prec_t ;
+#define DSS_PREC_DEFAULT dss_prec_secondary
+#define DSS_DEFAULT "secondary"
+
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
+extern const char *dss_prec_names[];
+
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
+dss_prec_t chunk_dss_prec_get(void);
+bool chunk_dss_prec_set(dss_prec_t dss_prec);
void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero);
bool chunk_in_dss(void *chunk);
bool chunk_dss_boot(void);
diff --git a/include/jemalloc/internal/chunk_mmap.h b/include/jemalloc/internal/chunk_mmap.h
index b29f39e..f24abac 100644
--- a/include/jemalloc/internal/chunk_mmap.h
+++ b/include/jemalloc/internal/chunk_mmap.h
@@ -9,7 +9,7 @@
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
-void pages_purge(void *addr, size_t length);
+bool pages_purge(void *addr, size_t length);
void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero);
bool chunk_dealloc_mmap(void *chunk, size_t size);
diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h
index adf3827..0ffecc5 100644
--- a/include/jemalloc/internal/ctl.h
+++ b/include/jemalloc/internal/ctl.h
@@ -33,6 +33,7 @@ struct ctl_indexed_node_s {
struct ctl_arena_stats_s {
bool initialized;
unsigned nthreads;
+ const char *dss;
size_t pactive;
size_t pdirty;
arena_stats_t astats;
@@ -61,6 +62,7 @@ struct ctl_stats_s {
uint64_t nmalloc; /* huge_nmalloc */
uint64_t ndalloc; /* huge_ndalloc */
} huge;
+ unsigned narenas;
ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */
};
@@ -75,6 +77,9 @@ int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp);
int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen);
bool ctl_boot(void);
+void ctl_prefork(void);
+void ctl_postfork_parent(void);
+void ctl_postfork_child(void);
#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \
if (je_mallctl(name, oldp, oldlenp, newp, newlen) \
diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h
index 36af8be..ba95ca8 100644
--- a/include/jemalloc/internal/extent.h
+++ b/include/jemalloc/internal/extent.h
@@ -23,6 +23,9 @@ struct extent_node_s {
/* Total region size. */
size_t size;
+
+ /* True if zero-filled; used by chunk recycling code. */
+ bool zeroed;
};
typedef rb_tree(extent_node_t) extent_tree_t;
diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h
index e8513c9..d987d37 100644
--- a/include/jemalloc/internal/huge.h
+++ b/include/jemalloc/internal/huge.h
@@ -22,7 +22,7 @@ void *huge_palloc(size_t size, size_t alignment, bool zero);
void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
size_t extra);
void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero);
+ size_t alignment, bool zero, bool try_tcache_dalloc);
void huge_dalloc(void *ptr, bool unmap);
size_t huge_salloc(const void *ptr);
prof_ctx_t *huge_prof_ctx_get(const void *ptr);
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 268cd14..475821a 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -270,6 +270,9 @@ static const bool config_ivsalloc =
# ifdef __arm__
# define LG_QUANTUM 3
# endif
+# ifdef __hppa__
+# define LG_QUANTUM 4
+# endif
# ifdef __mips__
# define LG_QUANTUM 3
# endif
@@ -424,6 +427,7 @@ static const bool config_ivsalloc =
VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \
} while (0)
#else
+#define RUNNING_ON_VALGRIND ((unsigned)0)
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
@@ -510,13 +514,19 @@ extern size_t opt_narenas;
/* Number of CPUs. */
extern unsigned ncpus;
-extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */
+/* Protects arenas initialization (arenas, arenas_total). */
+extern malloc_mutex_t arenas_lock;
/*
* Arenas that are used to service external requests. Not all elements of the
* arenas array are necessarily used; arenas are created lazily as needed.
+ *
+ * arenas[0..narenas_auto) are used for automatic multiplexing of threads and
+ * arenas. arenas[narenas_auto..narenas_total) are only used if the application
+ * takes some action to create them and allocate from them.
*/
extern arena_t **arenas;
-extern unsigned narenas;
+extern unsigned narenas_total;
+extern unsigned narenas_auto; /* Read-only after initialization. */
arena_t *arenas_extend(unsigned ind);
void arenas_cleanup(void *arg);
@@ -571,6 +581,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *)
size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
+unsigned narenas_total_get(void);
arena_t *choose_arena(arena_t *arena);
#endif
@@ -675,6 +686,18 @@ sa2u(size_t size, size_t alignment)
}
}
+JEMALLOC_INLINE unsigned
+narenas_total_get(void)
+{
+ unsigned narenas;
+
+ malloc_mutex_lock(&arenas_lock);
+ narenas = narenas_total;
+ malloc_mutex_unlock(&arenas_lock);
+
+ return (narenas);
+}
+
/* Choose an arena based on a per-thread value. */
JEMALLOC_INLINE arena_t *
choose_arena(arena_t *arena)
@@ -710,15 +733,24 @@ choose_arena(arena_t *arena)
#include "jemalloc/internal/quarantine.h"
#ifndef JEMALLOC_ENABLE_INLINE
+void *imallocx(size_t size, bool try_tcache, arena_t *arena);
void *imalloc(size_t size);
+void *icallocx(size_t size, bool try_tcache, arena_t *arena);
void *icalloc(size_t size);
+void *ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+ arena_t *arena);
void *ipalloc(size_t usize, size_t alignment, bool zero);
size_t isalloc(const void *ptr, bool demote);
size_t ivsalloc(const void *ptr, bool demote);
size_t u2rz(size_t usize);
size_t p2rz(const void *ptr);
+void idallocx(void *ptr, bool try_tcache);
void idalloc(void *ptr);
+void iqallocx(void *ptr, bool try_tcache);
void iqalloc(void *ptr);
+void *irallocx(void *ptr, size_t size, size_t extra, size_t alignment,
+ bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc,
+ arena_t *arena);
void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
bool zero, bool no_move);
malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
@@ -726,29 +758,44 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t)
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_INLINE void *
-imalloc(size_t size)
+imallocx(size_t size, bool try_tcache, arena_t *arena)
{
assert(size != 0);
if (size <= arena_maxclass)
- return (arena_malloc(NULL, size, false, true));
+ return (arena_malloc(arena, size, false, try_tcache));
else
return (huge_malloc(size, false));
}
JEMALLOC_INLINE void *
-icalloc(size_t size)
+imalloc(size_t size)
+{
+
+ return (imallocx(size, true, NULL));
+}
+
+JEMALLOC_INLINE void *
+icallocx(size_t size, bool try_tcache, arena_t *arena)
{
if (size <= arena_maxclass)
- return (arena_malloc(NULL, size, true, true));
+ return (arena_malloc(arena, size, true, try_tcache));
else
return (huge_malloc(size, true));
}
JEMALLOC_INLINE void *
-ipalloc(size_t usize, size_t alignment, bool zero)
+icalloc(size_t size)
+{
+
+ return (icallocx(size, true, NULL));
+}
+
+JEMALLOC_INLINE void *
+ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache,
+ arena_t *arena)
{
void *ret;
@@ -756,11 +803,11 @@ ipalloc(size_t usize, size_t alignment, bool zero)
assert(usize == sa2u(usize, alignment));
if (usize <= arena_maxclass && alignment <= PAGE)
- ret = arena_malloc(NULL, usize, zero, true);
+ ret = arena_malloc(arena, usize, zero, try_tcache);
else {
if (usize <= arena_maxclass) {
- ret = arena_palloc(choose_arena(NULL), usize, alignment,
- zero);
+ ret = arena_palloc(choose_arena(arena), usize,
+ alignment, zero);
} else if (alignment <= chunksize)
ret = huge_malloc(usize, zero);
else
@@ -771,6 +818,13 @@ ipalloc(size_t usize, size_t alignment, bool zero)
return (ret);
}
+JEMALLOC_INLINE void *
+ipalloc(size_t usize, size_t alignment, bool zero)
+{
+
+ return (ipallocx(usize, alignment, zero, true, NULL));
+}
+
/*
* Typical usage:
* void *ptr = [...]
@@ -829,7 +883,7 @@ p2rz(const void *ptr)
}
JEMALLOC_INLINE void
-idalloc(void *ptr)
+idallocx(void *ptr, bool try_tcache)
{
arena_chunk_t *chunk;
@@ -837,24 +891,38 @@ idalloc(void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk != ptr)
- arena_dalloc(chunk->arena, chunk, ptr, true);
+ arena_dalloc(chunk->arena, chunk, ptr, try_tcache);
else
huge_dalloc(ptr, true);
}
JEMALLOC_INLINE void
-iqalloc(void *ptr)
+idalloc(void *ptr)
+{
+
+ idallocx(ptr, true);
+}
+
+JEMALLOC_INLINE void
+iqallocx(void *ptr, bool try_tcache)
{
if (config_fill && opt_quarantine)
quarantine(ptr);
else
- idalloc(ptr);
+ idallocx(ptr, try_tcache);
+}
+
+JEMALLOC_INLINE void
+iqalloc(void *ptr)
+{
+
+ iqallocx(ptr, true);
}
JEMALLOC_INLINE void *
-iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
- bool no_move)
+irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+ bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena)
{
void *ret;
size_t oldsize;
@@ -877,7 +945,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
- ret = ipalloc(usize, alignment, zero);
+ ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena);
if (ret == NULL) {
if (extra == 0)
return (NULL);
@@ -885,7 +953,8 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
- ret = ipalloc(usize, alignment, zero);
+ ret = ipallocx(usize, alignment, zero, try_tcache_alloc,
+ arena);
if (ret == NULL)
return (NULL);
}
@@ -896,7 +965,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
*/
copysize = (size < oldsize) ? size : oldsize;
memcpy(ret, ptr, copysize);
- iqalloc(ptr);
+ iqallocx(ptr, try_tcache_dalloc);
return (ret);
}
@@ -910,15 +979,25 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
}
} else {
if (size + extra <= arena_maxclass) {
- return (arena_ralloc(ptr, oldsize, size, extra,
- alignment, zero, true));
+ return (arena_ralloc(arena, ptr, oldsize, size, extra,
+ alignment, zero, try_tcache_alloc,
+ try_tcache_dalloc));
} else {
return (huge_ralloc(ptr, oldsize, size, extra,
- alignment, zero));
+ alignment, zero, try_tcache_dalloc));
}
}
}
+JEMALLOC_INLINE void *
+iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
+ bool no_move)
+{
+
+ return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true,
+ NULL));
+}
+
malloc_tsd_externs(thread_allocated, thread_allocated_t)
malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup)
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index b816647..06241cd 100644
--- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h
@@ -12,6 +12,8 @@
#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
#define arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked)
#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small)
+#define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get)
+#define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set)
#define arena_malloc JEMALLOC_N(arena_malloc)
#define arena_malloc_large JEMALLOC_N(arena_malloc_large)
#define arena_malloc_small JEMALLOC_N(arena_malloc_small)
@@ -51,14 +53,13 @@
#define arena_stats_merge JEMALLOC_N(arena_stats_merge)
#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small)
#define arenas JEMALLOC_N(arenas)
-#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index)
#define arenas_booted JEMALLOC_N(arenas_booted)
#define arenas_cleanup JEMALLOC_N(arenas_cleanup)
#define arenas_extend JEMALLOC_N(arenas_extend)
#define arenas_initialized JEMALLOC_N(arenas_initialized)
#define arenas_lock JEMALLOC_N(arenas_lock)
-#define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index)
#define arenas_tls JEMALLOC_N(arenas_tls)
+#define arenas_tsd JEMALLOC_N(arenas_tsd)
#define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot)
#define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper)
#define arenas_tsd_get JEMALLOC_N(arenas_tsd_get)
@@ -101,9 +102,15 @@
#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child)
#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent)
+#define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get)
+#define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set)
#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork)
#define chunk_in_dss JEMALLOC_N(chunk_in_dss)
#define chunk_npages JEMALLOC_N(chunk_npages)
+#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child)
+#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent)
+#define chunk_prefork JEMALLOC_N(chunk_prefork)
+#define chunk_unmap JEMALLOC_N(chunk_unmap)
#define chunks_mtx JEMALLOC_N(chunks_mtx)
#define chunks_rtree JEMALLOC_N(chunks_rtree)
#define chunksize JEMALLOC_N(chunksize)
@@ -129,6 +136,10 @@
#define ctl_bymib JEMALLOC_N(ctl_bymib)
#define ctl_byname JEMALLOC_N(ctl_byname)
#define ctl_nametomib JEMALLOC_N(ctl_nametomib)
+#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child)
+#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent)
+#define ctl_prefork JEMALLOC_N(ctl_prefork)
+#define dss_prec_names JEMALLOC_N(dss_prec_names)
#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first)
#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert)
#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter)
@@ -161,6 +172,7 @@
#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse)
#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start)
#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search)
+#define get_errno JEMALLOC_N(get_errno)
#define hash JEMALLOC_N(hash)
#define huge_allocated JEMALLOC_N(huge_allocated)
#define huge_boot JEMALLOC_N(huge_boot)
@@ -180,11 +192,17 @@
#define huge_salloc JEMALLOC_N(huge_salloc)
#define iallocm JEMALLOC_N(iallocm)
#define icalloc JEMALLOC_N(icalloc)
+#define icallocx JEMALLOC_N(icallocx)
#define idalloc JEMALLOC_N(idalloc)
+#define idallocx JEMALLOC_N(idallocx)
#define imalloc JEMALLOC_N(imalloc)
+#define imallocx JEMALLOC_N(imallocx)
#define ipalloc JEMALLOC_N(ipalloc)
+#define ipallocx JEMALLOC_N(ipallocx)
#define iqalloc JEMALLOC_N(iqalloc)
+#define iqallocx JEMALLOC_N(iqallocx)
#define iralloc JEMALLOC_N(iralloc)
+#define irallocx JEMALLOC_N(irallocx)
#define isalloc JEMALLOC_N(isalloc)
#define isthreaded JEMALLOC_N(isthreaded)
#define ivsalloc JEMALLOC_N(ivsalloc)
@@ -212,7 +230,9 @@
#define map_bias JEMALLOC_N(map_bias)
#define mb_write JEMALLOC_N(mb_write)
#define mutex_boot JEMALLOC_N(mutex_boot)
-#define narenas JEMALLOC_N(narenas)
+#define narenas_auto JEMALLOC_N(narenas_auto)
+#define narenas_total JEMALLOC_N(narenas_total)
+#define narenas_total_get JEMALLOC_N(narenas_total_get)
#define ncpus JEMALLOC_N(ncpus)
#define nhbins JEMALLOC_N(nhbins)
#define opt_abort JEMALLOC_N(opt_abort)
@@ -254,6 +274,9 @@
#define prof_lookup JEMALLOC_N(prof_lookup)
#define prof_malloc JEMALLOC_N(prof_malloc)
#define prof_mdump JEMALLOC_N(prof_mdump)
+#define prof_postfork_child JEMALLOC_N(prof_postfork_child)
+#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent)
+#define prof_prefork JEMALLOC_N(prof_prefork)
#define prof_promote JEMALLOC_N(prof_promote)
#define prof_realloc JEMALLOC_N(prof_realloc)
#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
@@ -264,6 +287,7 @@
#define prof_tdata_init JEMALLOC_N(prof_tdata_init)
#define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized)
#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
+#define prof_tdata_tsd JEMALLOC_N(prof_tdata_tsd)
#define prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot)
#define prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper)
#define prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get)
@@ -278,12 +302,13 @@
#define rtree_get JEMALLOC_N(rtree_get)
#define rtree_get_locked JEMALLOC_N(rtree_get_locked)
#define rtree_new JEMALLOC_N(rtree_new)
+#define rtree_postfork_child JEMALLOC_N(rtree_postfork_child)
+#define rtree_postfork_parent JEMALLOC_N(rtree_postfork_parent)
+#define rtree_prefork JEMALLOC_N(rtree_prefork)
#define rtree_set JEMALLOC_N(rtree_set)
#define s2u JEMALLOC_N(s2u)
#define sa2u JEMALLOC_N(sa2u)
-#define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index)
-#define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index)
-#define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index)
+#define set_errno JEMALLOC_N(set_errno)
#define stats_cactive JEMALLOC_N(stats_cactive)
#define stats_cactive_add JEMALLOC_N(stats_cactive_add)
#define stats_cactive_get JEMALLOC_N(stats_cactive_get)
@@ -311,6 +336,7 @@
#define tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized)
#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set)
#define tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls)
+#define tcache_enabled_tsd JEMALLOC_N(tcache_enabled_tsd)
#define tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot)
#define tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper)
#define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get)
@@ -325,6 +351,7 @@
#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
#define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup)
#define tcache_tls JEMALLOC_N(tcache_tls)
+#define tcache_tsd JEMALLOC_N(tcache_tsd)
#define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot)
#define tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper)
#define tcache_tsd_get JEMALLOC_N(tcache_tsd_get)
@@ -332,6 +359,7 @@
#define thread_allocated_booted JEMALLOC_N(thread_allocated_booted)
#define thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized)
#define thread_allocated_tls JEMALLOC_N(thread_allocated_tls)
+#define thread_allocated_tsd JEMALLOC_N(thread_allocated_tsd)
#define thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot)
#define thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper)
#define thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get)
diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h
index c3e3f9e..47f22ad 100644
--- a/include/jemalloc/internal/prof.h
+++ b/include/jemalloc/internal/prof.h
@@ -223,6 +223,9 @@ void prof_tdata_cleanup(void *arg);
void prof_boot0(void);
void prof_boot1(void);
bool prof_boot2(void);
+void prof_prefork(void);
+void prof_postfork_parent(void);
+void prof_postfork_child(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
@@ -506,7 +509,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt,
if ((uintptr_t)cnt > (uintptr_t)1U) {
prof_ctx_set(ptr, cnt->ctx);
cnt->epoch++;
- } else
+ } else if (ptr != NULL)
prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U);
/*********/
mb_write();
diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h
index 95d6355..9bd9854 100644
--- a/include/jemalloc/internal/rtree.h
+++ b/include/jemalloc/internal/rtree.h
@@ -36,6 +36,9 @@ struct rtree_s {
#ifdef JEMALLOC_H_EXTERNS
rtree_t *rtree_new(unsigned bits);
+void rtree_prefork(rtree_t *rtree);
+void rtree_postfork_parent(rtree_t *rtree);
+void rtree_postfork_child(rtree_t *rtree);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
diff --git a/include/jemalloc/jemalloc.h.in b/include/jemalloc/jemalloc.h.in
index ad06948..31b1304 100644
--- a/include/jemalloc/jemalloc.h.in
+++ b/include/jemalloc/jemalloc.h.in
@@ -25,6 +25,8 @@ extern "C" {
#endif
#define ALLOCM_ZERO ((int)0x40)
#define ALLOCM_NO_MOVE ((int)0x80)
+/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */
+#define ALLOCM_ARENA(a) ((int)(((a)+1) << 8))
#define ALLOCM_SUCCESS 0
#define ALLOCM_ERR_OOM 1
@@ -59,7 +61,8 @@ JEMALLOC_EXPORT void * je_memalign(size_t alignment, size_t size)
JEMALLOC_EXPORT void * je_valloc(size_t size) JEMALLOC_ATTR(malloc);
#endif
-JEMALLOC_EXPORT size_t je_malloc_usable_size(const void *ptr);
+JEMALLOC_EXPORT size_t je_malloc_usable_size(
+ JEMALLOC_USABLE_SIZE_CONST void *ptr);
JEMALLOC_EXPORT void je_malloc_stats_print(void (*write_cb)(void *,
const char *), void *je_cbopaque, const char *opts);
JEMALLOC_EXPORT int je_mallctl(const char *name, void *oldp,
diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
index c469142..1cd6025 100644
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -222,6 +222,15 @@
#undef JEMALLOC_OVERRIDE_VALLOC
/*
+ * At least Linux omits the "const" in:
+ *
+ * size_t malloc_usable_size(const void *ptr);
+ *
+ * Match the operating system's prototype.
+ */
+#undef JEMALLOC_USABLE_SIZE_CONST
+
+/*
* Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings.
*/
#undef JEMALLOC_ZONE
diff --git a/src/arena.c b/src/arena.c
index 2a6150f..1e6964a 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -372,7 +372,7 @@ arena_chunk_alloc(arena_t *arena)
zero = false;
malloc_mutex_unlock(&arena->lock);
chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize,
- false, &zero);
+ false, &zero, arena->dss_prec);
malloc_mutex_lock(&arena->lock);
if (chunk == NULL)
return (NULL);
@@ -551,24 +551,12 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
{
ql_head(arena_chunk_map_t) mapelms;
arena_chunk_map_t *mapelm;
- size_t pageind, flag_unzeroed;
+ size_t pageind;
size_t ndirty;
size_t nmadvise;
ql_new(&mapelms);
- flag_unzeroed =
-#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
- /*
- * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous
- * mappings, but not for file-backed mappings.
- */
- 0
-#else
- CHUNK_MAP_UNZEROED
-#endif
- ;
-
/*
* If chunk is the spare, temporarily re-allocate it, 1) so that its
* run is reinserted into runs_avail_dirty, and 2) so that it cannot be
@@ -603,26 +591,12 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
assert(arena_mapbits_dirty_get(chunk, pageind) ==
arena_mapbits_dirty_get(chunk, pageind+npages-1));
if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
- size_t i;
-
arena_avail_tree_remove(
&arena->runs_avail_dirty, mapelm);
- arena_mapbits_unzeroed_set(chunk, pageind,
- flag_unzeroed);
arena_mapbits_large_set(chunk, pageind,
(npages << LG_PAGE), 0);
- /*
- * Update internal elements in the page map, so
- * that CHUNK_MAP_UNZEROED is properly set.
- */
- for (i = 1; i < npages - 1; i++) {
- arena_mapbits_unzeroed_set(chunk,
- pageind+i, flag_unzeroed);
- }
if (npages > 1) {
- arena_mapbits_unzeroed_set(chunk,
- pageind+npages-1, flag_unzeroed);
arena_mapbits_large_set(chunk,
pageind+npages-1, 0, 0);
}
@@ -685,14 +659,30 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
sizeof(arena_chunk_map_t)) + map_bias;
size_t npages = arena_mapbits_large_size_get(chunk, pageind) >>
LG_PAGE;
+ bool unzeroed;
+ size_t flag_unzeroed, i;
assert(pageind + npages <= chunk_npages);
assert(ndirty >= npages);
if (config_debug)
ndirty -= npages;
-
- pages_purge((void *)((uintptr_t)chunk + (pageind << LG_PAGE)),
- (npages << LG_PAGE));
+ unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind <<
+ LG_PAGE)), (npages << LG_PAGE));
+ flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0;
+ /*
+ * Set the unzeroed flag for all pages, now that pages_purge()
+ * has returned whether the pages were zeroed as a side effect
+ * of purging. This chunk map modification is safe even though
+ * the arena mutex isn't currently owned by this thread,
+ * because the run is marked as allocated, thus protecting it
+ * from being modified by any other thread. As long as these
+ * writes don't perturb the first and last elements'
+ * CHUNK_MAP_ALLOCATED bits, behavior is well defined.
+ */
+ for (i = 0; i < npages; i++) {
+ arena_mapbits_unzeroed_set(chunk, pageind+i,
+ flag_unzeroed);
+ }
if (config_stats)
nmadvise++;
}
@@ -1629,52 +1619,6 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
mapelm = arena_mapp_get(chunk, pageind);
arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm);
}
-void
-arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
- arena_stats_t *astats, malloc_bin_stats_t *bstats,
- malloc_large_stats_t *lstats)
-{
- unsigned i;
-
- malloc_mutex_lock(&arena->lock);
- *nactive += arena->nactive;
- *ndirty += arena->ndirty;
-
- astats->mapped += arena->stats.mapped;
- astats->npurge += arena->stats.npurge;
- astats->nmadvise += arena->stats.nmadvise;
- astats->purged += arena->stats.purged;
- astats->allocated_large += arena->stats.allocated_large;
- astats->nmalloc_large += arena->stats.nmalloc_large;
- astats->ndalloc_large += arena->stats.ndalloc_large;
- astats->nrequests_large += arena->stats.nrequests_large;
-
- for (i = 0; i < nlclasses; i++) {
- lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
- lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
- lstats[i].nrequests += arena->stats.lstats[i].nrequests;
- lstats[i].curruns += arena->stats.lstats[i].curruns;
- }
- malloc_mutex_unlock(&arena->lock);
-
- for (i = 0; i < NBINS; i++) {
- arena_bin_t *bin = &arena->bins[i];
-
- malloc_mutex_lock(&bin->lock);
- bstats[i].allocated += bin->stats.allocated;
- bstats[i].nmalloc += bin->stats.nmalloc;
- bstats[i].ndalloc += bin->stats.ndalloc;
- bstats[i].nrequests += bin->stats.nrequests;
- if (config_tcache) {
- bstats[i].nfills += bin->stats.nfills;
- bstats[i].nflushes += bin->stats.nflushes;
- }
- bstats[i].nruns += bin->stats.nruns;
- bstats[i].reruns += bin->stats.reruns;
- bstats[i].curruns += bin->stats.curruns;
- malloc_mutex_unlock(&bin->lock);
- }
-}
void
arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
@@ -1887,8 +1831,9 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra,
}
void *
-arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero, bool try_tcache)
+arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size,
+ size_t extra, size_t alignment, bool zero, bool try_tcache_alloc,
+ bool try_tcache_dalloc)
{
void *ret;
size_t copysize;
@@ -1907,9 +1852,9 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t usize = sa2u(size + extra, alignment);
if (usize == 0)
return (NULL);
- ret = ipalloc(usize, alignment, zero);
+ ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena);
} else
- ret = arena_malloc(NULL, size + extra, zero, try_tcache);
+ ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc);
if (ret == NULL) {
if (extra == 0)
@@ -1919,9 +1864,10 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
size_t usize = sa2u(size, alignment);
if (usize == 0)
return (NULL);
- ret = ipalloc(usize, alignment, zero);
+ ret = ipallocx(usize, alignment, zero, try_tcache_alloc,
+ arena);
} else
- ret = arena_malloc(NULL, size, zero, try_tcache);
+ ret = arena_malloc(arena, size, zero, try_tcache_alloc);
if (ret == NULL)
return (NULL);
@@ -1936,10 +1882,78 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
copysize = (size < oldsize) ? size : oldsize;
VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
memcpy(ret, ptr, copysize);
- iqalloc(ptr);
+ iqallocx(ptr, try_tcache_dalloc);
+ return (ret);
+}
+
+dss_prec_t
+arena_dss_prec_get(arena_t *arena)
+{
+ dss_prec_t ret;
+
+ malloc_mutex_lock(&arena->lock);
+ ret = arena->dss_prec;
+ malloc_mutex_unlock(&arena->lock);
return (ret);
}
+void
+arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec)
+{
+
+ malloc_mutex_lock(&arena->lock);
+ arena->dss_prec = dss_prec;
+ malloc_mutex_unlock(&arena->lock);
+}
+
+void
+arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive,
+ size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats,
+ malloc_large_stats_t *lstats)
+{
+ unsigned i;
+
+ malloc_mutex_lock(&arena->lock);
+ *dss = dss_prec_names[arena->dss_prec];
+ *nactive += arena->nactive;
+ *ndirty += arena->ndirty;
+
+ astats->mapped += arena->stats.mapped;
+ astats->npurge += arena->stats.npurge;
+ astats->nmadvise += arena->stats.nmadvise;
+ astats->purged += arena->stats.purged;
+ astats->allocated_large += arena->stats.allocated_large;
+ astats->nmalloc_large += arena->stats.nmalloc_large;
+ astats->ndalloc_large += arena->stats.ndalloc_large;
+ astats->nrequests_large += arena->stats.nrequests_large;
+
+ for (i = 0; i < nlclasses; i++) {
+ lstats[i].nmalloc += arena->stats.lstats[i].nmalloc;
+ lstats[i].ndalloc += arena->stats.lstats[i].ndalloc;
+ lstats[i].nrequests += arena->stats.lstats[i].nrequests;
+ lstats[i].curruns += arena->stats.lstats[i].curruns;
+ }
+ malloc_mutex_unlock(&arena->lock);
+
+ for (i = 0; i < NBINS; i++) {
+ arena_bin_t *bin = &arena->bins[i];
+
+ malloc_mutex_lock(&bin->lock);
+ bstats[i].allocated += bin->stats.allocated;
+ bstats[i].nmalloc += bin->stats.nmalloc;
+ bstats[i].ndalloc += bin->stats.ndalloc;
+ bstats[i].nrequests += bin->stats.nrequests;
+ if (config_tcache) {
+ bstats[i].nfills += bin->stats.nfills;
+ bstats[i].nflushes += bin->stats.nflushes;
+ }
+ bstats[i].nruns += bin->stats.nruns;
+ bstats[i].reruns += bin->stats.reruns;
+ bstats[i].curruns += bin->stats.curruns;
+ malloc_mutex_unlock(&bin->lock);
+ }
+}
+
bool
arena_new(arena_t *arena, unsigned ind)
{
@@ -1968,6 +1982,8 @@ arena_new(arena_t *arena, unsigned ind)
if (config_prof)
arena->prof_accumbytes = 0;
+ arena->dss_prec = chunk_dss_prec_get();
+
/* Initialize chunks. */
ql_new(&arena->chunks_dirty);
arena->spare = NULL;
diff --git a/src/base.c b/src/base.c
index bafaa74..b1a5945 100644
--- a/src/base.c
+++ b/src/base.c
@@ -32,7 +32,8 @@ base_pages_alloc(size_t minsize)
assert(minsize != 0);
csize = CHUNK_CEILING(minsize);
zero = false;
- base_pages = chunk_alloc(csize, chunksize, true, &zero);
+ base_pages = chunk_alloc(csize, chunksize, true, &zero,
+ chunk_dss_prec_get());
if (base_pages == NULL)
return (true);
base_next_addr = base_pages;
diff --git a/src/chunk.c b/src/chunk.c
index 6bc2454..5fc9e75 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -4,7 +4,8 @@
/******************************************************************************/
/* Data. */
-size_t opt_lg_chunk = LG_CHUNK_DEFAULT;
+const char *opt_dss = DSS_DEFAULT;
+size_t opt_lg_chunk = LG_CHUNK_DEFAULT;
malloc_mutex_t chunks_mtx;
chunk_stats_t stats_chunks;
@@ -15,8 +16,10 @@ chunk_stats_t stats_chunks;
* address space. Depending on function, different tree orderings are needed,
* which is why there are two trees with the same contents.
*/
-static extent_tree_t chunks_szad;
-static extent_tree_t chunks_ad;
+static extent_tree_t chunks_szad_mmap;
+static extent_tree_t chunks_ad_mmap;
+static extent_tree_t chunks_szad_dss;
+static extent_tree_t chunks_ad_dss;
rtree_t *chunks_rtree;
@@ -30,19 +33,23 @@ size_t arena_maxclass; /* Max size class for arenas. */
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
-static void *chunk_recycle(size_t size, size_t alignment, bool base,
+static void *chunk_recycle(extent_tree_t *chunks_szad,
+ extent_tree_t *chunks_ad, size_t size, size_t alignment, bool base,
bool *zero);
-static void chunk_record(void *chunk, size_t size);
+static void chunk_record(extent_tree_t *chunks_szad,
+ extent_tree_t *chunks_ad, void *chunk, size_t size);
/******************************************************************************/
static void *
-chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
+chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size,
+ size_t alignment, bool base, bool *zero)
{
void *ret;
extent_node_t *node;
extent_node_t key;
size_t alloc_size, leadsize, trailsize;
+ bool zeroed;
if (base) {
/*
@@ -61,7 +68,7 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
key.addr = NULL;
key.size = alloc_size;
malloc_mutex_lock(&chunks_mtx);
- node = extent_tree_szad_nsearch(&chunks_szad, &key);
+ node = extent_tree_szad_nsearch(chunks_szad, &key);
if (node == NULL) {
malloc_mutex_unlock(&chunks_mtx);
return (NULL);
@@ -72,13 +79,13 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
trailsize = node->size - leadsize - size;
ret = (void *)((uintptr_t)node->addr + leadsize);
/* Remove node from the tree. */
- extent_tree_szad_remove(&chunks_szad, node);
- extent_tree_ad_remove(&chunks_ad, node);
+ extent_tree_szad_remove(chunks_szad, node);
+ extent_tree_ad_remove(chunks_ad, node);
if (leadsize != 0) {
/* Insert the leading space as a smaller chunk. */
node->size = leadsize;
- extent_tree_szad_insert(&chunks_szad, node);
- extent_tree_ad_insert(&chunks_ad, node);
+ extent_tree_szad_insert(chunks_szad, node);
+ extent_tree_ad_insert(chunks_ad, node);
node = NULL;
}
if (trailsize != 0) {
@@ -101,23 +108,24 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
}
node->addr = (void *)((uintptr_t)(ret) + size);
node->size = trailsize;
- extent_tree_szad_insert(&chunks_szad, node);
- extent_tree_ad_insert(&chunks_ad, node);
+ extent_tree_szad_insert(chunks_szad, node);
+ extent_tree_ad_insert(chunks_ad, node);
node = NULL;
}
malloc_mutex_unlock(&chunks_mtx);
- if (node != NULL)
+ zeroed = false;
+ if (node != NULL) {
+ if (node->zeroed) {
+ zeroed = true;
+ *zero = true;
+ }
base_node_dealloc(node);
-#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
- /* Pages are zeroed as a side effect of pages_purge(). */
- *zero = true;
-#else
- if (*zero) {
+ }
+ if (zeroed == false && *zero) {
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
}
-#endif
return (ret);
}
@@ -128,7 +136,8 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
* advantage of them if they are returned.
*/
void *
-chunk_alloc(size_t size, size_t alignment, bool base, bool *zero)
+chunk_alloc(size_t size, size_t alignment, bool base, bool *zero,
+ dss_prec_t dss_prec)
{
void *ret;
@@ -137,19 +146,40 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero)
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
- ret = chunk_recycle(size, alignment, base, zero);
- if (ret != NULL)
- goto label_return;
+ /*
+ * Try to recycle an existing mapping.
+ */
- ret = chunk_alloc_mmap(size, alignment, zero);
- if (ret != NULL)
+ /* "primary" dss. */
+ if (config_dss && dss_prec == dss_prec_primary && (ret =
+ chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment,
+ base, zero)) != NULL)
+ goto label_return;
+ /* mmap. */
+ if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size,
+ alignment, base, zero)) != NULL)
+ goto label_return;
+ /* "secondary" dss. */
+ if (config_dss && dss_prec == dss_prec_secondary && (ret =
+ chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment,
+ base, zero)) != NULL)
goto label_return;
- if (config_dss) {
- ret = chunk_alloc_dss(size, alignment, zero);
- if (ret != NULL)
- goto label_return;
- }
+ /*
+ * Try to allocate a new mapping.
+ */
+
+ /* "primary" dss. */
+ if (config_dss && dss_prec == dss_prec_primary && (ret =
+ chunk_alloc_dss(size, alignment, zero)) != NULL)
+ goto label_return;
+ /* mmap. */
+ if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL)
+ goto label_return;
+ /* "secondary" dss. */
+ if (config_dss && dss_prec == dss_prec_secondary && (ret =
+ chunk_alloc_dss(size, alignment, zero)) != NULL)
+ goto label_return;
/* All strategies for allocation failed. */
ret = NULL;
@@ -189,11 +219,13 @@ label_return:
}
static void
-chunk_record(void *chunk, size_t size)
+chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk,
+ size_t size)
{
+ bool unzeroed;
extent_node_t *xnode, *node, *prev, key;
- pages_purge(chunk, size);
+ unzeroed = pages_purge(chunk, size);
/*
* Allocate a node before acquiring chunks_mtx even though it might not
@@ -205,7 +237,7 @@ chunk_record(void *chunk, size_t size)
malloc_mutex_lock(&chunks_mtx);
key.addr = (void *)((uintptr_t)chunk + size);
- node = extent_tree_ad_nsearch(&chunks_ad, &key);
+ node = extent_tree_ad_nsearch(chunks_ad, &key);
/* Try to coalesce forward. */
if (node != NULL && node->addr == key.addr) {
/*
@@ -213,10 +245,11 @@ chunk_record(void *chunk, size_t size)
* not change the position within chunks_ad, so only
* remove/insert from/into chunks_szad.
*/
- extent_tree_szad_remove(&chunks_szad, node);
+ extent_tree_szad_remove(chunks_szad, node);
node->addr = chunk;
node->size += size;
- extent_tree_szad_insert(&chunks_szad, node);
+ node->zeroed = (node->zeroed && (unzeroed == false));
+ extent_tree_szad_insert(chunks_szad, node);
if (xnode != NULL)
base_node_dealloc(xnode);
} else {
@@ -234,12 +267,13 @@ chunk_record(void *chunk, size_t size)
node = xnode;
node->addr = chunk;
node->size = size;
- extent_tree_ad_insert(&chunks_ad, node);
- extent_tree_szad_insert(&chunks_szad, node);
+ node->zeroed = (unzeroed == false);
+ extent_tree_ad_insert(chunks_ad, node);
+ extent_tree_szad_insert(chunks_szad, node);
}
/* Try to coalesce backward. */
- prev = extent_tree_ad_prev(&chunks_ad, node);
+ prev = extent_tree_ad_prev(chunks_ad, node);
if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
chunk) {
/*
@@ -247,13 +281,14 @@ chunk_record(void *chunk, size_t size)
* not change the position within chunks_ad, so only
* remove/insert node from/into chunks_szad.
*/
- extent_tree_szad_remove(&chunks_szad, prev);
- extent_tree_ad_remove(&chunks_ad, prev);
+ extent_tree_szad_remove(chunks_szad, prev);
+ extent_tree_ad_remove(chunks_ad, prev);
- extent_tree_szad_remove(&chunks_szad, node);
+ extent_tree_szad_remove(chunks_szad, node);
node->addr = prev->addr;
node->size += prev->size;
- extent_tree_szad_insert(&chunks_szad, node);
+ node->zeroed = (node->zeroed && prev->zeroed);
+ extent_tree_szad_insert(chunks_szad, node);
base_node_dealloc(prev);
}
@@ -261,6 +296,20 @@ chunk_record(void *chunk, size_t size)
}
void
+chunk_unmap(void *chunk, size_t size)
+{
+ assert(chunk != NULL);
+ assert(CHUNK_ADDR2BASE(chunk) == chunk);
+ assert(size != 0);
+ assert((size & chunksize_mask) == 0);
+
+ if (config_dss && chunk_in_dss(chunk))
+ chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size);
+ else if (chunk_dealloc_mmap(chunk, size))
+ chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size);
+}
+
+void
chunk_dealloc(void *chunk, size_t size, bool unmap)
{
@@ -273,15 +322,13 @@ chunk_dealloc(void *chunk, size_t size, bool unmap)
rtree_set(chunks_rtree, (uintptr_t)chunk, NULL);
if (config_stats || config_prof) {
malloc_mutex_lock(&chunks_mtx);
+ assert(stats_chunks.curchunks >= (size / chunksize));
stats_chunks.curchunks -= (size / chunksize);
malloc_mutex_unlock(&chunks_mtx);
}
- if (unmap) {
- if ((config_dss && chunk_in_dss(chunk)) ||
- chunk_dealloc_mmap(chunk, size))
- chunk_record(chunk, size);
- }
+ if (unmap)
+ chunk_unmap(chunk, size);
}
bool
@@ -301,8 +348,10 @@ chunk_boot(void)
}
if (config_dss && chunk_dss_boot())
return (true);
- extent_tree_szad_new(&chunks_szad);
- extent_tree_ad_new(&chunks_ad);
+ extent_tree_szad_new(&chunks_szad_mmap);
+ extent_tree_ad_new(&chunks_ad_mmap);
+ extent_tree_szad_new(&chunks_szad_dss);
+ extent_tree_ad_new(&chunks_ad_dss);
if (config_ivsalloc) {
chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) -
opt_lg_chunk);
@@ -312,3 +361,33 @@ chunk_boot(void)
return (false);
}
+
+void
+chunk_prefork(void)
+{
+
+ malloc_mutex_lock(&chunks_mtx);
+ if (config_ivsalloc)
+ rtree_prefork(chunks_rtree);
+ chunk_dss_prefork();
+}
+
+void
+chunk_postfork_parent(void)
+{
+
+ chunk_dss_postfork_parent();
+ if (config_ivsalloc)
+ rtree_postfork_parent(chunks_rtree);
+ malloc_mutex_postfork_parent(&chunks_mtx);
+}
+
+void
+chunk_postfork_child(void)
+{
+
+ chunk_dss_postfork_child();
+ if (config_ivsalloc)
+ rtree_postfork_child(chunks_rtree);
+ malloc_mutex_postfork_child(&chunks_mtx);
+}
diff --git a/src/chunk_dss.c b/src/chunk_dss.c
index 2d68e48..24781cc 100644
--- a/src/chunk_dss.c
+++ b/src/chunk_dss.c
@@ -3,6 +3,16 @@
/******************************************************************************/
/* Data. */
+const char *dss_prec_names[] = {
+ "disabled",
+ "primary",
+ "secondary",
+ "N/A"
+};
+
+/* Current dss precedence default, used when creating new arenas. */
+static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT;
+
/*
* Protects sbrk() calls. This avoids malloc races among threads, though it
* does not protect against races with threads that call sbrk() directly.
@@ -29,6 +39,31 @@ sbrk(intptr_t increment)
}
#endif
+dss_prec_t
+chunk_dss_prec_get(void)
+{
+ dss_prec_t ret;
+
+ if (config_dss == false)
+ return (dss_prec_disabled);
+ malloc_mutex_lock(&dss_mtx);
+ ret = dss_prec_default;
+ malloc_mutex_unlock(&dss_mtx);
+ return (ret);
+}
+
+bool
+chunk_dss_prec_set(dss_prec_t dss_prec)
+{
+
+ if (config_dss == false)
+ return (true);
+ malloc_mutex_lock(&dss_mtx);
+ dss_prec_default = dss_prec;
+ malloc_mutex_unlock(&dss_mtx);
+ return (false);
+}
+
void *
chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
{
@@ -88,7 +123,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero)
dss_max = dss_next;
malloc_mutex_unlock(&dss_mtx);
if (cpad_size != 0)
- chunk_dealloc(cpad, cpad_size, true);
+ chunk_unmap(cpad, cpad_size);
if (*zero) {
VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
memset(ret, 0, size);
diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c
index c8da655..8a42e75 100644
--- a/src/chunk_mmap.c
+++ b/src/chunk_mmap.c
@@ -113,22 +113,30 @@ pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
#endif
}
-void
+bool
pages_purge(void *addr, size_t length)
{
+ bool unzeroed;
#ifdef _WIN32
VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
+ unzeroed = true;
#else
# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
# define JEMALLOC_MADV_PURGE MADV_DONTNEED
+# define JEMALLOC_MADV_ZEROS true
# elif defined(JEMALLOC_PURGE_MADVISE_FREE)
# define JEMALLOC_MADV_PURGE MADV_FREE
+# define JEMALLOC_MADV_ZEROS false
# else
# error "No method defined for purging unused dirty pages."
# endif
- madvise(addr, length, JEMALLOC_MADV_PURGE);
+ int err = madvise(addr, length, JEMALLOC_MADV_PURGE);
+ unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0);
+# undef JEMALLOC_MADV_PURGE
+# undef JEMALLOC_MADV_ZEROS
#endif
+ return (unzeroed);
}
static void *
diff --git a/src/ctl.c b/src/ctl.c
index 55e7667..e3ab9da 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -48,8 +48,8 @@ static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen);
#define INDEX_PROTO(n) \
-const ctl_named_node_t *n##_index(const size_t *mib, size_t miblen, \
- size_t i);
+static const ctl_named_node_t *n##_index(const size_t *mib, \
+ size_t miblen, size_t i);
static bool ctl_arena_init(ctl_arena_stats_t *astats);
static void ctl_arena_clear(ctl_arena_stats_t *astats);
@@ -58,6 +58,7 @@ static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats,
static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats,
ctl_arena_stats_t *astats);
static void ctl_arena_refresh(arena_t *arena, unsigned i);
+static bool ctl_grow(void);
static void ctl_refresh(void);
static bool ctl_init(void);
static int ctl_lookup(const char *name, ctl_node_t const **nodesp,
@@ -88,6 +89,7 @@ CTL_PROTO(config_utrace)
CTL_PROTO(config_valgrind)
CTL_PROTO(config_xmalloc)
CTL_PROTO(opt_abort)
+CTL_PROTO(opt_dss)
CTL_PROTO(opt_lg_chunk)
CTL_PROTO(opt_narenas)
CTL_PROTO(opt_lg_dirty_mult)
@@ -110,6 +112,10 @@ CTL_PROTO(opt_prof_gdump)
CTL_PROTO(opt_prof_final)
CTL_PROTO(opt_prof_leak)
CTL_PROTO(opt_prof_accum)
+CTL_PROTO(arena_i_purge)
+static int arena_purge(unsigned arena_ind);
+CTL_PROTO(arena_i_dss)
+INDEX_PROTO(arena_i)
CTL_PROTO(arenas_bin_i_size)
CTL_PROTO(arenas_bin_i_nregs)
CTL_PROTO(arenas_bin_i_run_size)
@@ -125,6 +131,7 @@ CTL_PROTO(arenas_nbins)
CTL_PROTO(arenas_nhbins)
CTL_PROTO(arenas_nlruns)
CTL_PROTO(arenas_purge)
+CTL_PROTO(arenas_extend)
CTL_PROTO(prof_active)
CTL_PROTO(prof_dump)
CTL_PROTO(prof_interval)
@@ -158,6 +165,7 @@ CTL_PROTO(stats_arenas_i_lruns_j_nrequests)
CTL_PROTO(stats_arenas_i_lruns_j_curruns)
INDEX_PROTO(stats_arenas_i_lruns_j)
CTL_PROTO(stats_arenas_i_nthreads)
+CTL_PROTO(stats_arenas_i_dss)
CTL_PROTO(stats_arenas_i_pactive)
CTL_PROTO(stats_arenas_i_pdirty)
CTL_PROTO(stats_arenas_i_mapped)
@@ -223,6 +231,7 @@ static const ctl_named_node_t config_node[] = {
static const ctl_named_node_t opt_node[] = {
{NAME("abort"), CTL(opt_abort)},
+ {NAME("dss"), CTL(opt_dss)},
{NAME("lg_chunk"), CTL(opt_lg_chunk)},
{NAME("narenas"), CTL(opt_narenas)},
{NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)},
@@ -247,6 +256,18 @@ static const ctl_named_node_t opt_node[] = {
{NAME("prof_accum"), CTL(opt_prof_accum)}
};
+static const ctl_named_node_t arena_i_node[] = {
+ {NAME("purge"), CTL(arena_i_purge)},
+ {NAME("dss"), CTL(arena_i_dss)}
+};
+static const ctl_named_node_t super_arena_i_node[] = {
+ {NAME(""), CHILD(named, arena_i)}
+};
+
+static const ctl_indexed_node_t arena_node[] = {
+ {INDEX(arena_i)}
+};
+
static const ctl_named_node_t arenas_bin_i_node[] = {
{NAME("size"), CTL(arenas_bin_i_size)},
{NAME("nregs"), CTL(arenas_bin_i_nregs)},
@@ -282,7 +303,8 @@ static const ctl_named_node_t arenas_node[] = {
{NAME("bin"), CHILD(indexed, arenas_bin)},
{NAME("nlruns"), CTL(arenas_nlruns)},
{NAME("lrun"), CHILD(indexed, arenas_lrun)},
- {NAME("purge"), CTL(arenas_purge)}
+ {NAME("purge"), CTL(arenas_purge)},
+ {NAME("extend"), CTL(arenas_extend)}
};
static const ctl_named_node_t prof_node[] = {
@@ -352,6 +374,7 @@ static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = {
static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
+ {NAME("dss"), CTL(stats_arenas_i_dss)},
{NAME("pactive"), CTL(stats_arenas_i_pactive)},
{NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
{NAME("mapped"), CTL(stats_arenas_i_mapped)},
@@ -387,6 +410,7 @@ static const ctl_named_node_t root_node[] = {
{NAME("thread"), CHILD(named, thread)},
{NAME("config"), CHILD(named, config)},
{NAME("opt"), CHILD(named, opt)},
+ {NAME("arena"), CHILD(indexed, arena)},
{NAME("arenas"), CHILD(named, arenas)},
{NAME("prof"), CHILD(named, prof)},
{NAME("stats"), CHILD(named, stats)}
@@ -420,6 +444,7 @@ static void
ctl_arena_clear(ctl_arena_stats_t *astats)
{
+ astats->dss = dss_prec_names[dss_prec_limit];
astats->pactive = 0;
astats->pdirty = 0;
if (config_stats) {
@@ -439,8 +464,8 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena)
{
unsigned i;
- arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty,
- &cstats->astats, cstats->bstats, cstats->lstats);
+ arena_stats_merge(arena, &cstats->dss, &cstats->pactive,
+ &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats);
for (i = 0; i < NBINS; i++) {
cstats->allocated_small += cstats->bstats[i].allocated;
@@ -500,7 +525,7 @@ static void
ctl_arena_refresh(arena_t *arena, unsigned i)
{
ctl_arena_stats_t *astats = &ctl_stats.arenas[i];
- ctl_arena_stats_t *sstats = &ctl_stats.arenas[narenas];
+ ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas];
ctl_arena_clear(astats);
@@ -518,11 +543,72 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
}
}
+static bool
+ctl_grow(void)
+{
+ size_t astats_size;
+ ctl_arena_stats_t *astats;
+ arena_t **tarenas;
+
+ /* Extend arena stats and arenas arrays. */
+ astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t);
+ if (ctl_stats.narenas == narenas_auto) {
+ /* ctl_stats.arenas and arenas came from base_alloc(). */
+ astats = (ctl_arena_stats_t *)imalloc(astats_size);
+ if (astats == NULL)
+ return (true);
+ memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
+ sizeof(ctl_arena_stats_t));
+
+ tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
+ sizeof(arena_t *));
+ if (tarenas == NULL) {
+ idalloc(astats);
+ return (true);
+ }
+ memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *));
+ } else {
+ astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas,
+ astats_size, 0, 0, false, false);
+ if (astats == NULL)
+ return (true);
+
+ tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) *
+ sizeof(arena_t *), 0, 0, false, false);
+ if (tarenas == NULL)
+ return (true);
+ }
+ /* Initialize the new astats and arenas elements. */
+ memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
+ if (ctl_arena_init(&astats[ctl_stats.narenas + 1]))
+ return (true);
+ tarenas[ctl_stats.narenas] = NULL;
+ /* Swap merged stats to their new location. */
+ {
+ ctl_arena_stats_t tstats;
+ memcpy(&tstats, &astats[ctl_stats.narenas],
+ sizeof(ctl_arena_stats_t));
+ memcpy(&astats[ctl_stats.narenas],
+ &astats[ctl_stats.narenas + 1], sizeof(ctl_arena_stats_t));
+ memcpy(&astats[ctl_stats.narenas + 1], &tstats,
+ sizeof(ctl_arena_stats_t));
+ }
+ ctl_stats.arenas = astats;
+ ctl_stats.narenas++;
+ malloc_mutex_lock(&arenas_lock);
+ arenas = tarenas;
+ narenas_total++;
+ arenas_extend(narenas_total - 1);
+ malloc_mutex_unlock(&arenas_lock);
+
+ return (false);
+}
+
static void
ctl_refresh(void)
{
unsigned i;
- VARIABLE_ARRAY(arena_t *, tarenas, narenas);
+ VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
if (config_stats) {
malloc_mutex_lock(&chunks_mtx);
@@ -542,19 +628,19 @@ ctl_refresh(void)
* Clear sum stats, since they will be merged into by
* ctl_arena_refresh().
*/
- ctl_stats.arenas[narenas].nthreads = 0;
- ctl_arena_clear(&ctl_stats.arenas[narenas]);
+ ctl_stats.arenas[ctl_stats.narenas].nthreads = 0;
+ ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]);
malloc_mutex_lock(&arenas_lock);
- memcpy(tarenas, arenas, sizeof(arena_t *) * narenas);
- for (i = 0; i < narenas; i++) {
+ memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas);
+ for (i = 0; i < ctl_stats.narenas; i++) {
if (arenas[i] != NULL)
ctl_stats.arenas[i].nthreads = arenas[i]->nthreads;
else
ctl_stats.arenas[i].nthreads = 0;
}
malloc_mutex_unlock(&arenas_lock);
- for (i = 0; i < narenas; i++) {
+ for (i = 0; i < ctl_stats.narenas; i++) {
bool initialized = (tarenas[i] != NULL);
ctl_stats.arenas[i].initialized = initialized;
@@ -563,11 +649,13 @@ ctl_refresh(void)
}
if (config_stats) {
- ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small
- + ctl_stats.arenas[narenas].astats.allocated_large
+ ctl_stats.allocated =
+ ctl_stats.arenas[ctl_stats.narenas].allocated_small
+ + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large
+ + ctl_stats.huge.allocated;
+ ctl_stats.active =
+ (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE)
+ ctl_stats.huge.allocated;
- ctl_stats.active = (ctl_stats.arenas[narenas].pactive <<
- LG_PAGE) + ctl_stats.huge.allocated;
ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk);
}
@@ -585,13 +673,15 @@ ctl_init(void)
* Allocate space for one extra arena stats element, which
* contains summed stats across all arenas.
*/
+ assert(narenas_auto == narenas_total_get());
+ ctl_stats.narenas = narenas_auto;
ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc(
- (narenas + 1) * sizeof(ctl_arena_stats_t));
+ (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t));
if (ctl_stats.arenas == NULL) {
ret = true;
goto label_return;
}
- memset(ctl_stats.arenas, 0, (narenas + 1) *
+ memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) *
sizeof(ctl_arena_stats_t));
/*
@@ -601,14 +691,14 @@ ctl_init(void)
*/
if (config_stats) {
unsigned i;
- for (i = 0; i <= narenas; i++) {
+ for (i = 0; i <= ctl_stats.narenas; i++) {
if (ctl_arena_init(&ctl_stats.arenas[i])) {
ret = true;
goto label_return;
}
}
}
- ctl_stats.arenas[narenas].initialized = true;
+ ctl_stats.arenas[ctl_stats.narenas].initialized = true;
ctl_epoch = 0;
ctl_refresh();
@@ -827,6 +917,27 @@ ctl_boot(void)
return (false);
}
+void
+ctl_prefork(void)
+{
+
+ malloc_mutex_lock(&ctl_mtx);
+}
+
+void
+ctl_postfork_parent(void)
+{
+
+ malloc_mutex_postfork_parent(&ctl_mtx);
+}
+
+void
+ctl_postfork_child(void)
+{
+
+ malloc_mutex_postfork_child(&ctl_mtx);
+}
+
/******************************************************************************/
/* *_ctl() functions. */
@@ -1032,8 +1143,8 @@ thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp,
}
READ(oldval, bool);
-label_return:
ret = 0;
+label_return:
return (ret);
}
@@ -1063,13 +1174,14 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
int ret;
unsigned newind, oldind;
+ malloc_mutex_lock(&ctl_mtx);
newind = oldind = choose_arena(NULL)->ind;
WRITE(newind, unsigned);
READ(oldind, unsigned);
if (newind != oldind) {
arena_t *arena;
- if (newind >= narenas) {
+ if (newind >= ctl_stats.narenas) {
/* New arena index is out of range. */
ret = EFAULT;
goto label_return;
@@ -1102,6 +1214,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = 0;
label_return:
+ malloc_mutex_unlock(&ctl_mtx);
return (ret);
}
@@ -1135,6 +1248,7 @@ CTL_RO_BOOL_CONFIG_GEN(config_xmalloc)
/******************************************************************************/
CTL_RO_NL_GEN(opt_abort, opt_abort, bool)
+CTL_RO_NL_GEN(opt_dss, opt_dss, const char *)
CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t)
CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t)
CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t)
@@ -1160,10 +1274,126 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
/******************************************************************************/
+static int
+arena_purge(unsigned arena_ind)
+{
+ int ret;
+
+ malloc_mutex_lock(&ctl_mtx);
+ {
+ VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas);
+
+ malloc_mutex_lock(&arenas_lock);
+ memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas);
+ malloc_mutex_unlock(&arenas_lock);
+
+ if (arena_ind == ctl_stats.narenas) {
+ unsigned i;
+ for (i = 0; i < ctl_stats.narenas; i++) {
+ if (tarenas[i] != NULL)
+ arena_purge_all(tarenas[i]);
+ }
+ } else {
+ assert(arena_ind < ctl_stats.narenas);
+ if (tarenas[arena_ind] != NULL)
+ arena_purge_all(tarenas[arena_ind]);
+ }
+ }
+
+ ret = 0;
+ malloc_mutex_unlock(&ctl_mtx);
+ return (ret);
+}
+
+static int
+arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
+ void *newp, size_t newlen)
+{
+ int ret;
+
+ READONLY();
+ WRITEONLY();
+ ret = arena_purge(mib[1]);
+
+label_return:
+ return (ret);
+}
+
+static int
+arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
+ void *newp, size_t newlen)
+{
+ int ret, i;
+ bool match, err;
+ const char *dss;
+ unsigned arena_ind = mib[1];
+ dss_prec_t dss_prec_old = dss_prec_limit;
+ dss_prec_t dss_prec = dss_prec_limit;
+
+ malloc_mutex_lock(&ctl_mtx);
+ WRITE(dss, const char *);
+ match = false;
+ for (i = 0; i < dss_prec_limit; i++) {
+ if (strcmp(dss_prec_names[i], dss) == 0) {
+ dss_prec = i;
+ match = true;
+ break;
+ }
+ }
+ if (match == false) {
+ ret = EINVAL;
+ goto label_return;
+ }
+
+ if (arena_ind < ctl_stats.narenas) {
+ arena_t *arena = arenas[arena_ind];
+ if (arena != NULL) {
+ dss_prec_old = arena_dss_prec_get(arena);
+ arena_dss_prec_set(arena, dss_prec);
+ err = false;
+ } else
+ err = true;
+ } else {
+ dss_prec_old = chunk_dss_prec_get();
+ err = chunk_dss_prec_set(dss_prec);
+ }
+ dss = dss_prec_names[dss_prec_old];
+ READ(dss, const char *);
+ if (err) {
+ ret = EFAULT;
+ goto label_return;
+ }
+
+ ret = 0;
+label_return:
+ malloc_mutex_unlock(&ctl_mtx);
+ return (ret);
+}
+
+static const ctl_named_node_t *
+arena_i_index(const size_t *mib, size_t miblen, size_t i)
+{
+ const ctl_named_node_t * ret;
+
+ malloc_mutex_lock(&ctl_mtx);
+ if (i > ctl_stats.narenas) {
+ ret = NULL;
+ goto label_return;
+ }
+
+ ret = super_arena_i_node;
+label_return:
+ malloc_mutex_unlock(&ctl_mtx);
+ return (ret);
+}
+
+
+/******************************************************************************/
+
CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t)
-const ctl_named_node_t *
+static const ctl_named_node_t *
arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
{
@@ -1173,7 +1403,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
}
CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t)
-const ctl_named_node_t *
+static const ctl_named_node_t *
arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
{
@@ -1182,7 +1412,27 @@ arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
return (super_arenas_lrun_i_node);
}
-CTL_RO_NL_GEN(arenas_narenas, narenas, unsigned)
+static int
+arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp,
+ size_t *oldlenp, void *newp, size_t newlen)
+{
+ int ret;
+ unsigned narenas;
+
+ malloc_mutex_lock(&ctl_mtx);
+ READONLY();
+ if (*oldlenp != sizeof(unsigned)) {
+ ret = EINVAL;
+ goto label_return;
+ }
+ narenas = ctl_stats.narenas;
+ READ(narenas, unsigned);
+
+ ret = 0;
+label_return:
+ malloc_mutex_unlock(&ctl_mtx);
+ return (ret);
+}
static int
arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp,
@@ -1193,13 +1443,13 @@ arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp,
malloc_mutex_lock(&ctl_mtx);
READONLY();
- if (*oldlenp != narenas * sizeof(bool)) {
+ if (*oldlenp != ctl_stats.narenas * sizeof(bool)) {
ret = EINVAL;
- nread = (*oldlenp < narenas * sizeof(bool))
- ? (*oldlenp / sizeof(bool)) : narenas;
+ nread = (*oldlenp < ctl_stats.narenas * sizeof(bool))
+ ? (*oldlenp / sizeof(bool)) : ctl_stats.narenas;
} else {
ret = 0;
- nread = narenas;
+ nread = ctl_stats.narenas;
}
for (i = 0; i < nread; i++)
@@ -1222,36 +1472,42 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen)
{
int ret;
- unsigned arena;
+ unsigned arena_ind;
+ malloc_mutex_lock(&ctl_mtx);
WRITEONLY();
- arena = UINT_MAX;
- WRITE(arena, unsigned);
- if (newp != NULL && arena >= narenas) {
+ arena_ind = UINT_MAX;
+ WRITE(arena_ind, unsigned);
+ if (newp != NULL && arena_ind >= ctl_stats.narenas)
ret = EFAULT;
- goto label_return;
- } else {
- VARIABLE_ARRAY(arena_t *, tarenas, narenas);
+ else {
+ if (arena_ind == UINT_MAX)
+ arena_ind = ctl_stats.narenas;
+ ret = arena_purge(arena_ind);
+ }
- malloc_mutex_lock(&arenas_lock);
- memcpy(tarenas, arenas, sizeof(arena_t *) * narenas);
- malloc_mutex_unlock(&arenas_lock);
+label_return:
+ malloc_mutex_unlock(&ctl_mtx);
+ return (ret);
+}
- if (arena == UINT_MAX) {
- unsigned i;
- for (i = 0; i < narenas; i++) {
- if (tarenas[i] != NULL)
- arena_purge_all(tarenas[i]);
- }
- } else {
- assert(arena < narenas);
- if (tarenas[arena] != NULL)
- arena_purge_all(tarenas[arena]);
- }
+static int
+arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
+ void *newp, size_t newlen)
+{
+ int ret;
+
+ malloc_mutex_lock(&ctl_mtx);
+ READONLY();
+ if (ctl_grow()) {
+ ret = EAGAIN;
+ goto label_return;
}
+ READ(ctl_stats.narenas - 1, unsigned);
ret = 0;
label_return:
+ malloc_mutex_unlock(&ctl_mtx);
return (ret);
}
@@ -1356,7 +1612,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns,
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns,
ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t)
-const ctl_named_node_t *
+static const ctl_named_node_t *
stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j)
{
@@ -1374,7 +1630,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests,
CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns,
ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t)
-const ctl_named_node_t *
+static const ctl_named_node_t *
stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j)
{
@@ -1384,6 +1640,7 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j)
}
CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned)
+CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *)
CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t)
CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_mapped,
@@ -1395,13 +1652,13 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise,
CTL_RO_CGEN(config_stats, stats_arenas_i_purged,
ctl_stats.arenas[mib[2]].astats.purged, uint64_t)
-const ctl_named_node_t *
+static const ctl_named_node_t *
stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i)
{
const ctl_named_node_t * ret;
malloc_mutex_lock(&ctl_mtx);
- if (ctl_stats.arenas[i].initialized == false) {
+ if (i > ctl_stats.narenas || ctl_stats.arenas[i].initialized == false) {
ret = NULL;
goto label_return;
}
diff --git a/src/huge.c b/src/huge.c
index 8a4ec94..aa08d43 100644
--- a/src/huge.c
+++ b/src/huge.c
@@ -48,7 +48,8 @@ huge_palloc(size_t size, size_t alignment, bool zero)
* it is possible to make correct junk/zero fill decisions below.
*/
is_zeroed = zero;
- ret = chunk_alloc(csize, alignment, false, &is_zeroed);
+ ret = chunk_alloc(csize, alignment, false, &is_zeroed,
+ chunk_dss_prec_get());
if (ret == NULL) {
base_node_dealloc(node);
return (NULL);
@@ -101,7 +102,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra)
void *
huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero)
+ size_t alignment, bool zero, bool try_tcache_dalloc)
{
void *ret;
size_t copysize;
@@ -180,7 +181,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
#endif
{
memcpy(ret, ptr, copysize);
- iqalloc(ptr);
+ iqallocx(ptr, try_tcache_dalloc);
}
return (ret);
}
diff --git a/src/jemalloc.c b/src/jemalloc.c
index bc54cd7..8a667b6 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -33,7 +33,8 @@ unsigned ncpus;
malloc_mutex_t arenas_lock;
arena_t **arenas;
-unsigned narenas;
+unsigned narenas_total;
+unsigned narenas_auto;
/* Set to true once the allocator has been initialized. */
static bool malloc_initialized = false;
@@ -144,14 +145,14 @@ choose_arena_hard(void)
{
arena_t *ret;
- if (narenas > 1) {
+ if (narenas_auto > 1) {
unsigned i, choose, first_null;
choose = 0;
- first_null = narenas;
+ first_null = narenas_auto;
malloc_mutex_lock(&arenas_lock);
assert(arenas[0] != NULL);
- for (i = 1; i < narenas; i++) {
+ for (i = 1; i < narenas_auto; i++) {
if (arenas[i] != NULL) {
/*
* Choose the first arena that has the lowest
@@ -160,7 +161,7 @@ choose_arena_hard(void)
if (arenas[i]->nthreads <
arenas[choose]->nthreads)
choose = i;
- } else if (first_null == narenas) {
+ } else if (first_null == narenas_auto) {
/*
* Record the index of the first uninitialized
* arena, in case all extant arenas are in use.
@@ -174,7 +175,8 @@ choose_arena_hard(void)
}
}
- if (arenas[choose]->nthreads == 0 || first_null == narenas) {
+ if (arenas[choose]->nthreads == 0
+ || first_null == narenas_auto) {
/*
* Use an unloaded arena, or the least loaded arena if
* all arenas are already initialized.
@@ -203,7 +205,7 @@ stats_print_atexit(void)
{
if (config_tcache && config_stats) {
- unsigned i;
+ unsigned narenas, i;
/*
* Merge stats from extant threads. This is racy, since
@@ -212,7 +214,7 @@ stats_print_atexit(void)
* out of date by the time they are reported, if other threads
* continue to allocate.
*/
- for (i = 0; i < narenas; i++) {
+ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena = arenas[i];
if (arena != NULL) {
tcache_t *tcache;
@@ -254,12 +256,13 @@ malloc_ncpus(void)
result = si.dwNumberOfProcessors;
#else
result = sysconf(_SC_NPROCESSORS_ONLN);
+#endif
if (result == -1) {
/* Error. */
ret = 1;
- }
-#endif
- ret = (unsigned)result;
+ } else {
+ ret = (unsigned)result;
+ }
return (ret);
}
@@ -377,6 +380,22 @@ malloc_conf_init(void)
const char *opts, *k, *v;
size_t klen, vlen;
+ /*
+ * Automatically configure valgrind before processing options. The
+ * valgrind option remains in jemalloc 3.x for compatibility reasons.
+ */
+ if (config_valgrind) {
+ opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false;
+ if (config_fill && opt_valgrind) {
+ opt_junk = false;
+ assert(opt_zero == false);
+ opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
+ opt_redzone = true;
+ }
+ if (config_tcache && opt_valgrind)
+ opt_tcache = false;
+ }
+
for (i = 0; i < 3; i++) {
/* Get runtime configuration. */
switch (i) {
@@ -537,6 +556,30 @@ malloc_conf_init(void)
*/
CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE +
(config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1)
+ if (strncmp("dss", k, klen) == 0) {
+ int i;
+ bool match = false;
+ for (i = 0; i < dss_prec_limit; i++) {
+ if (strncmp(dss_prec_names[i], v, vlen)
+ == 0) {
+ if (chunk_dss_prec_set(i)) {
+ malloc_conf_error(
+ "Error setting dss",
+ k, klen, v, vlen);
+ } else {
+ opt_dss =
+ dss_prec_names[i];
+ match = true;
+ break;
+ }
+ }
+ }
+ if (match == false) {
+ malloc_conf_error("Invalid conf value",
+ k, klen, v, vlen);
+ }
+ continue;
+ }
CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1,
SIZE_T_MAX)
CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult",
@@ -553,20 +596,7 @@ malloc_conf_init(void)
CONF_HANDLE_BOOL(opt_utrace, "utrace")
}
if (config_valgrind) {
- bool hit;
- CONF_HANDLE_BOOL_HIT(opt_valgrind,
- "valgrind", hit)
- if (config_fill && opt_valgrind && hit) {
- opt_junk = false;
- opt_zero = false;
- if (opt_quarantine == 0) {
- opt_quarantine =
- JEMALLOC_VALGRIND_QUARANTINE_DEFAULT;
- }
- opt_redzone = true;
- }
- if (hit)
- continue;
+ CONF_HANDLE_BOOL(opt_valgrind, "valgrind")
}
if (config_xmalloc) {
CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc")
@@ -695,9 +725,9 @@ malloc_init_hard(void)
* Create enough scaffolding to allow recursive allocation in
* malloc_ncpus().
*/
- narenas = 1;
+ narenas_total = narenas_auto = 1;
arenas = init_arenas;
- memset(arenas, 0, sizeof(arena_t *) * narenas);
+ memset(arenas, 0, sizeof(arena_t *) * narenas_auto);
/*
* Initialize one arena here. The rest are lazily created in
@@ -755,20 +785,21 @@ malloc_init_hard(void)
else
opt_narenas = 1;
}
- narenas = opt_narenas;
+ narenas_auto = opt_narenas;
/*
* Make sure that the arenas array can be allocated. In practice, this
* limit is enough to allow the allocator to function, but the ctl
* machinery will fail to allocate memory at far lower limits.
*/
- if (narenas > chunksize / sizeof(arena_t *)) {
- narenas = chunksize / sizeof(arena_t *);
+ if (narenas_auto > chunksize / sizeof(arena_t *)) {
+ narenas_auto = chunksize / sizeof(arena_t *);
malloc_printf("<jemalloc>: Reducing narenas to limit (%d)\n",
- narenas);
+ narenas_auto);
}
+ narenas_total = narenas_auto;
/* Allocate and initialize arenas. */
- arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
+ arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total);
if (arenas == NULL) {
malloc_mutex_unlock(&init_lock);
return (true);
@@ -777,7 +808,7 @@ malloc_init_hard(void)
* Zero the array. In practice, this should always be pre-zeroed,
* since it was just mmap()ed, but let's be sure.
*/
- memset(arenas, 0, sizeof(arena_t *) * narenas);
+ memset(arenas, 0, sizeof(arena_t *) * narenas_total);
/* Copy the pointer to the one arena that was already initialized. */
arenas[0] = init_arenas[0];
@@ -1262,11 +1293,10 @@ je_valloc(size_t size)
* passed an extra argument for the caller return address, which will be
* ignored.
*/
-JEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free;
-JEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc;
-JEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) =
- je_realloc;
-JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) =
+JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free;
+JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc;
+JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc;
+JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) =
je_memalign;
#endif
@@ -1279,7 +1309,7 @@ JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) =
*/
size_t
-je_malloc_usable_size(const void *ptr)
+je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
{
size_t ret;
@@ -1343,18 +1373,19 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
#ifdef JEMALLOC_EXPERIMENTAL
JEMALLOC_INLINE void *
-iallocm(size_t usize, size_t alignment, bool zero)
+iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache,
+ arena_t *arena)
{
assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize,
alignment)));
if (alignment != 0)
- return (ipalloc(usize, alignment, zero));
+ return (ipallocx(usize, alignment, zero, try_tcache, arena));
else if (zero)
- return (icalloc(usize));
+ return (icallocx(usize, try_tcache, arena));
else
- return (imalloc(usize));
+ return (imallocx(usize, try_tcache, arena));
}
int
@@ -1365,6 +1396,9 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
& (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO;
+ unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
+ arena_t *arena;
+ bool try_tcache;
assert(ptr != NULL);
assert(size != 0);
@@ -1372,6 +1406,14 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
if (malloc_init())
goto label_oom;
+ if (arena_ind != UINT_MAX) {
+ arena = arenas[arena_ind];
+ try_tcache = false;
+ } else {
+ arena = NULL;
+ try_tcache = true;
+ }
+
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
if (usize == 0)
goto label_oom;
@@ -1388,18 +1430,19 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1,
alignment);
assert(usize_promoted != 0);
- p = iallocm(usize_promoted, alignment, zero);
+ p = iallocm(usize_promoted, alignment, zero,
+ try_tcache, arena);
if (p == NULL)
goto label_oom;
arena_prof_promoted(p, usize);
} else {
- p = iallocm(usize, alignment, zero);
+ p = iallocm(usize, alignment, zero, try_tcache, arena);
if (p == NULL)
goto label_oom;
}
prof_malloc(p, usize, cnt);
} else {
- p = iallocm(usize, alignment, zero);
+ p = iallocm(usize, alignment, zero, try_tcache, arena);
if (p == NULL)
goto label_oom;
}
@@ -1436,6 +1479,9 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
& (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO;
bool no_move = flags & ALLOCM_NO_MOVE;
+ unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
+ bool try_tcache_alloc, try_tcache_dalloc;
+ arena_t *arena;
assert(ptr != NULL);
assert(*ptr != NULL);
@@ -1443,6 +1489,19 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
assert(SIZE_T_MAX - size >= extra);
assert(malloc_initialized || IS_INITIALIZER);
+ if (arena_ind != UINT_MAX) {
+ arena_chunk_t *chunk;
+ try_tcache_alloc = true;
+ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(*ptr);
+ try_tcache_dalloc = (chunk == *ptr || chunk->arena !=
+ arenas[arena_ind]);
+ arena = arenas[arena_ind];
+ } else {
+ try_tcache_alloc = true;
+ try_tcache_dalloc = true;
+ arena = NULL;
+ }
+
p = *ptr;
if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
@@ -1469,9 +1528,10 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U
&& ((alignment == 0) ? s2u(size) : sa2u(size, alignment))
<= SMALL_MAXCLASS) {
- q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
+ q = irallocx(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >=
size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1),
- alignment, zero, no_move);
+ alignment, zero, no_move, try_tcache_alloc,
+ try_tcache_dalloc, arena);
if (q == NULL)
goto label_err;
if (max_usize < PAGE) {
@@ -1480,7 +1540,8 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
} else
usize = isalloc(q, config_prof);
} else {
- q = iralloc(p, size, extra, alignment, zero, no_move);
+ q = irallocx(p, size, extra, alignment, zero, no_move,
+ try_tcache_alloc, try_tcache_dalloc, arena);
if (q == NULL)
goto label_err;
usize = isalloc(q, config_prof);
@@ -1497,7 +1558,8 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
old_size = isalloc(p, false);
old_rzsize = u2rz(old_size);
}
- q = iralloc(p, size, extra, alignment, zero, no_move);
+ q = irallocx(p, size, extra, alignment, zero, no_move,
+ try_tcache_alloc, try_tcache_dalloc, arena);
if (q == NULL)
goto label_err;
if (config_stats)
@@ -1558,10 +1620,19 @@ je_dallocm(void *ptr, int flags)
{
size_t usize;
size_t rzsize JEMALLOC_CC_SILENCE_INIT(0);
+ unsigned arena_ind = ((unsigned)(flags >> 8)) - 1;
+ bool try_tcache;
assert(ptr != NULL);
assert(malloc_initialized || IS_INITIALIZER);
+ if (arena_ind != UINT_MAX) {
+ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
+ try_tcache = (chunk == ptr || chunk->arena !=
+ arenas[arena_ind]);
+ } else
+ try_tcache = true;
+
UTRACE(ptr, 0, 0);
if (config_stats || config_valgrind)
usize = isalloc(ptr, config_prof);
@@ -1574,7 +1645,7 @@ je_dallocm(void *ptr, int flags)
thread_allocated_tsd_get()->deallocated += usize;
if (config_valgrind && opt_valgrind)
rzsize = p2rz(ptr);
- iqalloc(ptr);
+ iqallocx(ptr, try_tcache);
JEMALLOC_VALGRIND_FREE(ptr, rzsize);
return (ALLOCM_SUCCESS);
@@ -1611,6 +1682,27 @@ je_nallocm(size_t *rsize, size_t size, int flags)
* malloc during fork().
*/
+/*
+ * If an application creates a thread before doing any allocation in the main
+ * thread, then calls fork(2) in the main thread followed by memory allocation
+ * in the child process, a race can occur that results in deadlock within the
+ * child: the main thread may have forked while the created thread had
+ * partially initialized the allocator. Ordinarily jemalloc prevents
+ * fork/malloc races via the following functions it registers during
+ * initialization using pthread_atfork(), but of course that does no good if
+ * the allocator isn't fully initialized at fork time. The following library
+ * constructor is a partial solution to this problem. It may still possible to
+ * trigger the deadlock described above, but doing so would involve forking via
+ * a library constructor that runs before jemalloc's runs.
+ */
+JEMALLOC_ATTR(constructor)
+static void
+jemalloc_constructor(void)
+{
+
+ malloc_init();
+}
+
#ifndef JEMALLOC_MUTEX_INIT_CB
void
jemalloc_prefork(void)
@@ -1628,14 +1720,16 @@ _malloc_prefork(void)
assert(malloc_initialized);
/* Acquire all mutexes in a safe order. */
+ ctl_prefork();
malloc_mutex_prefork(&arenas_lock);
- for (i = 0; i < narenas; i++) {
+ for (i = 0; i < narenas_total; i++) {
if (arenas[i] != NULL)
arena_prefork(arenas[i]);
}
+ prof_prefork();
+ chunk_prefork();
base_prefork();
huge_prefork();
- chunk_dss_prefork();
}
#ifndef JEMALLOC_MUTEX_INIT_CB
@@ -1655,14 +1749,16 @@ _malloc_postfork(void)
assert(malloc_initialized);
/* Release all mutexes, now that fork() has completed. */
- chunk_dss_postfork_parent();
huge_postfork_parent();
base_postfork_parent();
- for (i = 0; i < narenas; i++) {
+ chunk_postfork_parent();
+ prof_postfork_parent();
+ for (i = 0; i < narenas_total; i++) {
if (arenas[i] != NULL)
arena_postfork_parent(arenas[i]);
}
malloc_mutex_postfork_parent(&arenas_lock);
+ ctl_postfork_parent();
}
void
@@ -1673,14 +1769,16 @@ jemalloc_postfork_child(void)
assert(malloc_initialized);
/* Release all mutexes, now that fork() has completed. */
- chunk_dss_postfork_child();
huge_postfork_child();
base_postfork_child();
- for (i = 0; i < narenas; i++) {
+ chunk_postfork_child();
+ prof_postfork_child();
+ for (i = 0; i < narenas_total; i++) {
if (arenas[i] != NULL)
arena_postfork_child(arenas[i]);
}
malloc_mutex_postfork_child(&arenas_lock);
+ ctl_postfork_child();
}
/******************************************************************************/
diff --git a/src/mutex.c b/src/mutex.c
index 37a843e..55e18c2 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -64,7 +64,7 @@ pthread_create(pthread_t *__restrict thread,
/******************************************************************************/
#ifdef JEMALLOC_MUTEX_INIT_CB
-int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
+JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void *(calloc_cb)(size_t, size_t));
#endif
diff --git a/src/prof.c b/src/prof.c
index de1d392..04964ef 100644
--- a/src/prof.c
+++ b/src/prof.c
@@ -1270,4 +1270,46 @@ prof_boot2(void)
return (false);
}
+void
+prof_prefork(void)
+{
+
+ if (opt_prof) {
+ unsigned i;
+
+ malloc_mutex_lock(&bt2ctx_mtx);
+ malloc_mutex_lock(&prof_dump_seq_mtx);
+ for (i = 0; i < PROF_NCTX_LOCKS; i++)
+ malloc_mutex_lock(&ctx_locks[i]);
+ }
+}
+
+void
+prof_postfork_parent(void)
+{
+
+ if (opt_prof) {
+ unsigned i;
+
+ for (i = 0; i < PROF_NCTX_LOCKS; i++)
+ malloc_mutex_postfork_parent(&ctx_locks[i]);
+ malloc_mutex_postfork_parent(&prof_dump_seq_mtx);
+ malloc_mutex_postfork_parent(&bt2ctx_mtx);
+ }
+}
+
+void
+prof_postfork_child(void)
+{
+
+ if (opt_prof) {
+ unsigned i;
+
+ for (i = 0; i < PROF_NCTX_LOCKS; i++)
+ malloc_mutex_postfork_child(&ctx_locks[i]);
+ malloc_mutex_postfork_child(&prof_dump_seq_mtx);
+ malloc_mutex_postfork_child(&bt2ctx_mtx);
+ }
+}
+
/******************************************************************************/
diff --git a/src/rtree.c b/src/rtree.c
index eb0ff1e..90c6935 100644
--- a/src/rtree.c
+++ b/src/rtree.c
@@ -44,3 +44,24 @@ rtree_new(unsigned bits)
return (ret);
}
+
+void
+rtree_prefork(rtree_t *rtree)
+{
+
+ malloc_mutex_prefork(&rtree->mutex);
+}
+
+void
+rtree_postfork_parent(rtree_t *rtree)
+{
+
+ malloc_mutex_postfork_parent(&rtree->mutex);
+}
+
+void
+rtree_postfork_child(rtree_t *rtree)
+{
+
+ malloc_mutex_postfork_child(&rtree->mutex);
+}
diff --git a/src/stats.c b/src/stats.c
index 433b80d..43f87af 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -206,6 +206,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
unsigned i, bool bins, bool large)
{
unsigned nthreads;
+ const char *dss;
size_t page, pactive, pdirty, mapped;
uint64_t npurge, nmadvise, purged;
size_t small_allocated;
@@ -218,6 +219,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned);
malloc_cprintf(write_cb, cbopaque,
"assigned threads: %u\n", nthreads);
+ CTL_I_GET("stats.arenas.0.dss", &dss, const char *);
+ malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n",
+ dss);
CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t);
CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t);
CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t);
@@ -370,6 +374,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
"Run-time option settings:\n");
OPT_WRITE_BOOL(abort)
OPT_WRITE_SIZE_T(lg_chunk)
+ OPT_WRITE_CHAR_P(dss)
OPT_WRITE_SIZE_T(narenas)
OPT_WRITE_SSIZE_T(lg_dirty_mult)
OPT_WRITE_BOOL(stats_print)
@@ -400,7 +405,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
malloc_cprintf(write_cb, cbopaque, "CPUs: %u\n", ncpus);
CTL_GET("arenas.narenas", &uv, unsigned);
- malloc_cprintf(write_cb, cbopaque, "Max arenas: %u\n", uv);
+ malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv);
malloc_cprintf(write_cb, cbopaque, "Pointer size: %zu\n",
sizeof(void *));
@@ -472,7 +477,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("stats.chunks.current", &chunks_current, size_t);
malloc_cprintf(write_cb, cbopaque, "chunks: nchunks "
"highchunks curchunks\n");
- malloc_cprintf(write_cb, cbopaque, " %13"PRIu64"%13zu%13zu\n",
+ malloc_cprintf(write_cb, cbopaque,
+ " %13"PRIu64" %12zu %12zu\n",
chunks_total, chunks_high, chunks_current);
/* Print huge stats. */
diff --git a/src/tcache.c b/src/tcache.c
index 60244c4..47e14f3 100644
--- a/src/tcache.c
+++ b/src/tcache.c
@@ -288,7 +288,7 @@ tcache_create(arena_t *arena)
else if (size <= tcache_maxclass)
tcache = (tcache_t *)arena_malloc_large(arena, size, true);
else
- tcache = (tcache_t *)icalloc(size);
+ tcache = (tcache_t *)icallocx(size, false, arena);
if (tcache == NULL)
return (NULL);
@@ -364,7 +364,7 @@ tcache_destroy(tcache_t *tcache)
arena_dalloc_large(arena, chunk, tcache);
} else
- idalloc(tcache);
+ idallocx(tcache, false);
}
void
diff --git a/src/util.c b/src/util.c
index 9b73c3e..b3a0114 100644
--- a/src/util.c
+++ b/src/util.c
@@ -377,7 +377,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
case '\0': goto label_out;
case '%': {
bool alt_form = false;
- bool zero_pad = false;
bool left_justify = false;
bool plus_space = false;
bool plus_plus = false;
@@ -398,10 +397,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
assert(alt_form == false);
alt_form = true;
break;
- case '0':
- assert(zero_pad == false);
- zero_pad = true;
- break;
case '-':
assert(left_justify == false);
left_justify = true;
diff --git a/test/ALLOCM_ARENA.c b/test/ALLOCM_ARENA.c
new file mode 100644
index 0000000..1585690
--- /dev/null
+++ b/test/ALLOCM_ARENA.c
@@ -0,0 +1,66 @@
+#define JEMALLOC_MANGLE
+#include "jemalloc_test.h"
+
+#define NTHREADS 10
+
+void *
+je_thread_start(void *arg)
+{
+ unsigned thread_ind = (unsigned)(uintptr_t)arg;
+ unsigned arena_ind;
+ int r;
+ void *p;
+ size_t rsz, sz;
+
+ sz = sizeof(arena_ind);
+ if (mallctl("arenas.extend", &arena_ind, &sz, NULL, 0)
+ != 0) {
+ malloc_printf("Error in arenas.extend\n");
+ abort();
+ }
+
+ if (thread_ind % 4 != 3) {
+ size_t mib[3];
+ size_t miblen = sizeof(mib) / sizeof(size_t);
+ const char *dss_precs[] = {"disabled", "primary", "secondary"};
+ const char *dss = dss_precs[thread_ind % 4];
+ if (mallctlnametomib("arena.0.dss", mib, &miblen) != 0) {
+ malloc_printf("Error in mallctlnametomib()\n");
+ abort();
+ }
+ mib[1] = arena_ind;
+ if (mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss,
+ sizeof(const char *))) {
+ malloc_printf("Error in mallctlbymib()\n");
+ abort();
+ }
+ }
+
+ r = allocm(&p, &rsz, 1, ALLOCM_ARENA(arena_ind));
+ if (r != ALLOCM_SUCCESS) {
+ malloc_printf("Unexpected allocm() error\n");
+ abort();
+ }
+
+ return (NULL);
+}
+
+int
+main(void)
+{
+ je_thread_t threads[NTHREADS];
+ unsigned i;
+
+ malloc_printf("Test begin\n");
+
+ for (i = 0; i < NTHREADS; i++) {
+ je_thread_create(&threads[i], je_thread_start,
+ (void *)(uintptr_t)i);
+ }
+
+ for (i = 0; i < NTHREADS; i++)
+ je_thread_join(threads[i], NULL);
+
+ malloc_printf("Test end\n");
+ return (0);
+}
diff --git a/test/ALLOCM_ARENA.exp b/test/ALLOCM_ARENA.exp
new file mode 100644
index 0000000..369a88d
--- /dev/null
+++ b/test/ALLOCM_ARENA.exp
@@ -0,0 +1,2 @@
+Test begin
+Test end
diff --git a/test/thread_arena.c b/test/thread_arena.c
index 2020d99..2ffdb5e 100644
--- a/test/thread_arena.c
+++ b/test/thread_arena.c
@@ -1,7 +1,7 @@
#define JEMALLOC_MANGLE
#include "jemalloc_test.h"
-#define NTHREADS 10
+#define NTHREADS 10
void *
je_thread_start(void *arg)
@@ -66,8 +66,10 @@ main(void)
goto label_return;
}
- for (i = 0; i < NTHREADS; i++)
- je_thread_create(&threads[i], je_thread_start, (void *)&arena_ind);
+ for (i = 0; i < NTHREADS; i++) {
+ je_thread_create(&threads[i], je_thread_start,
+ (void *)&arena_ind);
+ }
for (i = 0; i < NTHREADS; i++)
je_thread_join(threads[i], (void *)&ret);