diff options
| -rw-r--r-- | .travis.yml | 14 | ||||
| -rw-r--r-- | INSTALL | 5 | ||||
| -rw-r--r-- | configure.ac | 17 | ||||
| -rw-r--r-- | doc/jemalloc.xml.in | 34 | ||||
| -rw-r--r-- | include/jemalloc/internal/arena_inlines_a.h | 2 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_defs.h.in | 7 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_inlines_a.h | 19 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_internal_inlines_b.h | 2 | ||||
| -rw-r--r-- | include/jemalloc/internal/jemalloc_preamble.h.in | 7 | ||||
| -rw-r--r-- | include/jemalloc/internal/tcache_inlines.h | 5 | ||||
| -rw-r--r-- | include/jemalloc/internal/tcache_structs.h | 9 | ||||
| -rwxr-xr-x | scripts/gen_run_tests.py | 2 | ||||
| -rwxr-xr-x | scripts/gen_travis.py | 15 | ||||
| -rw-r--r-- | src/arena.c | 56 | ||||
| -rw-r--r-- | src/ctl.c | 66 | ||||
| -rw-r--r-- | src/jemalloc.c | 13 | ||||
| -rw-r--r-- | src/stats.c | 114 | ||||
| -rw-r--r-- | src/tcache.c | 44 | ||||
| -rw-r--r-- | test/integration/thread_tcache_enabled.c | 25 | ||||
| -rw-r--r-- | test/test.sh.in | 1 | ||||
| -rw-r--r-- | test/unit/decay.c | 109 | ||||
| -rw-r--r-- | test/unit/decay.sh | 5 | ||||
| -rw-r--r-- | test/unit/mallctl.c | 9 | ||||
| -rw-r--r-- | test/unit/prof_idump.sh | 8 | ||||
| -rw-r--r-- | test/unit/stats.c | 20 |
25 files changed, 192 insertions, 416 deletions
diff --git a/.travis.yml b/.travis.yml index 2235206..4838cb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ matrix: - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx @@ -31,7 +31,7 @@ matrix: - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" addons: @@ -45,7 +45,7 @@ matrix: - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux - env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" addons: @@ -65,7 +65,7 @@ matrix: packages: - gcc-multilib - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" addons: apt: packages: @@ -75,13 +75,13 @@ matrix: - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-tcache" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" before_script: @@ -153,11 +153,6 @@ any of the following arguments (not a definitive list) to 'configure': Statically link against the specified libunwind.a rather than dynamically linking with -lunwind. ---disable-tcache - Disable thread-specific caches for small objects. Objects are cached and - released in bulk, thus reducing the total number of mutex operations. See - the "opt.tcache" option for usage details. - --disable-munmap Disable virtual memory deallocation via munmap(2); instead keep track of the virtual memory for later use. munmap() is disabled by default (i.e. diff --git a/configure.ac b/configure.ac index f6d08cc..669c1b3 100644 --- a/configure.ac +++ b/configure.ac @@ -1137,22 +1137,6 @@ if test "x$enable_prof" = "x1" ; then fi AC_SUBST([enable_prof]) -dnl Enable thread-specific caching by default. -AC_ARG_ENABLE([tcache], - [AS_HELP_STRING([--disable-tcache], [Disable per thread caches])], -[if test "x$enable_tcache" = "xno" ; then - enable_tcache="0" -else - enable_tcache="1" -fi -], -[enable_tcache="1"] -) -if test "x$enable_tcache" = "x1" ; then - AC_DEFINE([JEMALLOC_TCACHE], [ ]) -fi -AC_SUBST([enable_tcache]) - dnl Indicate whether adjacent virtual memory mappings automatically coalesce dnl (and fragment on demand). if test "x${maps_coalesce}" = "x1" ; then @@ -2181,7 +2165,6 @@ AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) -AC_MSG_RESULT([tcache : ${enable_tcache}]) AC_MSG_RESULT([fill : ${enable_fill}]) AC_MSG_RESULT([utrace : ${enable_utrace}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 3b98395..2b321a7 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -510,13 +510,12 @@ for (i = 0; i < nbins; i++) { sense to reduce the number of arenas if an application does not make much use of the allocation functions.</para> - <para>In addition to multiple arenas, unless - <option>--disable-tcache</option> is specified during configuration, this - allocator supports thread-specific caching, in order to make it possible to - completely avoid synchronization for most allocation requests. Such caching - allows very fast allocation in the common case, but it increases memory - usage and fragmentation, since a bounded number of objects can remain - allocated in each thread cache.</para> + <para>In addition to multiple arenas, this allocator supports + thread-specific caching, in order to make it possible to completely avoid + synchronization for most allocation requests. Such caching allows very fast + allocation in the common case, but it increases memory usage and + fragmentation, since a bounded number of objects can remain allocated in + each thread cache.</para> <para>Memory is conceptually broken into extents. Extents are always aligned to multiples of the page size. This alignment makes it possible to @@ -839,16 +838,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", build configuration.</para></listitem> </varlistentry> - <varlistentry id="config.tcache"> - <term> - <mallctl>config.tcache</mallctl> - (<type>bool</type>) - <literal>r-</literal> - </term> - <listitem><para><option>--disable-tcache</option> was not specified - during build configuration.</para></listitem> - </varlistentry> - <varlistentry id="config.tls"> <term> <mallctl>config.tls</mallctl> @@ -1095,7 +1084,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>opt.tcache</mallctl> (<type>bool</type>) <literal>r-</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Thread-specific caching (tcache) enabled/disabled. When there are multiple threads, each thread uses a tcache for objects up to @@ -1112,7 +1100,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>opt.lg_tcache_max</mallctl> (<type>size_t</type>) <literal>r-</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Maximum size class (log base 2) to cache in the thread-specific cache (tcache). At a minimum, all small size classes @@ -1370,7 +1357,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>thread.tcache.enabled</mallctl> (<type>bool</type>) <literal>rw</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Enable/disable calling thread's tcache. The tcache is implicitly flushed as a side effect of becoming @@ -1384,7 +1370,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>thread.tcache.flush</mallctl> (<type>void</type>) <literal>--</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Flush calling thread's thread-specific cache (tcache). This interface releases all cached objects and internal data structures @@ -1440,7 +1425,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>tcache.create</mallctl> (<type>unsigned</type>) <literal>r-</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Create an explicit thread-specific cache (tcache) and return an identifier that can be passed to the <link @@ -1457,7 +1441,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>tcache.flush</mallctl> (<type>unsigned</type>) <literal>-w</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Flush the specified thread-specific cache (tcache). The same considerations apply to this interface as to <link @@ -1471,7 +1454,6 @@ malloc_conf = "xmalloc:true";]]></programlisting> <mallctl>tcache.destroy</mallctl> (<type>unsigned</type>) <literal>-w</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Flush the specified thread-specific cache (tcache) and make the identifier available for use during a future tcache creation. @@ -1873,7 +1855,6 @@ struct extent_hooks_s { <mallctl>arenas.tcache_max</mallctl> (<type>size_t</type>) <literal>r-</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Maximum thread-cached size class.</para></listitem> </varlistentry> @@ -1892,7 +1873,6 @@ struct extent_hooks_s { <mallctl>arenas.nhbins</mallctl> (<type>unsigned</type>) <literal>r-</literal> - [<option>--enable-tcache</option>] </term> <listitem><para>Total number of thread cache bin size classes.</para></listitem> @@ -2575,7 +2555,6 @@ struct extent_hooks_s { <mallctl>stats.arenas.<i>.bins.<j>.nfills</mallctl> (<type>uint64_t</type>) <literal>r-</literal> - [<option>--enable-stats</option> <option>--enable-tcache</option>] </term> <listitem><para>Cumulative number of tcache fills.</para></listitem> </varlistentry> @@ -2585,7 +2564,6 @@ struct extent_hooks_s { <mallctl>stats.arenas.<i>.bins.<j>.nflushes</mallctl> (<type>uint64_t</type>) <literal>r-</literal> - [<option>--enable-stats</option> <option>--enable-tcache</option>] </term> <listitem><para>Cumulative number of tcache flushes.</para></listitem> </varlistentry> diff --git a/include/jemalloc/internal/arena_inlines_a.h b/include/jemalloc/internal/arena_inlines_a.h index cf92342..2bd5ce7 100644 --- a/include/jemalloc/internal/arena_inlines_a.h +++ b/include/jemalloc/internal/arena_inlines_a.h @@ -58,7 +58,7 @@ percpu_arena_update(tsd_t *tsd, unsigned cpu) { /* Set new arena/tcache associations. */ arena_migrate(tsd, oldind, newind); tcache_t *tcache = tcache_get(tsd); - if (config_tcache && tcache) { + if (tcache != NULL) { tcache_arena_reassociate(tsd_tsdn(tsd), tcache, newarena); } diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 28eb0b3..d3d7694 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -155,13 +155,6 @@ #undef JEMALLOC_PROF_GCC /* - * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. - * This makes it possible to allocate/deallocate objects without any locking - * when the cache is in the steady state. - */ -#undef JEMALLOC_TCACHE - -/* * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage * segment (DSS). */ diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 38fa3c7..c28bd7c 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -323,7 +323,8 @@ malloc_getcpu(void) { JEMALLOC_ALWAYS_INLINE unsigned percpu_arena_choose(void) { unsigned arena_ind; - assert(have_percpu_arena && (percpu_arena_mode != percpu_arena_disabled)); + assert(have_percpu_arena && (percpu_arena_mode != + percpu_arena_disabled)); malloc_cpuid_t cpuid = malloc_getcpu(); assert(cpuid >= 0); @@ -420,19 +421,16 @@ tcache_large_bin_get(tcache_t *tcache, szind_t binind) { JEMALLOC_ALWAYS_INLINE bool tcache_available(tsd_t *tsd) { - cassert(config_tcache); - /* * Thread specific auto tcache might be unavailable if: 1) during tcache * initialization, or 2) disabled through thread.tcache.enabled mallctl * or config options. This check covers all cases. */ - if (likely(tsd_tcache_enabled_get(tsd) == true)) { - /* Associated arena == null implies tcache init in progress. */ - if (tsd_tcachep_get(tsd)->arena != NULL) { - assert(tcache_small_bin_get(tsd_tcachep_get(tsd), - 0)->avail != NULL); - } + if (likely(tsd_tcache_enabled_get(tsd))) { + /* Associated arena == NULL implies tcache init in progress. */ + assert(tsd_tcachep_get(tsd)->arena == NULL || + tcache_small_bin_get(tsd_tcachep_get(tsd), 0)->avail != + NULL); return true; } @@ -441,9 +439,6 @@ tcache_available(tsd_t *tsd) { JEMALLOC_ALWAYS_INLINE tcache_t * tcache_get(tsd_t *tsd) { - if (!config_tcache) { - return NULL; - } if (!tcache_available(tsd)) { return NULL; } diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/include/jemalloc/internal/jemalloc_internal_inlines_b.h index ab54a59..2fd371c 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -24,7 +24,7 @@ arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal) { if (unlikely(ret == NULL)) { ret = arena_choose_hard(tsd, internal); assert(ret); - if (config_tcache && tcache_available(tsd)) { + if (tcache_available(tsd)) { tcache_t *tcache = tcache_get(tsd); if (tcache->arena != NULL) { /* See comments in tcache_data_init().*/ diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in index 7c796c6..0e2ce31 100644 --- a/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/include/jemalloc/internal/jemalloc_preamble.h.in @@ -111,13 +111,6 @@ static const bool config_stats = false #endif ; -static const bool config_tcache = -#ifdef JEMALLOC_TCACHE - true -#else - false -#endif - ; static const bool config_tls = #ifdef JEMALLOC_TLS true diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index d425b82..67d35b5 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -6,7 +6,6 @@ #ifndef JEMALLOC_ENABLE_INLINE void tcache_event(tsd_t *tsd, tcache_t *tcache); -void tcache_flush(void); bool tcache_enabled_get(tsd_t *tsd); tcache_t *tcache_get(tsd_t *tsd); void tcache_enabled_set(tsd_t *tsd, bool enabled); @@ -25,15 +24,11 @@ tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) JEMALLOC_INLINE bool tcache_enabled_get(tsd_t *tsd) { - cassert(config_tcache); - return tsd_tcache_enabled_get(tsd); } JEMALLOC_INLINE void tcache_enabled_set(tsd_t *tsd, bool enabled) { - cassert(config_tcache); - bool was_enabled = tsd_tcache_enabled_get(tsd); if (!was_enabled && enabled) { diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index c43e59b..fe27f36 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -40,23 +40,14 @@ struct tcache_s { * element of tbins is initialized to point to the proper offset within * this array. */ -#ifdef JEMALLOC_TCACHE tcache_bin_t tbins_small[NBINS]; -#else - tcache_bin_t tbins_small[0]; -#endif /* Data accessed less often below. */ ql_elm(tcache_t) link; /* Used for aggregating stats. */ arena_t *arena; /* Associated arena. */ szind_t next_gc_bin; /* Next bin to GC. */ -#ifdef JEMALLOC_TCACHE /* For small bins, fill (ncached_max >> lg_fill_div). */ uint8_t lg_fill_div[NBINS]; tcache_bin_t tbins_large[NSIZES-NBINS]; -#else - uint8_t lg_fill_div[0]; - tcache_bin_t tbins_large[0]; -#endif }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/scripts/gen_run_tests.py b/scripts/gen_run_tests.py index 729ecb1..9e46ba9 100755 --- a/scripts/gen_run_tests.py +++ b/scripts/gen_run_tests.py @@ -18,7 +18,7 @@ possible_config_opts = [ '--enable-debug', '--enable-prof', '--disable-stats', - '--disable-tcache', + '--with-malloc-conf=tcache:false', ] print 'set -e' diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py index 35a10ee..4649cb7 100755 --- a/scripts/gen_travis.py +++ b/scripts/gen_travis.py @@ -24,11 +24,11 @@ script: # The 'default' configuration is gcc, on linux, with no compiler or configure # flags. We also test with clang, -m32, --enable-debug, --enable-prof, -# --disable-stats, and --disable-tcache. To avoid abusing travis though, we -# don't test all 2**7 = 128 possible combinations of these; instead, we only -# test combinations of up to 2 'unusual' settings, under the hope that bugs -# involving interactions of such settings are rare. -# things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29 +# --disable-stats, and --with-malloc-conf=tcache:false. To avoid abusing +# travis though, we don't test all 2**7 = 128 possible combinations of these; +# instead, we only test combinations of up to 2 'unusual' settings, under the +# hope that bugs involving interactions of such settings are rare. +# Things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29 MAX_UNUSUAL_OPTIONS = 2 os_default = 'linux' @@ -40,7 +40,10 @@ compilers_unusual = 'CC=clang CXX=clang++' compiler_flag_unusuals = ['-m32'] configure_flag_unusuals = [ - '--enable-debug', '--enable-prof', '--disable-stats', '--disable-tcache', + '--enable-debug', + '--enable-prof', + '--disable-stats', + '--with-malloc-conf=tcache:false', ] all_unusuals = ( diff --git a/src/arena.c b/src/arena.c index 94a4b5e..c2eca44 100644 --- a/src/arena.c +++ b/src/arena.c @@ -283,31 +283,27 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_unlock(tsdn, &arena->stats); - if (config_tcache) { - tcache_bin_t *tbin; - tcache_t *tcache; - - /* tcache_bytes counts currently cached bytes. */ - atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED); - malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); - ql_foreach(tcache, &arena->tcache_ql, link) { - szind_t i = 0; - for (; i < NBINS; i++) { - tbin = tcache_small_bin_get(tcache, i); - arena_stats_accum_zu(&astats->tcache_bytes, - tbin->ncached * index2size(i)); - } - for (; i < nhbins; i++) { - tbin = tcache_large_bin_get(tcache, i); - arena_stats_accum_zu(&astats->tcache_bytes, - tbin->ncached * index2size(i)); - } + /* tcache_bytes counts currently cached bytes. */ + atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED); + malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); + tcache_t *tcache; + ql_foreach(tcache, &arena->tcache_ql, link) { + szind_t i = 0; + for (; i < NBINS; i++) { + tcache_bin_t *tbin = tcache_small_bin_get(tcache, i); + arena_stats_accum_zu(&astats->tcache_bytes, + tbin->ncached * index2size(i)); + } + for (; i < nhbins; i++) { + tcache_bin_t *tbin = tcache_large_bin_get(tcache, i); + arena_stats_accum_zu(&astats->tcache_bytes, + tbin->ncached * index2size(i)); } - malloc_mutex_prof_read(tsdn, - &astats->mutex_prof_data[arena_prof_mutex_tcache_list], - &arena->tcache_ql_mtx); - malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } + malloc_mutex_prof_read(tsdn, + &astats->mutex_prof_data[arena_prof_mutex_tcache_list], + &arena->tcache_ql_mtx); + malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); #define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \ malloc_mutex_lock(tsdn, &arena->mtx); \ @@ -342,10 +338,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, bstats[i].ndalloc += bin->stats.ndalloc; bstats[i].nrequests += bin->stats.nrequests; bstats[i].curregs += bin->stats.curregs; - if (config_tcache) { - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; - } + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; bstats[i].nslabs += bin->stats.nslabs; bstats[i].reslabs += bin->stats.reslabs; bstats[i].curslabs += bin->stats.curslabs; @@ -1867,9 +1861,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { if (arena_stats_init(tsdn, &arena->stats)) { goto label_error; } - } - if (config_stats && config_tcache) { ql_new(&arena->tcache_ql); if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", WITNESS_RANK_TCACHE_QL)) { @@ -2007,7 +1999,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) { void arena_prefork1(tsdn_t *tsdn, arena_t *arena) { - if (config_stats && config_tcache) { + if (config_stats) { malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx); } } @@ -2056,7 +2048,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { extents_postfork_parent(tsdn, &arena->extents_retained); malloc_mutex_postfork_parent(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &arena->decay_muzzy.mtx); - if (config_stats && config_tcache) { + if (config_stats) { malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); } } @@ -2076,7 +2068,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { extents_postfork_child(tsdn, &arena->extents_retained); malloc_mutex_postfork_child(tsdn, &arena->decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &arena->decay_muzzy.mtx); - if (config_stats && config_tcache) { + if (config_stats) { malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); } } @@ -70,7 +70,6 @@ CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) CTL_PROTO(config_stats) -CTL_PROTO(config_tcache) CTL_PROTO(config_tls) CTL_PROTO(config_utrace) CTL_PROTO(config_xmalloc) @@ -255,7 +254,6 @@ static const ctl_named_node_t config_node[] = { {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, {NAME("stats"), CTL(config_stats)}, - {NAME("tcache"), CTL(config_tcache)}, {NAME("tls"), CTL(config_tls)}, {NAME("utrace"), CTL(config_utrace)}, {NAME("xmalloc"), CTL(config_xmalloc)} @@ -777,10 +775,8 @@ ARENA_PROF_MUTEXES accum_arena_stats_u64(&sdstats->astats.nrequests_large, &astats->astats.nrequests_large); - if (config_tcache) { - accum_atomic_zu(&sdstats->astats.tcache_bytes, - &astats->astats.tcache_bytes); - } + accum_atomic_zu(&sdstats->astats.tcache_bytes, + &astats->astats.tcache_bytes); for (i = 0; i < NBINS; i++) { sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; @@ -793,12 +789,9 @@ ARENA_PROF_MUTEXES } else { assert(astats->bstats[i].curregs == 0); } - if (config_tcache) { - sdstats->bstats[i].nfills += - astats->bstats[i].nfills; - sdstats->bstats[i].nflushes += - astats->bstats[i].nflushes; - } + sdstats->bstats[i].nfills += astats->bstats[i].nfills; + sdstats->bstats[i].nflushes += + astats->bstats[i].nflushes; sdstats->bstats[i].nslabs += astats->bstats[i].nslabs; sdstats->bstats[i].reslabs += astats->bstats[i].reslabs; if (!destroyed) { @@ -1457,7 +1450,6 @@ CTL_RO_CONFIG_GEN(config_prof, bool) CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) CTL_RO_CONFIG_GEN(config_stats, bool) -CTL_RO_CONFIG_GEN(config_tcache, bool) CTL_RO_CONFIG_GEN(config_tls, bool) CTL_RO_CONFIG_GEN(config_utrace, bool) CTL_RO_CONFIG_GEN(config_xmalloc, bool) @@ -1475,8 +1467,8 @@ CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) -CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) -CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) +CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) +CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) @@ -1536,12 +1528,9 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } /* Set new arena/tcache associations. */ arena_migrate(tsd, oldind, newind); - if (config_tcache) { - tcache_t *tcache = tsd_tcachep_get(tsd); - if (tcache != NULL) { - tcache_arena_reassociate(tsd_tsdn(tsd), tcache, - newarena); - } + if (tcache_available(tsd)) { + tcache_arena_reassociate(tsd_tsdn(tsd), + tsd_tcachep_get(tsd), newarena); } } @@ -1565,10 +1554,6 @@ thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, int ret; bool oldval; - if (!config_tcache) { - return ENOENT; - } - oldval = tcache_enabled_get(tsd); if (newp != NULL) { if (newlen != sizeof(bool)) { @@ -1589,8 +1574,9 @@ thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - if (!config_tcache) { - return ENOENT; + if (!tcache_available(tsd)) { + ret = EFAULT; + goto label_return; } READONLY(); @@ -1670,10 +1656,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, int ret; unsigned tcache_ind; - if (!config_tcache) { - return ENOENT; - } - READONLY(); if (tcaches_create(tsd, &tcache_ind)) { ret = EFAULT; @@ -1692,10 +1674,6 @@ tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, int ret; unsigned tcache_ind; - if (!config_tcache) { - return ENOENT; - } - WRITEONLY(); tcache_ind = UINT_MAX; WRITE(tcache_ind, unsigned); @@ -1716,10 +1694,6 @@ tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, int ret; unsigned tcache_ind; - if (!config_tcache) { - return ENOENT; - } - WRITEONLY(); tcache_ind = UINT_MAX; WRITE(tcache_ind, unsigned); @@ -2150,9 +2124,9 @@ arenas_muzzy_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) -CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) +CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) -CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) +CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t) @@ -2380,7 +2354,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_base, CTL_RO_CGEN(config_stats, stats_arenas_i_internal, atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), size_t) -CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_tcache_bytes, +CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes, ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_resident, @@ -2480,9 +2454,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, MUTEX_PROF_RESET(arena->extents_retained.mtx); MUTEX_PROF_RESET(arena->decay_dirty.mtx); MUTEX_PROF_RESET(arena->decay_muzzy.mtx); - if (config_tcache) { - MUTEX_PROF_RESET(arena->tcache_ql_mtx); - } + MUTEX_PROF_RESET(arena->tcache_ql_mtx); MUTEX_PROF_RESET(arena->base->mtx); for (szind_t i = 0; i < NBINS; i++) { @@ -2502,9 +2474,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, arenas_i(mib[2])->astats->bstats[mib[4]].nrequests, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, arenas_i(mib[2])->astats->bstats[mib[4]].curregs, size_t) -CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nfills, arenas_i(mib[2])->astats->bstats[mib[4]].nfills, uint64_t) -CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nflushes, arenas_i(mib[2])->astats->bstats[mib[4]].nflushes, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nslabs, arenas_i(mib[2])->astats->bstats[mib[4]].nslabs, uint64_t) diff --git a/src/jemalloc.c b/src/jemalloc.c index ea632c2..e08226c 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -682,7 +682,7 @@ arenas_tdata_cleanup(tsd_t *tsd) { static void stats_print_atexit(void) { - if (config_tcache && config_stats) { + if (config_stats) { tsdn_t *tsdn; unsigned narenas, i; @@ -1106,12 +1106,9 @@ malloc_conf_init(void) { if (config_xmalloc) { CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) } - if (config_tcache) { - CONF_HANDLE_BOOL(opt_tcache, "tcache", true) - CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, - "lg_tcache_max", -1, - (sizeof(size_t) << 3) - 1) - } + CONF_HANDLE_BOOL(opt_tcache, "tcache", true) + CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", + -1, (sizeof(size_t) << 3) - 1) if (strncmp("percpu_arena", k, klen) == 0) { int i; bool match = false; @@ -1236,7 +1233,7 @@ malloc_init_hard_a0_locked() { prof_boot1(); } arena_boot(); - if (config_tcache && tcache_boot(TSDN_NULL)) { + if (tcache_boot(TSDN_NULL)) { return true; } if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS)) { diff --git a/src/stats.c b/src/stats.c index 435dfb9..4074c94 100644 --- a/src/stats.c +++ b/src/stats.c @@ -128,20 +128,11 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } else { char *mutex_counters = " n_lock_ops n_waiting" " n_spin_acq max_wait_ns\n"; - if (config_tcache) { - malloc_cprintf(write_cb, cbopaque, - "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs" - " curslabs regs pgs util nfills" - " nflushes newslabs reslabs%s", - mutex ? mutex_counters : "\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs" - " curslabs regs pgs util newslabs" - " reslabs%s", mutex ? mutex_counters : "\n"); - } + malloc_cprintf(write_cb, cbopaque, + "bins: size ind allocated nmalloc" + " ndalloc nrequests curregs curslabs regs" + " pgs util nfills nflushes newslabs" + " reslabs%s", mutex ? mutex_counters : "\n"); } for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nslabs; @@ -173,12 +164,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nrequests", i, j, &nrequests, uint64_t); - if (config_tcache) { - CTL_M2_M4_GET("stats.arenas.0.bins.0.nfills", i, j, - &nfills, uint64_t); - CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", i, j, - &nflushes, uint64_t); - } + CTL_M2_M4_GET("stats.arenas.0.bins.0.nfills", i, j, &nfills, + uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", i, j, &nflushes, + uint64_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nreslabs", i, j, &nreslabs, uint64_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs, @@ -190,23 +179,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, "\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n" "\t\t\t\t\t\t\"ndalloc\": %"FMTu64",\n" "\t\t\t\t\t\t\"curregs\": %zu,\n" - "\t\t\t\t\t\t\"nrequests\": %"FMTu64",\n", - nmalloc, - ndalloc, - curregs, - nrequests); - if (config_tcache) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\t\"nfills\": %"FMTu64",\n" - "\t\t\t\t\t\t\"nflushes\": %"FMTu64",\n", - nfills, - nflushes); - } - malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\t\t\"nrequests\": %"FMTu64",\n" + "\t\t\t\t\t\t\"nfills\": %"FMTu64",\n" + "\t\t\t\t\t\t\"nflushes\": %"FMTu64",\n" "\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n" "\t\t\t\t\t\t\"curslabs\": %zu%s\n", - nreslabs, curslabs, mutex ? "," : ""); - + nmalloc, ndalloc, curregs, nrequests, nfills, + nflushes, nreslabs, curslabs, mutex ? "," : ""); if (mutex) { uint64_t mutex_stats[num_mutex_prof_counters]; read_arena_bin_mutex_stats(i, j, mutex_stats); @@ -260,27 +239,13 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } } - if (config_tcache) { - malloc_cprintf(write_cb, cbopaque, - "%20zu %3u %12zu %12"FMTu64 - " %12"FMTu64" %12"FMTu64" %12zu" - " %12zu %4u %3zu %-5s %12"FMTu64 - " %12"FMTu64" %12"FMTu64" %12"FMTu64, - reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, curslabs, - nregs, slab_size / page, util, nfills, - nflushes, nslabs, nreslabs); - } else { - malloc_cprintf(write_cb, cbopaque, - "%20zu %3u %12zu %12"FMTu64 - " %12"FMTu64" %12"FMTu64" %12zu" - " %12zu %4u %3zu %-5s %12"FMTu64 - " %12"FMTu64, - reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, curslabs, - nregs, slab_size / page, util, nslabs, - nreslabs); - } + malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12" + FMTu64" %12"FMTu64" %12"FMTu64" %12zu %12zu %4u" + " %3zu %-5s %12"FMTu64" %12"FMTu64" %12"FMTu64 + " %12"FMTu64, reg_size, j, curregs * reg_size, + nmalloc, ndalloc, nrequests, curregs, curslabs, + nregs, slab_size / page, util, nfills, nflushes, + nslabs, nreslabs); if (mutex) { malloc_cprintf(write_cb, cbopaque, " %12"FMTu64" %12"FMTu64" %12"FMTu64 @@ -423,14 +388,7 @@ stats_arena_mutexes_print(void (*write_cb)(void *, const char *), malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n"); arena_prof_mutex_ind_t i, last_mutex; last_mutex = num_arena_prof_mutexes - 1; - if (!config_tcache) { - last_mutex--; - } for (i = 0; i < num_arena_prof_mutexes; i++) { - if (!config_tcache && - i == arena_prof_mutex_tcache_list) { - continue; - } mutex_stats_output_json(write_cb, cbopaque, arena_mutex_names[i], mutex_stats[i], "\t\t\t\t\t", (i == last_mutex)); @@ -440,10 +398,6 @@ stats_arena_mutexes_print(void (*write_cb)(void *, const char *), } else { arena_prof_mutex_ind_t i; for (i = 0; i < num_arena_prof_mutexes; i++) { - if (!config_tcache && - i == arena_prof_mutex_tcache_list) { - continue; - } mutex_stats_output(write_cb, cbopaque, arena_mutex_names[i], mutex_stats[i], i == 0); } @@ -659,16 +613,13 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, "internal: %12zu\n", internal); } - if (config_tcache) { - CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &tcache_bytes, - size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"tcache\": %zu,\n", tcache_bytes); - } else { - malloc_cprintf(write_cb, cbopaque, - "tcache: %12zu\n", tcache_bytes); - } + CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &tcache_bytes, size_t); + if (json) { + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\t\"tcache\": %zu,\n", tcache_bytes); + } else { + malloc_cprintf(write_cb, cbopaque, + "tcache: %12zu\n", tcache_bytes); } CTL_M2_GET("stats.arenas.0.resident", i, &resident, size_t); @@ -761,7 +712,6 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, CONFIG_WRITE_BOOL_JSON(prof_libgcc, ",") CONFIG_WRITE_BOOL_JSON(prof_libunwind, ",") CONFIG_WRITE_BOOL_JSON(stats, ",") - CONFIG_WRITE_BOOL_JSON(tcache, ",") CONFIG_WRITE_BOOL_JSON(tls, ",") CONFIG_WRITE_BOOL_JSON(utrace, ",") CONFIG_WRITE_BOOL_JSON(xmalloc, "") @@ -959,11 +909,9 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\t\t\t\"nbins\": %u,\n", nbins); - if (config_tcache) { - CTL_GET("arenas.nhbins", &uv, unsigned); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"nhbins\": %u,\n", uv); - } + CTL_GET("arenas.nhbins", &uv, unsigned); + malloc_cprintf(write_cb, cbopaque, "\t\t\t\"nhbins\": %u,\n", + uv); malloc_cprintf(write_cb, cbopaque, "\t\t\t\"bin\": [\n"); diff --git a/src/tcache.c b/src/tcache.c index 971c016..72d1e47 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -7,13 +7,7 @@ /******************************************************************************/ /* Data. */ -bool opt_tcache = -#ifdef JEMALLOC_TCACHE - true -#else - false -#endif - ; +bool opt_tcache = true; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; tcache_bin_info_t *tcache_bin_info; @@ -93,7 +87,7 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, tcache_bin_t *tbin, szind_t binind, bool *tcache_success) { void *ret; - assert(tcache->arena); + assert(tcache->arena != NULL); arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind, config_prof ? tcache->prof_accumbytes : 0); if (config_prof) { @@ -304,7 +298,7 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { static void tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { arena_t *arena = tcache->arena; - assert(arena); + assert(arena != NULL); if (config_stats) { /* Unlink from list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); @@ -383,10 +377,6 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { /* Initialize auto tcache (embedded in TSD). */ bool tsd_tcache_data_init(tsd_t *tsd) { - if (!config_tcache) { - return false; - } - tcache_t *tcache = &tsd->tcache; assert(tcache_small_bin_get(tcache, 0)->avail == NULL); size_t size = stack_nelms * sizeof(void *); @@ -458,9 +448,9 @@ tcache_create_explicit(tsd_t *tsd) { static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { - unsigned i; + assert(tcache->arena != NULL); - for (i = 0; i < NBINS; i++) { + for (unsigned i = 0; i < NBINS; i++) { tcache_bin_t *tbin = tcache_small_bin_get(tcache, i); tcache_bin_flush_small(tsd, tcache, tbin, i, 0); @@ -468,7 +458,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tbin->tstats.nrequests == 0); } } - for (; i < nhbins; i++) { + for (unsigned i = NBINS; i < nhbins; i++) { tcache_bin_t *tbin = tcache_large_bin_get(tcache, i); tcache_bin_flush_large(tsd, tbin, i, 0, tcache); @@ -477,20 +467,17 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { } } - arena_t *arena = tcache->arena; - if (config_prof && arena && tcache->prof_accumbytes > 0 && - arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes)) { + if (config_prof && tcache->prof_accumbytes > 0 && + arena_prof_accum(tsd_tsdn(tsd), tcache->arena, + tcache->prof_accumbytes)) { prof_idump(tsd_tsdn(tsd)); } } void tcache_flush(void) { - tsd_t *tsd; - - cassert(config_tcache); - - tsd = tsd_fetch(); + tsd_t *tsd = tsd_fetch(); + assert(tcache_available(tsd)); tcache_flush_cache(tsd, tsd_tcachep_get(tsd)); } @@ -514,10 +501,6 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { /* For auto tcache (embedded in TSD) only. */ void tcache_cleanup(tsd_t *tsd) { - if (!config_tcache) { - return; - } - tcache_t *tcache = tsd_tcachep_get(tsd); if (!tcache_available(tsd)) { assert(tsd_tcache_enabled_get(tsd) == false); @@ -660,10 +643,6 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) { bool tcache_boot(tsdn_t *tsdn) { - cassert(config_tcache); - - unsigned i; - /* If necessary, clamp opt_lg_tcache_max. */ if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < SMALL_MAXCLASS) { @@ -685,6 +664,7 @@ tcache_boot(tsdn_t *tsdn) { return true; } stack_nelms = 0; + unsigned i; for (i = 0; i < NBINS; i++) { if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { tcache_bin_info[i].ncached_max = diff --git a/test/integration/thread_tcache_enabled.c b/test/integration/thread_tcache_enabled.c index a0ba56b..0c343a6 100644 --- a/test/integration/thread_tcache_enabled.c +++ b/test/integration/thread_tcache_enabled.c @@ -1,29 +1,11 @@ #include "test/jemalloc_test.h" -static const bool config_tcache = -#ifdef JEMALLOC_TCACHE - true -#else - false -#endif - ; - void * thd_start(void *arg) { - int err; - size_t sz; bool e0, e1; - - sz = sizeof(bool); - if ((err = mallctl("thread.tcache.enabled", (void *)&e0, &sz, NULL, - 0))) { - if (err == ENOENT) { - assert_false(config_tcache, - "ENOENT should only be returned if tcache is " - "disabled"); - } - goto label_ENOENT; - } + size_t sz = sizeof(bool); + assert_d_eq(mallctl("thread.tcache.enabled", (void *)&e0, &sz, NULL, + 0), 0, "Unexpected mallctl failure"); if (e0) { e1 = false; @@ -78,7 +60,6 @@ thd_start(void *arg) { free(malloc(1)); return NULL; -label_ENOENT: test_skip("\"thread.tcache.enabled\" mallctl not available"); return NULL; } diff --git a/test/test.sh.in b/test/test.sh.in index f0f0f97..4d0e0df 100644 --- a/test/test.sh.in +++ b/test/test.sh.in @@ -43,7 +43,6 @@ for t in $@; do # per test shell script to ignore the @JEMALLOC_CPREFIX@ detail). $(enable_fill=@enable_fill@ \ enable_prof=@enable_prof@ \ - enable_tcache=@enable_tcache@ \ . @srcroot@${t}.sh && \ export_malloc_conf && \ ${t}@exe@ @abs_srcroot@ @abs_objroot@) diff --git a/test/unit/decay.c b/test/unit/decay.c index 471a558..26359fa 100644 --- a/test/unit/decay.c +++ b/test/unit/decay.c @@ -174,16 +174,15 @@ TEST_BEGIN(test_decay_ticks) { assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL, 0), 0, "Unexpected mallctl failure"); - int err; /* Set up a manually managed arena for test. */ arena_ind = do_arena_create(0, 0); /* Migrate to the new arena, and get the ticker. */ unsigned old_arena_ind; size_t sz_arena_ind = sizeof(old_arena_ind); - err = mallctl("thread.arena", (void *)&old_arena_ind, &sz_arena_ind, - (void *)&arena_ind, sizeof(arena_ind)); - assert_d_eq(err, 0, "Unexpected mallctl() failure"); + assert_d_eq(mallctl("thread.arena", (void *)&old_arena_ind, + &sz_arena_ind, (void *)&arena_ind, sizeof(arena_ind)), 0, + "Unexpected mallctl() failure"); decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind); assert_ptr_not_null(decay_ticker, "Unexpected failure getting decay ticker"); @@ -310,51 +309,48 @@ TEST_BEGIN(test_decay_ticks) { * Test tcache fill/flush interactions for large and small size classes, * using an explicit tcache. */ - if (config_tcache) { - unsigned tcache_ind, i; - size_t tcache_sizes[2]; - tcache_sizes[0] = large0; - tcache_sizes[1] = 1; - - size_t tcache_max, sz_tcache_max; - sz_tcache_max = sizeof(tcache_max); - err = mallctl("arenas.tcache_max", (void *)&tcache_max, - &sz_tcache_max, NULL, 0); - assert_d_eq(err, 0, "Unexpected mallctl() failure"); - - sz = sizeof(unsigned); - assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz, - NULL, 0), 0, "Unexpected mallctl failure"); - - for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) { - sz = tcache_sizes[i]; - - /* tcache fill. */ - tick0 = ticker_read(decay_ticker); - p = mallocx(sz, MALLOCX_TCACHE(tcache_ind)); - assert_ptr_not_null(p, "Unexpected mallocx() failure"); - tick1 = ticker_read(decay_ticker); + unsigned tcache_ind, i; + size_t tcache_sizes[2]; + tcache_sizes[0] = large0; + tcache_sizes[1] = 1; + + size_t tcache_max, sz_tcache_max; + sz_tcache_max = sizeof(tcache_max); + assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max, + &sz_tcache_max, NULL, 0), 0, "Unexpected mallctl() failure"); + + sz = sizeof(unsigned); + assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz, + NULL, 0), 0, "Unexpected mallctl failure"); + + for (i = 0; i < sizeof(tcache_sizes) / sizeof(size_t); i++) { + sz = tcache_sizes[i]; + + /* tcache fill. */ + tick0 = ticker_read(decay_ticker); + p = mallocx(sz, MALLOCX_TCACHE(tcache_ind)); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + tick1 = ticker_read(decay_ticker); + assert_u32_ne(tick1, tick0, + "Expected ticker to tick during tcache fill " + "(sz=%zu)", sz); + /* tcache flush. */ + dallocx(p, MALLOCX_TCACHE(tcache_ind)); + tick0 = ticker_read(decay_ticker); + assert_d_eq(mallctl("tcache.flush", NULL, NULL, + (void *)&tcache_ind, sizeof(unsigned)), 0, + "Unexpected mallctl failure"); + tick1 = ticker_read(decay_ticker); + + /* Will only tick if it's in tcache. */ + if (sz <= tcache_max) { assert_u32_ne(tick1, tick0, - "Expected ticker to tick during tcache fill " - "(sz=%zu)", sz); - /* tcache flush. */ - dallocx(p, MALLOCX_TCACHE(tcache_ind)); - tick0 = ticker_read(decay_ticker); - assert_d_eq(mallctl("tcache.flush", NULL, NULL, - (void *)&tcache_ind, sizeof(unsigned)), 0, - "Unexpected mallctl failure"); - tick1 = ticker_read(decay_ticker); - - /* Will only tick if it's in tcache. */ - if (sz <= tcache_max) { - assert_u32_ne(tick1, tick0, - "Expected ticker to tick during tcache " - "flush (sz=%zu)", sz); - } else { - assert_u32_eq(tick1, tick0, - "Unexpected ticker tick during tcache " - "flush (sz=%zu)", sz); - } + "Expected ticker to tick during tcache " + "flush (sz=%zu)", sz); + } else { + assert_u32_eq(tick1, tick0, + "Unexpected ticker tick during tcache " + "flush (sz=%zu)", sz); } } } @@ -422,18 +418,11 @@ TEST_BEGIN(test_decay_ticker) { * the ticker triggers purging. */ - if (config_tcache) { - size_t tcache_max; - - size_t sz = sizeof(size_t); - assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max, - &sz, NULL, 0), 0, "Unexpected mallctl failure"); - large = nallocx(tcache_max + 1, flags); - } else { - size_t sz = sizeof(size_t); - assert_d_eq(mallctl("arenas.lextent.0.size", &large, &sz, NULL, - 0), 0, "Unexpected mallctl failure"); - } + size_t tcache_max; + size_t sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.tcache_max", (void *)&tcache_max, &sz, NULL, + 0), 0, "Unexpected mallctl failure"); + large = nallocx(tcache_max + 1, flags); do_purge(arena_ind); uint64_t dirty_npurge0 = get_arena_dirty_npurge(arena_ind); diff --git a/test/unit/decay.sh b/test/unit/decay.sh index 0df1788..a41489b 100644 --- a/test/unit/decay.sh +++ b/test/unit/decay.sh @@ -1,6 +1,3 @@ #!/bin/sh -export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1" -if [ "x${enable_tcache}" = "x1" ] ; then - export MALLOC_CONF="${MALLOC_CONF},lg_tcache_max:0" -fi +export MALLOC_CONF="dirty_decay_time:1,muzzy_decay_time:1,lg_tcache_max:0" diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index b8c6a25..945d829 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -136,7 +136,6 @@ TEST_BEGIN(test_mallctl_config) { TEST_MALLCTL_CONFIG(prof_libgcc, bool); TEST_MALLCTL_CONFIG(prof_libunwind, bool); TEST_MALLCTL_CONFIG(stats, bool); - TEST_MALLCTL_CONFIG(tcache, bool); TEST_MALLCTL_CONFIG(tls, bool); TEST_MALLCTL_CONFIG(utrace, bool); TEST_MALLCTL_CONFIG(xmalloc, bool); @@ -170,8 +169,8 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, zero, fill); TEST_MALLCTL_OPT(bool, utrace, utrace); TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); - TEST_MALLCTL_OPT(bool, tcache, tcache); - TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache); + TEST_MALLCTL_OPT(bool, tcache, always); + TEST_MALLCTL_OPT(size_t, lg_tcache_max, always); TEST_MALLCTL_OPT(bool, prof, prof); TEST_MALLCTL_OPT(const char *, prof_prefix, prof); TEST_MALLCTL_OPT(bool, prof_active, prof); @@ -213,8 +212,6 @@ TEST_END TEST_BEGIN(test_tcache_none) { void *p0, *q, *p1; - test_skip_if(!config_tcache); - /* Allocate p and q. */ p0 = mallocx(42, 0); assert_ptr_not_null(p0, "Unexpected mallocx() failure"); @@ -243,8 +240,6 @@ TEST_BEGIN(test_tcache) { unsigned i; size_t sz, psz, qsz; - test_skip_if(!config_tcache); - psz = 42; qsz = nallocx(psz, 0) + 1; diff --git a/test/unit/prof_idump.sh b/test/unit/prof_idump.sh index fdb5813..4dc599a 100644 --- a/test/unit/prof_idump.sh +++ b/test/unit/prof_idump.sh @@ -1,12 +1,8 @@ #!/bin/sh +export MALLOC_CONF="tcache:false" if [ "x${enable_prof}" = "x1" ] ; then - export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0" - if [ "x${enable_tcache}" = "x1" ] ; then - export MALLOC_CONF="${MALLOC_CONF},tcache:false" - fi -elif [ "x${enable_tcache}" = "x1" ] ; then - export MALLOC_CONF="tcache:false" + export MALLOC_CONF="${MALLOC_CONF},prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0" fi diff --git a/test/unit/stats.c b/test/unit/stats.c index 1619f5b..f5ee128 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -83,7 +83,7 @@ TEST_BEGIN(test_stats_arenas_summary) { dallocx(large, 0); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), - config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result"); assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, "Unexpected mallctl() failure"); @@ -150,7 +150,7 @@ TEST_BEGIN(test_stats_arenas_small) { assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), - config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), 0, "Unexpected mallctl() failure"); @@ -230,6 +230,10 @@ TEST_BEGIN(test_stats_arenas_bins) { uint64_t nslabs, nreslabs; int expected = config_stats ? 0 : ENOENT; + /* Make sure allocation below isn't satisfied by tcache. */ + assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), + opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result"); + unsigned arena_ind, old_arena_ind; sz = sizeof(unsigned); assert_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), @@ -243,7 +247,7 @@ TEST_BEGIN(test_stats_arenas_bins) { assert_ptr_not_null(p, "Unexpected malloc() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), - config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + opt_tcache ? 0 : EFAULT, "Unexpected mallctl() result"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), 0, "Unexpected mallctl() failure"); @@ -266,11 +270,11 @@ TEST_BEGIN(test_stats_arenas_bins) { sz = sizeof(uint64_t); gen_mallctl_str(cmd, "nfills", arena_ind); - assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0), - config_tcache ? expected : ENOENT, "Unexpected mallctl() result"); + assert_d_eq(mallctl(cmd, (void *)&nfills, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); gen_mallctl_str(cmd, "nflushes", arena_ind); - assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0), - config_tcache ? expected : ENOENT, "Unexpected mallctl() result"); + assert_d_eq(mallctl(cmd, (void *)&nflushes, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); gen_mallctl_str(cmd, "nslabs", arena_ind); assert_d_eq(mallctl(cmd, (void *)&nslabs, &sz, NULL, 0), expected, @@ -292,7 +296,7 @@ TEST_BEGIN(test_stats_arenas_bins) { "nrequests should be greater than zero"); assert_zu_gt(curregs, 0, "allocated should be greater than zero"); - if (config_tcache) { + if (opt_tcache) { assert_u64_gt(nfills, 0, "At least one fill should have occurred"); assert_u64_gt(nflushes, 0, |