From 9389335b866e5920a3af9c1545c14931c1a9ef1a Mon Sep 17 00:00:00 2001 From: John Paul Adrian Glaubitz Date: Wed, 21 Dec 2016 09:38:54 +0100 Subject: Use better pre-processor defines for sparc64 Currently, jemalloc detects sparc64 targets by checking whether __sparc64__ is defined. However, this definition is used on BSD targets only. Linux targets define both __sparc__ and __arch64__ for sparc64. Since this also works on BSD, rather use __sparc__ and __arch64__ instead of __sparc64__ to detect sparc64 targets. --- include/jemalloc/internal/mb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/jemalloc/internal/mb.h b/include/jemalloc/internal/mb.h index 5384728..e58da5c 100644 --- a/include/jemalloc/internal/mb.h +++ b/include/jemalloc/internal/mb.h @@ -76,7 +76,7 @@ mb_write(void) : "memory" /* Clobbers. */ ); } -#elif defined(__sparc64__) +#elif defined(__sparc__) && defined(__arch64__) JEMALLOC_INLINE void mb_write(void) { -- cgit v0.12 From c68bb4179312665e22d375aecf9f4306607c7c1a Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 17 Jan 2017 15:54:36 +0900 Subject: Don't rely on OSX SDK malloc/malloc.h for malloc_zone struct definitions The SDK jemalloc is built against might be not be the latest for various reasons, but the resulting binary ought to work on newer versions of OSX. In order to ensure this, we need the fullest definitions possible, so copy what we need from the latest version of malloc/malloc.h available on opensource.apple.com. --- configure.ac | 31 ------ include/jemalloc/internal/jemalloc_internal.h.in | 1 - .../jemalloc/internal/jemalloc_internal_defs.h.in | 1 - src/zone.c | 122 +++++++++++++++------ 4 files changed, 86 insertions(+), 69 deletions(-) diff --git a/configure.ac b/configure.ac index 9573c30..4996406 100644 --- a/configure.ac +++ b/configure.ac @@ -1774,37 +1774,6 @@ if test "x${enable_zone_allocator}" = "x1" ; then AC_MSG_ERROR([--enable-zone-allocator is only supported on Darwin]) fi AC_DEFINE([JEMALLOC_ZONE], [ ]) - - dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 - dnl releases. malloc_zone_t and malloc_introspection_t have new fields in - dnl 10.6, which is the only source-level indication of the change. - AC_MSG_CHECKING([malloc zone version]) - AC_DEFUN([JE_ZONE_PROGRAM], - [AC_LANG_PROGRAM( - [#include ], - [static int foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]] - )]) - - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,14)],[JEMALLOC_ZONE_VERSION=3],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,15)],[JEMALLOC_ZONE_VERSION=5],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,16)],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,9)],[JEMALLOC_ZONE_VERSION=6],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,13)],[JEMALLOC_ZONE_VERSION=7],[JEMALLOC_ZONE_VERSION=] - )])],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,17)],[JEMALLOC_ZONE_VERSION=8],[ - AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,>,17)],[JEMALLOC_ZONE_VERSION=9],[JEMALLOC_ZONE_VERSION=] - )])])])]) - if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then - AC_MSG_RESULT([unsupported]) - AC_MSG_ERROR([Unsupported malloc zone version]) - fi - if test "${JEMALLOC_ZONE_VERSION}" = 9; then - JEMALLOC_ZONE_VERSION=8 - AC_MSG_RESULT([> 8]) - else - AC_MSG_RESULT([$JEMALLOC_ZONE_VERSION]) - fi - AC_DEFINE_UNQUOTED(JEMALLOC_ZONE_VERSION, [$JEMALLOC_ZONE_VERSION]) fi dnl ============================================================================ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index e7ace7d..6213dd8 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -158,7 +158,6 @@ static const bool config_cache_oblivious = #include #include #include -#include #endif #include "jemalloc/internal/ph.h" diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index def4ba5..b7ae3b7 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -239,7 +239,6 @@ * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ #undef JEMALLOC_ZONE -#undef JEMALLOC_ZONE_VERSION /* * Methods for determining whether the OS overcommits. diff --git a/src/zone.c b/src/zone.c index 0571920..d4805c5 100644 --- a/src/zone.c +++ b/src/zone.c @@ -3,6 +3,75 @@ # error "This source file is for zones on Darwin (OS X)." #endif +/* Definitions of the following structs in malloc/malloc.h might be too old + * for the built binary to run on newer versions of OSX. So use the newest + * possible version of those structs. + */ +typedef struct _malloc_zone_t { + void *reserved1; + void *reserved2; + size_t (*size)(struct _malloc_zone_t *, const void *); + void *(*malloc)(struct _malloc_zone_t *, size_t); + void *(*calloc)(struct _malloc_zone_t *, size_t, size_t); + void *(*valloc)(struct _malloc_zone_t *, size_t); + void (*free)(struct _malloc_zone_t *, void *); + void *(*realloc)(struct _malloc_zone_t *, void *, size_t); + void (*destroy)(struct _malloc_zone_t *); + const char *zone_name; + unsigned (*batch_malloc)(struct _malloc_zone_t *, size_t, void **, unsigned); + void (*batch_free)(struct _malloc_zone_t *, void **, unsigned); + struct malloc_introspection_t *introspect; + unsigned version; + void *(*memalign)(struct _malloc_zone_t *, size_t, size_t); + void (*free_definite_size)(struct _malloc_zone_t *, void *, size_t); + size_t (*pressure_relief)(struct _malloc_zone_t *, size_t); +} malloc_zone_t; + +typedef struct { + vm_address_t address; + vm_size_t size; +} vm_range_t; + +typedef struct malloc_statistics_t { + unsigned blocks_in_use; + size_t size_in_use; + size_t max_size_in_use; + size_t size_allocated; +} malloc_statistics_t; + +typedef kern_return_t memory_reader_t(task_t, vm_address_t, vm_size_t, void **); + +typedef void vm_range_recorder_t(task_t, void *, unsigned type, vm_range_t *, unsigned); + +typedef struct malloc_introspection_t { + kern_return_t (*enumerator)(task_t, void *, unsigned, vm_address_t, memory_reader_t, vm_range_recorder_t); + size_t (*good_size)(malloc_zone_t *, size_t); + boolean_t (*check)(malloc_zone_t *); + void (*print)(malloc_zone_t *, boolean_t); + void (*log)(malloc_zone_t *, void *); + void (*force_lock)(malloc_zone_t *); + void (*force_unlock)(malloc_zone_t *); + void (*statistics)(malloc_zone_t *, malloc_statistics_t *); + boolean_t (*zone_locked)(malloc_zone_t *); + boolean_t (*enable_discharge_checking)(malloc_zone_t *); + boolean_t (*disable_discharge_checking)(malloc_zone_t *); + void (*discharge)(malloc_zone_t *, void *); +#ifdef __BLOCKS__ + void (*enumerate_discharged_pointers)(malloc_zone_t *, void (^)(void *, void *)); +#else + void *enumerate_unavailable_without_blocks; +#endif + void (*reinit_lock)(malloc_zone_t *); +} malloc_introspection_t; + +extern kern_return_t malloc_get_all_zones(task_t, memory_reader_t, vm_address_t **, unsigned *); + +extern malloc_zone_t *malloc_default_zone(void); + +extern void malloc_zone_register(malloc_zone_t *zone); + +extern void malloc_zone_unregister(malloc_zone_t *zone); + /* * The malloc_default_purgeable_zone() function is only available on >= 10.6. * We need to check whether it is present at runtime, thus the weak_import. @@ -20,21 +89,17 @@ static struct malloc_introspection_t jemalloc_zone_introspect; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static size_t zone_size(malloc_zone_t *zone, void *ptr); +static size_t zone_size(malloc_zone_t *zone, const void *ptr); static void *zone_malloc(malloc_zone_t *zone, size_t size); static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); static void *zone_valloc(malloc_zone_t *zone, size_t size); static void zone_free(malloc_zone_t *zone, void *ptr); static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -#if (JEMALLOC_ZONE_VERSION >= 5) static void *zone_memalign(malloc_zone_t *zone, size_t alignment, -#endif -#if (JEMALLOC_ZONE_VERSION >= 6) size_t size); static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size); -#endif -static void *zone_destroy(malloc_zone_t *zone); +static void zone_destroy(malloc_zone_t *zone); static size_t zone_good_size(malloc_zone_t *zone, size_t size); static void zone_force_lock(malloc_zone_t *zone); static void zone_force_unlock(malloc_zone_t *zone); @@ -45,7 +110,7 @@ static void zone_force_unlock(malloc_zone_t *zone); */ static size_t -zone_size(malloc_zone_t *zone, void *ptr) +zone_size(malloc_zone_t *zone, const void *ptr) { /* @@ -106,7 +171,6 @@ zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) return (realloc(ptr, size)); } -#if (JEMALLOC_ZONE_VERSION >= 5) static void * zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) { @@ -116,9 +180,7 @@ zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) return (ret); } -#endif -#if (JEMALLOC_ZONE_VERSION >= 6) static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { @@ -133,15 +195,13 @@ zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) free(ptr); } -#endif -static void * +static void zone_destroy(malloc_zone_t *zone) { /* This function should never be called. */ not_reached(); - return (NULL); } static size_t @@ -180,48 +240,38 @@ static void zone_init(void) { - jemalloc_zone.size = (void *)zone_size; - jemalloc_zone.malloc = (void *)zone_malloc; - jemalloc_zone.calloc = (void *)zone_calloc; - jemalloc_zone.valloc = (void *)zone_valloc; - jemalloc_zone.free = (void *)zone_free; - jemalloc_zone.realloc = (void *)zone_realloc; - jemalloc_zone.destroy = (void *)zone_destroy; + jemalloc_zone.size = zone_size; + jemalloc_zone.malloc = zone_malloc; + jemalloc_zone.calloc = zone_calloc; + jemalloc_zone.valloc = zone_valloc; + jemalloc_zone.free = zone_free; + jemalloc_zone.realloc = zone_realloc; + jemalloc_zone.destroy = zone_destroy; jemalloc_zone.zone_name = "jemalloc_zone"; jemalloc_zone.batch_malloc = NULL; jemalloc_zone.batch_free = NULL; jemalloc_zone.introspect = &jemalloc_zone_introspect; - jemalloc_zone.version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 5) + jemalloc_zone.version = 8; jemalloc_zone.memalign = zone_memalign; -#endif -#if (JEMALLOC_ZONE_VERSION >= 6) jemalloc_zone.free_definite_size = zone_free_definite_size; -#endif -#if (JEMALLOC_ZONE_VERSION >= 8) jemalloc_zone.pressure_relief = NULL; -#endif jemalloc_zone_introspect.enumerator = NULL; - jemalloc_zone_introspect.good_size = (void *)zone_good_size; + jemalloc_zone_introspect.good_size = zone_good_size; jemalloc_zone_introspect.check = NULL; jemalloc_zone_introspect.print = NULL; jemalloc_zone_introspect.log = NULL; - jemalloc_zone_introspect.force_lock = (void *)zone_force_lock; - jemalloc_zone_introspect.force_unlock = (void *)zone_force_unlock; + jemalloc_zone_introspect.force_lock = zone_force_lock; + jemalloc_zone_introspect.force_unlock = zone_force_unlock; jemalloc_zone_introspect.statistics = NULL; -#if (JEMALLOC_ZONE_VERSION >= 6) jemalloc_zone_introspect.zone_locked = NULL; -#endif -#if (JEMALLOC_ZONE_VERSION >= 7) jemalloc_zone_introspect.enable_discharge_checking = NULL; jemalloc_zone_introspect.disable_discharge_checking = NULL; jemalloc_zone_introspect.discharge = NULL; -# ifdef __BLOCKS__ +#ifdef __BLOCKS__ jemalloc_zone_introspect.enumerate_discharged_pointers = NULL; -# else +#else jemalloc_zone_introspect.enumerate_unavailable_without_blocks = NULL; -# endif #endif } -- cgit v0.12 From c6943acb3c56d1b3d1e82dd43b3fcfeae7771990 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 17 Jan 2017 16:20:05 +0900 Subject: Add dummy implementations for most remaining OSX zone allocator functions Some system libraries are using malloc_default_zone() and then using some of the malloc_zone_* API. Under normal conditions, those functions check the malloc_zone_t/malloc_introspection_t struct for the values that are allowed to be NULL, so that a NULL deref doesn't happen. As of OSX 10.12, malloc_default_zone() doesn't return the actual default zone anymore, but returns a fake, wrapper zone. The wrapper zone defines all the possible functions in the malloc_zone_t/malloc_introspection_t struct (almost), and calls the function from the registered default zone (jemalloc in our case) on its own. Without checking whether the pointers are NULL. This means that a system library that calls e.g. malloc_zone_batch_malloc(malloc_default_zone(), ...) ends up trying to call jemalloc_zone.batch_malloc, which is NULL, and crash follows. So as of OSX 10.12, the default zone is required to have all the functions available (really, the same as the wrapper zone), even if they do nothing. This is arguably a bug in libsystem_malloc in OSX 10.12, but jemalloc still needs to work in that case. --- src/zone.c | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 108 insertions(+), 10 deletions(-) diff --git a/src/zone.c b/src/zone.c index d4805c5..6215133 100644 --- a/src/zone.c +++ b/src/zone.c @@ -100,9 +100,24 @@ static void *zone_memalign(malloc_zone_t *zone, size_t alignment, static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size); static void zone_destroy(malloc_zone_t *zone); +static unsigned zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, + void **results, unsigned num_requested); +static void zone_batch_free(struct _malloc_zone_t *zone, + void **to_be_freed, unsigned num_to_be_freed); +static size_t zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal); static size_t zone_good_size(malloc_zone_t *zone, size_t size); +static kern_return_t zone_enumerator(task_t task, void *data, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, + vm_range_recorder_t recorder); +static boolean_t zone_check(malloc_zone_t *zone); +static void zone_print(malloc_zone_t *zone, boolean_t verbose); +static void zone_log(malloc_zone_t *zone, void *address); static void zone_force_lock(malloc_zone_t *zone); static void zone_force_unlock(malloc_zone_t *zone); +static void zone_statistics(malloc_zone_t *zone, + malloc_statistics_t *stats); +static boolean_t zone_locked(malloc_zone_t *zone); +static void zone_reinit_lock(malloc_zone_t *zone); /******************************************************************************/ /* @@ -204,6 +219,39 @@ zone_destroy(malloc_zone_t *zone) not_reached(); } +static unsigned +zone_batch_malloc(struct _malloc_zone_t *zone, size_t size, void **results, + unsigned num_requested) +{ + unsigned i; + + for (i = 0; i < num_requested; i++) { + results[i] = je_malloc(size); + if (!results[i]) + break; + } + + return i; +} + +static void +zone_batch_free(struct _malloc_zone_t *zone, void **to_be_freed, + unsigned num_to_be_freed) +{ + unsigned i; + + for (i = 0; i < num_to_be_freed; i++) { + zone_free(zone, to_be_freed[i]); + to_be_freed[i] = NULL; + } +} + +static size_t +zone_pressure_relief(struct _malloc_zone_t *zone, size_t goal) +{ + return 0; +} + static size_t zone_good_size(malloc_zone_t *zone, size_t size) { @@ -213,6 +261,30 @@ zone_good_size(malloc_zone_t *zone, size_t size) return (s2u(size)); } +static kern_return_t +zone_enumerator(task_t task, void *data, unsigned type_mask, + vm_address_t zone_address, memory_reader_t reader, + vm_range_recorder_t recorder) +{ + return KERN_SUCCESS; +} + +static boolean_t +zone_check(malloc_zone_t *zone) +{ + return true; +} + +static void +zone_print(malloc_zone_t *zone, boolean_t verbose) +{ +} + +static void +zone_log(malloc_zone_t *zone, void *address) +{ +} + static void zone_force_lock(malloc_zone_t *zone) { @@ -237,6 +309,31 @@ zone_force_unlock(malloc_zone_t *zone) } static void +zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) +{ + /* We make no effort to actually fill the values */ + stats->blocks_in_use = 0; + stats->size_in_use = 0; + stats->max_size_in_use = 0; + stats->size_allocated = 0; +} + +static boolean_t +zone_locked(malloc_zone_t *zone) +{ + /* Pretend no lock is being held */ + return false; +} + +static void +zone_reinit_lock(malloc_zone_t *zone) +{ + /* As of OSX 10.12, this function is only used when force_unlock would + * be used if the zone version were < 9. So just use force_unlock. */ + zone_force_unlock(zone); +} + +static void zone_init(void) { @@ -248,23 +345,23 @@ zone_init(void) jemalloc_zone.realloc = zone_realloc; jemalloc_zone.destroy = zone_destroy; jemalloc_zone.zone_name = "jemalloc_zone"; - jemalloc_zone.batch_malloc = NULL; - jemalloc_zone.batch_free = NULL; + jemalloc_zone.batch_malloc = zone_batch_malloc; + jemalloc_zone.batch_free = zone_batch_free; jemalloc_zone.introspect = &jemalloc_zone_introspect; - jemalloc_zone.version = 8; + jemalloc_zone.version = 9; jemalloc_zone.memalign = zone_memalign; jemalloc_zone.free_definite_size = zone_free_definite_size; - jemalloc_zone.pressure_relief = NULL; + jemalloc_zone.pressure_relief = zone_pressure_relief; - jemalloc_zone_introspect.enumerator = NULL; + jemalloc_zone_introspect.enumerator = zone_enumerator; jemalloc_zone_introspect.good_size = zone_good_size; - jemalloc_zone_introspect.check = NULL; - jemalloc_zone_introspect.print = NULL; - jemalloc_zone_introspect.log = NULL; + jemalloc_zone_introspect.check = zone_check; + jemalloc_zone_introspect.print = zone_print; + jemalloc_zone_introspect.log = zone_log; jemalloc_zone_introspect.force_lock = zone_force_lock; jemalloc_zone_introspect.force_unlock = zone_force_unlock; - jemalloc_zone_introspect.statistics = NULL; - jemalloc_zone_introspect.zone_locked = NULL; + jemalloc_zone_introspect.statistics = zone_statistics; + jemalloc_zone_introspect.zone_locked = zone_locked; jemalloc_zone_introspect.enable_discharge_checking = NULL; jemalloc_zone_introspect.disable_discharge_checking = NULL; jemalloc_zone_introspect.discharge = NULL; @@ -273,6 +370,7 @@ zone_init(void) #else jemalloc_zone_introspect.enumerate_unavailable_without_blocks = NULL; #endif + jemalloc_zone_introspect.reinit_lock = zone_reinit_lock; } static malloc_zone_t * -- cgit v0.12 From dad74bd3c811ca2b1af1fd57b28f2456da5ba08b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 21 Jan 2017 15:12:03 -0800 Subject: Convert witness_assert_lockless() to witness_assert_lock_depth(). This makes it possible to make lock state assertions about precisely which locks are held. --- include/jemalloc/internal/private_symbols.txt | 4 +- include/jemalloc/internal/witness.h | 23 ++++++--- src/jemalloc.c | 74 +++++++++++++-------------- src/witness.c | 19 +++---- test/unit/witness.c | 63 +++++++++++++---------- 5 files changed, 102 insertions(+), 81 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index c1c6c40..4dfe442 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -612,14 +612,14 @@ valgrind_freelike_block valgrind_make_mem_defined valgrind_make_mem_noaccess valgrind_make_mem_undefined -witness_assert_lockless +witness_assert_lock_depth witness_assert_not_owner witness_assert_owner witness_fork_cleanup witness_init witness_lock witness_lock_error -witness_lockless_error +witness_lock_depth_error witness_not_owner_error witness_owner witness_owner_error diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index cdf15d7..dfd827f 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -91,10 +91,12 @@ extern witness_not_owner_error_t *witness_not_owner_error; void witness_not_owner_error(const witness_t *witness); #endif #ifdef JEMALLOC_JET -typedef void (witness_lockless_error_t)(const witness_list_t *); -extern witness_lockless_error_t *witness_lockless_error; +typedef void (witness_lock_depth_error_t)(const witness_list_t *, + unsigned depth); +extern witness_lock_depth_error_t *witness_lock_depth_error; #else -void witness_lockless_error(const witness_list_t *witnesses); +void witness_lock_depth_error(const witness_list_t *witnesses, + unsigned depth); #endif void witnesses_cleanup(tsd_t *tsd); @@ -111,7 +113,7 @@ void witness_postfork_child(tsd_t *tsd); bool witness_owner(tsd_t *tsd, const witness_t *witness); void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_lockless(tsdn_t *tsdn); +void witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth); void witness_lock(tsdn_t *tsdn, witness_t *witness); void witness_unlock(tsdn_t *tsdn, witness_t *witness); #endif @@ -175,9 +177,10 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) } JEMALLOC_INLINE void -witness_assert_lockless(tsdn_t *tsdn) +witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth) { tsd_t *tsd; + unsigned d; witness_list_t *witnesses; witness_t *w; @@ -188,10 +191,16 @@ witness_assert_lockless(tsdn_t *tsdn) return; tsd = tsdn_tsd(tsdn); + d = 0; witnesses = tsd_witnessesp_get(tsd); w = ql_last(witnesses, link); - if (w != NULL) - witness_lockless_error(witnesses); + if (w != NULL) { + ql_foreach(w, witnesses, link) { + d++; + } + } + if (d != depth) + witness_lock_depth_error(witnesses, depth); } JEMALLOC_INLINE void diff --git a/src/jemalloc.c b/src/jemalloc.c index baead66..c08f7e2 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1581,7 +1581,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ind = size2index(size); if (unlikely(ind >= NSIZES)) @@ -1619,7 +1619,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, assert(usize == isalloc(tsdn, ret, config_prof)); *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; } - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN @@ -1704,7 +1704,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) goto label_oom; } tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (size == 0) size = 1; @@ -1745,7 +1745,7 @@ label_return: UTRACE(0, size, result); JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, false); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); label_oom: assert(result == NULL); @@ -1755,7 +1755,7 @@ label_oom: abort(); } ret = ENOMEM; - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); goto label_return; } @@ -1873,7 +1873,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) size_t usize; UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1901,7 +1901,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1947,7 +1947,7 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) { @@ -1994,7 +1994,7 @@ je_realloc(void *ptr, size_t size) UTRACE(ptr, size, ret); JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, old_usize, old_rzsize, maybe, false); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } @@ -2005,12 +2005,12 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { tsd_t *tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (likely(!malloc_slow)) ifree(tsd, ptr, tcache_get(tsd, false), false); else ifree(tsd, ptr, tcache_get(tsd, false), true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } } @@ -2239,7 +2239,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (likely(flags == 0)) { szind_t ind = size2index(size); @@ -2374,7 +2374,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -2421,7 +2421,7 @@ je_rallocx(void *ptr, size_t size, int flags) UTRACE(ptr, size, p); JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, old_usize, old_rzsize, no, zero); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (p); label_oom: if (config_xmalloc && unlikely(opt_xmalloc)) { @@ -2429,7 +2429,7 @@ label_oom: abort(); } UTRACE(ptr, size, 0); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (NULL); } @@ -2525,7 +2525,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); @@ -2566,7 +2566,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) old_usize, old_rzsize, no, zero); label_not_resized: UTRACE(ptr, size, ptr); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (usize); } @@ -2581,14 +2581,14 @@ je_sallocx(const void *ptr, int flags) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (config_ivsalloc) usize = ivsalloc(tsdn, ptr, config_prof); else usize = isalloc(tsdn, ptr, config_prof); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2602,7 +2602,7 @@ je_dallocx(void *ptr, int flags) assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2616,7 +2616,7 @@ je_dallocx(void *ptr, int flags) ifree(tsd, ptr, tcache, false); else ifree(tsd, ptr, tcache, true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -2624,13 +2624,13 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { size_t usize; - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) usize = s2u(size); else usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2647,7 +2647,7 @@ je_sdallocx(void *ptr, size_t size, int flags) usize = inallocx(tsd_tsdn(tsd), size, flags); assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2661,7 +2661,7 @@ je_sdallocx(void *ptr, size_t size, int flags) isfree(tsd, ptr, usize, tcache, false); else isfree(tsd, ptr, usize, tcache, true); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2677,13 +2677,13 @@ je_nallocx(size_t size, int flags) return (0); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); usize = inallocx(tsdn, size, flags); if (unlikely(usize > HUGE_MAXCLASS)) return (0); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (usize); } @@ -2698,9 +2698,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); } @@ -2714,9 +2714,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) return (EAGAIN); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); ret = ctl_nametomib(tsdn, name, mibp, miblenp); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } @@ -2731,9 +2731,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); return (ret); } @@ -2744,9 +2744,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, tsdn_t *tsdn; tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); stats_print(write_cb, cbopaque, opts); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2759,14 +2759,14 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); if (config_ivsalloc) ret = ivsalloc(tsdn, ptr, config_prof); else ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); return (ret); } diff --git a/src/witness.c b/src/witness.c index 23753f2..aaea88d 100644 --- a/src/witness.c +++ b/src/witness.c @@ -71,15 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error = #endif #ifdef JEMALLOC_JET -#undef witness_lockless_error -#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error) +#undef witness_lock_depth_error +#define witness_lock_depth_error JEMALLOC_N(n_witness_lock_depth_error) #endif void -witness_lockless_error(const witness_list_t *witnesses) +witness_lock_depth_error(const witness_list_t *witnesses, unsigned depth) { witness_t *w; - malloc_printf(": Should not own any locks:"); + malloc_printf(": Should own %u lock%s:", depth, (depth != 1) ? + "s" : ""); ql_foreach(w, witnesses, link) { malloc_printf(" %s(%u)", w->name, w->rank); } @@ -87,17 +88,17 @@ witness_lockless_error(const witness_list_t *witnesses) abort(); } #ifdef JEMALLOC_JET -#undef witness_lockless_error -#define witness_lockless_error JEMALLOC_N(witness_lockless_error) -witness_lockless_error_t *witness_lockless_error = - JEMALLOC_N(n_witness_lockless_error); +#undef witness_lock_depth_error +#define witness_lock_depth_error JEMALLOC_N(witness_lock_depth_error) +witness_lock_depth_error_t *witness_lock_depth_error = + JEMALLOC_N(n_witness_lock_depth_error); #endif void witnesses_cleanup(tsd_t *tsd) { - witness_assert_lockless(tsd_tsdn(tsd)); + witness_assert_lock_depth(tsd_tsdn(tsd), 0); /* Do nothing. */ } diff --git a/test/unit/witness.c b/test/unit/witness.c index ed17275..9d4a171 100644 --- a/test/unit/witness.c +++ b/test/unit/witness.c @@ -3,12 +3,12 @@ static witness_lock_error_t *witness_lock_error_orig; static witness_owner_error_t *witness_owner_error_orig; static witness_not_owner_error_t *witness_not_owner_error_orig; -static witness_lockless_error_t *witness_lockless_error_orig; +static witness_lock_depth_error_t *witness_lock_depth_error_orig; static bool saw_lock_error; static bool saw_owner_error; static bool saw_not_owner_error; -static bool saw_lockless_error; +static bool saw_lock_depth_error; static void witness_lock_error_intercept(const witness_list_t *witnesses, @@ -33,10 +33,11 @@ witness_not_owner_error_intercept(const witness_t *witness) } static void -witness_lockless_error_intercept(const witness_list_t *witnesses) +witness_lock_depth_error_intercept(const witness_list_t *witnesses, + unsigned depth) { - saw_lockless_error = true; + saw_lock_depth_error = true; } static int @@ -66,22 +67,25 @@ TEST_BEGIN(test_witness) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_init(&b, "b", 2, NULL); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); + witness_assert_lock_depth(tsdn, 2); witness_unlock(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &b); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); } TEST_END @@ -94,18 +98,21 @@ TEST_BEGIN(test_witness_comp) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, witness_comp); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_init(&b, "b", 1, witness_comp); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); + witness_assert_lock_depth(tsdn, 2); witness_unlock(tsdn, &b); + witness_assert_lock_depth(tsdn, 1); witness_lock_error_orig = witness_lock_error; witness_lock_error = witness_lock_error_intercept; @@ -117,6 +124,7 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &c); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &c); + witness_assert_lock_depth(tsdn, 1); saw_lock_error = false; @@ -126,10 +134,11 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &d); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &d); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_lock_error = witness_lock_error_orig; } @@ -148,20 +157,22 @@ TEST_BEGIN(test_witness_reversal) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); witness_init(&b, "b", 2, NULL); witness_lock(tsdn, &b); + witness_assert_lock_depth(tsdn, 1); assert_false(saw_lock_error, "Unexpected witness lock error"); witness_lock(tsdn, &a); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &a); + witness_assert_lock_depth(tsdn, 1); witness_unlock(tsdn, &b); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_lock_error = witness_lock_error_orig; } @@ -184,7 +195,7 @@ TEST_BEGIN(test_witness_recursive) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); @@ -197,7 +208,7 @@ TEST_BEGIN(test_witness_recursive) witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_owner_error = witness_owner_error_orig; witness_lock_error = witness_lock_error_orig; @@ -218,7 +229,7 @@ TEST_BEGIN(test_witness_unlock_not_owned) tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); @@ -226,41 +237,41 @@ TEST_BEGIN(test_witness_unlock_not_owned) witness_unlock(tsdn, &a); assert_true(saw_owner_error, "Expected owner error"); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_owner_error = witness_owner_error_orig; } TEST_END -TEST_BEGIN(test_witness_lockful) +TEST_BEGIN(test_witness_lock_depth) { witness_t a; tsdn_t *tsdn; test_skip_if(!config_debug); - witness_lockless_error_orig = witness_lockless_error; - witness_lockless_error = witness_lockless_error_intercept; - saw_lockless_error = false; + witness_lock_depth_error_orig = witness_lock_depth_error; + witness_lock_depth_error = witness_lock_depth_error_intercept; + saw_lock_depth_error = false; tsdn = tsdn_fetch(); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); - assert_false(saw_lockless_error, "Unexpected lockless error"); - witness_assert_lockless(tsdn); + assert_false(saw_lock_depth_error, "Unexpected lock_depth error"); + witness_assert_lock_depth(tsdn, 0); witness_lock(tsdn, &a); - witness_assert_lockless(tsdn); - assert_true(saw_lockless_error, "Expected lockless error"); + witness_assert_lock_depth(tsdn, 0); + assert_true(saw_lock_depth_error, "Expected lock_depth error"); witness_unlock(tsdn, &a); - witness_assert_lockless(tsdn); + witness_assert_lock_depth(tsdn, 0); - witness_lockless_error = witness_lockless_error_orig; + witness_lock_depth_error = witness_lock_depth_error_orig; } TEST_END @@ -274,5 +285,5 @@ main(void) test_witness_reversal, test_witness_recursive, test_witness_unlock_not_owned, - test_witness_lockful)); + test_witness_lock_depth)); } -- cgit v0.12 From b49c649bc18fff4bd10a1c8adbaf1f25f6453cb6 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 20 Jan 2017 17:36:56 -0800 Subject: Fix lock order reversal during gdump. --- include/jemalloc/internal/chunk.h | 4 ++-- src/arena.c | 41 ++++++++++++++++++++++++++------------- src/chunk.c | 5 ++--- src/huge.c | 41 +++++++++++++++++++++++++++------------ 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 50b9904..55df9ac 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -52,8 +52,8 @@ chunk_hooks_t chunk_hooks_get(tsdn_t *tsdn, arena_t *arena); chunk_hooks_t chunk_hooks_set(tsdn_t *tsdn, arena_t *arena, const chunk_hooks_t *chunk_hooks); -bool chunk_register(tsdn_t *tsdn, const void *chunk, - const extent_node_t *node); +bool chunk_register(const void *chunk, const extent_node_t *node, + bool *gdump); void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, diff --git a/src/arena.c b/src/arena.c index 648a8da..193a4a2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -568,8 +568,8 @@ arena_chunk_init_spare(arena_t *arena) } static bool -arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, - size_t sn, bool zero) +arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, size_t sn, bool zero, + bool *gdump) { /* @@ -580,7 +580,7 @@ arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, */ extent_node_init(&chunk->node, arena, chunk, chunksize, sn, zero, true); extent_node_achunk_set(&chunk->node, true); - return (chunk_register(tsdn, chunk, &chunk->node)); + return (chunk_register(chunk, &chunk->node, gdump)); } static arena_chunk_t * @@ -591,6 +591,7 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, size_t sn; malloc_mutex_unlock(tsdn, &arena->lock); + witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, chunksize, chunksize, &sn, zero, commit); @@ -603,16 +604,20 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, chunk = NULL; } } - if (chunk != NULL && arena_chunk_register(tsdn, arena, chunk, sn, - *zero)) { - if (!*commit) { - /* Undo commit of header. */ - chunk_hooks->decommit(chunk, chunksize, 0, map_bias << - LG_PAGE, arena->ind); + if (chunk != NULL) { + bool gdump; + if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) { + if (!*commit) { + /* Undo commit of header. */ + chunk_hooks->decommit(chunk, chunksize, 0, + map_bias << LG_PAGE, arena->ind); + } + chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, + (void *)chunk, chunksize, sn, *zero, *commit); + chunk = NULL; } - chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, (void *)chunk, - chunksize, sn, *zero, *commit); - chunk = NULL; + if (config_prof && opt_prof && gdump) + prof_gdump(tsdn); } malloc_mutex_lock(tsdn, &arena->lock); @@ -627,14 +632,24 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; size_t sn; + /* prof_gdump() requirement. */ + witness_assert_lock_depth(tsdn, 1); + malloc_mutex_assert_owner(tsdn, &arena->lock); + chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, chunksize, &sn, zero, commit, true); if (chunk != NULL) { - if (arena_chunk_register(tsdn, arena, chunk, sn, *zero)) { + bool gdump; + if (arena_chunk_register(arena, chunk, sn, *zero, &gdump)) { chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk, chunksize, sn, true); return (NULL); } + if (config_prof && opt_prof && gdump) { + malloc_mutex_unlock(tsdn, &arena->lock); + prof_gdump(tsdn); + malloc_mutex_lock(tsdn, &arena->lock); + } } if (chunk == NULL) { chunk = arena_chunk_alloc_internal_hard(tsdn, arena, diff --git a/src/chunk.c b/src/chunk.c index c1c514a..de3bf4c 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -141,7 +141,7 @@ chunk_hooks_assure_initialized(tsdn_t *tsdn, arena_t *arena, } bool -chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node) +chunk_register(const void *chunk, const extent_node_t *node, bool *gdump) { assert(extent_node_addr_get(node) == chunk); @@ -160,8 +160,7 @@ chunk_register(tsdn_t *tsdn, const void *chunk, const extent_node_t *node) */ high = atomic_read_z(&highchunks); } - if (cur > high && prof_gdump_get_unlocked()) - prof_gdump(tsdn); + *gdump = (cur > high && prof_gdump_get_unlocked()); } return (false); diff --git a/src/huge.c b/src/huge.c index 8abd8c0..9a91bed 100644 --- a/src/huge.c +++ b/src/huge.c @@ -15,20 +15,20 @@ huge_node_get(const void *ptr) } static bool -huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node) +huge_node_set(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump) { assert(extent_node_addr_get(node) == ptr); assert(!extent_node_achunk_get(node)); - return (chunk_register(tsdn, ptr, node)); + return (chunk_register(ptr, node, gdump)); } static void -huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node) +huge_node_reset(tsdn_t *tsdn, const void *ptr, extent_node_t *node, bool *gdump) { bool err; - err = huge_node_set(tsdn, ptr, node); + err = huge_node_set(tsdn, ptr, node, gdump); assert(!err); } @@ -57,11 +57,12 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, arena_t *iarena; extent_node_t *node; size_t sn; - bool is_zeroed; + bool is_zeroed, gdump; /* Allocate one or more contiguous chunks for this request. */ assert(!tsdn_null(tsdn) || arena != NULL); + witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ ausize = sa2u(usize, alignment); if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) @@ -91,11 +92,13 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, extent_node_init(node, arena, ret, usize, sn, is_zeroed, true); - if (huge_node_set(tsdn, ret, node)) { + if (huge_node_set(tsdn, ret, node, &gdump)) { arena_chunk_dalloc_huge(tsdn, arena, ret, usize, sn); idalloctm(tsdn, node, NULL, true, true); return (NULL); } + if (config_prof && opt_prof && gdump) + prof_gdump(tsdn); /* Insert node into huge. */ malloc_mutex_lock(tsdn, &arena->huge_mtx); @@ -144,7 +147,9 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, extent_node_t *node; arena_t *arena; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; - bool pre_zeroed, post_zeroed; + bool pre_zeroed, post_zeroed, gdump; + + witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ /* Increase usize to incorporate extra. */ for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) @@ -178,10 +183,13 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, huge_node_unset(ptr, node); assert(extent_node_size_get(node) != usize); extent_node_size_set(node, usize); - huge_node_reset(tsdn, ptr, node); + huge_node_reset(tsdn, ptr, node, &gdump); /* Update zeroed. */ extent_node_zeroed_set(node, post_zeroed); malloc_mutex_unlock(tsdn, &arena->huge_mtx); + /* gdump without any locks held. */ + if (config_prof && opt_prof && gdump) + prof_gdump(tsdn); arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize); @@ -207,7 +215,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, arena_t *arena; chunk_hooks_t chunk_hooks; size_t cdiff; - bool pre_zeroed, post_zeroed; + bool pre_zeroed, post_zeroed, gdump; node = huge_node_get(ptr); arena = extent_node_arena_get(node); @@ -215,6 +223,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, chunk_hooks = chunk_hooks_get(tsdn, arena); assert(oldsize > usize); + witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ /* Split excess chunks. */ cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); @@ -241,10 +250,13 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, /* Update the size of the huge allocation. */ huge_node_unset(ptr, node); extent_node_size_set(node, usize); - huge_node_reset(tsdn, ptr, node); + huge_node_reset(tsdn, ptr, node, &gdump); /* Update zeroed. */ extent_node_zeroed_set(node, post_zeroed); malloc_mutex_unlock(tsdn, &arena->huge_mtx); + /* gdump without any locks held. */ + if (config_prof && opt_prof && gdump) + prof_gdump(tsdn); /* Zap the excess chunks. */ arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize, @@ -258,7 +270,7 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize, bool zero) { extent_node_t *node; arena_t *arena; - bool is_zeroed_subchunk, is_zeroed_chunk; + bool is_zeroed_subchunk, is_zeroed_chunk, gdump; node = huge_node_get(ptr); arena = extent_node_arena_get(node); @@ -266,6 +278,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, is_zeroed_subchunk = extent_node_zeroed_get(node); malloc_mutex_unlock(tsdn, &arena->huge_mtx); + witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + /* * Use is_zeroed_chunk to detect whether the trailing memory is zeroed, * update extent's zeroed field, and zero as necessary. @@ -280,8 +294,11 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, extent_node_size_set(node, usize); extent_node_zeroed_set(node, extent_node_zeroed_get(node) && is_zeroed_chunk); - huge_node_reset(tsdn, ptr, node); + huge_node_reset(tsdn, ptr, node, &gdump); malloc_mutex_unlock(tsdn, &arena->huge_mtx); + /* gdump without any locks held. */ + if (config_prof && opt_prof && gdump) + prof_gdump(tsdn); if (zero || (config_fill && unlikely(opt_zero))) { if (!is_zeroed_subchunk) { -- cgit v0.12 From b973ec797587778c6bb35f51c8f837a2ae6366cc Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Tue, 24 Jan 2017 14:54:18 -0500 Subject: Avoid redeclaring glibc's secure_getenv Avoid the name secure_getenv to avoid redeclaring secure_getenv when secure_getenv is present but its use is manually disabled via ac_cv_func_secure_getenv=no. --- src/jemalloc.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index c08f7e2..92813b6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -790,18 +790,19 @@ stats_print_atexit(void) * Begin initialization functions. */ -#ifndef JEMALLOC_HAVE_SECURE_GETENV static char * -secure_getenv(const char *name) +jemalloc_secure_getenv(const char *name) { - +#ifdef JEMALLOC_HAVE_SECURE_GETENV + return secure_getenv(name); +#else # ifdef JEMALLOC_HAVE_ISSETUGID if (issetugid() != 0) return (NULL); # endif return (getenv(name)); -} #endif +} static unsigned malloc_ncpus(void) @@ -1018,7 +1019,7 @@ malloc_conf_init(void) #endif ; - if ((opts = secure_getenv(envname)) != NULL) { + if ((opts = jemalloc_secure_getenv(envname)) != NULL) { /* * Do nothing; opts is already initialized to * the value of the MALLOC_CONF environment -- cgit v0.12 From d4f3f9a03f0ba199a7b51c93bdebe8236e0105da Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Wed, 25 Jan 2017 12:58:50 -0800 Subject: Beef up travis CI integration testing Introduces gen_travis.py, which generates .travis.yml, and updates .travis.yml to be the generated version. The travis build matrix approach doesn't play well with mixing and matching various different environment settings, so we generate every build explicitly, rather than letting them do it for us. To avoid abusing travis resources (and save us time waiting for CI results), we don't test every possible combination of options; we only check up to 2 unusual settings at a time. --- .travis.yml | 88 ++++++++++++++++++++++++++++++++++++++++++++------- scripts/gen_travis.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 11 deletions(-) create mode 100755 scripts/gen_travis.py diff --git a/.travis.yml b/.travis.yml index 1fed4f8..b563928 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,95 @@ -language: c +language: generic matrix: include: - os: linux - compiler: gcc + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="" - os: linux - compiler: gcc - env: - - EXTRA_FLAGS=-m32 + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" addons: apt: packages: - - gcc-multilib + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" - os: osx - compiler: clang + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="" - os: osx - compiler: clang - env: - - EXTRA_FLAGS=-m32 + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: osx + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" + - os: linux + env: CC=clang COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" + - os: linux + env: CC=clang COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-tcache" + addons: + apt: + packages: + - gcc-multilib + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-tcache" + - os: linux + env: CC=gcc COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-tcache" + before_script: - autoconf - - ./configure${EXTRA_FLAGS:+ CC="$CC $EXTRA_FLAGS"} + - ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" } $CONFIGURE_FLAGS - make -j3 - make -j3 tests script: - make check + diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py new file mode 100755 index 0000000..ccbcaf8 --- /dev/null +++ b/scripts/gen_travis.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +from itertools import combinations + +travis_template = """\ +language: generic + +matrix: + include: +%s + +before_script: + - autoconf + - ./configure ${COMPILER_FLAGS:+ \ + CC="$CC $COMPILER_FLAGS" } \ + $CONFIGURE_FLAGS + - make -j3 + - make -j3 tests + +script: + - make check +""" + +# The 'default' configuration is gcc, on linux, with no compiler or configure +# flags. We also test with clang, -m32, --enable-debug, --enable-prof, +# --disable-stats, and --disable-tcache. To avoid abusing travis though, we +# don't test all 2**7 = 128 possible combinations of these; instead, we only +# test combinations of up to 2 'unusual' settings, under the hope that bugs +# involving interactions of such settings are rare. +# things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29 +MAX_UNUSUAL_OPTIONS = 2 + +os_default = 'linux' +os_unusual = 'osx' + +compilers_default = 'CC=gcc' +compilers_unusual = 'CC=clang' + +compiler_flag_unusuals = ['-m32'] + +configure_flag_unusuals = [ + '--enable-debug', '--enable-prof', '--disable-stats', '--disable-tcache', +] + +all_unusuals = ( + [os_unusual] + [compilers_unusual] + compiler_flag_unusuals + + configure_flag_unusuals +) + +unusual_combinations_to_test = [] +for i in xrange(MAX_UNUSUAL_OPTIONS + 1): + unusual_combinations_to_test += combinations(all_unusuals, i) + +include_rows = "" +for unusual_combination in unusual_combinations_to_test: + os = os_default + if os_unusual in unusual_combination: + os = os_unusual + + compilers = compilers_default + if compilers_unusual in unusual_combination: + compilers = compilers_unusual + + compiler_flags = [ + x for x in unusual_combination if x in compiler_flag_unusuals] + + configure_flags = [ + x for x in unusual_combination if x in configure_flag_unusuals] + + # Filter out an unsupported configuration - heap profiling on OS X. + if os == 'osx' and '--enable-prof' in configure_flags: + continue + + env_string = '{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}"'.format( + compilers, " ".join(compiler_flags), " ".join(configure_flags)) + + include_rows += ' - os: %s\n' % os + include_rows += ' env: %s\n' % env_string + if '-m32' in unusual_combination and os == 'linux': + include_rows += ' addons:\n' + include_rows += ' apt:\n' + include_rows += ' packages:\n' + include_rows += ' - gcc-multilib\n' + +print travis_template % include_rows -- cgit v0.12 From fdba5ad5cc67cac8bfc247b407df1fc43e6551f9 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 21 Feb 2017 13:15:55 -0800 Subject: Repair file permissions. This regression was caused by 8f61fdedb908c29905103b22dda32ceb29cd8ede (Uniformly cast mallctl[bymib]() oldp/newp arguments to (void *).). This resolves #538. --- msvc/projects/vc2015/test_threads/test_threads.cpp | 0 src/stats.c | 0 src/tcache.c | 0 src/util.c | 0 test/integration/MALLOCX_ARENA.c | 0 test/integration/allocated.c | 0 test/integration/mallocx.c | 0 test/integration/overflow.c | 0 test/integration/rallocx.c | 0 test/integration/thread_arena.c | 0 test/integration/thread_tcache_enabled.c | 0 test/integration/xallocx.c | 0 test/unit/arena_reset.c | 0 test/unit/decay.c | 0 test/unit/mallctl.c | 0 test/unit/prof_accum.c | 0 test/unit/prof_active.c | 0 test/unit/prof_gdump.c | 0 test/unit/prof_idump.c | 0 test/unit/prof_reset.c | 0 test/unit/prof_thread_name.c | 0 test/unit/size_classes.c | 0 test/unit/stats.c | 0 23 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 msvc/projects/vc2015/test_threads/test_threads.cpp mode change 100755 => 100644 src/stats.c mode change 100755 => 100644 src/tcache.c mode change 100755 => 100644 src/util.c mode change 100755 => 100644 test/integration/MALLOCX_ARENA.c mode change 100755 => 100644 test/integration/allocated.c mode change 100755 => 100644 test/integration/mallocx.c mode change 100755 => 100644 test/integration/overflow.c mode change 100755 => 100644 test/integration/rallocx.c mode change 100755 => 100644 test/integration/thread_arena.c mode change 100755 => 100644 test/integration/thread_tcache_enabled.c mode change 100755 => 100644 test/integration/xallocx.c mode change 100755 => 100644 test/unit/arena_reset.c mode change 100755 => 100644 test/unit/decay.c mode change 100755 => 100644 test/unit/mallctl.c mode change 100755 => 100644 test/unit/prof_accum.c mode change 100755 => 100644 test/unit/prof_active.c mode change 100755 => 100644 test/unit/prof_gdump.c mode change 100755 => 100644 test/unit/prof_idump.c mode change 100755 => 100644 test/unit/prof_reset.c mode change 100755 => 100644 test/unit/prof_thread_name.c mode change 100755 => 100644 test/unit/size_classes.c mode change 100755 => 100644 test/unit/stats.c diff --git a/msvc/projects/vc2015/test_threads/test_threads.cpp b/msvc/projects/vc2015/test_threads/test_threads.cpp old mode 100755 new mode 100644 diff --git a/src/stats.c b/src/stats.c old mode 100755 new mode 100644 diff --git a/src/tcache.c b/src/tcache.c old mode 100755 new mode 100644 diff --git a/src/util.c b/src/util.c old mode 100755 new mode 100644 diff --git a/test/integration/MALLOCX_ARENA.c b/test/integration/MALLOCX_ARENA.c old mode 100755 new mode 100644 diff --git a/test/integration/allocated.c b/test/integration/allocated.c old mode 100755 new mode 100644 diff --git a/test/integration/mallocx.c b/test/integration/mallocx.c old mode 100755 new mode 100644 diff --git a/test/integration/overflow.c b/test/integration/overflow.c old mode 100755 new mode 100644 diff --git a/test/integration/rallocx.c b/test/integration/rallocx.c old mode 100755 new mode 100644 diff --git a/test/integration/thread_arena.c b/test/integration/thread_arena.c old mode 100755 new mode 100644 diff --git a/test/integration/thread_tcache_enabled.c b/test/integration/thread_tcache_enabled.c old mode 100755 new mode 100644 diff --git a/test/integration/xallocx.c b/test/integration/xallocx.c old mode 100755 new mode 100644 diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c old mode 100755 new mode 100644 diff --git a/test/unit/decay.c b/test/unit/decay.c old mode 100755 new mode 100644 diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_accum.c b/test/unit/prof_accum.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_active.c b/test/unit/prof_active.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_gdump.c b/test/unit/prof_gdump.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_idump.c b/test/unit/prof_idump.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c old mode 100755 new mode 100644 diff --git a/test/unit/prof_thread_name.c b/test/unit/prof_thread_name.c old mode 100755 new mode 100644 diff --git a/test/unit/size_classes.c b/test/unit/size_classes.c old mode 100755 new mode 100644 diff --git a/test/unit/stats.c b/test/unit/stats.c old mode 100755 new mode 100644 -- cgit v0.12 From 3ecc3c84862ef3e66b20be8213b0301c06c692cc Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 29 Jan 2017 21:32:39 -0800 Subject: Fix/refactor tcaches synchronization. Synchronize tcaches with tcaches_mtx rather than ctl_mtx. Add missing synchronization for tcache flushing. This bug was introduced by 1cb181ed632e7573fb4eab194e4d216867222d27 (Implement explicit tcache support.), which was first released in 4.0.0. --- include/jemalloc/internal/private_symbols.txt | 3 + include/jemalloc/internal/tcache.h | 3 + include/jemalloc/internal/witness.h | 21 ++--- src/ctl.c | 4 +- src/jemalloc.c | 3 + src/tcache.c | 114 ++++++++++++++++++++------ 6 files changed, 110 insertions(+), 38 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 4dfe442..6111eac 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -529,6 +529,9 @@ tcache_flush tcache_get tcache_get_hard tcache_maxclass +tcache_prefork +tcache_postfork_child +tcache_postfork_parent tcache_salloc tcache_stats_merge tcaches diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 01ba062..5fe5ebf 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -149,6 +149,9 @@ bool tcaches_create(tsd_t *tsd, unsigned *r_ind); void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(tsdn_t *tsdn); +void tcache_prefork(tsdn_t *tsdn); +void tcache_postfork_parent(tsdn_t *tsdn); +void tcache_postfork_child(tsdn_t *tsdn); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index dfd827f..e64e56e 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -14,19 +14,20 @@ typedef int witness_comp_t (const witness_t *, const witness_t *); #define WITNESS_RANK_INIT 1U #define WITNESS_RANK_CTL 1U -#define WITNESS_RANK_ARENAS 2U +#define WITNESS_RANK_TCACHES 2U +#define WITNESS_RANK_ARENAS 3U -#define WITNESS_RANK_PROF_DUMP 3U -#define WITNESS_RANK_PROF_BT2GCTX 4U -#define WITNESS_RANK_PROF_TDATAS 5U -#define WITNESS_RANK_PROF_TDATA 6U -#define WITNESS_RANK_PROF_GCTX 7U +#define WITNESS_RANK_PROF_DUMP 4U +#define WITNESS_RANK_PROF_BT2GCTX 5U +#define WITNESS_RANK_PROF_TDATAS 6U +#define WITNESS_RANK_PROF_TDATA 7U +#define WITNESS_RANK_PROF_GCTX 8U -#define WITNESS_RANK_ARENA 8U -#define WITNESS_RANK_ARENA_CHUNKS 9U -#define WITNESS_RANK_ARENA_NODE_CACHE 10 +#define WITNESS_RANK_ARENA 9U +#define WITNESS_RANK_ARENA_CHUNKS 10U +#define WITNESS_RANK_ARENA_NODE_CACHE 11U -#define WITNESS_RANK_BASE 11U +#define WITNESS_RANK_BASE 12U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF diff --git a/src/ctl.c b/src/ctl.c index bc78b20..1e62e2d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1476,7 +1476,6 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, if (!config_tcache) return (ENOENT); - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); READONLY(); if (tcaches_create(tsd, &tcache_ind)) { ret = EFAULT; @@ -1486,8 +1485,7 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, ret = 0; label_return: - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); - return (ret); + return ret; } static int diff --git a/src/jemalloc.c b/src/jemalloc.c index 92813b6..a376e14 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2828,6 +2828,7 @@ _malloc_prefork(void) witness_prefork(tsd); /* Acquire all mutexes in a safe order. */ ctl_prefork(tsd_tsdn(tsd)); + tcache_prefork(tsd_tsdn(tsd)); malloc_mutex_prefork(tsd_tsdn(tsd), &arenas_lock); prof_prefork0(tsd_tsdn(tsd)); for (i = 0; i < 3; i++) { @@ -2887,6 +2888,7 @@ _malloc_postfork(void) } prof_postfork_parent(tsd_tsdn(tsd)); malloc_mutex_postfork_parent(tsd_tsdn(tsd), &arenas_lock); + tcache_postfork_parent(tsd_tsdn(tsd)); ctl_postfork_parent(tsd_tsdn(tsd)); } @@ -2911,6 +2913,7 @@ jemalloc_postfork_child(void) } prof_postfork_child(tsd_tsdn(tsd)); malloc_mutex_postfork_child(tsd_tsdn(tsd), &arenas_lock); + tcache_postfork_child(tsd_tsdn(tsd)); ctl_postfork_child(tsd_tsdn(tsd)); } diff --git a/src/tcache.c b/src/tcache.c index 21540ff..e3b04be 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -21,6 +21,9 @@ static unsigned tcaches_past; /* Head of singly linked list tracking available tcaches elements. */ static tcaches_t *tcaches_avail; +/* Protects tcaches{,_past,_avail}. */ +static malloc_mutex_t tcaches_mtx; + /******************************************************************************/ size_t @@ -444,29 +447,56 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) } } +static bool +tcaches_create_prep(tsd_t *tsd) { + bool err; + + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); + + if (tcaches == NULL) { + tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) * + (MALLOCX_TCACHE_MAX+1)); + if (tcaches == NULL) { + err = true; + goto label_return; + } + } + + if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) { + err = true; + goto label_return; + } + + err = false; +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); + return err; +} + bool -tcaches_create(tsd_t *tsd, unsigned *r_ind) -{ +tcaches_create(tsd_t *tsd, unsigned *r_ind) { + bool err; arena_t *arena; tcache_t *tcache; tcaches_t *elm; - if (tcaches == NULL) { - tcaches = base_alloc(tsd_tsdn(tsd), sizeof(tcache_t *) * - (MALLOCX_TCACHE_MAX+1)); - if (tcaches == NULL) - return (true); + if (tcaches_create_prep(tsd)) { + err = true; + goto label_return; } - if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) - return (true); arena = arena_ichoose(tsd, NULL); - if (unlikely(arena == NULL)) - return (true); + if (unlikely(arena == NULL)) { + err = true; + goto label_return; + } tcache = tcache_create(tsd_tsdn(tsd), arena); - if (tcache == NULL) - return (true); + if (tcache == NULL) { + err = true; + goto label_return; + } + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); if (tcaches_avail != NULL) { elm = tcaches_avail; tcaches_avail = tcaches_avail->next; @@ -478,41 +508,50 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) *r_ind = tcaches_past; tcaches_past++; } + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); - return (false); + err = false; +label_return: + malloc_mutex_assert_not_owner(tsd_tsdn(tsd), &tcaches_mtx); + return err; } static void -tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) -{ +tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) { + malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); - if (elm->tcache == NULL) + if (elm->tcache == NULL) { return; + } tcache_destroy(tsd, elm->tcache); elm->tcache = NULL; } void -tcaches_flush(tsd_t *tsd, unsigned ind) -{ - +tcaches_flush(tsd_t *tsd, unsigned ind) { + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_elm_flush(tsd, &tcaches[ind]); + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); } void -tcaches_destroy(tsd_t *tsd, unsigned ind) -{ - tcaches_t *elm = &tcaches[ind]; +tcaches_destroy(tsd_t *tsd, unsigned ind) { + tcaches_t *elm; + + malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); + elm = &tcaches[ind]; tcaches_elm_flush(tsd, elm); elm->next = tcaches_avail; tcaches_avail = elm; + malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); } bool -tcache_boot(tsdn_t *tsdn) -{ +tcache_boot(tsdn_t *tsdn) { unsigned i; + cassert(config_tcache); + /* * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is * known. @@ -524,6 +563,10 @@ tcache_boot(tsdn_t *tsdn) else tcache_maxclass = (ZU(1) << opt_lg_tcache_max); + if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES)) { + return true; + } + nhbins = size2index(tcache_maxclass) + 1; /* Initialize tcache_bin_info. */ @@ -553,3 +596,24 @@ tcache_boot(tsdn_t *tsdn) return (false); } + +void +tcache_prefork(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_prefork(tsdn, &tcaches_mtx); + } +} + +void +tcache_postfork_parent(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_postfork_parent(tsdn, &tcaches_mtx); + } +} + +void +tcache_postfork_child(tsdn_t *tsdn) { + if (!config_prof && opt_tcache) { + malloc_mutex_postfork_child(tsdn, &tcaches_mtx); + } +} -- cgit v0.12 From e85e588e45fd3bac1ddc3778e6f8bfe3f668f634 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 21 Feb 2017 23:40:06 -0800 Subject: Use MALLOC_CONF rather than malloc_conf for tests. malloc_conf does not reliably work with MSVC, which complains of "inconsistent dll linkage", i.e. its inability to support the application overriding malloc_conf when dynamically linking/loading. Work around this limitation by adding test harness support for per test shell script sourcing, and converting all tests to use MALLOC_CONF instead of malloc_conf. --- test/integration/chunk.c | 4 ---- test/integration/chunk.sh | 5 +++++ test/integration/mallocx.c | 4 ---- test/integration/mallocx.sh | 5 +++++ test/integration/xallocx.c | 4 ---- test/integration/xallocx.sh | 5 +++++ test/test.sh.in | 29 ++++++++++++++++++++++++++++- test/unit/arena_reset.c | 4 ---- test/unit/arena_reset.sh | 5 +++++ test/unit/decay.c | 2 -- test/unit/decay.sh | 3 +++ test/unit/junk.c | 8 -------- test/unit/junk.sh | 5 +++++ test/unit/junk_alloc.c | 2 -- test/unit/junk_alloc.sh | 5 +++++ test/unit/junk_free.c | 2 -- test/unit/junk_free.sh | 5 +++++ test/unit/lg_chunk.c | 7 ------- test/unit/lg_chunk.sh | 6 ++++++ test/unit/pack.c | 8 -------- test/unit/pack.sh | 5 +++++ test/unit/prof_accum.c | 5 ----- test/unit/prof_accum.sh | 5 +++++ test/unit/prof_active.c | 5 ----- test/unit/prof_active.sh | 5 +++++ test/unit/prof_gdump.c | 4 ---- test/unit/prof_gdump.sh | 6 ++++++ test/unit/prof_idump.c | 6 ------ test/unit/prof_idump.sh | 7 +++++++ test/unit/prof_reset.c | 5 ----- test/unit/prof_reset.sh | 5 +++++ test/unit/prof_tctx.sh | 5 +++++ test/unit/prof_thread_name.c | 4 ---- test/unit/prof_thread_name.sh | 5 +++++ test/unit/quarantine.c | 8 +------- test/unit/quarantine.sh | 8 ++++++++ test/unit/zero.c | 5 ----- test/unit/zero.sh | 5 +++++ 38 files changed, 129 insertions(+), 87 deletions(-) create mode 100644 test/integration/chunk.sh create mode 100644 test/integration/mallocx.sh create mode 100644 test/integration/xallocx.sh create mode 100644 test/unit/arena_reset.sh create mode 100644 test/unit/decay.sh create mode 100644 test/unit/junk.sh create mode 100644 test/unit/junk_alloc.sh create mode 100644 test/unit/junk_free.sh create mode 100644 test/unit/lg_chunk.sh create mode 100644 test/unit/pack.sh create mode 100644 test/unit/prof_accum.sh create mode 100644 test/unit/prof_active.sh create mode 100644 test/unit/prof_gdump.sh create mode 100644 test/unit/prof_idump.sh create mode 100644 test/unit/prof_reset.sh create mode 100644 test/unit/prof_tctx.sh create mode 100644 test/unit/prof_thread_name.sh create mode 100644 test/unit/quarantine.sh create mode 100644 test/unit/zero.sh diff --git a/test/integration/chunk.c b/test/integration/chunk.c index 94cf002..997567a 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_FILL -const char *malloc_conf = "junk:false"; -#endif - static chunk_hooks_t orig_hooks; static chunk_hooks_t old_hooks; diff --git a/test/integration/chunk.sh b/test/integration/chunk.sh new file mode 100644 index 0000000..0cc2187 --- /dev/null +++ b/test/integration/chunk.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="junk:false" +fi diff --git a/test/integration/mallocx.c b/test/integration/mallocx.c index d709eb3..5a9058d 100644 --- a/test/integration/mallocx.c +++ b/test/integration/mallocx.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_FILL -const char *malloc_conf = "junk:false"; -#endif - static unsigned get_nsizes_impl(const char *cmd) { diff --git a/test/integration/mallocx.sh b/test/integration/mallocx.sh new file mode 100644 index 0000000..0cc2187 --- /dev/null +++ b/test/integration/mallocx.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="junk:false" +fi diff --git a/test/integration/xallocx.c b/test/integration/xallocx.c index 67e0a0e..2517a81 100644 --- a/test/integration/xallocx.c +++ b/test/integration/xallocx.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_FILL -const char *malloc_conf = "junk:false"; -#endif - /* * Use a separate arena for xallocx() extension/contraction tests so that * internal allocation e.g. by heap profiling can't interpose allocations where diff --git a/test/integration/xallocx.sh b/test/integration/xallocx.sh new file mode 100644 index 0000000..0cc2187 --- /dev/null +++ b/test/integration/xallocx.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="junk:false" +fi diff --git a/test/test.sh.in b/test/test.sh.in index a39f99f..f0f0f97 100644 --- a/test/test.sh.in +++ b/test/test.sh.in @@ -11,6 +11,18 @@ case @abi@ in ;; esac +# Make a copy of the @JEMALLOC_CPREFIX@MALLOC_CONF passed in to this script, so +# it can be repeatedly concatenated with per test settings. +export MALLOC_CONF_ALL=${@JEMALLOC_CPREFIX@MALLOC_CONF} +# Concatenate the individual test's MALLOC_CONF and MALLOC_CONF_ALL. +export_malloc_conf() { + if [ "x${MALLOC_CONF}" != "x" -a "x${MALLOC_CONF_ALL}" != "x" ] ; then + export @JEMALLOC_CPREFIX@MALLOC_CONF="${MALLOC_CONF},${MALLOC_CONF_ALL}" + else + export @JEMALLOC_CPREFIX@MALLOC_CONF="${MALLOC_CONF}${MALLOC_CONF_ALL}" + fi +} + # Corresponds to test_status_t. pass_code=0 skip_code=1 @@ -24,7 +36,22 @@ for t in $@; do echo fi echo "=== ${t} ===" - ${t}@exe@ @abs_srcroot@ @abs_objroot@ + if [ -e "@srcroot@${t}.sh" ] ; then + # Source the shell script corresponding to the test in a subshell and + # execute the test. This allows the shell script to set MALLOC_CONF, which + # is then used to set @JEMALLOC_CPREFIX@MALLOC_CONF (thus allowing the + # per test shell script to ignore the @JEMALLOC_CPREFIX@ detail). + $(enable_fill=@enable_fill@ \ + enable_prof=@enable_prof@ \ + enable_tcache=@enable_tcache@ \ + . @srcroot@${t}.sh && \ + export_malloc_conf && \ + ${t}@exe@ @abs_srcroot@ @abs_objroot@) + else + $(export MALLOC_CONF= && \ + export_malloc_conf && + ${t}@exe@ @abs_srcroot@ @abs_objroot@) + fi result_code=$? case ${result_code} in ${pass_code}) diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c index adf9baa..ec1c214 100644 --- a/test/unit/arena_reset.c +++ b/test/unit/arena_reset.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = "prof:true,lg_prof_sample:0"; -#endif - static unsigned get_nsizes_impl(const char *cmd) { diff --git a/test/unit/arena_reset.sh b/test/unit/arena_reset.sh new file mode 100644 index 0000000..8fcc7d8 --- /dev/null +++ b/test/unit/arena_reset.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi diff --git a/test/unit/decay.c b/test/unit/decay.c index 5af8f80..2d8d69d 100644 --- a/test/unit/decay.c +++ b/test/unit/decay.c @@ -1,7 +1,5 @@ #include "test/jemalloc_test.h" -const char *malloc_conf = "purge:decay,decay_time:1"; - static nstime_monotonic_t *nstime_monotonic_orig; static nstime_update_t *nstime_update_orig; diff --git a/test/unit/decay.sh b/test/unit/decay.sh new file mode 100644 index 0000000..7b8f470 --- /dev/null +++ b/test/unit/decay.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export MALLOC_CONF="purge:decay,decay_time:1" diff --git a/test/unit/junk.c b/test/unit/junk.c index 460bd52..bbd83fb 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -1,13 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_FILL -# ifndef JEMALLOC_TEST_JUNK_OPT -# define JEMALLOC_TEST_JUNK_OPT "junk:true" -# endif -const char *malloc_conf = - "abort:false,zero:false,redzone:true,quarantine:0," JEMALLOC_TEST_JUNK_OPT; -#endif - static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; static arena_dalloc_junk_large_t *arena_dalloc_junk_large_orig; static huge_dalloc_junk_t *huge_dalloc_junk_orig; diff --git a/test/unit/junk.sh b/test/unit/junk.sh new file mode 100644 index 0000000..e19c313 --- /dev/null +++ b/test/unit/junk.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:true" +fi diff --git a/test/unit/junk_alloc.c b/test/unit/junk_alloc.c index a5895b5..a442a0c 100644 --- a/test/unit/junk_alloc.c +++ b/test/unit/junk_alloc.c @@ -1,3 +1 @@ -#define JEMALLOC_TEST_JUNK_OPT "junk:alloc" #include "junk.c" -#undef JEMALLOC_TEST_JUNK_OPT diff --git a/test/unit/junk_alloc.sh b/test/unit/junk_alloc.sh new file mode 100644 index 0000000..984387d --- /dev/null +++ b/test/unit/junk_alloc.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:alloc" +fi diff --git a/test/unit/junk_free.c b/test/unit/junk_free.c index bb5183c..a442a0c 100644 --- a/test/unit/junk_free.c +++ b/test/unit/junk_free.c @@ -1,3 +1 @@ -#define JEMALLOC_TEST_JUNK_OPT "junk:free" #include "junk.c" -#undef JEMALLOC_TEST_JUNK_OPT diff --git a/test/unit/junk_free.sh b/test/unit/junk_free.sh new file mode 100644 index 0000000..a5c21a5 --- /dev/null +++ b/test/unit/junk_free.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="abort:false,zero:false,redzone:true,quarantine:0,junk:free" +fi diff --git a/test/unit/lg_chunk.c b/test/unit/lg_chunk.c index 7e5df38..d4f77b7 100644 --- a/test/unit/lg_chunk.c +++ b/test/unit/lg_chunk.c @@ -1,12 +1,5 @@ #include "test/jemalloc_test.h" -/* - * Make sure that opt.lg_chunk clamping is sufficient. In practice, this test - * program will fail a debug assertion during initialization and abort (rather - * than the test soft-failing) if clamping is insufficient. - */ -const char *malloc_conf = "lg_chunk:0"; - TEST_BEGIN(test_lg_chunk_clamp) { void *p; diff --git a/test/unit/lg_chunk.sh b/test/unit/lg_chunk.sh new file mode 100644 index 0000000..103eef1 --- /dev/null +++ b/test/unit/lg_chunk.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# Make sure that opt.lg_chunk clamping is sufficient. In practice, this test +# program will fail a debug assertion during initialization and abort (rather +# than the test soft-failing) if clamping is insufficient. +export MALLOC_CONF="lg_chunk:0" diff --git a/test/unit/pack.c b/test/unit/pack.c index 0b6ffcd..991faa6 100644 --- a/test/unit/pack.c +++ b/test/unit/pack.c @@ -1,13 +1,5 @@ #include "test/jemalloc_test.h" -const char *malloc_conf = - /* Use smallest possible chunk size. */ - "lg_chunk:0" - /* Immediately purge to minimize fragmentation. */ - ",lg_dirty_mult:-1" - ",decay_time:-1" - ; - /* * Size class that is a divisor of the page size, ideally 4+ regions per run. */ diff --git a/test/unit/pack.sh b/test/unit/pack.sh new file mode 100644 index 0000000..a58151d --- /dev/null +++ b/test/unit/pack.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +# Use smallest possible chunk size. Immediately purge to minimize +# fragmentation. +export MALLOC_CONF="lg_chunk:0,lg_dirty_mult:-1,decay_time:-1" diff --git a/test/unit/prof_accum.c b/test/unit/prof_accum.c index d941b5b..031f083 100644 --- a/test/unit/prof_accum.c +++ b/test/unit/prof_accum.c @@ -5,11 +5,6 @@ #define DUMP_INTERVAL 1 #define BT_COUNT_CHECK_INTERVAL 5 -#ifdef JEMALLOC_PROF -const char *malloc_conf = - "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0"; -#endif - static int prof_dump_open_intercept(bool propagate_err, const char *filename) { diff --git a/test/unit/prof_accum.sh b/test/unit/prof_accum.sh new file mode 100644 index 0000000..b3e13fc --- /dev/null +++ b/test/unit/prof_accum.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0" +fi diff --git a/test/unit/prof_active.c b/test/unit/prof_active.c index d00943a..a906beb 100644 --- a/test/unit/prof_active.c +++ b/test/unit/prof_active.c @@ -1,10 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = - "prof:true,prof_thread_active_init:false,lg_prof_sample:0"; -#endif - static void mallctl_bool_get(const char *name, bool expected, const char *func, int line) { diff --git a/test/unit/prof_active.sh b/test/unit/prof_active.sh new file mode 100644 index 0000000..0167cb1 --- /dev/null +++ b/test/unit/prof_active.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_thread_active_init:false,lg_prof_sample:0" +fi diff --git a/test/unit/prof_gdump.c b/test/unit/prof_gdump.c index 996cb67..b88a74c 100644 --- a/test/unit/prof_gdump.c +++ b/test/unit/prof_gdump.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = "prof:true,prof_active:false,prof_gdump:true"; -#endif - static bool did_prof_dump_open; static int diff --git a/test/unit/prof_gdump.sh b/test/unit/prof_gdump.sh new file mode 100644 index 0000000..3f600d2 --- /dev/null +++ b/test/unit/prof_gdump.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_active:false,prof_gdump:true" +fi + diff --git a/test/unit/prof_idump.c b/test/unit/prof_idump.c index 16c6462..87734a4 100644 --- a/test/unit/prof_idump.c +++ b/test/unit/prof_idump.c @@ -1,11 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = - "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0," - "lg_prof_interval:0"; -#endif - static bool did_prof_dump_open; static int diff --git a/test/unit/prof_idump.sh b/test/unit/prof_idump.sh new file mode 100644 index 0000000..08a1b62 --- /dev/null +++ b/test/unit/prof_idump.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0,lg_prof_interval:0" +fi + + diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c index 59d7079..87b0d0c 100644 --- a/test/unit/prof_reset.c +++ b/test/unit/prof_reset.c @@ -1,10 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = - "prof:true,prof_active:false,lg_prof_sample:0"; -#endif - static int prof_dump_open_intercept(bool propagate_err, const char *filename) { diff --git a/test/unit/prof_reset.sh b/test/unit/prof_reset.sh new file mode 100644 index 0000000..43c516a --- /dev/null +++ b/test/unit/prof_reset.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_active:false,lg_prof_sample:0" +fi diff --git a/test/unit/prof_tctx.sh b/test/unit/prof_tctx.sh new file mode 100644 index 0000000..8fcc7d8 --- /dev/null +++ b/test/unit/prof_tctx.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi diff --git a/test/unit/prof_thread_name.c b/test/unit/prof_thread_name.c index 9ec5497..3251853 100644 --- a/test/unit/prof_thread_name.c +++ b/test/unit/prof_thread_name.c @@ -1,9 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_PROF -const char *malloc_conf = "prof:true,prof_active:false"; -#endif - static void mallctl_thread_name_get_impl(const char *thread_name_expected, const char *func, int line) diff --git a/test/unit/prof_thread_name.sh b/test/unit/prof_thread_name.sh new file mode 100644 index 0000000..298c105 --- /dev/null +++ b/test/unit/prof_thread_name.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,prof_active:false" +fi diff --git a/test/unit/quarantine.c b/test/unit/quarantine.c index bbd48a5..6068768 100644 --- a/test/unit/quarantine.c +++ b/test/unit/quarantine.c @@ -1,13 +1,7 @@ #include "test/jemalloc_test.h" +/* Keep in sync with definition in quarantine.sh. */ #define QUARANTINE_SIZE 8192 -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - -#ifdef JEMALLOC_FILL -const char *malloc_conf = "abort:false,junk:true,redzone:true,quarantine:" - STRINGIFY(QUARANTINE_SIZE); -#endif void quarantine_clear(void) diff --git a/test/unit/quarantine.sh b/test/unit/quarantine.sh new file mode 100644 index 0000000..e3c6932 --- /dev/null +++ b/test/unit/quarantine.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Keep in sync with definition in quarantine.c. +export QUARANTINE_SIZE=8192 + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="abort:false,junk:true,redzone:true,quarantine:${QUARANTINE_SIZE}" +fi diff --git a/test/unit/zero.c b/test/unit/zero.c index 30ebe37..573993a 100644 --- a/test/unit/zero.c +++ b/test/unit/zero.c @@ -1,10 +1,5 @@ #include "test/jemalloc_test.h" -#ifdef JEMALLOC_FILL -const char *malloc_conf = - "abort:false,junk:false,zero:true,redzone:false,quarantine:0"; -#endif - static void test_zero(size_t sz_min, size_t sz_max) { diff --git a/test/unit/zero.sh b/test/unit/zero.sh new file mode 100644 index 0000000..24488f0 --- /dev/null +++ b/test/unit/zero.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="abort:false,junk:false,zero:true,redzone:false,quarantine:0" +fi -- cgit v0.12 From 44e50041dc89c3aed4d03e231fd8ce6cb061f982 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 31 Jan 2017 16:44:57 -0800 Subject: CI: Run --enable-debug builds on windows This will hopefully catch some windows-specific bugs. --- .appveyor.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index ddd5c57..510815d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,6 +12,20 @@ environment: CPU: x86_64 - MSYSTEM: MINGW32 CPU: i686 + - MSYSTEM: MINGW64 + CPU: x86_64 + MSVC: amd64 + CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW32 + CPU: i686 + MSVC: x86 + CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW64 + CPU: x86_64 + CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW32 + CPU: i686 + CONFIG_FLAGS: --enable-debug install: - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH% @@ -21,7 +35,7 @@ install: build_script: - bash -c "autoconf" - - bash -c "./configure" + - bash -c "./configure $CONFIG_FLAGS" - mingw32-make -j3 - file lib/jemalloc.dll - mingw32-make -j3 tests -- cgit v0.12 From 7034e6baa10163b3c6d7866562c0b8bd4d80904a Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 21 Feb 2017 20:52:44 -0800 Subject: Enable mutex witnesses even when !isthreaded. This fixes interactions with witness_assert_depth[_to_rank](), which was added in dad74bd3c811ca2b1af1fd57b28f2456da5ba08b (Convert witness_assert_lockless() to witness_assert_lock_depth().). --- include/jemalloc/internal/mutex.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index b442d2d..2b4b1c3 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -85,8 +85,8 @@ JEMALLOC_INLINE void malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { + witness_assert_not_owner(tsdn, &mutex->witness); if (isthreaded) { - witness_assert_not_owner(tsdn, &mutex->witness); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 AcquireSRWLockExclusive(&mutex->lock); @@ -100,16 +100,16 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) #else pthread_mutex_lock(&mutex->lock); #endif - witness_lock(tsdn, &mutex->witness); } + witness_lock(tsdn, &mutex->witness); } JEMALLOC_INLINE void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { + witness_unlock(tsdn, &mutex->witness); if (isthreaded) { - witness_unlock(tsdn, &mutex->witness); #ifdef _WIN32 # if _WIN32_WINNT >= 0x0600 ReleaseSRWLockExclusive(&mutex->lock); @@ -130,16 +130,14 @@ JEMALLOC_INLINE void malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - if (isthreaded) - witness_assert_owner(tsdn, &mutex->witness); + witness_assert_owner(tsdn, &mutex->witness); } JEMALLOC_INLINE void malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { - if (isthreaded) - witness_assert_not_owner(tsdn, &mutex->witness); + witness_assert_not_owner(tsdn, &mutex->witness); } #endif -- cgit v0.12 From f56cb9a68e9cc95d23af0809ab4cf3e288c7e448 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 21 Jan 2017 15:12:03 -0800 Subject: Add witness_assert_depth[_to_rank](). This makes it possible to make lock state assertions about precisely which locks are held. --- include/jemalloc/internal/private_symbols.txt | 6 +- include/jemalloc/internal/witness.h | 40 ++++++++--- src/arena.c | 4 +- src/huge.c | 8 +-- src/jemalloc.c | 74 ++++++++++----------- src/witness.c | 21 +++--- test/unit/witness.c | 95 +++++++++++++++------------ 7 files changed, 141 insertions(+), 107 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 6111eac..8a9e32f 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -615,14 +615,16 @@ valgrind_freelike_block valgrind_make_mem_defined valgrind_make_mem_noaccess valgrind_make_mem_undefined -witness_assert_lock_depth +witness_assert_depth +witness_assert_depth_to_rank +witness_assert_lockless witness_assert_not_owner witness_assert_owner +witness_depth_error witness_fork_cleanup witness_init witness_lock witness_lock_error -witness_lock_depth_error witness_not_owner_error witness_owner witness_owner_error diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index e64e56e..b89d12a 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -12,6 +12,8 @@ typedef int witness_comp_t (const witness_t *, const witness_t *); */ #define WITNESS_RANK_OMIT 0U +#define WITNESS_RANK_MIN 1U + #define WITNESS_RANK_INIT 1U #define WITNESS_RANK_CTL 1U #define WITNESS_RANK_TCACHES 2U @@ -92,12 +94,12 @@ extern witness_not_owner_error_t *witness_not_owner_error; void witness_not_owner_error(const witness_t *witness); #endif #ifdef JEMALLOC_JET -typedef void (witness_lock_depth_error_t)(const witness_list_t *, - unsigned depth); -extern witness_lock_depth_error_t *witness_lock_depth_error; +typedef void (witness_depth_error_t)(const witness_list_t *, + witness_rank_t rank_inclusive, unsigned depth); +extern witness_depth_error_t *witness_depth_error; #else -void witness_lock_depth_error(const witness_list_t *witnesses, - unsigned depth); +void witness_depth_error(const witness_list_t *witnesses, + witness_rank_t rank_inclusive, unsigned depth); #endif void witnesses_cleanup(tsd_t *tsd); @@ -114,7 +116,10 @@ void witness_postfork_child(tsd_t *tsd); bool witness_owner(tsd_t *tsd, const witness_t *witness); void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); -void witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth); +void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, + unsigned depth); +void witness_assert_depth(tsdn_t *tsdn, unsigned depth); +void witness_assert_lockless(tsdn_t *tsdn); void witness_lock(tsdn_t *tsdn, witness_t *witness); void witness_unlock(tsdn_t *tsdn, witness_t *witness); #endif @@ -126,6 +131,8 @@ witness_owner(tsd_t *tsd, const witness_t *witness) witness_list_t *witnesses; witness_t *w; + cassert(config_debug); + witnesses = tsd_witnessesp_get(tsd); ql_foreach(w, witnesses, link) { if (w == witness) @@ -178,8 +185,8 @@ witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) } JEMALLOC_INLINE void -witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth) -{ +witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive, + unsigned depth) { tsd_t *tsd; unsigned d; witness_list_t *witnesses; @@ -196,12 +203,25 @@ witness_assert_lock_depth(tsdn_t *tsdn, unsigned depth) witnesses = tsd_witnessesp_get(tsd); w = ql_last(witnesses, link); if (w != NULL) { - ql_foreach(w, witnesses, link) { + ql_reverse_foreach(w, witnesses, link) { + if (w->rank < rank_inclusive) { + break; + } d++; } } if (d != depth) - witness_lock_depth_error(witnesses, depth); + witness_depth_error(witnesses, rank_inclusive, depth); +} + +JEMALLOC_INLINE void +witness_assert_depth(tsdn_t *tsdn, unsigned depth) { + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth); +} + +JEMALLOC_INLINE void +witness_assert_lockless(tsdn_t *tsdn) { + witness_assert_depth(tsdn, 0); } JEMALLOC_INLINE void diff --git a/src/arena.c b/src/arena.c index 193a4a2..c3d2622 100644 --- a/src/arena.c +++ b/src/arena.c @@ -591,7 +591,7 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, size_t sn; malloc_mutex_unlock(tsdn, &arena->lock); - witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, chunksize, chunksize, &sn, zero, commit); @@ -633,7 +633,7 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, size_t sn; /* prof_gdump() requirement. */ - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); malloc_mutex_assert_owner(tsdn, &arena->lock); chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, diff --git a/src/huge.c b/src/huge.c index 9a91bed..f712fd8 100644 --- a/src/huge.c +++ b/src/huge.c @@ -62,7 +62,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, /* Allocate one or more contiguous chunks for this request. */ assert(!tsdn_null(tsdn) || arena != NULL); - witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ ausize = sa2u(usize, alignment); if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) @@ -149,7 +149,7 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; bool pre_zeroed, post_zeroed, gdump; - witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ /* Increase usize to incorporate extra. */ for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) @@ -223,7 +223,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, chunk_hooks = chunk_hooks_get(tsdn, arena); assert(oldsize > usize); - witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ /* Split excess chunks. */ cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); @@ -278,7 +278,7 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, is_zeroed_subchunk = extent_node_zeroed_get(node); malloc_mutex_unlock(tsdn, &arena->huge_mtx); - witness_assert_lock_depth(tsdn, 0); /* prof_gdump() requirement. */ + witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ /* * Use is_zeroed_chunk to detect whether the trailing memory is zeroed, diff --git a/src/jemalloc.c b/src/jemalloc.c index a376e14..029fe52 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1582,7 +1582,7 @@ ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); ind = size2index(size); if (unlikely(ind >= NSIZES)) @@ -1620,7 +1620,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func, assert(usize == isalloc(tsdn, ret, config_prof)); *tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize; } - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); } JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN @@ -1705,7 +1705,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) goto label_oom; } tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (size == 0) size = 1; @@ -1746,7 +1746,7 @@ label_return: UTRACE(0, size, result); JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize, false); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (ret); label_oom: assert(result == NULL); @@ -1756,7 +1756,7 @@ label_oom: abort(); } ret = ENOMEM; - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); goto label_return; } @@ -1874,7 +1874,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) size_t usize; UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1902,7 +1902,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -1948,7 +1948,7 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) { @@ -1995,7 +1995,7 @@ je_realloc(void *ptr, size_t size) UTRACE(ptr, size, ret); JEMALLOC_VALGRIND_REALLOC(maybe, tsdn, ret, usize, maybe, ptr, old_usize, old_rzsize, maybe, false); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (ret); } @@ -2006,12 +2006,12 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { tsd_t *tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (likely(!malloc_slow)) ifree(tsd, ptr, tcache_get(tsd, false), false); else ifree(tsd, ptr, tcache_get(tsd, false), true); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); } } @@ -2240,7 +2240,7 @@ imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize, tsd = tsd_fetch(); *tsdn = tsd_tsdn(tsd); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (likely(flags == 0)) { szind_t ind = size2index(size); @@ -2375,7 +2375,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -2422,7 +2422,7 @@ je_rallocx(void *ptr, size_t size, int flags) UTRACE(ptr, size, p); JEMALLOC_VALGRIND_REALLOC(maybe, tsd_tsdn(tsd), p, usize, no, ptr, old_usize, old_rzsize, no, zero); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (p); label_oom: if (config_xmalloc && unlikely(opt_xmalloc)) { @@ -2430,7 +2430,7 @@ label_oom: abort(); } UTRACE(ptr, size, 0); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (NULL); } @@ -2526,7 +2526,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof); @@ -2567,7 +2567,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) old_usize, old_rzsize, no, zero); label_not_resized: UTRACE(ptr, size, ptr); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (usize); } @@ -2582,14 +2582,14 @@ je_sallocx(const void *ptr, int flags) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); if (config_ivsalloc) usize = ivsalloc(tsdn, ptr, config_prof); else usize = isalloc(tsdn, ptr, config_prof); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (usize); } @@ -2603,7 +2603,7 @@ je_dallocx(void *ptr, int flags) assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2617,7 +2617,7 @@ je_dallocx(void *ptr, int flags) ifree(tsd, ptr, tcache, false); else ifree(tsd, ptr, tcache, true); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -2625,13 +2625,13 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { size_t usize; - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) usize = s2u(size); else usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (usize); } @@ -2648,7 +2648,7 @@ je_sdallocx(void *ptr, size_t size, int flags) usize = inallocx(tsd_tsdn(tsd), size, flags); assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof)); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) tcache = NULL; @@ -2662,7 +2662,7 @@ je_sdallocx(void *ptr, size_t size, int flags) isfree(tsd, ptr, usize, tcache, false); else isfree(tsd, ptr, usize, tcache, true); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2678,13 +2678,13 @@ je_nallocx(size_t size, int flags) return (0); tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); usize = inallocx(tsdn, size, flags); if (unlikely(usize > HUGE_MAXCLASS)) return (0); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (usize); } @@ -2699,9 +2699,9 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (ret); } @@ -2715,9 +2715,9 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) return (EAGAIN); tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); ret = ctl_nametomib(tsdn, name, mibp, miblenp); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (ret); } @@ -2732,9 +2732,9 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, return (EAGAIN); tsd = tsd_fetch(); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); return (ret); } @@ -2745,9 +2745,9 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, tsdn_t *tsdn; tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); stats_print(write_cb, cbopaque, opts); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2760,14 +2760,14 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) malloc_thread_init(); tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); if (config_ivsalloc) ret = ivsalloc(tsdn, ptr, config_prof); else ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); return (ret); } diff --git a/src/witness.c b/src/witness.c index aaea88d..c3a65f7 100644 --- a/src/witness.c +++ b/src/witness.c @@ -71,16 +71,16 @@ witness_not_owner_error_t *witness_not_owner_error = #endif #ifdef JEMALLOC_JET -#undef witness_lock_depth_error -#define witness_lock_depth_error JEMALLOC_N(n_witness_lock_depth_error) +#undef witness_depth_error +#define witness_depth_error JEMALLOC_N(n_witness_depth_error) #endif void -witness_lock_depth_error(const witness_list_t *witnesses, unsigned depth) -{ +witness_depth_error(const witness_list_t *witnesses, + witness_rank_t rank_inclusive, unsigned depth) { witness_t *w; - malloc_printf(": Should own %u lock%s:", depth, (depth != 1) ? - "s" : ""); + malloc_printf(": Should own %u lock%s of rank >= %u:", depth, + (depth != 1) ? "s" : "", rank_inclusive); ql_foreach(w, witnesses, link) { malloc_printf(" %s(%u)", w->name, w->rank); } @@ -88,17 +88,16 @@ witness_lock_depth_error(const witness_list_t *witnesses, unsigned depth) abort(); } #ifdef JEMALLOC_JET -#undef witness_lock_depth_error -#define witness_lock_depth_error JEMALLOC_N(witness_lock_depth_error) -witness_lock_depth_error_t *witness_lock_depth_error = - JEMALLOC_N(n_witness_lock_depth_error); +#undef witness_depth_error +#define witness_depth_error JEMALLOC_N(witness_depth_error) +witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error); #endif void witnesses_cleanup(tsd_t *tsd) { - witness_assert_lock_depth(tsd_tsdn(tsd), 0); + witness_assert_lockless(tsd_tsdn(tsd)); /* Do nothing. */ } diff --git a/test/unit/witness.c b/test/unit/witness.c index 9d4a171..8b99413 100644 --- a/test/unit/witness.c +++ b/test/unit/witness.c @@ -3,12 +3,12 @@ static witness_lock_error_t *witness_lock_error_orig; static witness_owner_error_t *witness_owner_error_orig; static witness_not_owner_error_t *witness_not_owner_error_orig; -static witness_lock_depth_error_t *witness_lock_depth_error_orig; +static witness_depth_error_t *witness_depth_error_orig; static bool saw_lock_error; static bool saw_owner_error; static bool saw_not_owner_error; -static bool saw_lock_depth_error; +static bool saw_depth_error; static void witness_lock_error_intercept(const witness_list_t *witnesses, @@ -33,11 +33,9 @@ witness_not_owner_error_intercept(const witness_t *witness) } static void -witness_lock_depth_error_intercept(const witness_list_t *witnesses, - unsigned depth) -{ - - saw_lock_depth_error = true; +witness_depth_error_intercept(const witness_list_t *witnesses, + witness_rank_t rank_inclusive, unsigned depth) { + saw_depth_error = true; } static int @@ -67,25 +65,37 @@ TEST_BEGIN(test_witness) tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0); witness_init(&a, "a", 1, NULL); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0); witness_init(&b, "b", 2, NULL); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); - witness_assert_lock_depth(tsdn, 2); + witness_assert_depth(tsdn, 2); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); witness_unlock(tsdn, &a); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0); witness_unlock(tsdn, &b); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); + witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0); } TEST_END @@ -98,21 +108,21 @@ TEST_BEGIN(test_witness_comp) tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_init(&a, "a", 1, witness_comp); witness_assert_not_owner(tsdn, &a); witness_lock(tsdn, &a); witness_assert_owner(tsdn, &a); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); witness_init(&b, "b", 1, witness_comp); witness_assert_not_owner(tsdn, &b); witness_lock(tsdn, &b); witness_assert_owner(tsdn, &b); - witness_assert_lock_depth(tsdn, 2); + witness_assert_depth(tsdn, 2); witness_unlock(tsdn, &b); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); witness_lock_error_orig = witness_lock_error; witness_lock_error = witness_lock_error_intercept; @@ -124,7 +134,7 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &c); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &c); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); saw_lock_error = false; @@ -134,11 +144,11 @@ TEST_BEGIN(test_witness_comp) witness_lock(tsdn, &d); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &d); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); witness_unlock(tsdn, &a); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_lock_error = witness_lock_error_orig; } @@ -157,22 +167,22 @@ TEST_BEGIN(test_witness_reversal) tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_init(&a, "a", 1, NULL); witness_init(&b, "b", 2, NULL); witness_lock(tsdn, &b); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); assert_false(saw_lock_error, "Unexpected witness lock error"); witness_lock(tsdn, &a); assert_true(saw_lock_error, "Expected witness lock error"); witness_unlock(tsdn, &a); - witness_assert_lock_depth(tsdn, 1); + witness_assert_depth(tsdn, 1); witness_unlock(tsdn, &b); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_lock_error = witness_lock_error_orig; } @@ -195,7 +205,7 @@ TEST_BEGIN(test_witness_recursive) tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_init(&a, "a", 1, NULL); @@ -208,7 +218,7 @@ TEST_BEGIN(test_witness_recursive) witness_unlock(tsdn, &a); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_owner_error = witness_owner_error_orig; witness_lock_error = witness_lock_error_orig; @@ -229,7 +239,7 @@ TEST_BEGIN(test_witness_unlock_not_owned) tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_init(&a, "a", 1, NULL); @@ -237,41 +247,44 @@ TEST_BEGIN(test_witness_unlock_not_owned) witness_unlock(tsdn, &a); assert_true(saw_owner_error, "Expected owner error"); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); witness_owner_error = witness_owner_error_orig; } TEST_END -TEST_BEGIN(test_witness_lock_depth) -{ +TEST_BEGIN(test_witness_depth) { witness_t a; tsdn_t *tsdn; test_skip_if(!config_debug); - witness_lock_depth_error_orig = witness_lock_depth_error; - witness_lock_depth_error = witness_lock_depth_error_intercept; - saw_lock_depth_error = false; + witness_depth_error_orig = witness_depth_error; + witness_depth_error = witness_depth_error_intercept; + saw_depth_error = false; tsdn = tsdn_fetch(); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); witness_init(&a, "a", 1, NULL); - assert_false(saw_lock_depth_error, "Unexpected lock_depth error"); - witness_assert_lock_depth(tsdn, 0); + assert_false(saw_depth_error, "Unexpected depth error"); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); witness_lock(tsdn, &a); - witness_assert_lock_depth(tsdn, 0); - assert_true(saw_lock_depth_error, "Expected lock_depth error"); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); + assert_true(saw_depth_error, "Expected depth error"); witness_unlock(tsdn, &a); - witness_assert_lock_depth(tsdn, 0); + witness_assert_lockless(tsdn); + witness_assert_depth(tsdn, 0); - witness_lock_depth_error = witness_lock_depth_error_orig; + witness_depth_error = witness_depth_error_orig; } TEST_END @@ -279,11 +292,11 @@ int main(void) { - return (test( + return test( test_witness, test_witness_comp, test_witness_reversal, test_witness_recursive, test_witness_unlock_not_owned, - test_witness_lock_depth)); + test_witness_depth); } -- cgit v0.12 From 08c24e7c1a034fc43353f47450f395a7272ccf02 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 22 Feb 2017 20:58:42 -0800 Subject: Relax witness assertions related to prof_gdump(). In some cases the prof machinery allocates (in order to modify the bt2gctx hash table), and such operations are synchronized via bt2gctx_mtx. Rather than asserting that no locks are held on entry into functions that may call prof_gdump(), make the weaker assertion that no "core" locks are held. The prof machinery enqueues dumps triggered by prof_gdump() calls when bt2gctx_mtx is held, so this weakened assertion avoids false failures in such cases. --- include/jemalloc/internal/witness.h | 8 ++++++++ src/arena.c | 5 +++-- src/huge.c | 12 ++++++++---- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/include/jemalloc/internal/witness.h b/include/jemalloc/internal/witness.h index b89d12a..30d8c7e 100644 --- a/include/jemalloc/internal/witness.h +++ b/include/jemalloc/internal/witness.h @@ -25,6 +25,14 @@ typedef int witness_comp_t (const witness_t *, const witness_t *); #define WITNESS_RANK_PROF_TDATA 7U #define WITNESS_RANK_PROF_GCTX 8U +/* + * Used as an argument to witness_assert_depth_to_rank() in order to validate + * depth excluding non-core locks with lower ranks. Since the rank argument to + * witness_assert_depth_to_rank() is inclusive rather than exclusive, this + * definition can have the same value as the minimally ranked core lock. + */ +#define WITNESS_RANK_CORE 9U + #define WITNESS_RANK_ARENA 9U #define WITNESS_RANK_ARENA_CHUNKS 10U #define WITNESS_RANK_ARENA_NODE_CACHE 11U diff --git a/src/arena.c b/src/arena.c index c3d2622..6d178d2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -591,7 +591,8 @@ arena_chunk_alloc_internal_hard(tsdn_t *tsdn, arena_t *arena, size_t sn; malloc_mutex_unlock(tsdn, &arena->lock); - witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); chunk = (arena_chunk_t *)chunk_alloc_wrapper(tsdn, arena, chunk_hooks, NULL, chunksize, chunksize, &sn, zero, commit); @@ -633,7 +634,7 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero, size_t sn; /* prof_gdump() requirement. */ - witness_assert_depth(tsdn, 1); + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 1); malloc_mutex_assert_owner(tsdn, &arena->lock); chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize, diff --git a/src/huge.c b/src/huge.c index f712fd8..0fbaa41 100644 --- a/src/huge.c +++ b/src/huge.c @@ -62,7 +62,8 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, /* Allocate one or more contiguous chunks for this request. */ assert(!tsdn_null(tsdn) || arena != NULL); - witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); ausize = sa2u(usize, alignment); if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS)) @@ -149,7 +150,8 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize, chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; bool pre_zeroed, post_zeroed, gdump; - witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); /* Increase usize to incorporate extra. */ for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) @@ -223,7 +225,8 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize, chunk_hooks = chunk_hooks_get(tsdn, arena); assert(oldsize > usize); - witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); /* Split excess chunks. */ cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); @@ -278,7 +281,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize, is_zeroed_subchunk = extent_node_zeroed_get(node); malloc_mutex_unlock(tsdn, &arena->huge_mtx); - witness_assert_lockless(tsdn); /* prof_gdump() requirement. */ + /* prof_gdump() requirement. */ + witness_assert_depth_to_rank(tsdn, WITNESS_RANK_CORE, 0); /* * Use is_zeroed_chunk to detect whether the trailing memory is zeroed, -- cgit v0.12 From adae7cfc4a2ac66c96b0dcc83b3837ac668fc44e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 24 Feb 2017 09:45:33 -0800 Subject: Fix chunk_alloc_dss() regression. Fix chunk_alloc_dss() to account for bytes that are not a multiple of the chunk size. This regression was introduced by e2bcf037d445a84a71c7997670819ebd0a893b4a (Make dss operations lockless.), which was first released in 4.3.0. --- src/chunk_dss.c | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/chunk_dss.c b/src/chunk_dss.c index ee3f838..8c67939 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -115,8 +115,9 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, * malloc. */ while (true) { - void *ret, *cpad, *max_cur, *dss_next, *dss_prev; - size_t gap_size, cpad_size; + void *ret, *max_cur, *dss_next, *dss_prev; + void *gap_addr_chunk, *gap_addr_subchunk; + size_t gap_size_chunk, gap_size_subchunk; intptr_t incr; max_cur = chunk_dss_max_update(new_addr); @@ -124,25 +125,32 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, goto label_oom; /* - * Calculate how much padding is necessary to - * chunk-align the end of the DSS. - */ - gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & - chunksize_mask; - /* - * Compute how much chunk-aligned pad space (if any) is + * Compute how much chunk-aligned gap space (if any) is * necessary to satisfy alignment. This space can be * recycled for later use. */ - cpad = (void *)((uintptr_t)dss_max + gap_size); - ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, - alignment); - cpad_size = (uintptr_t)ret - (uintptr_t)cpad; + gap_addr_chunk = (void *)(CHUNK_CEILING( + (uintptr_t)max_cur)); + ret = (void *)ALIGNMENT_CEILING( + (uintptr_t)gap_addr_chunk, alignment); + gap_size_chunk = (uintptr_t)ret - + (uintptr_t)gap_addr_chunk; + /* + * Compute the address just past the end of the desired + * allocation space. + */ dss_next = (void *)((uintptr_t)ret + size); - if ((uintptr_t)ret < (uintptr_t)dss_max || - (uintptr_t)dss_next < (uintptr_t)dss_max) + if ((uintptr_t)ret < (uintptr_t)max_cur || + (uintptr_t)dss_next < (uintptr_t)max_cur) goto label_oom; /* Wrap-around. */ - incr = gap_size + cpad_size + size; + /* Compute the increment, including subchunk bytes. */ + gap_addr_subchunk = max_cur; + gap_size_subchunk = (uintptr_t)ret - + (uintptr_t)gap_addr_subchunk; + incr = gap_size_subchunk + size; + + assert((uintptr_t)max_cur + incr == (uintptr_t)ret + + size); /* * Optimistically update dss_max, and roll back below if @@ -157,11 +165,12 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, dss_prev = chunk_dss_sbrk(incr); if (dss_prev == max_cur) { /* Success. */ - if (cpad_size != 0) { + if (gap_size_chunk != 0) { chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_dalloc_wrapper(tsdn, arena, - &chunk_hooks, cpad, cpad_size, + &chunk_hooks, gap_addr_chunk, + gap_size_chunk, arena_extent_sn_next(arena), false, true); } -- cgit v0.12 From 61d26425e53d74f31e9f2ef3a423bf730f832f68 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 24 Feb 2017 10:40:23 -0800 Subject: Fix JSON-mode output for !config_stats and/or !config_prof cases. These bugs were introduced by b599b32280e1142856b0b96293a71e1684b1ccfb (Add "J" (JSON) support to malloc_stats_print().), which was first released in 4.3.0. This resolves #615. --- src/stats.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/stats.c b/src/stats.c index 1360f3b..6b71158 100644 --- a/src/stats.c +++ b/src/stats.c @@ -555,7 +555,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, static void stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, bool merged, bool unmerged) + bool json, bool more) { const char *cpv; bool bv; @@ -907,11 +907,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, "\t\t\t]\n"); malloc_cprintf(write_cb, cbopaque, - "\t\t},\n"); + "\t\t}%s\n", (config_prof || more) ? "," : ""); } /* prof. */ - if (json) { + if (config_prof && json) { malloc_cprintf(write_cb, cbopaque, "\t\t\"prof\": {\n"); @@ -937,8 +937,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, "\t\t\t\"lg_sample\": %zd\n", ssv); malloc_cprintf(write_cb, cbopaque, - "\t\t}%s\n", (config_stats || merged || unmerged) ? "," : - ""); + "\t\t}%s\n", more ? "," : ""); } } @@ -1069,8 +1068,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t u64sz; bool json = false; bool general = true; - bool merged = true; - bool unmerged = true; + bool merged = config_stats; + bool unmerged = config_stats; bool bins = true; bool large = true; bool huge = true; @@ -1137,8 +1136,10 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, "___ Begin jemalloc statistics ___\n"); } - if (general) - stats_general_print(write_cb, cbopaque, json, merged, unmerged); + if (general) { + bool more = (merged || unmerged); + stats_general_print(write_cb, cbopaque, json, more); + } if (config_stats) { stats_print_helper(write_cb, cbopaque, json, merged, unmerged, bins, large, huge); -- cgit v0.12 From 54d2d697b21af7a5553d0c9de2e9174bfa0fc7a5 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Jan 2017 01:01:19 -0800 Subject: Test JSON output of malloc_stats_print() and fix bugs. Implement and test a JSON validation parser. Use the parser to validate JSON output from malloc_stats_print(), with a significant subset of supported output options. This resolves #583. --- Makefile.in | 1 + src/stats.c | 61 +-- test/unit/stats_print.c | 1000 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1035 insertions(+), 27 deletions(-) create mode 100644 test/unit/stats_print.c diff --git a/Makefile.in b/Makefile.in index c705363..675e4cb 100644 --- a/Makefile.in +++ b/Makefile.in @@ -186,6 +186,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/size_classes.c \ $(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/stats.c \ + $(srcroot)test/unit/stats_print.c \ $(srcroot)test/unit/ticker.c \ $(srcroot)test/unit/nstime.c \ $(srcroot)test/unit/tsd.c \ diff --git a/src/stats.c b/src/stats.c index 6b71158..8d579c7 100644 --- a/src/stats.c +++ b/src/stats.c @@ -39,7 +39,7 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, bool json, bool large, bool huge, unsigned i) { size_t page; - bool config_tcache, in_gap, in_gap_prev; + bool in_gap, in_gap_prev; unsigned nbins, j; CTL_GET("arenas.page", &page, size_t); @@ -49,7 +49,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"bins\": [\n"); } else { - CTL_GET("config.tcache", &config_tcache, bool); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, "bins: size ind allocated nmalloc" @@ -536,7 +535,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, "\t\t\t\t\t\"allocated\": %zu\n", metadata_allocated); malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t},\n"); + "\t\t\t\t}%s\n", (bins || large || huge) ? "," : ""); } else { malloc_cprintf(write_cb, cbopaque, "metadata: mapped: %zu, allocated: %zu\n", @@ -838,9 +837,11 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\t\t\t\"nbins\": %u,\n", nbins); - CTL_GET("arenas.nhbins", &uv, unsigned); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"nhbins\": %u,\n", uv); + if (config_tcache) { + CTL_GET("arenas.nhbins", &uv, unsigned); + malloc_cprintf(write_cb, cbopaque, + "\t\t\t\"nhbins\": %u,\n", uv); + } malloc_cprintf(write_cb, cbopaque, "\t\t\t\"bin\": [\n"); @@ -1022,31 +1023,37 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, narenas, bins, large, huge); if (json) { malloc_cprintf(write_cb, cbopaque, - "\t\t\t}%s\n", (ninitialized > 1) ? - "," : ""); + "\t\t\t}%s\n", unmerged ? "," : + ""); } } /* Unmerged stats. */ - for (i = j = 0; i < narenas; i++) { - if (initialized[i]) { - if (json) { - j++; - malloc_cprintf(write_cb, - cbopaque, - "\t\t\t\"%u\": {\n", i); - } else { - malloc_cprintf(write_cb, - cbopaque, "\narenas[%u]:\n", - i); - } - stats_arena_print(write_cb, cbopaque, - json, i, bins, large, huge); - if (json) { - malloc_cprintf(write_cb, - cbopaque, - "\t\t\t}%s\n", (j < - ninitialized) ? "," : ""); + if (unmerged) { + for (i = j = 0; i < narenas; i++) { + if (initialized[i]) { + if (json) { + j++; + malloc_cprintf(write_cb, + cbopaque, + "\t\t\t\"%u\": {\n", + i); + } else { + malloc_cprintf(write_cb, + cbopaque, + "\narenas[%u]:\n", + i); + } + stats_arena_print(write_cb, + cbopaque, json, i, bins, + large, huge); + if (json) { + malloc_cprintf(write_cb, + cbopaque, + "\t\t\t}%s\n", (j < + ninitialized) ? "," + : ""); + } } } } diff --git a/test/unit/stats_print.c b/test/unit/stats_print.c new file mode 100644 index 0000000..91cfdf2 --- /dev/null +++ b/test/unit/stats_print.c @@ -0,0 +1,1000 @@ +#include "test/jemalloc_test.h" + +typedef enum { + TOKEN_TYPE_NONE, + TOKEN_TYPE_ERROR, + TOKEN_TYPE_EOI, + TOKEN_TYPE_NULL, + TOKEN_TYPE_FALSE, + TOKEN_TYPE_TRUE, + TOKEN_TYPE_LBRACKET, + TOKEN_TYPE_RBRACKET, + TOKEN_TYPE_LBRACE, + TOKEN_TYPE_RBRACE, + TOKEN_TYPE_COLON, + TOKEN_TYPE_COMMA, + TOKEN_TYPE_STRING, + TOKEN_TYPE_NUMBER +} token_type_t; + +typedef struct parser_s parser_t; +typedef struct { + parser_t *parser; + token_type_t token_type; + size_t pos; + size_t len; + size_t line; + size_t col; +} token_t; + +struct parser_s { + bool verbose; + char *buf; /* '\0'-terminated. */ + size_t len; /* Number of characters preceding '\0' in buf. */ + size_t pos; + size_t line; + size_t col; + token_t token; +}; + +static void +token_init(token_t *token, parser_t *parser, token_type_t token_type, + size_t pos, size_t len, size_t line, size_t col) +{ + token->parser = parser; + token->token_type = token_type; + token->pos = pos; + token->len = len; + token->line = line; + token->col = col; +} + +static void +token_error(token_t *token) +{ + if (!token->parser->verbose) { + return; + } + switch (token->token_type) { + case TOKEN_TYPE_NONE: + not_reached(); + case TOKEN_TYPE_ERROR: + malloc_printf("%zu:%zu: Unexpected character in token: ", + token->line, token->col); + break; + default: + malloc_printf("%zu:%zu: Unexpected token: ", token->line, + token->col); + break; + } + write(STDERR_FILENO, &token->parser->buf[token->pos], token->len); + malloc_printf("\n"); +} + +static void +parser_init(parser_t *parser, bool verbose) +{ + parser->verbose = verbose; + parser->buf = NULL; + parser->len = 0; + parser->pos = 0; + parser->line = 1; + parser->col = 0; +} + +static void +parser_fini(parser_t *parser) +{ + if (parser->buf != NULL) { + dallocx(parser->buf, MALLOCX_TCACHE_NONE); + } +} + +static bool +parser_append(parser_t *parser, const char *str) +{ + size_t len = strlen(str); + char *buf = (parser->buf == NULL) ? mallocx(len + 1, + MALLOCX_TCACHE_NONE) : rallocx(parser->buf, parser->len + len + 1, + MALLOCX_TCACHE_NONE); + if (buf == NULL) { + return true; + } + memcpy(&buf[parser->len], str, len + 1); + parser->buf = buf; + parser->len += len; + return false; +} + +static bool +parser_tokenize(parser_t *parser) +{ + enum { + STATE_START, + STATE_EOI, + STATE_N, STATE_NU, STATE_NUL, STATE_NULL, + STATE_F, STATE_FA, STATE_FAL, STATE_FALS, STATE_FALSE, + STATE_T, STATE_TR, STATE_TRU, STATE_TRUE, + STATE_LBRACKET, + STATE_RBRACKET, + STATE_LBRACE, + STATE_RBRACE, + STATE_COLON, + STATE_COMMA, + STATE_CHARS, + STATE_CHAR_ESCAPE, + STATE_CHAR_U, STATE_CHAR_UD, STATE_CHAR_UDD, STATE_CHAR_UDDD, + STATE_STRING, + STATE_MINUS, + STATE_LEADING_ZERO, + STATE_DIGITS, + STATE_DECIMAL, + STATE_FRAC_DIGITS, + STATE_EXP, + STATE_EXP_SIGN, + STATE_EXP_DIGITS, + STATE_ACCEPT + } state = STATE_START; + size_t token_pos, token_line, token_col; + + assert_zu_le(parser->pos, parser->len, + "Position is past end of buffer"); + + while (state != STATE_ACCEPT) { + char c = parser->buf[parser->pos]; + + switch (state) { + case STATE_START: + token_pos = parser->pos; + token_line = parser->line; + token_col = parser->col; + switch (c) { + case ' ': case '\b': case '\n': case '\r': case '\t': + break; + case '\0': + state = STATE_EOI; + break; + case 'n': + state = STATE_N; + break; + case 'f': + state = STATE_F; + break; + case 't': + state = STATE_T; + break; + case '[': + state = STATE_LBRACKET; + break; + case ']': + state = STATE_RBRACKET; + break; + case '{': + state = STATE_LBRACE; + break; + case '}': + state = STATE_RBRACE; + break; + case ':': + state = STATE_COLON; + break; + case ',': + state = STATE_COMMA; + break; + case '"': + state = STATE_CHARS; + break; + case '-': + state = STATE_MINUS; + break; + case '0': + state = STATE_LEADING_ZERO; + break; + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + state = STATE_DIGITS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_EOI: + token_init(&parser->token, parser, + TOKEN_TYPE_EOI, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + case STATE_N: + switch (c) { + case 'u': + state = STATE_NU; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_NU: + switch (c) { + case 'l': + state = STATE_NUL; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_NUL: + switch (c) { + case 'l': + state = STATE_NULL; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_NULL: + switch (c) { + case ' ': case '\b': case '\n': case '\r': case '\t': + case '\0': + case '[': case ']': case '{': case '}': case ':': + case ',': + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + token_init(&parser->token, parser, TOKEN_TYPE_NULL, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_F: + switch (c) { + case 'a': + state = STATE_FA; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_FA: + switch (c) { + case 'l': + state = STATE_FAL; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_FAL: + switch (c) { + case 's': + state = STATE_FALS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_FALS: + switch (c) { + case 'e': + state = STATE_FALSE; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_FALSE: + switch (c) { + case ' ': case '\b': case '\n': case '\r': case '\t': + case '\0': + case '[': case ']': case '{': case '}': case ':': + case ',': + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + token_init(&parser->token, parser, + TOKEN_TYPE_FALSE, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + case STATE_T: + switch (c) { + case 'r': + state = STATE_TR; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_TR: + switch (c) { + case 'u': + state = STATE_TRU; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_TRU: + switch (c) { + case 'e': + state = STATE_TRUE; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_TRUE: + switch (c) { + case ' ': case '\b': case '\n': case '\r': case '\t': + case '\0': + case '[': case ']': case '{': case '}': case ':': + case ',': + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + token_init(&parser->token, parser, TOKEN_TYPE_TRUE, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_LBRACKET: + token_init(&parser->token, parser, TOKEN_TYPE_LBRACKET, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_RBRACKET: + token_init(&parser->token, parser, TOKEN_TYPE_RBRACKET, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_LBRACE: + token_init(&parser->token, parser, TOKEN_TYPE_LBRACE, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_RBRACE: + token_init(&parser->token, parser, TOKEN_TYPE_RBRACE, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_COLON: + token_init(&parser->token, parser, TOKEN_TYPE_COLON, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_COMMA: + token_init(&parser->token, parser, TOKEN_TYPE_COMMA, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_CHARS: + switch (c) { + case '\\': + state = STATE_CHAR_ESCAPE; + break; + case '"': + state = STATE_STRING; + break; + case 0x00: case 0x01: case 0x02: case 0x03: case 0x04: + case 0x05: case 0x06: case 0x07: case 0x08: case 0x09: + case 0x0a: case 0x0b: case 0x0c: case 0x0d: case 0x0e: + case 0x0f: case 0x10: case 0x11: case 0x12: case 0x13: + case 0x14: case 0x15: case 0x16: case 0x17: case 0x18: + case 0x19: case 0x1a: case 0x1b: case 0x1c: case 0x1d: + case 0x1e: case 0x1f: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + default: + break; + } + break; + case STATE_CHAR_ESCAPE: + switch (c) { + case '"': case '\\': case '/': case 'b': case 'n': + case 'r': case 't': + state = STATE_CHARS; + break; + case 'u': + state = STATE_CHAR_U; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_CHAR_U: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + state = STATE_CHAR_UD; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_CHAR_UD: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + state = STATE_CHAR_UDD; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_CHAR_UDD: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + state = STATE_CHAR_UDDD; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_CHAR_UDDD: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + state = STATE_CHARS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_STRING: + token_init(&parser->token, parser, TOKEN_TYPE_STRING, + token_pos, parser->pos - token_pos, token_line, + token_col); + state = STATE_ACCEPT; + break; + case STATE_MINUS: + switch (c) { + case '0': + state = STATE_LEADING_ZERO; + break; + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + state = STATE_DIGITS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_LEADING_ZERO: + switch (c) { + case '.': + state = STATE_DECIMAL; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_NUMBER, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + } + break; + case STATE_DIGITS: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + break; + case '.': + state = STATE_DECIMAL; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_NUMBER, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + } + break; + case STATE_DECIMAL: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + state = STATE_FRAC_DIGITS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_FRAC_DIGITS: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + break; + case 'e': case 'E': + state = STATE_EXP; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_NUMBER, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + } + break; + case STATE_EXP: + switch (c) { + case '-': case '+': + state = STATE_EXP_SIGN; + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + state = STATE_EXP_DIGITS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_EXP_SIGN: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + state = STATE_EXP_DIGITS; + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_ERROR, token_pos, parser->pos + 1 + - token_pos, token_line, token_col); + return true; + } + break; + case STATE_EXP_DIGITS: + switch (c) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + break; + default: + token_init(&parser->token, parser, + TOKEN_TYPE_NUMBER, token_pos, parser->pos - + token_pos, token_line, token_col); + state = STATE_ACCEPT; + break; + } + break; + default: + not_reached(); + } + + if (state != STATE_ACCEPT) { + if (c == '\n') { + parser->line++; + parser->col = 0; + } else { + parser->col++; + } + parser->pos++; + } + } + return false; +} + +static bool parser_parse_array(parser_t *parser); +static bool parser_parse_object(parser_t *parser); + +static bool +parser_parse_value(parser_t *parser) +{ + switch (parser->token.token_type) { + case TOKEN_TYPE_NULL: + case TOKEN_TYPE_FALSE: + case TOKEN_TYPE_TRUE: + case TOKEN_TYPE_STRING: + case TOKEN_TYPE_NUMBER: + return false; + case TOKEN_TYPE_LBRACE: + return parser_parse_object(parser); + case TOKEN_TYPE_LBRACKET: + return parser_parse_array(parser); + default: + return true; + } + not_reached(); +} + +static bool +parser_parse_pair(parser_t *parser) +{ + assert_d_eq(parser->token.token_type, TOKEN_TYPE_STRING, + "Pair should start with string"); + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_COLON: + if (parser_tokenize(parser)) { + return true; + } + return parser_parse_value(parser); + default: + return true; + } +} + +static bool +parser_parse_values(parser_t *parser) +{ + if (parser_parse_value(parser)) { + return true; + } + + while (true) { + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_COMMA: + if (parser_tokenize(parser)) { + return true; + } + if (parser_parse_value(parser)) { + return true; + } + break; + case TOKEN_TYPE_RBRACKET: + return false; + default: + return true; + } + } +} + +static bool +parser_parse_array(parser_t *parser) +{ + assert_d_eq(parser->token.token_type, TOKEN_TYPE_LBRACKET, + "Array should start with ["); + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_RBRACKET: + return false; + default: + return parser_parse_values(parser); + } + not_reached(); +} + +static bool +parser_parse_pairs(parser_t *parser) +{ + assert_d_eq(parser->token.token_type, TOKEN_TYPE_STRING, + "Object should start with string"); + if (parser_parse_pair(parser)) { + return true; + } + + while (true) { + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_COMMA: + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_STRING: + if (parser_parse_pair(parser)) { + return true; + } + break; + default: + return true; + } + break; + case TOKEN_TYPE_RBRACE: + return false; + default: + return true; + } + } +} + +static bool +parser_parse_object(parser_t *parser) +{ + assert_d_eq(parser->token.token_type, TOKEN_TYPE_LBRACE, + "Object should start with {"); + if (parser_tokenize(parser)) { + return true; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_STRING: + return parser_parse_pairs(parser); + case TOKEN_TYPE_RBRACE: + return false; + default: + return true; + } + not_reached(); +} + +static bool +parser_parse(parser_t *parser) +{ + if (parser_tokenize(parser)) { + goto label_error; + } + if (parser_parse_value(parser)) { + goto label_error; + } + + if (parser_tokenize(parser)) { + goto label_error; + } + switch (parser->token.token_type) { + case TOKEN_TYPE_EOI: + return false; + default: + goto label_error; + } + not_reached(); + +label_error: + token_error(&parser->token); + return true; +} + +TEST_BEGIN(test_json_parser) +{ + size_t i; + const char *invalid_inputs[] = { + /* Tokenizer error case tests. */ + "{ \"string\": X }", + "{ \"string\": nXll }", + "{ \"string\": nuXl }", + "{ \"string\": nulX }", + "{ \"string\": nullX }", + "{ \"string\": fXlse }", + "{ \"string\": faXse }", + "{ \"string\": falXe }", + "{ \"string\": falsX }", + "{ \"string\": falseX }", + "{ \"string\": tXue }", + "{ \"string\": trXe }", + "{ \"string\": truX }", + "{ \"string\": trueX }", + "{ \"string\": \"\n\" }", + "{ \"string\": \"\\z\" }", + "{ \"string\": \"\\uX000\" }", + "{ \"string\": \"\\u0X00\" }", + "{ \"string\": \"\\u00X0\" }", + "{ \"string\": \"\\u000X\" }", + "{ \"string\": -X }", + "{ \"string\": 0.X }", + "{ \"string\": 0.0eX }", + "{ \"string\": 0.0e+X }", + + /* Parser error test cases. */ + "{\"string\": }", + "{\"string\" }", + "{\"string\": [ 0 }", + "{\"string\": {\"a\":0, 1 } }", + "{\"string\": {\"a\":0: } }", + "{", + "{}{", + }; + const char *valid_inputs[] = { + /* Token tests. */ + "null", + "false", + "true", + "{}", + "{\"a\": 0}", + "[]", + "[0, 1]", + "0", + "1", + "10", + "-10", + "10.23", + "10.23e4", + "10.23e-4", + "10.23e+4", + "10.23E4", + "10.23E-4", + "10.23E+4", + "-10.23", + "-10.23e4", + "-10.23e-4", + "-10.23e+4", + "-10.23E4", + "-10.23E-4", + "-10.23E+4", + "\"value\"", + "\" \\\" \\/ \\b \\n \\r \\t \\u0abc \\u1DEF \"", + + /* Parser test with various nesting. */ + "{\"a\":null, \"b\":[1,[{\"c\":2},3]], \"d\":{\"e\":true}}", + }; + + for (i = 0; i < sizeof(invalid_inputs)/sizeof(const char *); i++) { + const char *input = invalid_inputs[i]; + parser_t parser; + parser_init(&parser, false); + assert_false(parser_append(&parser, input), + "Unexpected input appending failure"); + assert_true(parser_parse(&parser), + "Unexpected parse success for input: %s", input); + parser_fini(&parser); + } + + for (i = 0; i < sizeof(valid_inputs)/sizeof(const char *); i++) { + const char *input = valid_inputs[i]; + parser_t parser; + parser_init(&parser, true); + assert_false(parser_append(&parser, input), + "Unexpected input appending failure"); + assert_false(parser_parse(&parser), + "Unexpected parse error for input: %s", input); + parser_fini(&parser); + } +} +TEST_END + +void +write_cb(void *opaque, const char *str) +{ + parser_t *parser = (parser_t *)opaque; + if (parser_append(parser, str)) { + test_fail("Unexpected input appending failure"); + } +} + +TEST_BEGIN(test_stats_print_json) +{ + const char *opts[] = { + "J", + "Jg", + "Jm", + "Jgm", + "Ja", + "Jb", + "Jab", + "Jl", + "Jal", + "Jbl", + "Jabl", + "Jh", + "Jah", + "Jbh", + "Jabh", + "Jlh", + "Jalh", + "Jblh", + "Jablh", + "Jgmablh", + }; + unsigned arena_ind, i; + + for (i = 0; i < 2; i++) { + unsigned j; + + switch (i) { + case 0: + break; + case 1: { + size_t sz = sizeof(arena_ind); + assert_d_eq(mallctl("arenas.extend", (void *)&arena_ind, + &sz, NULL, 0), 0, "Unexpected mallctl failure"); + break; + } default: + not_reached(); + } + + for (j = 0; j < sizeof(opts)/sizeof(const char *); j++) { + parser_t parser; + + parser_init(&parser, true); + malloc_stats_print(write_cb, (void *)&parser, opts[j]); + assert_false(parser_parse(&parser), + "Unexpected parse error, opts=\"%s\"", opts[j]); + parser_fini(&parser); + } + } +} +TEST_END + +int +main(void) +{ + return (test( + test_json_parser, + test_stats_print_json)); +} -- cgit v0.12 From ed19a4892861e40ba231c3be4e299819ce30ff3d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 1 Feb 2017 10:03:04 -0800 Subject: Silence harmless warnings discovered via run_tests.sh. --- test/unit/stats_print.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/unit/stats_print.c b/test/unit/stats_print.c index 91cfdf2..4f412dc 100644 --- a/test/unit/stats_print.c +++ b/test/unit/stats_print.c @@ -67,7 +67,10 @@ token_error(token_t *token) token->col); break; } - write(STDERR_FILENO, &token->parser->buf[token->pos], token->len); + { + UNUSED ssize_t err = write(STDERR_FILENO, + &token->parser->buf[token->pos], token->len); + } malloc_printf("\n"); } @@ -135,7 +138,9 @@ parser_tokenize(parser_t *parser) STATE_EXP_DIGITS, STATE_ACCEPT } state = STATE_START; - size_t token_pos, token_line, token_col; + size_t token_pos JEMALLOC_CC_SILENCE_INIT(0); + size_t token_line JEMALLOC_CC_SILENCE_INIT(1); + size_t token_col JEMALLOC_CC_SILENCE_INIT(0); assert_zu_le(parser->pos, parser->len, "Position is past end of buffer"); -- cgit v0.12 From 1e2c9ef8d6778669657a057979f2a7049012e879 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 26 Feb 2017 12:58:15 -0800 Subject: Fix huge-aligned allocation. This regression was caused by b9408d77a63a54fd331f9b81c884f68e6d57f2e5 (Fix/simplify chunk_recycle() allocation size computations.). This resolves #647. --- Makefile.in | 1 + include/jemalloc/internal/extent.h | 5 ++ include/jemalloc/internal/private_symbols.txt | 2 + src/chunk.c | 9 ++- src/extent.c | 37 +++++++--- test/unit/extent_quantize.c | 98 +++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 test/unit/extent_quantize.c diff --git a/Makefile.in b/Makefile.in index 675e4cb..8f1fb55 100644 --- a/Makefile.in +++ b/Makefile.in @@ -156,6 +156,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/bitmap.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/decay.c \ + $(srcroot)test/unit/extent_quantize.c \ $(srcroot)test/unit/fork.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/junk.c \ diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 168ffe6..fc77f9f 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -75,6 +75,11 @@ typedef rb_tree(extent_node_t) extent_tree_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +#ifdef JEMALLOC_JET +size_t extent_size_quantize_floor(size_t size); +#endif +size_t extent_size_quantize_ceil(size_t size); + rb_proto(, extent_tree_szsnad_, extent_tree_t, extent_node_t) rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 8a9e32f..a83d984 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -223,6 +223,8 @@ extent_node_sn_get extent_node_sn_set extent_node_zeroed_get extent_node_zeroed_set +extent_size_quantize_ceil +extent_size_quantize_floor extent_tree_ad_destroy extent_tree_ad_destroy_recurse extent_tree_ad_empty diff --git a/src/chunk.c b/src/chunk.c index de3bf4c..94f28f2 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -188,12 +188,17 @@ chunk_deregister(const void *chunk, const extent_node_t *node) static extent_node_t * chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szsnad, size_t size) { + extent_node_t *node; + size_t qsize; extent_node_t key; assert(size == CHUNK_CEILING(size)); - extent_node_init(&key, arena, NULL, size, 0, false, false); - return (extent_tree_szsnad_nsearch(chunks_szsnad, &key)); + qsize = extent_size_quantize_ceil(size); + extent_node_init(&key, arena, NULL, qsize, 0, false, false); + node = extent_tree_szsnad_nsearch(chunks_szsnad, &key); + assert(node == NULL || extent_node_size_get(node) >= size); + return node; } static void * diff --git a/src/extent.c b/src/extent.c index 218156c..ff8de2f 100644 --- a/src/extent.c +++ b/src/extent.c @@ -3,13 +3,11 @@ /******************************************************************************/ -/* - * Round down to the nearest chunk size that can actually be requested during - * normal huge allocation. - */ -JEMALLOC_INLINE_C size_t -extent_quantize(size_t size) -{ +#ifndef JEMALLOC_JET +static +#endif +size_t +extent_size_quantize_floor(size_t size) { size_t ret; szind_t ind; @@ -25,11 +23,32 @@ extent_quantize(size_t size) return (ret); } +size_t +extent_size_quantize_ceil(size_t size) { + size_t ret; + + assert(size > 0); + + ret = extent_size_quantize_floor(size); + if (ret < size) { + /* + * Skip a quantization that may have an adequately large extent, + * because under-sized extents may be mixed in. This only + * happens when an unusual size is requested, i.e. for aligned + * allocation, and is just one of several places where linear + * search would potentially find sufficiently aligned available + * memory somewhere lower. + */ + ret = index2size(size2index(ret + 1)); + } + return ret; +} + JEMALLOC_INLINE_C int extent_sz_comp(const extent_node_t *a, const extent_node_t *b) { - size_t a_qsize = extent_quantize(extent_node_size_get(a)); - size_t b_qsize = extent_quantize(extent_node_size_get(b)); + size_t a_qsize = extent_size_quantize_floor(extent_node_size_get(a)); + size_t b_qsize = extent_size_quantize_floor(extent_node_size_get(b)); return ((a_qsize > b_qsize) - (a_qsize < b_qsize)); } diff --git a/test/unit/extent_quantize.c b/test/unit/extent_quantize.c new file mode 100644 index 0000000..d2eb6d7 --- /dev/null +++ b/test/unit/extent_quantize.c @@ -0,0 +1,98 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_huge_extent_size) { + unsigned nhchunks, i; + size_t sz, extent_size_prev, ceil_prev; + size_t mib[4]; + size_t miblen = sizeof(mib) / sizeof(size_t); + + /* + * Iterate over all huge size classes, get their extent sizes, and + * verify that the quantized size is the same as the extent size. + */ + + sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.nhchunks", (void *)&nhchunks, &sz, NULL, + 0), 0, "Unexpected mallctl failure"); + + assert_d_eq(mallctlnametomib("arenas.hchunk.0.size", mib, &miblen), 0, + "Unexpected mallctlnametomib failure"); + for (i = 0; i < nhchunks; i++) { + size_t extent_size, floor, ceil; + + + mib[2] = i; + sz = sizeof(size_t); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&extent_size, + &sz, NULL, 0), 0, "Unexpected mallctlbymib failure"); + floor = extent_size_quantize_floor(extent_size); + ceil = extent_size_quantize_ceil(extent_size); + + assert_zu_eq(extent_size, floor, + "Extent quantization should be a no-op for precise size " + "(extent_size=%zu)", extent_size); + assert_zu_eq(extent_size, ceil, + "Extent quantization should be a no-op for precise size " + "(extent_size=%zu)", extent_size); + + if (i > 0) { + assert_zu_eq(extent_size_prev, + extent_size_quantize_floor(extent_size - PAGE), + "Floor should be a precise size"); + if (extent_size_prev < ceil_prev) { + assert_zu_eq(ceil_prev, extent_size, + "Ceiling should be a precise size " + "(extent_size_prev=%zu, ceil_prev=%zu, " + "extent_size=%zu)", extent_size_prev, + ceil_prev, extent_size); + } + } + if (i + 1 < nhchunks) { + extent_size_prev = floor; + ceil_prev = extent_size_quantize_ceil(extent_size + + PAGE); + } + } +} +TEST_END + +TEST_BEGIN(test_monotonic) { +#define SZ_MAX ZU(4 * 1024 * 1024) + unsigned i; + size_t floor_prev, ceil_prev; + + floor_prev = 0; + ceil_prev = 0; + for (i = 1; i <= SZ_MAX >> LG_PAGE; i++) { + size_t extent_size, floor, ceil; + + extent_size = i << LG_PAGE; + floor = extent_size_quantize_floor(extent_size); + ceil = extent_size_quantize_ceil(extent_size); + + assert_zu_le(floor, extent_size, + "Floor should be <= (floor=%zu, extent_size=%zu, ceil=%zu)", + floor, extent_size, ceil); + assert_zu_ge(ceil, extent_size, + "Ceiling should be >= (floor=%zu, extent_size=%zu, " + "ceil=%zu)", floor, extent_size, ceil); + + assert_zu_le(floor_prev, floor, "Floor should be monotonic " + "(floor_prev=%zu, floor=%zu, extent_size=%zu, ceil=%zu)", + floor_prev, floor, extent_size, ceil); + assert_zu_le(ceil_prev, ceil, "Ceiling should be monotonic " + "(floor=%zu, extent_size=%zu, ceil_prev=%zu, ceil=%zu)", + floor, extent_size, ceil_prev, ceil); + + floor_prev = floor; + ceil_prev = ceil; + } +} +TEST_END + +int +main(void) { + return test( + test_huge_extent_size, + test_monotonic); +} -- cgit v0.12 From 1027a2682bf02204265f2a2403d7701a3778a8a2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 13 Dec 2016 13:38:11 -0800 Subject: Add some missing explicit casts. This resolves #614. --- include/jemalloc/internal/tsd.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 9055aca..9f37433 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -479,13 +479,14 @@ a_name##tsd_wrapper_get(bool init) \ \ if (init && unlikely(wrapper == NULL)) { \ tsd_init_block_t block; \ - wrapper = tsd_init_check_recursion( \ - &a_name##tsd_init_head, &block); \ + wrapper = (a_name##tsd_wrapper_t *) \ + tsd_init_check_recursion(&a_name##tsd_init_head, \ + &block); \ if (wrapper) \ return (wrapper); \ wrapper = (a_name##tsd_wrapper_t *) \ malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ - block.data = wrapper; \ + block.data = (void *)wrapper; \ if (wrapper == NULL) { \ malloc_write(": Error allocating" \ " TSD for "#a_name"\n"); \ -- cgit v0.12 From 7c124830a1f542b5b8d386aa33fab5aa320eb975 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 27 Feb 2017 10:56:23 -0800 Subject: Fix lg_chunk clamping for config_cache_oblivious. Fix lg_chunk clamping to take into account cache-oblivious large allocation. This regression only resulted in incorrect behavior if !config_fill (false unless --disable-fill specified) and config_cache_oblivious (true unless --disable-cache-oblivious specified). This regression was introduced by 8a03cf039cd06f9fa6972711195055d865673966 (Implement cache index randomization for large allocations.), which was first released in 4.0.0. This resolves #555. --- src/arena.c | 10 ++-------- src/jemalloc.c | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/arena.c b/src/arena.c index 6d178d2..ca992f7 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2710,6 +2710,7 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, return (arena_malloc_small(tsdn, arena, ind, zero)); if (likely(size <= large_maxclass)) return (arena_malloc_large(tsdn, arena, ind, zero)); + assert(index2size(ind) >= chunksize); return (huge_malloc(tsdn, arena, index2size(ind), zero)); } @@ -3806,15 +3807,8 @@ arena_boot(void) arena_maxrun = chunksize - (map_bias << LG_PAGE); assert(arena_maxrun > 0); large_maxclass = index2size(size2index(chunksize)-1); - if (large_maxclass > arena_maxrun) { - /* - * For small chunk sizes it's possible for there to be fewer - * non-header pages available than are necessary to serve the - * size classes just below chunksize. - */ - large_maxclass = arena_maxrun; - } assert(large_maxclass > 0); + assert(large_maxclass + large_pad <= arena_maxrun); nlclasses = size2index(large_maxclass) - size2index(SMALL_MAXCLASS); nhclasses = NSIZES - nlclasses - NBINS; diff --git a/src/jemalloc.c b/src/jemalloc.c index 029fe52..e9d8352 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1136,16 +1136,18 @@ malloc_conf_init(void) CONF_HANDLE_BOOL(opt_abort, "abort", true) /* - * Chunks always require at least one header page, - * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and - * possibly an additional page in the presence of - * redzones. In order to simplify options processing, - * use a conservative bound that accommodates all these - * constraints. + * Chunks always require at least one header page, as + * many as 2^(LG_SIZE_CLASS_GROUP+1) data pages (plus an + * additional page in the presence of cache-oblivious + * large), and possibly an additional page in the + * presence of redzones. In order to simplify options + * processing, use a conservative bound that + * accommodates all these constraints. */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + - LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), - (sizeof(size_t) << 3) - 1, yes, yes, true) + LG_SIZE_CLASS_GROUP + 1 + (config_cache_oblivious || + config_fill ? 1 : 0), (sizeof(size_t) << 3) - 1, + yes, yes, true) if (strncmp("dss", k, klen) == 0) { int i; bool match = false; -- cgit v0.12 From 7b53fe928ee857d9401ea1a4fb77285b6fa91e7a Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Mon, 30 Jan 2017 15:54:16 -0800 Subject: Handle race in stats_arena_bins_print When multiple threads calling stats_print, race could happen as we read the counters in separate mallctl calls; and the removed assertion could fail when other operations happened in between the mallctl calls. For simplicity, output "race" in the utilization field in this case. This resolves #616. --- src/stats.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/stats.c b/src/stats.c index 8d579c7..92b8086 100644 --- a/src/stats.c +++ b/src/stats.c @@ -136,8 +136,16 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, availregs = nregs * curruns; milli = (availregs != 0) ? (1000 * curregs) / availregs : 1000; - assert(milli <= 1000); - if (milli < 10) { + + if (milli > 1000) { + /* + * Race detected: the counters were read in + * separate mallctl calls and concurrent + * operations happened in between. In this case + * no meaningful utilization can be computed. + */ + malloc_snprintf(util, sizeof(util), " race"); + } else if (milli < 10) { malloc_snprintf(util, sizeof(util), "0.00%zu", milli); } else if (milli < 100) { @@ -146,8 +154,10 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } else if (milli < 1000) { malloc_snprintf(util, sizeof(util), "0.%zu", milli); - } else + } else { + assert(milli == 1000); malloc_snprintf(util, sizeof(util), "1"); + } if (config_tcache) { malloc_cprintf(write_cb, cbopaque, -- cgit v0.12 From 766ddcd0f20715799042b7e24ea489f24f7121f0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 16 Dec 2016 07:18:55 -0800 Subject: restructure *CFLAGS configuration. Convert CFLAGS to be a concatenation: CFLAGS := CONFIGURE_CFLAGS SPECIFIED_CFLAGS EXTRA_CFLAGS This ordering makes it possible to override the flags set by the configure script both during and after configuration, with CFLAGS and EXTRA_CFLAGS, respectively. This resolves #619. --- INSTALL | 17 +++-- Makefile.in | 4 +- configure.ac | 231 +++++++++++++++++++++++++++++++++-------------------------- 3 files changed, 141 insertions(+), 111 deletions(-) diff --git a/INSTALL b/INSTALL index cce3ed7..08b3624 100644 --- a/INSTALL +++ b/INSTALL @@ -306,17 +306,16 @@ The following environment variables (not a definitive list) impact configure's behavior: CFLAGS="?" - Pass these flags to the compiler. You probably shouldn't define this unless - you know what you are doing. (Use EXTRA_CFLAGS instead.) + Pass these flags to the C compiler. Any flags set by the configure script + are prepended, which means explicitly set flags generally take precedence. + Take care when specifying flags such as -Werror, because configure tests may + be affected in undesirable ways. EXTRA_CFLAGS="?" - Append these flags to CFLAGS. This makes it possible to add flags such as - -Werror, while allowing the configure script to determine what other flags - are appropriate for the specified configuration. - - The configure script specifically checks whether an optimization flag (-O*) - is specified in EXTRA_CFLAGS, and refrains from specifying an optimization - level if it finds that one has already been specified. + Append these flags to CFLAGS, without passing them to the compiler during + configuration. This makes it possible to add flags such as -Werror, while + allowing the configure script to determine what other flags are appropriate + for the specified configuration. CPPFLAGS="?" Pass these flags to the C preprocessor. Note that CFLAGS is not passed to diff --git a/Makefile.in b/Makefile.in index 8f1fb55..e49a871 100644 --- a/Makefile.in +++ b/Makefile.in @@ -24,8 +24,10 @@ abs_objroot := @abs_objroot@ # Build parameters. CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include +CONFIGURE_CFLAGS := @CONFIGURE_CFLAGS@ +SPECIFIED_CFLAGS := @SPECIFIED_CFLAGS@ EXTRA_CFLAGS := @EXTRA_CFLAGS@ -CFLAGS := @CFLAGS@ $(EXTRA_CFLAGS) +CFLAGS := $(strip $(CONFIGURE_CFLAGS) $(SPECIFIED_CFLAGS) $(EXTRA_CFLAGS)) LDFLAGS := @LDFLAGS@ EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ LIBS := @LIBS@ diff --git a/configure.ac b/configure.ac index 4996406..db9e722 100644 --- a/configure.ac +++ b/configure.ac @@ -6,29 +6,66 @@ AC_CONFIG_AUX_DIR([build-aux]) dnl ============================================================================ dnl Custom macro definitions. -dnl JE_CFLAGS_APPEND(cflag) -AC_DEFUN([JE_CFLAGS_APPEND], -[ -AC_MSG_CHECKING([whether compiler supports $1]) -TCFLAGS="${CFLAGS}" -if test "x${CFLAGS}" = "x" ; then - CFLAGS="$1" +dnl JE_CONCAT_VVV(r, a, b) +dnl +dnl Set $r to the concatenation of $a and $b, with a space separating them iff +dnl both $a and $b are non-emty. +AC_DEFUN([JE_CONCAT_VVV], +if test "x[$]{$2}" = "x" -o "x[$]{$3}" = "x" ; then + $1="[$]{$2}[$]{$3}" else - CFLAGS="${CFLAGS} $1" + $1="[$]{$2} [$]{$3}" fi +) + +dnl JE_APPEND_VS(a, b) +dnl +dnl Set $a to the concatenation of $a and b, with a space separating them iff +dnl both $a and b are non-empty. +AC_DEFUN([JE_APPEND_VS], + T_APPEND_V=$2 + JE_CONCAT_VVV($1, $1, T_APPEND_V) +) + +CONFIGURE_CFLAGS= +SPECIFIED_CFLAGS="${CFLAGS}" +dnl JE_CFLAGS_ADD(cflag) +dnl +dnl CFLAGS is the concatenation of CONFIGURE_CFLAGS and SPECIFIED_CFLAGS +dnl (ignoring EXTRA_CFLAGS, which does not impact configure tests. This macro +dnl appends to CONFIGURE_CFLAGS and regenerates CFLAGS. +AC_DEFUN([JE_CFLAGS_ADD], +[ +AC_MSG_CHECKING([whether compiler supports $1]) +T_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" +JE_APPEND_VS(CONFIGURE_CFLAGS, $1) +JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS) AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ ]], [[ return 0; ]])], - [je_cv_cflags_appended=$1] + [je_cv_cflags_added=$1] AC_MSG_RESULT([yes]), - [je_cv_cflags_appended=] + [je_cv_cflags_added=] AC_MSG_RESULT([no]) - [CFLAGS="${TCFLAGS}"] + [CONFIGURE_CFLAGS="${T_CONFIGURE_CFLAGS}"] ) +JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS) ]) +dnl JE_CFLAGS_SAVE() +dnl JE_CFLAGS_RESTORE() +dnl +dnl Save/restore CFLAGS. Nesting is not supported. +AC_DEFUN([JE_CFLAGS_SAVE], +SAVED_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" +) +AC_DEFUN([JE_CFLAGS_RESTORE], +CONFIGURE_CFLAGS="${SAVED_CONFIGURE_CFLAGS}" +JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS) +) + dnl JE_COMPILABLE(label, hcode, mcode, rvar) dnl dnl Use AC_LINK_IFELSE() rather than AC_COMPILE_IFELSE() so that linker errors @@ -168,46 +205,45 @@ if test "x${je_cv_cray}" = "xyes" ; then [je_cv_cray_84=no])]) fi -if test "x$CFLAGS" = "x" ; then - no_CFLAGS="yes" - if test "x$GCC" = "xyes" ; then - JE_CFLAGS_APPEND([-std=gnu11]) - if test "x$je_cv_cflags_appended" = "x-std=gnu11" ; then +if test "x$GCC" = "xyes" ; then + JE_CFLAGS_ADD([-std=gnu11]) + if test "x$je_cv_cflags_added" = "x-std=gnu11" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT]) + else + JE_CFLAGS_ADD([-std=gnu99]) + if test "x$je_cv_cflags_added" = "x-std=gnu99" ; then AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT]) - else - JE_CFLAGS_APPEND([-std=gnu99]) - if test "x$je_cv_cflags_appended" = "x-std=gnu99" ; then - AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT]) - fi fi - JE_CFLAGS_APPEND([-Wall]) - JE_CFLAGS_APPEND([-Werror=declaration-after-statement]) - JE_CFLAGS_APPEND([-Wshorten-64-to-32]) - JE_CFLAGS_APPEND([-Wsign-compare]) - JE_CFLAGS_APPEND([-pipe]) - JE_CFLAGS_APPEND([-g3]) - elif test "x$je_cv_msvc" = "xyes" ; then - CC="$CC -nologo" - JE_CFLAGS_APPEND([-Zi]) - JE_CFLAGS_APPEND([-MT]) - JE_CFLAGS_APPEND([-W3]) - JE_CFLAGS_APPEND([-FS]) - CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat" fi - if test "x$je_cv_cray" = "xyes" ; then - dnl cray compiler 8.4 has an inlining bug - if test "x$je_cv_cray_84" = "xyes" ; then - JE_CFLAGS_APPEND([-hipa2]) - JE_CFLAGS_APPEND([-hnognu]) - fi - if test "x$enable_cc_silence" != "xno" ; then - dnl ignore unreachable code warning - JE_CFLAGS_APPEND([-hnomessage=128]) - dnl ignore redefinition of "malloc", "free", etc warning - JE_CFLAGS_APPEND([-hnomessage=1357]) - fi + JE_CFLAGS_ADD([-Wall]) + JE_CFLAGS_ADD([-Werror=declaration-after-statement]) + JE_CFLAGS_ADD([-Wshorten-64-to-32]) + JE_CFLAGS_ADD([-Wsign-compare]) + JE_CFLAGS_ADD([-pipe]) + JE_CFLAGS_ADD([-g3]) +elif test "x$je_cv_msvc" = "xyes" ; then + CC="$CC -nologo" + JE_CFLAGS_ADD([-Zi]) + JE_CFLAGS_ADD([-MT]) + JE_CFLAGS_ADD([-W3]) + JE_CFLAGS_ADD([-FS]) + JE_APPEND_VS(CPPFLAGS, -I${srcdir}/include/msvc_compat) +fi +if test "x$je_cv_cray" = "xyes" ; then + dnl cray compiler 8.4 has an inlining bug + if test "x$je_cv_cray_84" = "xyes" ; then + JE_CFLAGS_ADD([-hipa2]) + JE_CFLAGS_ADD([-hnognu]) + fi + if test "x$enable_cc_silence" != "xno" ; then + dnl ignore unreachable code warning + JE_CFLAGS_ADD([-hnomessage=128]) + dnl ignore redefinition of "malloc", "free", etc warning + JE_CFLAGS_ADD([-hnomessage=1357]) fi fi +AC_SUBST([CONFIGURE_CFLAGS]) +AC_SUBST([SPECIFIED_CFLAGS]) AC_SUBST([EXTRA_CFLAGS]) AC_PROG_CPP @@ -217,7 +253,7 @@ if test "x${ac_cv_big_endian}" = "x1" ; then fi if test "x${je_cv_msvc}" = "xyes" -a "x${ac_cv_header_inttypes_h}" = "xno"; then - CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat/C99" + JE_APPEND_VS(CPPFLAGS, -I${srcdir}/include/msvc_compat/C99) fi if test "x${je_cv_msvc}" = "xyes" ; then @@ -348,7 +384,6 @@ dnl dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. -CFLAGS="$CFLAGS" default_munmap="1" maps_coalesce="1" case "${host}" in @@ -380,7 +415,7 @@ case "${host}" in ;; *-*-linux-android) dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE. - CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" + JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE) abi="elf" AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ]) @@ -391,7 +426,7 @@ case "${host}" in ;; *-*-linux* | *-*-kfreebsd*) dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE. - CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" + JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE) abi="elf" AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ]) @@ -416,8 +451,8 @@ case "${host}" in abi="elf" RPATH='-Wl,-R,$(1)' dnl Solaris needs this for sigwait(). - CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" - LIBS="$LIBS -lposix4 -lsocket -lnsl" + JE_APPEND_VS(CPPFLAGS, -D_POSIX_PTHREAD_SEMANTICS) + JE_APPEND_VS(LIBS, -lposix4 -lsocket -lnsl) ;; *-ibm-aix*) if "$LG_SIZEOF_PTR" = "8"; then @@ -515,19 +550,19 @@ JE_COMPILABLE([__attribute__ syntax], if test "x${je_cv_attribute}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ]) if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then - JE_CFLAGS_APPEND([-fvisibility=hidden]) + JE_CFLAGS_ADD([-fvisibility=hidden]) fi fi dnl Check for tls_model attribute support (clang 3.0 still lacks support). -SAVED_CFLAGS="${CFLAGS}" -JE_CFLAGS_APPEND([-Werror]) -JE_CFLAGS_APPEND([-herror_on_warning]) +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) JE_COMPILABLE([tls_model attribute], [], [static __thread int __attribute__((tls_model("initial-exec"), unused)) foo; foo = 0;], [je_cv_tls_model]) -CFLAGS="${SAVED_CFLAGS}" +JE_CFLAGS_RESTORE() if test "x${je_cv_tls_model}" = "xyes" ; then AC_DEFINE([JEMALLOC_TLS_MODEL], [__attribute__((tls_model("initial-exec")))]) @@ -535,35 +570,35 @@ else AC_DEFINE([JEMALLOC_TLS_MODEL], [ ]) fi dnl Check for alloc_size attribute support. -SAVED_CFLAGS="${CFLAGS}" -JE_CFLAGS_APPEND([-Werror]) -JE_CFLAGS_APPEND([-herror_on_warning]) +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) JE_COMPILABLE([alloc_size attribute], [#include ], [void *foo(size_t size) __attribute__((alloc_size(1)));], [je_cv_alloc_size]) -CFLAGS="${SAVED_CFLAGS}" +JE_CFLAGS_RESTORE() if test "x${je_cv_alloc_size}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR_ALLOC_SIZE], [ ]) fi dnl Check for format(gnu_printf, ...) attribute support. -SAVED_CFLAGS="${CFLAGS}" -JE_CFLAGS_APPEND([-Werror]) -JE_CFLAGS_APPEND([-herror_on_warning]) +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) JE_COMPILABLE([format(gnu_printf, ...) attribute], [#include ], [void *foo(const char *format, ...) __attribute__((format(gnu_printf, 1, 2)));], [je_cv_format_gnu_printf]) -CFLAGS="${SAVED_CFLAGS}" +JE_CFLAGS_RESTORE() if test "x${je_cv_format_gnu_printf}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF], [ ]) fi dnl Check for format(printf, ...) attribute support. -SAVED_CFLAGS="${CFLAGS}" -JE_CFLAGS_APPEND([-Werror]) -JE_CFLAGS_APPEND([-herror_on_warning]) +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) JE_COMPILABLE([format(printf, ...) attribute], [#include ], [void *foo(const char *format, ...) __attribute__((format(printf, 1, 2)));], [je_cv_format_printf]) -CFLAGS="${SAVED_CFLAGS}" +JE_CFLAGS_RESTORE() if test "x${je_cv_format_printf}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_PRINTF], [ ]) fi @@ -625,9 +660,9 @@ if test "x$enable_code_coverage" = "x1" ; then deoptimize="no" echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || deoptimize="yes" if test "x${deoptimize}" = "xyes" ; then - JE_CFLAGS_APPEND([-O0]) + JE_CFLAGS_ADD([-O0]) fi - JE_CFLAGS_APPEND([-fprofile-arcs -ftest-coverage]) + JE_CFLAGS_ADD([-fprofile-arcs -ftest-coverage]) EXTRA_LDFLAGS="$EXTRA_LDFLAGS -fprofile-arcs -ftest-coverage" AC_DEFINE([JEMALLOC_CODE_COVERAGE], [ ]) fi @@ -817,19 +852,14 @@ if test "x$enable_ivsalloc" = "x1" ; then fi dnl Only optimize if not debugging. -if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then - dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. - optimize="no" - echo "$CFLAGS $EXTRA_CFLAGS" | grep '\-O' >/dev/null || optimize="yes" - if test "x${optimize}" = "xyes" ; then - if test "x$GCC" = "xyes" ; then - JE_CFLAGS_APPEND([-O3]) - JE_CFLAGS_APPEND([-funroll-loops]) - elif test "x$je_cv_msvc" = "xyes" ; then - JE_CFLAGS_APPEND([-O2]) - else - JE_CFLAGS_APPEND([-O]) - fi +if test "x$enable_debug" = "x0" ; then + if test "x$GCC" = "xyes" ; then + JE_CFLAGS_ADD([-O3]) + JE_CFLAGS_ADD([-funroll-loops]) + elif test "x$je_cv_msvc" = "xyes" ; then + JE_CFLAGS_ADD([-O2]) + else + JE_CFLAGS_ADD([-O]) fi fi @@ -893,10 +923,10 @@ fi, if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"]) if test "x$LUNWIND" = "x-lunwind" ; then - AC_CHECK_LIB([unwind], [unw_backtrace], [LIBS="$LIBS $LUNWIND"], + AC_CHECK_LIB([unwind], [unw_backtrace], [JE_APPEND_VS(LIBS, $LUNWIND)], [enable_prof_libunwind="0"]) else - LIBS="$LIBS $LUNWIND" + JE_APPEND_VS(LIBS, $LUNWIND) fi if test "x${enable_prof_libunwind}" = "x1" ; then backtrace_method="libunwind" @@ -918,7 +948,7 @@ fi if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ -a "x$GCC" = "xyes" ; then AC_CHECK_HEADERS([unwind.h], , [enable_prof_libgcc="0"]) - AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [LIBS="$LIBS -lgcc"], [enable_prof_libgcc="0"]) + AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [JE_APPEND_VS(LIBS, -lgcc)], [enable_prof_libgcc="0"]) if test "x${enable_prof_libgcc}" = "x1" ; then backtrace_method="libgcc" AC_DEFINE([JEMALLOC_PROF_LIBGCC], [ ]) @@ -940,7 +970,7 @@ fi ) if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \ -a "x$GCC" = "xyes" ; then - JE_CFLAGS_APPEND([-fno-omit-frame-pointer]) + JE_CFLAGS_ADD([-fno-omit-frame-pointer]) backtrace_method="gcc intrinsics" AC_DEFINE([JEMALLOC_PROF_GCC], [ ]) else @@ -955,9 +985,7 @@ AC_MSG_CHECKING([configured backtracing method]) AC_MSG_RESULT([$backtrace_method]) if test "x$enable_prof" = "x1" ; then dnl Heap profiling uses the log(3) function. - if test "x$LM" != "x" ; then - LIBS="$LIBS $LM" - fi + JE_APPEND_VS(LIBS, $LM) AC_DEFINE([JEMALLOC_PROF], [ ]) fi @@ -1326,7 +1354,7 @@ if test "x$abi" != "xpecoff" ; then AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) dnl Some systems may embed pthreads functionality in libc; check for libpthread dnl first, but try libc too before failing. - AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], + AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)], [AC_SEARCH_LIBS([pthread_create], , , AC_MSG_ERROR([libpthread is missing]))]) JE_COMPILABLE([pthread_atfork(3)], [ @@ -1339,7 +1367,7 @@ if test "x$abi" != "xpecoff" ; then fi fi -CPPFLAGS="$CPPFLAGS -D_REENTRANT" +JE_APPEND_VS(CPPFLAGS, -D_REENTRANT) dnl Check whether clock_gettime(2) is in libc or librt. AC_SEARCH_LIBS([clock_gettime], [rt]) @@ -1348,13 +1376,13 @@ dnl Cray wrapper compiler often adds `-lrt` when using `-static`. Check with dnl `-dynamic` as well in case a user tries to dynamically link in jemalloc if test "x$je_cv_cray_prgenv_wrapper" = "xyes" ; then if test "$ac_cv_search_clock_gettime" != "-lrt"; then - SAVED_CFLAGS="${CFLAGS}" + JE_CFLAGS_SAVE() unset ac_cv_search_clock_gettime - JE_CFLAGS_APPEND([-dynamic]) + JE_CFLAGS_ADD([-dynamic]) AC_SEARCH_LIBS([clock_gettime], [rt]) - CFLAGS="${SAVED_CFLAGS}" + JE_CFLAGS_RESTORE() fi fi @@ -1410,8 +1438,8 @@ fi if test "x$enable_syscall" = "x1" ; then dnl Check if syscall(2) is usable. Treat warnings as errors, so that e.g. OS dnl X 10.12's deprecation warning prevents use. - SAVED_CFLAGS="${CFLAGS}" - JE_CFLAGS_APPEND([-Werror]) + JE_CFLAGS_SAVE() + JE_CFLAGS_ADD([-Werror]) JE_COMPILABLE([syscall(2)], [ #include #include @@ -1419,7 +1447,7 @@ if test "x$enable_syscall" = "x1" ; then syscall(SYS_write, 2, "hello", 5); ], [je_cv_syscall]) - CFLAGS="${SAVED_CFLAGS}" + JE_CFLAGS_RESTORE() if test "x$je_cv_syscall" = "xyes" ; then AC_DEFINE([JEMALLOC_USE_SYSCALL], [ ]) fi @@ -1495,7 +1523,7 @@ if test "x$enable_lazy_lock" = "x1" ; then if test "x$abi" != "xpecoff" ; then AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) AC_CHECK_FUNC([dlsym], [], - [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], + [AC_CHECK_LIB([dl], [dlsym], [JE_APPEND_VS(LIBS, -ldl)], [AC_MSG_ERROR([libdl is missing])]) ]) fi @@ -1947,7 +1975,8 @@ AC_MSG_RESULT([library revision : ${rev}]) AC_MSG_RESULT([]) AC_MSG_RESULT([CONFIG : ${CONFIG}]) AC_MSG_RESULT([CC : ${CC}]) -AC_MSG_RESULT([CFLAGS : ${CFLAGS}]) +AC_MSG_RESULT([CONFIGURE_CFLAGS : ${CONFIGURE_CFLAGS}]) +AC_MSG_RESULT([SPECIFIED_CFLAGS : ${SPECIFIED_CFLAGS}]) AC_MSG_RESULT([EXTRA_CFLAGS : ${EXTRA_CFLAGS}]) AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) -- cgit v0.12 From d84d2909c3132ee633c92fd0d720ec2aed80ff11 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 28 Feb 2017 01:08:28 -0800 Subject: Fix/enhance THP integration. Detect whether chunks start off as THP-capable by default (according to the state of /sys/kernel/mm/transparent_hugepage/enabled), and use this as the basis for whether to call pages_nohuge() once per chunk during first purge of any of the chunk's page runs. Add the --disable-thp configure option, as well as the the opt.thp mallctl. This resolves #541. --- INSTALL | 7 ++ configure.ac | 24 ++++- doc/jemalloc.xml.in | 26 +++++ include/jemalloc/internal/arena.h | 1 + include/jemalloc/internal/jemalloc_internal.h.in | 7 ++ .../jemalloc/internal/jemalloc_internal_defs.h.in | 11 ++- include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 105 ++++++++++++++++++--- src/ctl.c | 6 ++ src/jemalloc.c | 3 + src/pages.c | 4 +- src/stats.c | 1 + test/unit/mallctl.c | 2 + 13 files changed, 177 insertions(+), 21 deletions(-) diff --git a/INSTALL b/INSTALL index 08b3624..19196ec 100644 --- a/INSTALL +++ b/INSTALL @@ -157,6 +157,13 @@ any of the following arguments (not a definitive list) to 'configure': released in bulk, thus reducing the total number of mutex operations. See the "opt.tcache" option for usage details. +--disable-thp + Disable transparent huge page (THP) integration. On systems with THP + support, THPs are explicitly disabled as a side effect of unused dirty page + purging for chunks that back small and/or large allocations, because such + chunks typically comprise active, unused dirty, and untouched clean + pages. + --disable-munmap Disable virtual memory deallocation via munmap(2); instead keep track of the virtual memory for later use. munmap() is disabled by default (i.e. diff --git a/configure.ac b/configure.ac index db9e722..20a8a64 100644 --- a/configure.ac +++ b/configure.ac @@ -1683,10 +1683,31 @@ if test "x${je_cv_madvise}" = "xyes" ; then madvise((void *)0, 0, MADV_NOHUGEPAGE); ], [je_cv_thp]) if test "x${je_cv_thp}" = "xyes" ; then - AC_DEFINE([JEMALLOC_THP], [ ]) + AC_DEFINE([JEMALLOC_HAVE_MADVISE_HUGE], [ ]) fi fi +dnl Enable transparent huge page support by default. +AC_ARG_ENABLE([thp], + [AS_HELP_STRING([--disable-thp], + [Disable transparent huge page supprot])], +[if test "x$enable_thp" = "xno" -o "x${je_cv_thp}" != "xyes" ; then + enable_thp="0" +else + enable_thp="1" +fi +], +[if test "x${je_cv_thp}" = "xyes" ; then + enable_thp="1" +else + enable_thp="0" +fi +]) +if test "x$enable_thp" = "x1" ; then + AC_DEFINE([JEMALLOC_THP], [ ]) +fi +AC_SUBST([enable_thp]) + dnl ============================================================================ dnl Check whether __sync_{add,sub}_and_fetch() are available despite dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. @@ -2014,6 +2035,7 @@ AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) AC_MSG_RESULT([tcache : ${enable_tcache}]) +AC_MSG_RESULT([thp : ${enable_thp}]) AC_MSG_RESULT([fill : ${enable_fill}]) AC_MSG_RESULT([utrace : ${enable_utrace}]) AC_MSG_RESULT([valgrind : ${enable_valgrind}]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index d9c8345..c97ab0f 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -850,6 +850,17 @@ for (i = 0; i < nbins; i++) { during build configuration. + + + config.thp + (bool) + r- + + was not specified + during build configuration, and the system supports transparent huge + page manipulation. + + config.tls @@ -1162,6 +1173,21 @@ malloc_conf = "xmalloc:true";]]> forcefully disabled. + + + opt.thp + (bool) + r- + [] + + Transparent huge page (THP) integration + enabled/disabled. When enabled, THPs are explicitly disabled as a side + effect of unused dirty page purging for chunks that back small and/or + large allocations, because such chunks typically comprise active, + unused dirty, and untouched clean pages. This option is enabled by + default. + + opt.lg_tcache_max diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index ce4e602..119e3a5 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -506,6 +506,7 @@ static const size_t large_pad = #endif ; +extern bool opt_thp; extern purge_mode_t opt_purge; extern const char *purge_mode_names[]; extern ssize_t opt_lg_dirty_mult; diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 6213dd8..e3b499a 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -99,6 +99,13 @@ static const bool config_tcache = false #endif ; +static const bool config_thp = +#ifdef JEMALLOC_THP + true +#else + false +#endif + ; static const bool config_tls = #ifdef JEMALLOC_TLS true diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index b7ae3b7..7c88b0d 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -253,6 +253,12 @@ #undef JEMALLOC_HAVE_MADVISE /* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#undef JEMALLOC_HAVE_MADVISE_HUGE + +/* * Methods for purging unused pages differ between operating systems. * * madvise(..., MADV_FREE) : This marks pages as being unused, such that they @@ -264,10 +270,7 @@ #undef JEMALLOC_PURGE_MADVISE_FREE #undef JEMALLOC_PURGE_MADVISE_DONTNEED -/* - * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE - * arguments to madvise(2). - */ +/* Defined if transparent huge page support is enabled. */ #undef JEMALLOC_THP /* Define if operating system has alloca.h header. */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index a83d984..0aa9b01 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -392,6 +392,7 @@ opt_quarantine opt_redzone opt_stats_print opt_tcache +opt_thp opt_utrace opt_xmalloc opt_zero diff --git a/src/arena.c b/src/arena.c index ca992f7..a9dff0b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -4,6 +4,8 @@ /******************************************************************************/ /* Data. */ +bool opt_thp = true; +static bool thp_initially_huge; purge_mode_t opt_purge = PURGE_DEFAULT; const char *purge_mode_names[] = { "ratio", @@ -680,7 +682,9 @@ arena_chunk_init_hard(tsdn_t *tsdn, arena_t *arena) if (chunk == NULL) return (NULL); - chunk->hugepage = true; + if (config_thp && opt_thp) { + chunk->hugepage = thp_initially_huge; + } /* * Initialize the map to contain one maximal free untouched run. Mark @@ -745,14 +749,17 @@ arena_chunk_alloc(tsdn_t *tsdn, arena_t *arena) static void arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) { - size_t sn, hugepage; + size_t sn; + UNUSED bool hugepage JEMALLOC_CC_SILENCE_INIT(false); bool committed; chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; chunk_deregister(chunk, &chunk->node); sn = extent_node_sn_get(&chunk->node); - hugepage = chunk->hugepage; + if (config_thp && opt_thp) { + hugepage = chunk->hugepage; + } committed = (arena_mapbits_decommitted_get(chunk, map_bias) == 0); if (!committed) { /* @@ -765,13 +772,16 @@ arena_chunk_discard(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk) chunk_hooks.decommit(chunk, chunksize, 0, map_bias << LG_PAGE, arena->ind); } - if (!hugepage) { + if (config_thp && opt_thp && hugepage != thp_initially_huge) { /* - * Convert chunk back to the default state, so that all - * subsequent chunk allocations start out with chunks that can - * be backed by transparent huge pages. + * Convert chunk back to initial THP state, so that all + * subsequent chunk allocations start out in a consistent state. */ - pages_huge(chunk, chunksize); + if (thp_initially_huge) { + pages_huge(chunk, chunksize); + } else { + pages_nohuge(chunk, chunksize); + } } chunk_dalloc_cache(tsdn, arena, &chunk_hooks, (void *)chunk, chunksize, @@ -1711,13 +1721,13 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, /* * If this is the first run purged within chunk, mark - * the chunk as non-huge. This will prevent all use of - * transparent huge pages for this chunk until the chunk - * as a whole is deallocated. + * the chunk as non-THP-capable. This will prevent all + * use of THPs for this chunk until the chunk as a whole + * is deallocated. */ - if (chunk->hugepage) { - pages_nohuge(chunk, chunksize); - chunk->hugepage = false; + if (config_thp && opt_thp && chunk->hugepage) { + chunk->hugepage = pages_nohuge(chunk, + chunksize); } assert(pageind + npages <= chunk_npages); @@ -3772,11 +3782,78 @@ bin_info_init(void) #undef SC } +static void +init_thp_initially_huge(void) { + int fd; + char buf[sizeof("[always] madvise never\n")]; + ssize_t nread; + static const char *enabled_states[] = { + "[always] madvise never\n", + "always [madvise] never\n", + "always madvise [never]\n" + }; + static const bool thp_initially_huge_states[] = { + true, + false, + false + }; + unsigned i; + + if (config_debug) { + for (i = 0; i < sizeof(enabled_states)/sizeof(const char *); + i++) { + assert(sizeof(buf) > strlen(enabled_states[i])); + } + } + assert(sizeof(enabled_states)/sizeof(const char *) == + sizeof(thp_initially_huge_states)/sizeof(bool)); + +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) + fd = (int)syscall(SYS_open, + "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); +#else + fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); +#endif + if (fd == -1) { + goto label_error; + } + +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) + nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf)); +#else + nread = read(fd, &buf, sizeof(buf)); +#endif + +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) + syscall(SYS_close, fd); +#else + close(fd); +#endif + + if (nread < 1) { + goto label_error; + } + for (i = 0; i < sizeof(enabled_states)/sizeof(const char *); + i++) { + if (strncmp(buf, enabled_states[i], (size_t)nread) == 0) { + thp_initially_huge = thp_initially_huge_states[i]; + return; + } + } + +label_error: + thp_initially_huge = false; +} + void arena_boot(void) { unsigned i; + if (config_thp && opt_thp) { + init_thp_initially_huge(); + } + arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); arena_decay_time_default_set(opt_decay_time); diff --git a/src/ctl.c b/src/ctl.c index 1e62e2d..56bc4f4 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -84,6 +84,7 @@ CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) CTL_PROTO(config_stats) CTL_PROTO(config_tcache) +CTL_PROTO(config_thp) CTL_PROTO(config_tls) CTL_PROTO(config_utrace) CTL_PROTO(config_valgrind) @@ -104,6 +105,7 @@ CTL_PROTO(opt_utrace) CTL_PROTO(opt_xmalloc) CTL_PROTO(opt_tcache) CTL_PROTO(opt_lg_tcache_max) +CTL_PROTO(opt_thp) CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) @@ -258,6 +260,7 @@ static const ctl_named_node_t config_node[] = { {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, {NAME("stats"), CTL(config_stats)}, {NAME("tcache"), CTL(config_tcache)}, + {NAME("thp"), CTL(config_thp)}, {NAME("tls"), CTL(config_tls)}, {NAME("utrace"), CTL(config_utrace)}, {NAME("valgrind"), CTL(config_valgrind)}, @@ -281,6 +284,7 @@ static const ctl_named_node_t opt_node[] = { {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, + {NAME("thp"), CTL(opt_thp)}, {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, {NAME("prof_active"), CTL(opt_prof_active)}, @@ -1268,6 +1272,7 @@ CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) CTL_RO_CONFIG_GEN(config_stats, bool) CTL_RO_CONFIG_GEN(config_tcache, bool) +CTL_RO_CONFIG_GEN(config_thp, bool) CTL_RO_CONFIG_GEN(config_tls, bool) CTL_RO_CONFIG_GEN(config_utrace, bool) CTL_RO_CONFIG_GEN(config_valgrind, bool) @@ -1291,6 +1296,7 @@ CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) +CTL_RO_NL_CGEN(config_thp, opt_thp, opt_thp, bool) CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) diff --git a/src/jemalloc.c b/src/jemalloc.c index e9d8352..1cefd4c 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1272,6 +1272,9 @@ malloc_conf_init(void) "lg_tcache_max", -1, (sizeof(size_t) << 3) - 1) } + if (config_thp) { + CONF_HANDLE_BOOL(opt_thp, "thp", true) + } if (config_prof) { CONF_HANDLE_BOOL(opt_prof, "prof", true) CONF_HANDLE_CHAR_P(opt_prof_prefix, diff --git a/src/pages.c b/src/pages.c index 5f0c966..7698e49 100644 --- a/src/pages.c +++ b/src/pages.c @@ -199,7 +199,7 @@ pages_huge(void *addr, size_t size) assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); -#ifdef JEMALLOC_THP +#ifdef JEMALLOC_HAVE_MADVISE_HUGE return (madvise(addr, size, MADV_HUGEPAGE) != 0); #else return (false); @@ -213,7 +213,7 @@ pages_nohuge(void *addr, size_t size) assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); -#ifdef JEMALLOC_THP +#ifdef JEMALLOC_HAVE_MADVISE_HUGE return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); #else return (false); diff --git a/src/stats.c b/src/stats.c index 92b8086..b76afc5 100644 --- a/src/stats.c +++ b/src/stats.c @@ -750,6 +750,7 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_BOOL(xmalloc, ",") OPT_WRITE_BOOL(tcache, ",") OPT_WRITE_SSIZE_T(lg_tcache_max, ",") + OPT_WRITE_BOOL(thp, ",") OPT_WRITE_BOOL(prof, ",") OPT_WRITE_CHAR_P(prof_prefix, ",") OPT_WRITE_BOOL_MUTABLE(prof_active, prof.active, ",") diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 2353c92..3d1a740 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -142,6 +142,7 @@ TEST_BEGIN(test_mallctl_config) TEST_MALLCTL_CONFIG(prof_libunwind, bool); TEST_MALLCTL_CONFIG(stats, bool); TEST_MALLCTL_CONFIG(tcache, bool); + TEST_MALLCTL_CONFIG(thp, bool); TEST_MALLCTL_CONFIG(tls, bool); TEST_MALLCTL_CONFIG(utrace, bool); TEST_MALLCTL_CONFIG(valgrind, bool); @@ -182,6 +183,7 @@ TEST_BEGIN(test_mallctl_opt) TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); TEST_MALLCTL_OPT(bool, tcache, tcache); TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache); + TEST_MALLCTL_OPT(bool, thp, thp); TEST_MALLCTL_OPT(bool, prof, prof); TEST_MALLCTL_OPT(const char *, prof_prefix, prof); TEST_MALLCTL_OPT(bool, prof_active, prof); -- cgit v0.12 From e723f99decc1ef4001ef4b946024056f7664ff9f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 28 Feb 2017 14:29:54 -0800 Subject: Alphabetize private symbol names. --- include/jemalloc/internal/private_symbols.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 0aa9b01..60b57e5 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -532,9 +532,9 @@ tcache_flush tcache_get tcache_get_hard tcache_maxclass -tcache_prefork tcache_postfork_child tcache_postfork_parent +tcache_prefork tcache_salloc tcache_stats_merge tcaches -- cgit v0.12 From 2406c22f366e8d2c37f4d38b2cd857a2a8bf1aa7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 28 Feb 2017 14:57:10 -0800 Subject: Add casts to CONF_HANDLE_T_U(). This avoids signed/unsigned comparison warnings when specifying integer constants as inputs. Clean up whitespace and add clarifying parentheses for CONF_HANDLE_SIZE_T(opt_lg_chunk, ...). --- src/jemalloc.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 1cefd4c..f73a26c 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1075,18 +1075,18 @@ malloc_conf_init(void) k, klen, v, vlen); \ } else if (clip) { \ if (CONF_MIN_##check_min(um, \ - (min))) \ + (t)(min))) \ o = (t)(min); \ else if (CONF_MAX_##check_max( \ - um, (max))) \ + um, (t)(max))) \ o = (t)(max); \ else \ o = (t)um; \ } else { \ if (CONF_MIN_##check_min(um, \ - (min)) || \ + (t)(min)) || \ CONF_MAX_##check_max(um, \ - (max))) { \ + (t)(max))) { \ malloc_conf_error( \ "Out-of-range " \ "conf value", \ @@ -1145,8 +1145,8 @@ malloc_conf_init(void) * accommodates all these constraints. */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + - LG_SIZE_CLASS_GROUP + 1 + (config_cache_oblivious || - config_fill ? 1 : 0), (sizeof(size_t) << 3) - 1, + LG_SIZE_CLASS_GROUP + 1 + ((config_cache_oblivious + || config_fill) ? 1 : 0), (sizeof(size_t) << 3) - 1, yes, yes, true) if (strncmp("dss", k, klen) == 0) { int i; -- cgit v0.12 From 700253e1f2f40d4a74e361fa1e688986c361dba4 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 28 Feb 2017 12:59:22 -0800 Subject: Update ChangeLog for 4.5.0. --- ChangeLog | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ChangeLog b/ChangeLog index f75edd9..a940685 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,41 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc +* 4.5.0 (February 28, 2017) + + This is the first release to benefit from much broader continuous integration + testing, thanks to @davidtgoldblatt. Had we had this testing infrastructure + in place for prior releases, it would have caught all of the most serious + regressions fixed by this release. + + New features: + - Add --disable-thp and the opt.thp to provide opt-out mechanisms for + transparent huge page integration. (@jasone) + - Update zone allocator integration to work with macOS 10.12. (@glandium) + - Restructure *CFLAGS configuration, so that CFLAGS behaves typically, and + EXTRA_CFLAGS provides a way to specify e.g. -Werror during building, but not + during configuration. (@jasone, @ronawho) + + Bug fixes: + - Fix DSS (sbrk(2)-based) allocation. This regression was first released in + 4.3.0. (@jasone) + - Handle race in per size class utilization computation. This functionality + was first released in 4.0.0. (@interwq) + - Fix lock order reversal during gdump. (@jasone) + - Fix-refactor tcache synchronization. This regression was first released in + 4.0.0. (@jasone) + - Fix various JSON-formatted malloc_stats_print() bugs. This functionality + was first released in 4.3.0. (@jasone) + - Fix huge-aligned allocation. This regression was first released in 4.4.0. + (@jasone) + - When transparent huge page integration is enabled, detect what state pages + start in according to the kernel's current operating mode, and only convert + arena chunks to non-huge during purging if that is not their initial state. + This functionality was first released in 4.4.0. (@jasone) + - Fix lg_chunk clamping for the --enable-cache-oblivious --disable-fill case. + This regression was first released in 4.0.0. (@jasone, @428desmo) + - Properly detect sparc64 when building for Linux. (@glaubitz) + * 4.4.0 (December 3, 2016) New features: -- cgit v0.12