Merge branch 'dev'3.4.1

4 files changed, 95 insertions, 67 deletions

diff --git a/src/arena.c b/src/arena.c
index 05a787f..d28b629 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -369,13 +369,20 @@ arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
 }
 
 static inline void
+arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind)
+{
+
+	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
+	    LG_PAGE)), PAGE);
+}
+
+static inline void
 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
 {
 	size_t i;
 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
 
-	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
-	    LG_PAGE)), PAGE);
+	arena_run_page_mark_zeroed(chunk, run_ind);
 	for (i = 0; i < PAGE / sizeof(size_t); i++)
 		assert(p[i] == 0);
 }
@@ -458,6 +465,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 					} else if (config_debug) {
 						arena_run_page_validate_zeroed(
 						    chunk, run_ind+i);
+					} else {
+						arena_run_page_mark_zeroed(
+						    chunk, run_ind+i);
 					}
 				}
 			} else {
@@ -467,6 +477,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 				 */
 				arena_run_zero(chunk, run_ind, need_pages);
 			}
+		} else {
+			VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
+			    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
 		}
 
 		/*
@@ -508,9 +521,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
 			arena_run_page_validate_zeroed(chunk,
 			    run_ind+need_pages-1);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk +
+		    (run_ind << LG_PAGE)), (need_pages << LG_PAGE));
 	}
-	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
-	    LG_PAGE)), (need_pages << LG_PAGE));
 }
 
 static arena_chunk_t *
@@ -569,17 +582,24 @@ arena_chunk_alloc(arena_t *arena)
 		 * unless the chunk is not zeroed.
 		 */
 		if (zero == false) {
+			VALGRIND_MAKE_MEM_UNDEFINED(
+			    (void *)arena_mapp_get(chunk, map_bias+1),
+			    (size_t)((uintptr_t) arena_mapp_get(chunk,
+			    chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk,
+			    map_bias+1)));
 			for (i = map_bias+1; i < chunk_npages-1; i++)
 				arena_mapbits_unzeroed_set(chunk, i, unzeroed);
-		} else if (config_debug) {
+		} else {
 			VALGRIND_MAKE_MEM_DEFINED(
 			    (void *)arena_mapp_get(chunk, map_bias+1),
-			    (void *)((uintptr_t)
-			    arena_mapp_get(chunk, chunk_npages-1)
-			    - (uintptr_t)arena_mapp_get(chunk, map_bias+1)));
-			for (i = map_bias+1; i < chunk_npages-1; i++) {
-				assert(arena_mapbits_unzeroed_get(chunk, i) ==
-				    unzeroed);
+			    (size_t)((uintptr_t) arena_mapp_get(chunk,
+			    chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk,
+			    map_bias+1)));
+			if (config_debug) {
+				for (i = map_bias+1; i < chunk_npages-1; i++) {
+					assert(arena_mapbits_unzeroed_get(chunk,
+					    i) == unzeroed);
+				}
 			}
 		}
 		arena_mapbits_unallocated_set(chunk, chunk_npages-1,
@@ -1458,6 +1478,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 			} else if (opt_zero)
 				memset(ret, 0, size);
 		}
+		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 	} else {
 		if (config_fill && opt_junk) {
 			arena_alloc_junk_small(ret, &arena_bin_info[binind],
@@ -1466,7 +1487,6 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero)
 		VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 		memset(ret, 0, size);
 	}
-	VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
 
 	return (ret);
 }
diff --git a/src/chunk.c b/src/chunk.c
index aef3fed..b17f43f 100644
--- a/src/chunk.c
+++ b/src/chunk.c
@@ -294,7 +294,7 @@ label_return:
 	if (xnode != NULL)
 		base_node_dealloc(xnode);
 	if (xprev != NULL)
-		base_node_dealloc(prev);
+		base_node_dealloc(xprev);
 }
 
 void
diff --git a/src/ctl.c b/src/ctl.c
index f2ef4e6..ebba7c2 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -546,43 +546,30 @@ ctl_arena_refresh(arena_t *arena, unsigned i)
 static bool
 ctl_grow(void)
 {
-	size_t astats_size;
 	ctl_arena_stats_t *astats;
 	arena_t **tarenas;
 
-	/* Extend arena stats and arenas arrays. */
-	astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t);
-	if (ctl_stats.narenas == narenas_auto) {
-		/* ctl_stats.arenas and arenas came from base_alloc(). */
-		astats = (ctl_arena_stats_t *)imalloc(astats_size);
-		if (astats == NULL)
-			return (true);
-		memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
-		    sizeof(ctl_arena_stats_t));
-
-		tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
-		    sizeof(arena_t *));
-		if (tarenas == NULL) {
-			idalloc(astats);
-			return (true);
-		}
-		memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *));
-	} else {
-		astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas,
-		    astats_size, 0, 0, false, false);
-		if (astats == NULL)
-			return (true);
-
-		tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) *
-		    sizeof(arena_t *), 0, 0, false, false);
-		if (tarenas == NULL)
-			return (true);
+	/* Allocate extended arena stats and arenas arrays. */
+	astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) *
+	    sizeof(ctl_arena_stats_t));
+	if (astats == NULL)
+		return (true);
+	tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) *
+	    sizeof(arena_t *));
+	if (tarenas == NULL) {
+		idalloc(astats);
+		return (true);
 	}
-	/* Initialize the new astats and arenas elements. */
+
+	/* Initialize the new astats element. */
+	memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) *
+	    sizeof(ctl_arena_stats_t));
 	memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t));
-	if (ctl_arena_init(&astats[ctl_stats.narenas + 1]))
+	if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) {
+		idalloc(tarenas);
+		idalloc(astats);
 		return (true);
-	tarenas[ctl_stats.narenas] = NULL;
+	}
 	/* Swap merged stats to their new location. */
 	{
 		ctl_arena_stats_t tstats;
@@ -593,13 +580,34 @@ ctl_grow(void)
 		memcpy(&astats[ctl_stats.narenas + 1], &tstats,
 		    sizeof(ctl_arena_stats_t));
 	}
+	/* Initialize the new arenas element. */
+	tarenas[ctl_stats.narenas] = NULL;
+	{
+		arena_t **arenas_old = arenas;
+		/*
+		 * Swap extended arenas array into place.  Although ctl_mtx
+		 * protects this function from other threads extending the
+		 * array, it does not protect from other threads mutating it
+		 * (i.e. initializing arenas and setting array elements to
+		 * point to them).  Therefore, array copying must happen under
+		 * the protection of arenas_lock.
+		 */
+		malloc_mutex_lock(&arenas_lock);
+		arenas = tarenas;
+		memcpy(arenas, arenas_old, ctl_stats.narenas *
+		    sizeof(arena_t *));
+		narenas_total++;
+		arenas_extend(narenas_total - 1);
+		malloc_mutex_unlock(&arenas_lock);
+		/*
+		 * Deallocate arenas_old only if it came from imalloc() (not
+		 * base_alloc()).
+		 */
+		if (ctl_stats.narenas != narenas_auto)
+			idalloc(arenas_old);
+	}
 	ctl_stats.arenas = astats;
 	ctl_stats.narenas++;
-	malloc_mutex_lock(&arenas_lock);
-	arenas = tarenas;
-	narenas_total++;
-	arenas_extend(narenas_total - 1);
-	malloc_mutex_unlock(&arenas_lock);
 
 	return (false);
 }
@@ -1109,7 +1117,7 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
     void *newp, size_t newlen)
 {
 	int ret;
-	uint64_t newval;
+	UNUSED uint64_t newval;
 
 	malloc_mutex_lock(&ctl_mtx);
 	WRITE(newval, uint64_t);
diff --git a/src/jemalloc.c b/src/jemalloc.c
index bc350ed..ae56db6 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -282,7 +282,7 @@ arenas_cleanup(void *arg)
 	malloc_mutex_unlock(&arenas_lock);
 }
 
-static JEMALLOC_ATTR(always_inline) void
+JEMALLOC_ALWAYS_INLINE_C void
 malloc_thread_init(void)
 {
 
@@ -299,7 +299,7 @@ malloc_thread_init(void)
 		quarantine_alloc_hook();
 }
 
-static JEMALLOC_ATTR(always_inline) bool
+JEMALLOC_ALWAYS_INLINE_C bool
 malloc_init(void)
 {
 
@@ -436,8 +436,9 @@ malloc_conf_init(void)
 			}
 			break;
 		case 1: {
+			int linklen = 0;
 #ifndef _WIN32
-			int linklen;
+			int saved_errno = errno;
 			const char *linkname =
 #  ifdef JEMALLOC_PREFIX
 			    "/etc/"JEMALLOC_PREFIX"malloc.conf"
@@ -446,21 +447,20 @@ malloc_conf_init(void)
 #  endif
 			    ;
 
-			if ((linklen = readlink(linkname, buf,
-			    sizeof(buf) - 1)) != -1) {
-				/*
-				 * Use the contents of the "/etc/malloc.conf"
-				 * symbolic link's name.
-				 */
-				buf[linklen] = '\0';
-				opts = buf;
-			} else
-#endif
-			{
+			/*
+			 * Try to use the contents of the "/etc/malloc.conf"
+			 * symbolic link's name.
+			 */
+			linklen = readlink(linkname, buf, sizeof(buf) - 1);
+			if (linklen == -1) {
 				/* No configuration specified. */
-				buf[0] = '\0';
-				opts = buf;
+				linklen = 0;
+				/* restore errno */
+				set_errno(saved_errno);
 			}
+#endif
+			buf[linklen] = '\0';
+			opts = buf;
 			break;
 		} case 2: {
 			const char *envname =
@@ -1402,7 +1402,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
  */
 #ifdef JEMALLOC_EXPERIMENTAL
 
-static JEMALLOC_ATTR(always_inline) void *
+JEMALLOC_ALWAYS_INLINE_C void *
 iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache,
     arena_t *arena)
 {