From e5effef428b5bf941e1697f6000c97f1ce734756 Mon Sep 17 00:00:00 2001
From: Jason Evans <jasone@canonware.com>
Date: Thu, 13 Oct 2016 14:47:50 -0700
Subject: Add/use adaptive spinning.
Add spin_t and spin_{init,adaptive}(), which provide a simple
abstraction for adaptive spinning.
Adaptively spin during busy waits in bootstrapping and rtree node
initialization.
---
Makefile.in | 1 +
include/jemalloc/internal/jemalloc_internal.h.in | 4 ++
include/jemalloc/internal/spin.h | 51 ++++++++++++++++++++++++
src/jemalloc.c | 5 ++-
src/rtree.c | 5 ++-
src/spin.c | 2 +
6 files changed, 66 insertions(+), 2 deletions(-)
create mode 100644 include/jemalloc/internal/spin.h
create mode 100644 src/spin.c
diff --git a/Makefile.in b/Makefile.in
index f6f0621..9e06309 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -98,6 +98,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/prof.c \
$(srcroot)src/rtree.c \
$(srcroot)src/stats.c \
+ $(srcroot)src/spin.c \
$(srcroot)src/tcache.c \
$(srcroot)src/ticker.c \
$(srcroot)src/tsd.c \
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index b69ddb1..1d02c20 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -346,6 +346,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@@ -375,6 +376,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@@ -465,6 +467,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@@ -494,6 +497,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/nstime.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
+#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
diff --git a/include/jemalloc/internal/spin.h b/include/jemalloc/internal/spin.h
new file mode 100644
index 0000000..9ef5ceb
--- /dev/null
+++ b/include/jemalloc/internal/spin.h
@@ -0,0 +1,51 @@
+/******************************************************************************/
+#ifdef JEMALLOC_H_TYPES
+
+typedef struct spin_s spin_t;
+
+#endif /* JEMALLOC_H_TYPES */
+/******************************************************************************/
+#ifdef JEMALLOC_H_STRUCTS
+
+struct spin_s {
+ unsigned iteration;
+};
+
+#endif /* JEMALLOC_H_STRUCTS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_EXTERNS
+
+#endif /* JEMALLOC_H_EXTERNS */
+/******************************************************************************/
+#ifdef JEMALLOC_H_INLINES
+
+#ifndef JEMALLOC_ENABLE_INLINE
+void spin_init(spin_t *spin);
+void spin_adaptive(spin_t *spin);
+#endif
+
+#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_SPIN_C_))
+JEMALLOC_INLINE void
+spin_init(spin_t *spin)
+{
+
+ spin->iteration = 0;
+}
+
+JEMALLOC_INLINE void
+spin_adaptive(spin_t *spin)
+{
+ volatile uint64_t i;
+
+ for (i = 0; i < (KQU(1) << spin->iteration); i++)
+ CPU_SPINWAIT;
+
+ if (spin->iteration < 63)
+ spin->iteration++;
+}
+
+#endif
+
+#endif /* JEMALLOC_H_INLINES */
+/******************************************************************************/
+
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 95cd054..0348b8a 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -1142,10 +1142,13 @@ malloc_init_hard_needed(void)
}
#ifdef JEMALLOC_THREADED_INIT
if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
+ spin_t spinner;
+
/* Busy-wait until the initializing thread completes. */
+ spin_init(&spinner);
do {
malloc_mutex_unlock(TSDN_NULL, &init_lock);
- CPU_SPINWAIT;
+ spin_adaptive(&spinner);
malloc_mutex_lock(TSDN_NULL, &init_lock);
} while (!malloc_initialized());
return (false);
diff --git a/src/rtree.c b/src/rtree.c
index 421de3e..d4a705a 100644
--- a/src/rtree.c
+++ b/src/rtree.c
@@ -136,12 +136,15 @@ rtree_node_init(tsdn_t *tsdn, rtree_t *rtree, unsigned level,
rtree_elm_t *node;
if (atomic_cas_p((void **)elmp, NULL, RTREE_NODE_INITIALIZING)) {
+ spin_t spinner;
+
/*
* Another thread is already in the process of initializing.
* Spin-wait until initialization is complete.
*/
+ spin_init(&spinner);
do {
- CPU_SPINWAIT;
+ spin_adaptive(&spinner);
node = atomic_read_p((void **)elmp);
} while (node == RTREE_NODE_INITIALIZING);
} else {
diff --git a/src/spin.c b/src/spin.c
new file mode 100644
index 0000000..5242d95
--- /dev/null
+++ b/src/spin.c
@@ -0,0 +1,2 @@
+#define JEMALLOC_SPIN_C_
+#include "jemalloc/internal/jemalloc_internal.h"
--
cgit v0.12