Add atomic(9) implementations of atomic operations.

Add atomic(9) implementations of atomic operations.  These are used on
FreeBSD for non-x86 architectures.

4 files changed, 72 insertions, 6 deletions

diff --git a/configure.ac b/configure.ac
index 3a7a245..f4c2506 100644
--- a/configure.ac
+++ b/configure.ac
@@ -981,6 +981,29 @@ if test "x${je_cv_function_ffsl}" != "xyes" ; then
 fi
 
 dnl ============================================================================
+dnl Check for atomic(9) operations as provided on FreeBSD.
+
+JE_COMPILABLE([atomic(9)], [
+#include <sys/types.h>
+#include <machine/atomic.h>
+#include <inttypes.h>
+], [
+	{
+		uint32_t x32 = 0;
+		volatile uint32_t *x32p = &x32;
+		atomic_fetchadd_32(x32p, 1);
+	}
+	{
+		unsigned long xlong = 0;
+		volatile unsigned long *xlongp = &xlong;
+		atomic_fetchadd_long(xlongp, 1);
+	}
+], [je_cv_atomic9])
+if test "x${je_cv_atomic9}" = "xyes" ; then
+  AC_DEFINE([JEMALLOC_ATOMIC9])
+fi
+
+dnl ============================================================================
 dnl Check for atomic(3) operations as provided on Darwin.
 
 JE_COMPILABLE([Darwin OSAtomic*()], [
@@ -1031,7 +1054,7 @@ AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[
   fi
 ])
 
-if test "x${je_cv_osatomic}" != "xyes" ; then
+if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then
   JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4)
   JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8)
 fi
diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h
index d8f6ca5..016c472 100644
--- a/include/jemalloc/internal/atomic.h
+++ b/include/jemalloc/internal/atomic.h
@@ -32,7 +32,8 @@ unsigned	atomic_sub_u(unsigned *p, unsigned x);
 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_))
 /******************************************************************************/
 /* 64-bit operations. */
-#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
+#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3)
+#  ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
 {
@@ -60,7 +61,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p));
 }
-#elif (defined(__amd64__) || defined(__x86_64__))
+#  elif (defined(__amd64__) || defined(__x86_64__))
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
 {
@@ -87,7 +88,29 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (x);
 }
-#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
+#  elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint64_t
+atomic_add_uint64(uint64_t *p, uint64_t x)
+{
+
+	/*
+	 * atomic_fetchadd_64() doesn't exist, but we only ever use this
+	 * function on LP64 systems, so atomic_fetchadd_long() will do.
+	 */
+	assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+	return (atomic_fetchadd_long(p, (unsigned long)x) + x);
+}
+
+JEMALLOC_INLINE uint64_t
+atomic_sub_uint64(uint64_t *p, uint64_t x)
+{
+
+	assert(sizeof(uint64_t) == sizeof(unsigned long));
+
+	return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x);
+}
+#  elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8))
 JEMALLOC_INLINE uint64_t
 atomic_add_uint64(uint64_t *p, uint64_t x)
 {
@@ -101,8 +124,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
 
 	return (__sync_sub_and_fetch(p, x));
 }
-#else
-#  if (LG_SIZEOF_PTR == 3)
+#  else
 #    error "Missing implementation for 64-bit atomic operations"
 #  endif
 #endif
@@ -164,6 +186,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
 
 	return (x);
 }
+#elif (defined(JEMALLOC_ATOMIC9))
+JEMALLOC_INLINE uint32_t
+atomic_add_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (atomic_fetchadd_32(p, x) + x);
+}
+
+JEMALLOC_INLINE uint32_t
+atomic_sub_uint32(uint32_t *p, uint32_t x)
+{
+
+	return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x);
+}
 #elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4))
 JEMALLOC_INLINE uint32_t
 atomic_add_uint32(uint32_t *p, uint32_t x)
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 51d40fb..905653a 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -161,6 +161,10 @@ static const bool config_ivsalloc =
 #endif
     ;
 
+#ifdef JEMALLOC_ATOMIC9
+#include <machine/atomic.h>
+#endif
+
 #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
 #include <libkern/OSAtomic.h>
 #endif
diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in
index b6e5593..90baa35 100644
--- a/include/jemalloc/jemalloc_defs.h.in
+++ b/include/jemalloc/jemalloc_defs.h.in
@@ -47,6 +47,9 @@
  */
 #undef CPU_SPINWAIT
 
+/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */
+#undef JEMALLOC_ATOMIC9
+
 /*
  * Defined if OSAtomic*() functions are available, as provided by Darwin, and
  * documented in the atomic(3) manual page.