gh-108337: Add pyatomic.h header (#108701)

This adds a new header that provides atomic operations on common data
types. The intention is that this will be exposed through Python.h,
although that is not the case yet. The only immediate use is in
the test file.

Co-authored-by: Sam Gross <colesbury@gmail.com>

5 files changed, 2824 insertions, 3 deletions

diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
new file mode 100644
index 0000000..73712db
--- /dev/null
+++ b/Include/cpython/pyatomic.h
@@ -0,0 +1,506 @@
+// This header provides cross-platform low-level atomic operations
+// similar to C11 atomics.
+//
+// Operations are sequentially consistent unless they have a suffix indicating
+// otherwise. If in doubt, prefer the sequentially consistent operations.
+//
+// The "_relaxed" suffix for load and store operations indicates the "relaxed"
+// memory order. They don't provide synchronization, but (roughly speaking)
+// guarantee somewhat sane behavior for races instead of undefined behavior.
+// In practice, they correspond to "normal" hardware load and store
+// instructions, so they are almost as inexpensive as plain loads and stores
+// in C.
+//
+// Note that atomic read-modify-write operations like _Py_atomic_add_* return
+// the previous value of the atomic variable, not the new value.
+//
+// See https://en.cppreference.com/w/c/atomic for more information on C11
+// atomics.
+// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
+// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage
+// or relaxed atomics.
+//
+// Functions with pseudo Python code:
+//
+//   def _Py_atomic_load(obj):
+//        return obj  # sequential consistency
+//
+//   def _Py_atomic_load_relaxed(obj):
+//       return obj  # relaxed consistency
+//
+//   def _Py_atomic_store(obj, value):
+//       obj = value  # sequential consistency
+//
+//   def _Py_atomic_store_relaxed(obj, value):
+//       obj = value  # relaxed consistency
+//
+//   def _Py_atomic_exchange(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj = value
+//       return old_obj
+//
+//   def _Py_atomic_compare_exchange(obj, expected, desired):
+//       # sequential consistency
+//       if obj == expected:
+//           obj = desired
+//           return True
+//       else:
+//           expected = obj
+//           return False
+//
+//   def _Py_atomic_add(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj += value
+//       return old_obj
+//
+//   def _Py_atomic_and(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj &= value
+//       return old_obj
+//
+//   def _Py_atomic_or(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj |= value
+//       return old_obj
+//
+// Other functions:
+//
+//   def _Py_atomic_load_ptr_acquire(obj):
+//       return obj  # acquire
+//
+//   def _Py_atomic_store_ptr_release(obj):
+//       return obj  # release
+//
+//   def _Py_atomic_fence_seq_cst():
+//       # sequential consistency
+//       ...
+//
+//   def _Py_atomic_fence_release():
+//       # release
+//       ...
+
+#ifndef Py_ATOMIC_H
+#define Py_ATOMIC_H
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+// Atomically adds `value` to `obj` and returns the previous value
+
+static inline int
+_Py_atomic_add_int(int *obj, int value);
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value);
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value);
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value);
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value);
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value);
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value);
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+// Performs an atomic compare-and-exchange.
+//
+// - If `*obj` and `*expected` are equal, store `desired` into `*obj`
+//   and return 1 (success).
+// - Otherwise, store the `*obj` current value into `*expected`
+//   and return 0 (failure).
+//
+// These correspond to the C11 atomic_compare_exchange_strong() function.
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired);
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired);
+
+// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*`
+// so that we can pass types like `PyObject**` without a cast.
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value);
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`.
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value);
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value);
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value);
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value);
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value);
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value);
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value);
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value);
+
+static inline void *
+_Py_atomic_exchange_ptr(void *obj, void *value);
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+// Performs `*obj &= value` atomically and returns the previous value of `*obj`.
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value);
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+// Performs `*obj |= value` atomically and returns the previous value of `*obj`.
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value);
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+// Atomically loads `*obj` (sequential consistency)
+
+static inline int
+_Py_atomic_load_int(const int *obj);
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj);
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj);
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj);
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj);
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj);
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj);
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj);
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj);
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj);
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj);
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj);
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj);
+
+static inline void *
+_Py_atomic_load_ptr(const void *obj);
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+// Loads `*obj` (relaxed consistency, i.e., no ordering)
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj);
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj);
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj);
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj);
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj);
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj);
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj);
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj);
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj);
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj);
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj);
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj);
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);
+
+static inline void *
+_Py_atomic_load_ptr_relaxed(const void *obj);
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+// Atomically performs `*obj = value` (sequential consistency)
+
+static inline void
+_Py_atomic_store_int(int *obj, int value);
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value);
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value);
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value);
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value);
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value);
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value);
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value);
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value);
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value);
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value);
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value);
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value);
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+// Stores `*obj = value` (relaxed consistency, i.e., no ordering)
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value);
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value);
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value);
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value);
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value);
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value);
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value);
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value);
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value);
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value);
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value);
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value);
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value);
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+// Loads `*obj` (acquire operation)
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj);
+
+// Stores `*obj = value` (release operation)
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value);
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+// Sequential consistency fence. C11 fences have complex semantics. When
+// possible, use the atomic operations on variables defined above, which
+// generally do not require explicit use of a fence.
+// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence
+static inline void _Py_atomic_fence_seq_cst(void);
+
+// Release fence
+static inline void _Py_atomic_fence_release(void);
+
+
+#ifndef _Py_USE_GCC_BUILTIN_ATOMICS
+#  if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
+#    define _Py_USE_GCC_BUILTIN_ATOMICS 1
+#  elif defined(__clang__)
+#    if __has_builtin(__atomic_load)
+#      define _Py_USE_GCC_BUILTIN_ATOMICS 1
+#    endif
+#  endif
+#endif
+
+#if _Py_USE_GCC_BUILTIN_ATOMICS
+#  define Py_ATOMIC_GCC_H
+#  include "cpython/pyatomic_gcc.h"
+#  undef Py_ATOMIC_GCC_H
+#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
+#  define Py_ATOMIC_STD_H
+#  include "cpython/pyatomic_std.h"
+#  undef Py_ATOMIC_STD_H
+#elif defined(_MSC_VER)
+#  define Py_ATOMIC_MSC_H
+#  include "cpython/pyatomic_msc.h"
+#  undef Py_ATOMIC_MSC_H
+#else
+#  error "no available pyatomic implementation for this platform/compiler"
+#endif
+
+#endif  /* Py_ATOMIC_H */
+
diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h
new file mode 100644
index 0000000..f1a38c7
--- /dev/null
+++ b/Include/cpython/pyatomic_gcc.h
@@ -0,0 +1,499 @@
+// This is the implementation of Python atomic operations using GCC's built-in
+// functions that match the C+11 memory model. This implementation is preferred
+// for GCC compatible compilers, such as Clang. These functions are available
+// in GCC 4.8+ without needing to compile with --std=c11 or --std=gnu11.
+
+#ifndef Py_ATOMIC_GCC_H
+#  error "this header file must not be included directly"
+#endif
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
+{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void *
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{ return __atomic_exchange_n((void **)obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline void *
+_Py_atomic_load_ptr(const void *obj)
+{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline void *
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{ return (void *)__atomic_load_n((const void **)obj, __ATOMIC_RELAXED); }
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{ return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); }
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); }
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+static inline void
+_Py_atomic_fence_seq_cst(void)
+{ __atomic_thread_fence(__ATOMIC_SEQ_CST); }
+
+ static inline void
+_Py_atomic_fence_release(void)
+{ __atomic_thread_fence(__ATOMIC_RELEASE); }
diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h
new file mode 100644
index 0000000..c88bb03
--- /dev/null
+++ b/Include/cpython/pyatomic_msc.h
@@ -0,0 +1,944 @@
+// This is the implementation of Python atomic operations for MSVC if the
+// compiler does not support C11 or C++11 atomics.
+//
+// MSVC intrinsics are defined on char, short, long, __int64, and pointer
+// types. Note that long and int are both 32-bits even on 64-bit Windows,
+// so operations on int are cast to long.
+//
+// The volatile keyword has additional memory ordering semantics on MSVC. On
+// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64,
+// volatile accesses behave like C11's memory_order_relaxed.
+
+#ifndef Py_ATOMIC_MSC_H
+#  error "this header file must not be included directly"
+#endif
+
+#include <intrin.h>
+
+#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \
+    Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE))
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
+}
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
+}
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
+}
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
+#else
+    int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
+    for (;;) {
+        int64_t new_value = old_value + value;
+        if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{
+    return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value);
+}
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{
+    return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value);
+}
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{
+    return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{
+    return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
+}
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(int64_t);
+    return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+#endif
+}
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
+    return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
+    return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
+}
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    int8_t initial = (int8_t)_InterlockedCompareExchange8(
+                                       (volatile char *)obj,
+                                       (char)value,
+                                       (char)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    int16_t initial = (int16_t)_InterlockedCompareExchange16(
+                                       (volatile short *)obj,
+                                       (short)value,
+                                       (short)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    int32_t initial = (int32_t)_InterlockedCompareExchange(
+                                       (volatile long *)obj,
+                                       (long)value,
+                                       (long)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    int64_t initial = (int64_t)_InterlockedCompareExchange64(
+                                       (volatile __int64 *)obj,
+                                       (__int64)value,
+                                       (__int64)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value)
+{
+    void *initial = _InterlockedCompareExchangePointer(
+                                       (void**)obj,
+                                       value,
+                                       *(void**)expected);
+    if (initial == *(void**)expected) {
+        return 1;
+    }
+    *(void**)expected = initial;
+    return 0;
+}
+
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value)
+{
+    return _Py_atomic_compare_exchange_int8((int8_t *)obj,
+                                            (int8_t *)expected,
+                                            (int8_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value)
+{
+    return _Py_atomic_compare_exchange_int16((int16_t *)obj,
+                                             (int16_t *)expected,
+                                             (int16_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value)
+{
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value)
+{
+    return _Py_atomic_compare_exchange_int64((int64_t *)obj,
+                                             (int64_t *)expected,
+                                             (int64_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value);
+}
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value);
+}
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value);
+}
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value);
+#else
+    int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
+    for (;;) {
+        if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+static inline void*
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{
+    return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value);
+}
+
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{
+    return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj,
+                                             (int8_t)value);
+}
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{
+    return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj,
+                                               (int16_t)value);
+}
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{
+    return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj,
+                                               (int32_t)value);
+}
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (int)_Py_atomic_exchange_int32((int32_t *)obj,
+                                           (int32_t)value);
+}
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj,
+                                                   (int32_t)value);
+}
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{
+    return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj,
+                                               (int64_t)value);
+}
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (intptr_t)_Py_atomic_exchange_ptr((void**)obj,
+                                             (void*)value);
+}
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj,
+                                              (void*)value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj,
+                                               (void*)value);
+}
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value);
+}
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value);
+}
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value);
+}
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value);
+#else
+    uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
+    for (;;) {
+        uint64_t new_value = old_value & value;
+        if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
+    return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj,
+                                            (uint64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj,
+                                            (uint32_t)value);
+#endif
+}
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value);
+}
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value);
+}
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value);
+}
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value);
+#else
+    uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
+    for (;;) {
+        uint64_t new_value = old_value | value;
+        if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
+    return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj,
+                                           (uint64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj,
+                                           (uint32_t)value);
+#endif
+}
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint8_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint8_t)__ldar8((unsigned __int8 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint8"
+#endif
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint16_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint16_t)__ldar16((unsigned __int16 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint16"
+#endif
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint32_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint32_t)__ldar32((unsigned __int32 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint32"
+#endif
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint64_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint64"
+#endif
+}
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{
+    return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj);
+}
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{
+    return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj);
+}
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{
+    return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj);
+}
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (int)_Py_atomic_load_uint32((uint32_t *)obj);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj);
+}
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{
+    return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj);
+}
+
+static inline void*
+_Py_atomic_load_ptr(const void *obj)
+{
+#if SIZEOF_VOID_P == 8
+    return (void*)_Py_atomic_load_uint64((const uint64_t *)obj);
+#else
+    return (void*)_Py_atomic_load_uint32((const uint32_t *)obj);
+#endif
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (intptr_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (uintptr_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{
+    return *(volatile int *)obj;
+}
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{
+    return *(volatile int8_t *)obj;
+}
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{
+    return *(volatile int16_t *)obj;
+}
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{
+    return *(volatile int32_t *)obj;
+}
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{
+    return *(volatile int64_t *)obj;
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{
+    return *(volatile intptr_t *)obj;
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{
+    return *(volatile uint8_t *)obj;
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{
+    return *(volatile uint16_t *)obj;
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{
+    return *(volatile uint32_t *)obj;
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{
+    return *(volatile uint64_t *)obj;
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{
+    return *(volatile uintptr_t *)obj;
+}
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{
+    return *(volatile unsigned int *)obj;
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{
+    return *(volatile Py_ssize_t *)obj;
+}
+
+static inline void*
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{
+    return *(void * volatile *)obj;
+}
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{
+    (void)_Py_atomic_exchange_int(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{
+    (void)_Py_atomic_exchange_int8(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{
+    (void)_Py_atomic_exchange_int16(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{
+    (void)_Py_atomic_exchange_int32(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{
+    (void)_Py_atomic_exchange_int64(obj, value);
+}
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{
+    (void)_Py_atomic_exchange_intptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{
+    (void)_Py_atomic_exchange_uint8(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{
+    (void)_Py_atomic_exchange_uint16(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{
+    (void)_Py_atomic_exchange_uint32(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{
+    (void)_Py_atomic_exchange_uint64(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    (void)_Py_atomic_exchange_uintptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{
+    (void)_Py_atomic_exchange_uint(obj, value);
+}
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{
+    (void)_Py_atomic_exchange_ptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    (void)_Py_atomic_exchange_ssize(obj, value);
+}
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{
+    *(volatile int *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{
+    *(volatile int8_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{
+    *(volatile int16_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{
+    *(volatile int32_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{
+    *(volatile int64_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{
+    *(volatile intptr_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{
+    *(volatile uint8_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{
+    *(volatile uint16_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{
+    *(volatile uint32_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{
+    *(volatile uint64_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{
+    *(volatile uintptr_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{
+    *(volatile unsigned int *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void* value)
+{
+    *(void * volatile *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{
+    *(volatile Py_ssize_t *)obj = value;
+}
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(void * volatile *)obj;
+#elif defined(_M_ARM64)
+    return (void *)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_ptr_acquire"
+#endif
+}
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(void * volatile *)obj = value;
+#elif defined(_M_ARM64)
+    __stlr64(obj, (uintptr_t)value);
+#else
+#  error "no implementation of _Py_atomic_store_ptr_release"
+#endif
+}
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+ static inline void
+_Py_atomic_fence_seq_cst(void)
+{
+#if defined(_M_ARM64)
+    __dmb(_ARM64_BARRIER_ISH);
+#elif defined(_M_X64)
+    __faststorefence();
+#elif defined(_M_IX86)
+    _mm_mfence();
+#else
+#  error "no implementation of _Py_atomic_fence_seq_cst"
+#endif
+}
+
+ static inline void
+_Py_atomic_fence_release(void)
+{
+#if defined(_M_ARM64)
+    __dmb(_ARM64_BARRIER_ISH);
+#elif defined(_M_X64) || defined(_M_IX86)
+    _ReadWriteBarrier();
+#else
+#  error "no implementation of _Py_atomic_fence_release"
+#endif
+}
+
+#undef _Py_atomic_ASSERT_ARG_TYPE
diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h
new file mode 100644
index 0000000..bf74a90
--- /dev/null
+++ b/Include/cpython/pyatomic_std.h
@@ -0,0 +1,872 @@
+// This is the implementation of Python atomic operations using C++11 or C11
+// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC
+// compatible compilers, even if they support C++11 atomics.
+
+#ifndef Py_ATOMIC_STD_H
+#  error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C++" {
+#  include <atomic>
+}
+#  define _Py_USING_STD using namespace std
+#  define _Atomic(tp) atomic<tp>
+#else
+#  define  _Py_USING_STD
+#  include <stdatomic.h>
+#endif
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int)*)obj, value);
+}
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int8_t)*)obj, value);
+}
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int16_t)*)obj, value);
+}
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int32_t)*)obj, value);
+}
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int64_t)*)obj, value);
+}
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(void *)*)obj,
+                                          (void **)expected, desired);
+}
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int)*)obj, value);
+}
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int8_t)*)obj, value);
+}
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int16_t)*)obj, value);
+}
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int32_t)*)obj, value);
+}
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int64_t)*)obj, value);
+}
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+static inline void*
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(void *)*)obj, value);
+}
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int)*)obj);
+}
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int8_t)*)obj);
+}
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int16_t)*)obj);
+}
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int32_t)*)obj);
+}
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int64_t)*)obj);
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(intptr_t)*)obj);
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint8_t)*)obj);
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint32_t)*)obj);
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint32_t)*)obj);
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint64_t)*)obj);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uintptr_t)*)obj);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(unsigned int)*)obj);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(Py_ssize_t)*)obj);
+}
+
+static inline void*
+_Py_atomic_load_ptr(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(void*)*)obj);
+}
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int8_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int16_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int32_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int64_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(intptr_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint8_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint16_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(unsigned int)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline void*
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(void*)*)obj,
+                                memory_order_relaxed);
+}
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int8_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int16_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int32_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int64_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(void*)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int8_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int16_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int32_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int64_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(intptr_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint8_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint16_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(void*)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(void*)*)obj,
+                                memory_order_acquire);
+}
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(void*)*)obj, value,
+                          memory_order_release);
+}
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+ static inline void
+_Py_atomic_fence_seq_cst(void)
+{
+    _Py_USING_STD;
+    atomic_thread_fence(memory_order_seq_cst);
+}
+
+ static inline void
+_Py_atomic_fence_release(void)
+{
+    _Py_USING_STD;
+    atomic_thread_fence(memory_order_release);
+}
diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h
index 48d246e..22ce971 100644
--- a/Include/internal/pycore_atomic.h
+++ b/Include/internal/pycore_atomic.h
@@ -1,5 +1,5 @@
-#ifndef Py_ATOMIC_H
-#define Py_ATOMIC_H
+#ifndef Py_INTERNAL_ATOMIC_H
+#define Py_INTERNAL_ATOMIC_H
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -554,4 +554,4 @@ typedef struct _Py_atomic_int {
 #ifdef __cplusplus
 }
 #endif
-#endif  /* Py_ATOMIC_H */
+#endif  /* Py_INTERNAL_ATOMIC_H */