gh-117657: TSAN Fix races in `PyMember_Get` and `PyMember_Set` for C extensions (GH-123211)

5 files changed, 419 insertions, 1 deletions

diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
index 4ecef4f..6d106c1 100644
--- a/Include/cpython/pyatomic.h
+++ b/Include/cpython/pyatomic.h
@@ -321,6 +321,27 @@ _Py_atomic_load_ptr(const void *obj);
 static inline int
 _Py_atomic_load_int_relaxed(const int *obj);
 
+static inline char
+_Py_atomic_load_char_relaxed(const char *obj);
+
+static inline unsigned char
+_Py_atomic_load_uchar_relaxed(const unsigned char *obj);
+
+static inline short
+_Py_atomic_load_short_relaxed(const short *obj);
+
+static inline unsigned short
+_Py_atomic_load_ushort_relaxed(const unsigned short *obj);
+
+static inline long
+_Py_atomic_load_long_relaxed(const long *obj);
+
+static inline double
+_Py_atomic_load_double_relaxed(const double *obj);
+
+static inline long long
+_Py_atomic_load_llong_relaxed(const long long *obj);
+
 static inline int8_t
 _Py_atomic_load_int8_relaxed(const int8_t *obj);
 
@@ -458,6 +479,30 @@ static inline void
 _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
                                 unsigned long long value);
 
+static inline void
+_Py_atomic_store_char_relaxed(char *obj, char value);
+
+static inline void
+_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value);
+
+static inline void
+_Py_atomic_store_short_relaxed(short *obj, short value);
+
+static inline void
+_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value);
+
+static inline void
+_Py_atomic_store_long_relaxed(long *obj, long value);
+
+static inline void
+_Py_atomic_store_float_relaxed(float *obj, float value);
+
+static inline void
+_Py_atomic_store_double_relaxed(double *obj, double value);
+
+static inline void
+_Py_atomic_store_llong_relaxed(long long *obj, long long value);
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
 
diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h
index ef09954..b179e4c 100644
--- a/Include/cpython/pyatomic_gcc.h
+++ b/Include/cpython/pyatomic_gcc.h
@@ -306,6 +306,34 @@ static inline int
 _Py_atomic_load_int_relaxed(const int *obj)
 { return __atomic_load_n(obj, __ATOMIC_RELAXED); }
 
+static inline char
+_Py_atomic_load_char_relaxed(const char *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline unsigned char
+_Py_atomic_load_uchar_relaxed(const unsigned char *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline short
+_Py_atomic_load_short_relaxed(const short *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline unsigned short
+_Py_atomic_load_ushort_relaxed(const unsigned short *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline long
+_Py_atomic_load_long_relaxed(const long *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline float
+_Py_atomic_load_float_relaxed(const float *obj)
+{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; }
+
+static inline double
+_Py_atomic_load_double_relaxed(const double *obj)
+{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; }
+
 static inline int8_t
 _Py_atomic_load_int8_relaxed(const int8_t *obj)
 { return __atomic_load_n(obj, __ATOMIC_RELAXED); }
@@ -362,6 +390,10 @@ static inline unsigned long long
 _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
 { return __atomic_load_n(obj, __ATOMIC_RELAXED); }
 
+static inline long long
+_Py_atomic_load_llong_relaxed(const long long *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
 
 // --- _Py_atomic_store ------------------------------------------------------
 
@@ -485,6 +517,38 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
                                 unsigned long long value)
 { __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
 
+static inline void
+_Py_atomic_store_char_relaxed(char *obj, char value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline void
+_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_short_relaxed(short *obj, short value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_long_relaxed(long *obj, long value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_float_relaxed(float *obj, float value)
+{ __atomic_store(obj, &value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_double_relaxed(double *obj, double value)
+{ __atomic_store(obj, &value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_llong_relaxed(long long *obj, long long value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
 
diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h
index 84da21b..d155955 100644
--- a/Include/cpython/pyatomic_msc.h
+++ b/Include/cpython/pyatomic_msc.h
@@ -634,6 +634,48 @@ _Py_atomic_load_int_relaxed(const int *obj)
     return *(volatile int *)obj;
 }
 
+static inline char
+_Py_atomic_load_char_relaxed(const char *obj)
+{
+    return *(volatile char *)obj;
+}
+
+static inline unsigned char
+_Py_atomic_load_uchar_relaxed(const unsigned char *obj)
+{
+    return *(volatile unsigned char *)obj;
+}
+
+static inline short
+_Py_atomic_load_short_relaxed(const short *obj)
+{
+    return *(volatile short *)obj;
+}
+
+static inline unsigned short
+_Py_atomic_load_ushort_relaxed(const unsigned short *obj)
+{
+    return *(volatile unsigned short *)obj;
+}
+
+static inline long
+_Py_atomic_load_long_relaxed(const long *obj)
+{
+    return *(volatile long *)obj;
+}
+
+static inline float
+_Py_atomic_load_float_relaxed(const float *obj)
+{
+    return *(volatile float *)obj;
+}
+
+static inline double
+_Py_atomic_load_double_relaxed(const double *obj)
+{
+    return *(volatile double *)obj;
+}
+
 static inline int8_t
 _Py_atomic_load_int8_relaxed(const int8_t *obj)
 {
@@ -718,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
     return *(volatile unsigned long long *)obj;
 }
 
+static inline long long
+_Py_atomic_load_llong_relaxed(const long long *obj)
+{
+    return *(volatile long long *)obj;
+}
+
 
 // --- _Py_atomic_store ------------------------------------------------------
 
@@ -899,6 +947,60 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
     *(volatile unsigned long long *)obj = value;
 }
 
+static inline void
+_Py_atomic_store_char_relaxed(char *obj, char value)
+{
+    *(volatile char *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)
+{
+    *(volatile unsigned char *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_short_relaxed(short *obj, short value)
+{
+    *(volatile short *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)
+{
+    *(volatile unsigned short *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value)
+{
+    *(volatile unsigned int *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_long_relaxed(long *obj, long value)
+{
+    *(volatile long *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_float_relaxed(float *obj, float value)
+{
+    *(volatile float *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_double_relaxed(double *obj, double value)
+{
+    *(volatile double *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_llong_relaxed(long long *obj, long long value)
+{
+    *(volatile long long *)obj = value;
+}
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
 
diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h
index 7c71e94..69a8b9e 100644
--- a/Include/cpython/pyatomic_std.h
+++ b/Include/cpython/pyatomic_std.h
@@ -515,6 +515,62 @@ _Py_atomic_load_int_relaxed(const int *obj)
                                 memory_order_relaxed);
 }
 
+static inline char
+_Py_atomic_load_char_relaxed(const char *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(char)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline unsigned char
+_Py_atomic_load_uchar_relaxed(const unsigned char *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(unsigned char)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline short
+_Py_atomic_load_short_relaxed(const short *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(short)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline unsigned short
+_Py_atomic_load_ushort_relaxed(const unsigned short *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(unsigned short)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline long
+_Py_atomic_load_long_relaxed(const long *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(long)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline float
+_Py_atomic_load_float_relaxed(const float *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(float)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline double
+_Py_atomic_load_double_relaxed(const double *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(double)*)obj,
+                                memory_order_relaxed);
+}
+
 static inline int8_t
 _Py_atomic_load_int8_relaxed(const int8_t *obj)
 {
@@ -627,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
                                 memory_order_relaxed);
 }
 
+static inline long long
+_Py_atomic_load_llong_relaxed(const long long *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(long long)*)obj,
+                                memory_order_relaxed);
+}
+
 
 // --- _Py_atomic_store ------------------------------------------------------
 
@@ -852,6 +916,78 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
                           memory_order_relaxed);
 }
 
+static inline void
+_Py_atomic_store_char_relaxed(char *obj, char value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(char)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned char)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_short_relaxed(short *obj, short value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(short)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned short)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_long_relaxed(long *obj, long value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(long)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_float_relaxed(float *obj, float value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(float)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_double_relaxed(double *obj, double value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(double)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_llong_relaxed(long long *obj, long long value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(long long)*)obj, value,
+                          memory_order_relaxed);
+}
+
 
 // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
 
diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h
index a1bb383..d755d03 100644
--- a/Include/internal/pycore_pyatomic_ft_wrappers.h
+++ b/Include/internal/pycore_pyatomic_ft_wrappers.h
@@ -61,6 +61,54 @@ extern "C" {
     _Py_atomic_store_uint16_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \
     _Py_atomic_store_uint32_relaxed(&value, new_value)
+#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \
+    _Py_atomic_store_char_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \
+    _Py_atomic_load_char_relaxed(&value)
+#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \
+    _Py_atomic_store_uchar_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \
+    _Py_atomic_load_uchar_relaxed(&value)
+#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \
+    _Py_atomic_store_short_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \
+    _Py_atomic_load_short_relaxed(&value)
+#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \
+    _Py_atomic_store_ushort_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \
+    _Py_atomic_load_ushort_relaxed(&value)
+#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \
+    _Py_atomic_store_int_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_INT_RELAXED(value) \
+    _Py_atomic_load_int_relaxed(&value)
+#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \
+    _Py_atomic_store_uint_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \
+    _Py_atomic_load_uint_relaxed(&value)
+#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \
+    _Py_atomic_store_long_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \
+    _Py_atomic_load_long_relaxed(&value)
+#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \
+    _Py_atomic_store_ulong_relaxed(&value, new_value)
+#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \
+    _Py_atomic_store_ssize_relaxed(&value, new_value)
+#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \
+    _Py_atomic_store_float_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \
+    _Py_atomic_load_float_relaxed(&value)
+#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \
+    _Py_atomic_store_double_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \
+    _Py_atomic_load_double_relaxed(&value)
+#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \
+    _Py_atomic_store_llong_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \
+    _Py_atomic_load_llong_relaxed(&value)
+#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \
+    _Py_atomic_store_ullong_relaxed(&value, new_value)
+#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \
+    _Py_atomic_load_ullong_relaxed(&value)
 
 #else
 #define FT_ATOMIC_LOAD_PTR(value) value
@@ -68,7 +116,6 @@ extern "C" {
 #define FT_ATOMIC_LOAD_SSIZE(value) value
 #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value
 #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value
-#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value
 #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value
 #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value
 #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value
@@ -85,6 +132,30 @@ extern "C" {
 #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value
+#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value
+#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value
+#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value
+#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_INT_RELAXED(value) value
+#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value
+#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value
+#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value
+#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value
+#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value
+#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value
+#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value
+#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value
 
 #endif