gh-121153: Fix some errors with use of _PyLong_CompactValue() (GH-121154)

* The result has type Py_ssize_t, not intptr_t.
* Type cast between unsigned and signdet integer types should be explicit.
* Downcasting should be explicit.
* Fix integer overflow check in sum().

2 files changed, 57 insertions, 15 deletions

diff --git a/Objects/longobject.c b/Objects/longobject.c
index 4ca259f..ace01dd 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -483,11 +483,18 @@ PyLong_AsLongAndOverflow(PyObject *vv, int *overflow)
         do_decref = 1;
     }
     if (_PyLong_IsCompact(v)) {
-#if SIZEOF_LONG < SIZEOF_VOID_P
-        intptr_t tmp = _PyLong_CompactValue(v);
-        res = (long)tmp;
-        if (res != tmp) {
-            *overflow = tmp < 0 ? -1 : 1;
+#if SIZEOF_LONG < SIZEOF_SIZE_T
+        Py_ssize_t tmp = _PyLong_CompactValue(v);
+        if (tmp < LONG_MIN) {
+            *overflow = -1;
+            res = -1;
+        }
+        else if (tmp > LONG_MAX) {
+            *overflow = 1;
+            res = -1;
+        }
+        else {
+            res = (long)tmp;
         }
 #else
         res = _PyLong_CompactValue(v);
@@ -632,14 +639,15 @@ PyLong_AsUnsignedLong(PyObject *vv)
 
     v = (PyLongObject *)vv;
     if (_PyLong_IsNonNegativeCompact(v)) {
-#if SIZEOF_LONG < SIZEOF_VOID_P
-        intptr_t tmp = _PyLong_CompactValue(v);
+#if SIZEOF_LONG < SIZEOF_SIZE_T
+        size_t tmp = (size_t)_PyLong_CompactValue(v);
         unsigned long res = (unsigned long)tmp;
         if (res != tmp) {
             goto overflow;
         }
+        return res;
 #else
-        return _PyLong_CompactValue(v);
+        return (unsigned long)(size_t)_PyLong_CompactValue(v);
 #endif
     }
     if (_PyLong_IsNegative(v)) {
@@ -685,7 +693,7 @@ PyLong_AsSize_t(PyObject *vv)
 
     v = (PyLongObject *)vv;
     if (_PyLong_IsNonNegativeCompact(v)) {
-        return _PyLong_CompactValue(v);
+        return (size_t)_PyLong_CompactValue(v);
     }
     if (_PyLong_IsNegative(v)) {
         PyErr_SetString(PyExc_OverflowError,
@@ -722,7 +730,11 @@ _PyLong_AsUnsignedLongMask(PyObject *vv)
     }
     v = (PyLongObject *)vv;
     if (_PyLong_IsCompact(v)) {
-        return (unsigned long)_PyLong_CompactValue(v);
+#if SIZEOF_LONG < SIZEOF_SIZE_T
+        return (unsigned long)(size_t)_PyLong_CompactValue(v);
+#else
+        return (unsigned long)(long)_PyLong_CompactValue(v);
+#endif
     }
     i = _PyLong_DigitCount(v);
     int sign = _PyLong_NonCompactSign(v);
@@ -1540,7 +1552,18 @@ PyLong_AsUnsignedLongLong(PyObject *vv)
     v = (PyLongObject*)vv;
     if (_PyLong_IsNonNegativeCompact(v)) {
         res = 0;
-        bytes = _PyLong_CompactValue(v);
+#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T
+        size_t tmp = (size_t)_PyLong_CompactValue(v);
+        bytes = (unsigned long long)tmp;
+        if (bytes != tmp) {
+            PyErr_SetString(PyExc_OverflowError,
+                            "Python int too large to convert "
+                            "to C unsigned long long");
+            res = -1;
+        }
+#else
+        bytes = (unsigned long long)(size_t)_PyLong_CompactValue(v);
+#endif
     }
     else {
         res = _PyLong_AsByteArray((PyLongObject *)vv, (unsigned char *)&bytes,
@@ -1571,7 +1594,11 @@ _PyLong_AsUnsignedLongLongMask(PyObject *vv)
     }
     v = (PyLongObject *)vv;
     if (_PyLong_IsCompact(v)) {
-        return (unsigned long long)(signed long long)_PyLong_CompactValue(v);
+#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T
+        return (unsigned long long)(size_t)_PyLong_CompactValue(v);
+#else
+        return (unsigned long long)(long long)_PyLong_CompactValue(v);
+#endif
     }
     i = _PyLong_DigitCount(v);
     sign = _PyLong_NonCompactSign(v);
@@ -1643,7 +1670,22 @@ PyLong_AsLongLongAndOverflow(PyObject *vv, int *overflow)
         do_decref = 1;
     }
     if (_PyLong_IsCompact(v)) {
+#if SIZEOF_LONG_LONG < SIZEOF_SIZE_T
+        Py_ssize_t tmp = _PyLong_CompactValue(v);
+        if (tmp < LLONG_MIN) {
+            *overflow = -1;
+            res = -1;
+        }
+        else if (tmp > LLONG_MAX) {
+            *overflow = 1;
+            res = -1;
+        }
+        else {
+            res = (long long)tmp;
+        }
+#else
         res = _PyLong_CompactValue(v);
+#endif
     }
     else {
         i = _PyLong_DigitCount(v);
@@ -3579,7 +3621,7 @@ long_hash(PyLongObject *v)
     int sign;
 
     if (_PyLong_IsCompact(v)) {
-        x = _PyLong_CompactValue(v);
+        x = (Py_uhash_t)_PyLong_CompactValue(v);
         if (x == (Py_uhash_t)-1) {
             x = (Py_uhash_t)-2;
         }
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index a5b45e3..3f7bf4d 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -2644,8 +2644,8 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start)
                     b = PyLong_AsLongAndOverflow(item, &overflow);
                 }
                 if (overflow == 0 &&
-                    (i_result >= 0 ? (b <= LONG_MAX - i_result)
-                                   : (b >= LONG_MIN - i_result)))
+                    (i_result >= 0 ? (b <= PY_SSIZE_T_MAX - i_result)
+                                   : (b >= PY_SSIZE_T_MIN - i_result)))
                 {
                     i_result += b;
                     Py_DECREF(item);