[2.7] bpo-37170: Fix the cast on error in PyLong_AsUnsignedLongLongMask() (GH-13860) (GH-13898)

(cherry picked from commit dc2476500d91082f0c907772c83a044bf49af279)

Co-authored-by: Zackery Spytz <zspytz@gmail.com>

4 files changed, 30 insertions, 1 deletions

diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index 28fb589..4684d1b 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -217,6 +217,9 @@ Long Integer Objects
    Return a C :c:type:`unsigned long` from a Python long integer, without checking
    for overflow.
 
+   Returns ``(unsigned long)-1`` on error.  Use :c:func:`PyErr_Occurred` to
+   disambiguate.
+
    .. versionadded:: 2.3
 
 
@@ -225,6 +228,9 @@ Long Integer Objects
    Return a C :c:type:`unsigned long long` from a Python long integer, without
    checking for overflow.
 
+   Returns ``(unsigned PY_LONG_LONG)-1`` on error.  Use
+   :c:func:`PyErr_Occurred` to disambiguate.
+
    .. versionadded:: 2.3
 
 
diff --git a/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst b/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst
new file mode 100644
index 0000000..7a35c95
--- /dev/null
+++ b/Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst
@@ -0,0 +1 @@
+Fix the cast on error in :c:func:`PyLong_AsUnsignedLongLongMask()`.
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
index 67488e7..af93d00 100644
--- a/Modules/_testcapimodule.c
+++ b/Modules/_testcapimodule.c
@@ -888,6 +888,26 @@ test_long_long_and_overflow(PyObject *self)
     return Py_None;
 }
 
+static PyObject *
+test_long_as_unsigned_long_long_mask(PyObject *self)
+{
+    unsigned PY_LONG_LONG res = PyLong_AsUnsignedLongLongMask(NULL);
+
+    if (res != (unsigned PY_LONG_LONG)-1 || !PyErr_Occurred()) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) didn't "
+                              "complain");
+    }
+    if (!PyErr_ExceptionMatches(PyExc_SystemError)) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) raised "
+                              "something other than SystemError");
+    }
+    PyErr_Clear();
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
 /* Test the L code for PyArg_ParseTuple.  This should deliver a PY_LONG_LONG
    for both long and int arguments.  The test may leak a little memory if
    it fails.
@@ -2715,6 +2735,8 @@ static PyMethodDef TestMethods[] = {
     {"test_longlong_api",       test_longlong_api,               METH_NOARGS},
     {"test_long_long_and_overflow",
         (PyCFunction)test_long_long_and_overflow, METH_NOARGS},
+    {"test_long_as_unsigned_long_long_mask",
+        (PyCFunction)test_long_as_unsigned_long_long_mask, METH_NOARGS},
     {"test_L_code",             (PyCFunction)test_L_code,        METH_NOARGS},
 #endif
     {"getargs_f",               getargs_f,                       METH_VARARGS},
diff --git a/Objects/longobject.c b/Objects/longobject.c
index f40ad7a..c05f67c 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -1022,7 +1022,7 @@ PyLong_AsUnsignedLongLongMask(PyObject *vv)
 
     if (vv == NULL || !PyLong_Check(vv)) {
         PyErr_BadInternalCall();
-        return (unsigned long) -1;
+        return (unsigned PY_LONG_LONG) -1;
     }
     v = (PyLongObject *)vv;
     i = Py_SIZE(v);