Issue #22335: Fix crash when trying to enlarge a bytearray to 0x7fffffff bytes on a 32-bit platform.

2 files changed, 18 insertions, 11 deletions

diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
index 5b75705..d72abb7 100644
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -175,20 +175,22 @@ PyByteArray_AsString(PyObject *self)
 }
 
 int
-PyByteArray_Resize(PyObject *self, Py_ssize_t size)
+PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size)
 {
     void *sval;
     PyByteArrayObject *obj = ((PyByteArrayObject *)self);
-    Py_ssize_t alloc = obj->ob_alloc;
-    Py_ssize_t logical_offset = obj->ob_start - obj->ob_bytes;
+    /* All computations are done unsigned to avoid integer overflows
+       (see issue #22335). */
+    size_t alloc = (size_t) obj->ob_alloc;
+    size_t logical_offset = (size_t) (obj->ob_start - obj->ob_bytes);
+    size_t size = (size_t) requested_size;
 
     assert(self != NULL);
     assert(PyByteArray_Check(self));
-    assert(size >= 0);
-    assert(logical_offset >= 0);
     assert(logical_offset <= alloc);
+    assert(requested_size >= 0);
 
-    if (size == Py_SIZE(self)) {
+    if (requested_size == Py_SIZE(self)) {
         return 0;
     }
     if (!_canresize(obj)) {
@@ -220,6 +222,10 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t size)
             alloc = size + 1;
         }
     }
+    if (alloc > PY_SSIZE_T_MAX) {
+        PyErr_NoMemory();
+        return -1;
+    }
 
     if (logical_offset > 0) {
         sval = PyObject_Malloc(alloc);
@@ -227,7 +233,8 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t size)
             PyErr_NoMemory();
             return -1;
         }
-        memcpy(sval, PyByteArray_AS_STRING(self), Py_MIN(size, Py_SIZE(self)));
+        memcpy(sval, PyByteArray_AS_STRING(self),
+               Py_MIN(requested_size, Py_SIZE(self)));
         PyObject_Free(obj->ob_bytes);
     }
     else {
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index 004cfaa..3c33255 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1754,8 +1754,8 @@ _PyMem_DebugMalloc(void *ctx, size_t nbytes)
 
     bumpserialno();
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     p = (uchar *)api->alloc.malloc(api->alloc.ctx, total);
@@ -1817,8 +1817,8 @@ _PyMem_DebugRealloc(void *ctx, void *p, size_t nbytes)
     bumpserialno();
     original_nbytes = read_size_t(q - 2*SST);
     total = nbytes + 4*SST;
-    if (total < nbytes)
-        /* overflow:  can't represent total as a size_t */
+    if (nbytes > PY_SSIZE_T_MAX - 4*SST)
+        /* overflow:  can't represent total as a Py_ssize_t */
         return NULL;
 
     /* Resize and add decorations. We may get a new pointer here, in which