A review of overflow-detecting code in the 2.4 branch.

* unified the way intobject, longobject and mystrtoul handle
  values around -sys.maxint-1.

* in general, trying to entierely avoid overflows in any computation
  involving signed ints or longs is extremely involved.  Fixed a few
  simple cases where a compiler might be too clever (but that's all
  guesswork).

* more overflow checks against bad data in marshal.c.

2 files changed, 49 insertions, 25 deletions

diff --git a/Python/marshal.c b/Python/marshal.c
index df88bcb..4f4d6b3 100644
--- a/Python/marshal.c
+++ b/Python/marshal.c
@@ -451,6 +451,11 @@ r_object(RFILE *p)
 			int size;
 			PyLongObject *ob;
 			n = r_long(p);
+			if (n < -INT_MAX || n > INT_MAX) {
+				PyErr_SetString(PyExc_ValueError,
+						"bad marshal data");
+				return NULL;
+			}
 			size = n<0 ? -n : n;
 			ob = _PyLong_New(size);
 			if (ob == NULL)
@@ -518,7 +523,7 @@ r_object(RFILE *p)
 	case TYPE_INTERNED:
 	case TYPE_STRING:
 		n = r_long(p);
-		if (n < 0) {
+		if (n < 0 || n > INT_MAX) {
 			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 			return NULL;
 		}
@@ -539,6 +544,10 @@ r_object(RFILE *p)
 
 	case TYPE_STRINGREF:
 		n = r_long(p);
+		if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
+			PyErr_SetString(PyExc_ValueError, "bad marshal data");
+			return NULL;
+		}
 		v = PyList_GET_ITEM(p->strings, n);
 		Py_INCREF(v);
 		return v;
@@ -549,7 +558,7 @@ r_object(RFILE *p)
 		char *buffer;
 
 		n = r_long(p);
-		if (n < 0) {
+		if (n < 0 || n > INT_MAX) {
 			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 			return NULL;
 		}
@@ -570,7 +579,7 @@ r_object(RFILE *p)
 
 	case TYPE_TUPLE:
 		n = r_long(p);
-		if (n < 0) {
+		if (n < 0 || n > INT_MAX) {
 			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 			return NULL;
 		}
@@ -593,7 +602,7 @@ r_object(RFILE *p)
 
 	case TYPE_LIST:
 		n = r_long(p);
-		if (n < 0) {
+		if (n < 0 || n > INT_MAX) {
 			PyErr_SetString(PyExc_ValueError, "bad marshal data");
 			return NULL;
 		}
@@ -643,10 +652,11 @@ r_object(RFILE *p)
 			return NULL;
 		}
 		else {
-			int argcount = r_long(p);
-			int nlocals = r_long(p);
-			int stacksize = r_long(p);
-			int flags = r_long(p);
+			/* XXX ignore long->int overflows for now */
+			int argcount = (int)r_long(p);
+			int nlocals = (int)r_long(p);
+			int stacksize = (int)r_long(p);
+			int flags = (int)r_long(p);
 			PyObject *code = r_object(p);
 			PyObject *consts = r_object(p);
 			PyObject *names = r_object(p);
@@ -655,7 +665,7 @@ r_object(RFILE *p)
 			PyObject *cellvars = r_object(p);
 			PyObject *filename = r_object(p);
 			PyObject *name = r_object(p);
-			int firstlineno = r_long(p);
+			int firstlineno = (int)r_long(p);
 			PyObject *lnotab = r_object(p);
 
 			if (!PyErr_Occurred()) {
@@ -821,10 +831,16 @@ PyMarshal_WriteObjectToString(PyObject *x, int version)
 	wf.strings = (version > 0) ? PyDict_New() : NULL;
 	w_object(x, &wf);
 	Py_XDECREF(wf.strings);
-	if (wf.str != NULL)
-		_PyString_Resize(&wf.str,
-		    (int) (wf.ptr -
-			   PyString_AS_STRING((PyStringObject *)wf.str)));
+	if (wf.str != NULL) {
+		char *base = PyString_AS_STRING((PyStringObject *)wf.str);
+		if (wf.ptr - base > INT_MAX) {
+			Py_DECREF(wf.str);
+			PyErr_SetString(PyExc_OverflowError,
+					"too much marshall data for a string");
+			return NULL;
+		}
+		_PyString_Resize(&wf.str, (int)(wf.ptr - base));
+	}
 	if (wf.error) {
 		Py_XDECREF(wf.str);
 		PyErr_SetString(PyExc_ValueError,
diff --git a/Python/mystrtoul.c b/Python/mystrtoul.c
index 8e60c0c..d9b3a4d 100644
--- a/Python/mystrtoul.c
+++ b/Python/mystrtoul.c
@@ -122,30 +122,38 @@ PyOS_strtoul(register char *str, char **ptr, int base)
     return result;
 }
 
+/* Checking for overflow in PyOS_strtol is a PITA; see comments
+ * about Py_ABS_LONG_MIN in longobject.c.
+ */
+#define Py_ABS_LONG_MIN		(0-(unsigned long)LONG_MIN)
+
 long
 PyOS_strtol(char *str, char **ptr, int base)
 {
 	long result;
+	unsigned long uresult;
 	char sign;
-	
+
 	while (*str && isspace(Py_CHARMASK(*str)))
 		str++;
-	
+
 	sign = *str;
 	if (sign == '+' || sign == '-')
 		str++;
-	
-	result = (long) PyOS_strtoul(str, ptr, base);
-	
-	/* Signal overflow if the result appears negative,
-	   except for the largest negative integer */
-	if (result < 0 && !(sign == '-' && result == -result)) {
+
+	uresult = PyOS_strtoul(str, ptr, base);
+
+	if (uresult <= (unsigned long)LONG_MAX) {
+		result = (long)uresult;
+		if (sign == '-')
+			result = -result;
+	}
+	else if (sign == '-' && uresult == Py_ABS_LONG_MIN) {
+		result = LONG_MIN;
+	}
+	else {
 		errno = ERANGE;
 		result = 0x7fffffff;
 	}
-	
-	if (sign == '-')
-		result = -result;
-	
 	return result;
 }