Alternatve patch to replace UB treatment of int overflow.

1 files changed, 16 insertions, 13 deletions

diff --git a/generic/tclBinary.c b/generic/tclBinary.c
index 1c97728..c0569a5 100644
--- a/generic/tclBinary.c
+++ b/generic/tclBinary.c
@@ -179,11 +179,12 @@ const Tcl_ObjType tclByteArrayType = {
  */
 
 typedef struct ByteArray {
-    int used;			/* The number of bytes used in the byte
+    unsigned int used;		/* The number of bytes used in the byte
 				 * array. */
-    int allocated;		/* The amount of space actually allocated
-				 * minus 1 byte. */
-    unsigned char bytes[TCLFLEXARRAY];	/* The array of bytes. The actual size of this
+    unsigned int allocated;	/* The number of bytes allocated for storage
+				 * of the following "bytes" field. */
+    unsigned char bytes[TCLFLEXARRAY];
+				/* The array of bytes. The actual size of this
 				 * field depends on the 'allocated' field
 				 * above. */
 } ByteArray;
@@ -401,7 +402,9 @@ Tcl_SetByteArrayLength(
     if (objPtr->typePtr != &tclByteArrayType) {
 	SetByteArrayFromAny(NULL, objPtr);
     }
-
+    if (length < 0) {
+	length = 0;
+    }
     byteArrayPtr = GET_BYTEARRAY(objPtr);
     if (length > byteArrayPtr->allocated) {
 	byteArrayPtr = (ByteArray *)ckrealloc(byteArrayPtr, BYTEARRAY_SIZE(length));
@@ -507,7 +510,7 @@ DupByteArrayInternalRep(
     Tcl_Obj *srcPtr,		/* Object with internal rep to copy. */
     Tcl_Obj *copyPtr)		/* Object with internal rep to set. */
 {
-    int length;
+    unsigned int length;
     ByteArray *srcArrayPtr, *copyArrayPtr;
 
     srcArrayPtr = GET_BYTEARRAY(srcPtr);
@@ -549,7 +552,7 @@ UpdateStringOfByteArray(
     Tcl_Obj *objPtr)		/* ByteArray object whose string rep to
 				 * update. */
 {
-    int i, length, size;
+    unsigned int i, length, size;
     unsigned char *src;
     char *dst;
     ByteArray *byteArrayPtr;
@@ -563,16 +566,16 @@ UpdateStringOfByteArray(
      */
 
     size = length;
-    for (i = 0; i < length && size >= 0; i++) {
+    for (i = 0; i < length && size < INT_MAX; i++) {
 	if ((src[i] == 0) || (src[i] > 127)) {
-	    size = (int)((unsigned int)size + 1U);
+	    size++;
 	}
     }
-    if (size < 0) {
+    if (i < length) {
 	Tcl_Panic("max size for a Tcl value (%d bytes) exceeded", INT_MAX);
     }
 
-    dst = (char *)ckalloc((unsigned int)size + 1U);
+    dst = (char *)ckalloc(size + 1U);
     objPtr->bytes = dst;
     objPtr->length = size;
 
@@ -613,7 +616,7 @@ TclAppendBytesToByteArray(
     int len)
 {
     ByteArray *byteArrayPtr;
-    int needed;
+    unsigned int needed;
 
     if (Tcl_IsShared(objPtr)) {
 	Tcl_Panic("%s called with shared object","TclAppendBytesToByteArray");
@@ -645,7 +648,7 @@ TclAppendBytesToByteArray(
 
     if (needed > byteArrayPtr->allocated) {
 	ByteArray *ptr = NULL;
-	int attempt;
+	unsigned int attempt;
 
 	if (needed <= INT_MAX/2) {
 	    /*