From 14143230c875cfddb154eef9698c9344bbb905d5 Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 11:54:34 +0000
Subject: added (reworked) patch suggested by aku in [f70ce1fead]: recognize
 stream is done (and unget the unprocessed data back to input channel)

---
 generic/tclZlib.c | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 002c6ae..1ee8919 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -128,6 +128,10 @@ typedef struct {
     Tcl_Obj *compDictObj;	/* Byte-array object containing compression
 				 * dictionary (not dictObj!) to use if
 				 * necessary. */
+    int over;			/* Number of bytes we read to far in the
+				 * underlying stream */
+    int skip;			/* Bytes processed in the last read before
+				 * the overshot */
 } ZlibChannelData;
 
 /*
@@ -137,11 +141,14 @@ typedef struct {
  *			the input compressor.
  *	OUT_HEADER -	Whether the outputHeader field has been registered
  *			with the output decompressor.
+ *	STREAM_DONE -	Flag to signal stream end up to transform
+ *			input.
  */
 
-#define ASYNC			0x1
-#define IN_HEADER		0x2
-#define OUT_HEADER		0x4
+#define ASYNC			0x01
+#define IN_HEADER		0x02
+#define OUT_HEADER		0x04
+#define STREAM_DONE		0x10
 
 /*
  * Size of buffers allocated by default, and the range it can be set to.  The
@@ -2949,6 +2956,10 @@ ZlibTransformClose(
 	(void) inflateEnd(&cd->inStream);
     }
 
+    if (cd->over) {
+	Tcl_Ungets (cd->parent, cd->inBuffer + cd->skip, cd->over, 0);
+    }
+
     /*
      * Release all memory.
      */
@@ -3010,7 +3021,7 @@ ZlibTransformInput(
 	buf += copied;
 	gotBytes += copied;
 
-	if (toRead == 0) {
+	if (toRead == 0 || (cd->flags & STREAM_DONE)) {
 	    return gotBytes;
 	}
 
@@ -3830,8 +3841,13 @@ ResultGenerate(
 	 * The cases where we're definitely done.
 	 */
 
+	if (e == Z_STREAM_END) {
+	    cd->flags |= STREAM_DONE;
+	    cd->over = cd->inStream.avail_in;
+	    cd->skip = n - cd->over;
+	    return TCL_OK;
+	}
 	if (((flush == Z_SYNC_FLUSH) && (e == Z_BUF_ERROR))
-		|| (e == Z_STREAM_END)
 		|| (e == Z_OK && written == 0)) {
 	    return TCL_OK;
 	}
-- 
cgit v0.12


From b7f84c7bc0a87376cd9d151b61beadb68ab18e03 Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 13:28:49 +0000
Subject: zlib: performance - set default read ahead limit to
 DEFAULT_BUFFER_SIZE (4K); better fix avoiding BO (replacement for
 [54605d16b9])

---
 generic/tclZlib.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 1ee8919..23472cf 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -2371,7 +2371,7 @@ ZlibPushSubcmd(
     const char *const *pushOptions = pushDecompressOptions;
     enum pushOptions {poDictionary, poHeader, poLevel, poLimit};
     Tcl_Obj *headerObj = NULL, *compDictObj = NULL;
-    int limit = 1, dummy;
+    int limit = DEFAULT_BUFFER_SIZE, dummy;
 
     if (objc < 4) {
 	Tcl_WrongNumArgs(interp, 2, objv, "mode channel ?options...?");
@@ -3661,6 +3661,9 @@ ZlibStackChannelTransform(
 	    goto error;
 	}
 	cd->inAllocated = DEFAULT_BUFFER_SIZE;
+	if (cd->inAllocated < cd->readAheadLimit) {
+	    cd->inAllocated = cd->readAheadLimit;
+	}
 	cd->inBuffer = ckalloc(cd->inAllocated);
 	if (cd->flags & IN_HEADER) {
 	    if (inflateGetHeader(&cd->inStream, &cd->inHeader.header) != Z_OK) {
-- 
cgit v0.12


From 3577e36e79a96a06a2d61bb7c773987768fccdbb Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 13:43:07 +0000
Subject: zlib: optimize ZlibTransformInput, if inflate is done no read and no
 generate needed anymore

---
 generic/tclZlib.c | 50 +++++++++++++++++++++++++-------------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 23472cf..53949a5 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -3011,20 +3011,13 @@ ZlibTransformInput(
 
     gotBytes = 0;
     while (toRead > 0) {
-	/*
-	 * Loop until the request is satisfied (or no data available from
-	 * below, possibly EOF).
+	/* 
+	 * If done - no read (and no genearate) needed anymore, check we have
+	 * to copy decompressed data, otherwise return with size (or 0 for Eof)
 	 */
-
-	copied = ResultCopy(cd, buf, toRead);
-	toRead -= copied;
-	buf += copied;
-	gotBytes += copied;
-
-	if (toRead == 0 || (cd->flags & STREAM_DONE)) {
-	    return gotBytes;
+	if (cd->flags & STREAM_DONE) {
+	    goto copyDecompressed;
 	}
-
 	/*
 	 * The buffer is exhausted, but the caller wants even more. We now
 	 * have to go to the underlying channel, get more bytes and then
@@ -3032,10 +3025,6 @@ ZlibTransformInput(
 	 * or temporarily out of data).
 	 *
 	 * Length (cd->decompressed) == 0, toRead > 0 here.
-	 *
-	 * The zlib transform allows us to read at most one character from the
-	 * underlying channel to properly identify Z_STREAM_END without
-	 * reading over the border.
 	 */
 
 	readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, cd->readAheadLimit);
@@ -3054,7 +3043,7 @@ ZlibTransformInput(
 
 	    /* See ReflectInput() in tclIORTrans.c */
 	    if (Tcl_InputBlocked(cd->parent) && (gotBytes > 0)) {
-		return gotBytes;
+		break;
 	    }
 
 	    *errorCodePtr = Tcl_GetErrno();
@@ -3072,14 +3061,6 @@ ZlibTransformInput(
 		return -1;
 	    }
 
-	    if (Tcl_DStringLength(&cd->decompressed) == 0) {
-		/*
-		 * The drain delivered nothing. Time to deliver what we've
-		 * got.
-		 */
-
-		return gotBytes;
-	    }
 	} else /* readBytes > 0 */ {
 	    /*
 	     * Transform the read chunk, which was not empty. Anything we get
@@ -3092,6 +3073,25 @@ ZlibTransformInput(
 		return -1;
 	    }
 	}
+
+copyDecompressed:
+
+	if (Tcl_DStringLength(&cd->decompressed) == 0) {
+	    /*
+	     * The drain delivered nothing. Time to deliver what we've
+	     * got.
+	     */
+	    break;
+	}
+	/*
+	 * Loop until the request is satisfied (or no data available from
+	 * above, possibly EOF).
+	 */
+
+	copied = ResultCopy(cd, buf, toRead);
+	toRead -= copied;
+	buf += copied;
+	gotBytes += copied;
     }
     return gotBytes;
 }
-- 
cgit v0.12


From fb7b5ca9d3d2253238c7a1923dcb211d5cdee7fb Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 14:05:24 +0000
Subject: small amend combining similar parts

---
 generic/tclZlib.c | 30 ++++++++++--------------------
 1 file changed, 10 insertions(+), 20 deletions(-)

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 53949a5..b3a6ac2 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -3049,29 +3049,19 @@ ZlibTransformInput(
 	    *errorCodePtr = Tcl_GetErrno();
 	    return -1;
 	}
-	if (readBytes == 0) {
-	    /*
-	     * Eof in parent.
-	     *
-	     * Now this is a bit different. The partial data waiting is
-	     * converted and returned.
-	     */
 
-	    if (ResultGenerate(cd, 0, Z_SYNC_FLUSH, errorCodePtr) != TCL_OK) {
-		return -1;
-	    }
-
-	} else /* readBytes > 0 */ {
-	    /*
-	     * Transform the read chunk, which was not empty. Anything we get
-	     * back is a transformation result to be put into our buffers, and
-	     * the next iteration will put it into the result.
-	     */
+	/*
+	 * Transform the read chunk, if not empty. Anything we get
+	 * back is a transformation result to be put into our buffers, and
+	 * the next iteration will put it into the result.
+	 * For the case readBytes is 0 which signaling Eof in parent, the
+	 * partial data waiting is converted and returned.
+	 */
 
-	    if (ResultGenerate(cd, readBytes, Z_NO_FLUSH,
+	if (ResultGenerate(cd, readBytes,
+		    (readBytes != 0) ? Z_NO_FLUSH : Z_SYNC_FLUSH,
 		    errorCodePtr) != TCL_OK) {
-		return -1;
-	    }
+	    return -1;
 	}
 
 copyDecompressed:
-- 
cgit v0.12


From becf4bcc18eb7b7114f65c271b21d9d7d6c94d2a Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 14:37:05 +0000
Subject: few documentation changes (limit default and handling is changed)

---
 doc/zlib.n | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/doc/zlib.n b/doc/zlib.n
index fd29e0d..994a063 100644
--- a/doc/zlib.n
+++ b/doc/zlib.n
@@ -193,10 +193,12 @@ How hard to compress the data. Must be an integer from 0 (uncompressed) to 9
 .TP
 \fB\-limit\fI readaheadLimit\fR
 .
-The maximum number of bytes ahead to read when decompressing. This defaults to
-1, which ensures that data is always decompressed correctly, but may be
-increased to improve performance. This is more useful when the channel is
-non-blocking.
+The maximum number of bytes ahead to read when decompressing. This can be set
+to 1 to find the border for multi-stream inflate or mixed data, to ensure it
+would handle correctly e. g. some operations on following streams after first
+caused to due time.
+Otherwise Tcl would try to automatically return previously read extra data
+to input stream back.
 .PP
 Both compressing and decompressing channel transformations add extra
 configuration options that may be accessed through \fBchan configure\fR. The
@@ -238,10 +240,8 @@ off the data stream.
 \fB\-limit\fI readaheadLimit\fR
 .
 This read-write option is used by decompressing channels to control the
-maximum number of bytes ahead to read from the underlying data source. This
-defaults to 1, which ensures that data is always decompressed correctly, but
-may be increased to improve performance. This is more useful when the channel
-is non-blocking.
+maximum number of bytes ahead to read from the underlying data source. See
+above for more information.
 .RE
 .SS "STREAMING SUBCOMMAND"
 .TP
-- 
cgit v0.12


From b0791473e99d375f6f3b65b13f99728e552f2b5a Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Thu, 16 Apr 2020 19:08:57 +0000
Subject: code simplification, no interim copy/move buffers, no decompression
 d-string, the inflate occurring in single step (ResultDecompress combines
 ResultGenerate and ResultCopy); better handling in non-blocking mode (also
 recognizes or distinguish no data/EAGAIN cases)

---
 generic/tclZlib.c | 230 ++++++++++++++++++++++++------------------------------
 1 file changed, 103 insertions(+), 127 deletions(-)

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index b3a6ac2..68bdf92 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -124,14 +124,9 @@ typedef struct {
     GzipHeader outHeader;	/* Header to write to an output stream, when
 				 * compressing a gzip stream. */
     Tcl_TimerToken timer;	/* Timer used for keeping events fresh. */
-    Tcl_DString decompressed;	/* Buffer for decompression results. */
     Tcl_Obj *compDictObj;	/* Byte-array object containing compression
 				 * dictionary (not dictObj!) to use if
 				 * necessary. */
-    int over;			/* Number of bytes we read to far in the
-				 * underlying stream */
-    int skip;			/* Bytes processed in the last read before
-				 * the overshot */
 } ZlibChannelData;
 
 /*
@@ -141,13 +136,14 @@ typedef struct {
  *			the input compressor.
  *	OUT_HEADER -	Whether the outputHeader field has been registered
  *			with the output decompressor.
- *	STREAM_DONE -	Flag to signal stream end up to transform
- *			input.
+ *	STREAM_DECOMPRESS - Signal decompress pending data.
+ *	STREAM_DONE -	Flag to signal stream end up to transform input.
  */
 
 #define ASYNC			0x01
 #define IN_HEADER		0x02
 #define OUT_HEADER		0x04
+#define STREAM_DECOMPRESS	0x08
 #define STREAM_DONE		0x10
 
 /*
@@ -191,10 +187,8 @@ static int		GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj,
 			    GzipHeader *headerPtr, int *extraSizePtr);
 static int		ZlibPushSubcmd(Tcl_Interp *interp, int objc,
 			    Tcl_Obj *const objv[]);
-static inline int	ResultCopy(ZlibChannelData *cd, char *buf,
-			    int toRead);
-static int		ResultGenerate(ZlibChannelData *cd, int n, int flush,
-			    int *errorCodePtr);
+static int		ResultDecompress(ZlibChannelData *cd, char *buf,
+			    int toRead, int flush, int *errorCodePtr);
 static Tcl_Channel	ZlibStackChannelTransform(Tcl_Interp *interp,
 			    int mode, int format, int level, int limit,
 			    Tcl_Channel channel, Tcl_Obj *gzipHeaderDictPtr,
@@ -2953,11 +2947,15 @@ ZlibTransformClose(
 	} while (e != Z_STREAM_END);
 	(void) deflateEnd(&cd->outStream);
     } else {
-	(void) inflateEnd(&cd->inStream);
-    }
+	/*
+	 * If we have rest of read input (overshot by Z_STREAM_END or on possible error),
+	 * unget this part of buffer back to the parent channel.
+	 */
+	if (cd->inStream.avail_in) {
+	    Tcl_Ungets (cd->parent, (char *)cd->inStream.next_in, cd->inStream.avail_in, 0);
+	}
 
-    if (cd->over) {
-	Tcl_Ungets (cd->parent, cd->inBuffer + cd->skip, cd->over, 0);
+	(void) inflateEnd(&cd->inStream);
     }
 
     /*
@@ -2968,7 +2966,6 @@ ZlibTransformClose(
 	Tcl_DecrRefCount(cd->compDictObj);
 	cd->compDictObj = NULL;
     }
-    Tcl_DStringFree(&cd->decompressed);
 
     if (cd->inBuffer) {
 	ckfree(cd->inBuffer);
@@ -3002,7 +2999,7 @@ ZlibTransformInput(
     ZlibChannelData *cd = instanceData;
     Tcl_DriverInputProc *inProc =
 	    Tcl_ChannelInputProc(Tcl_GetChannelType(cd->parent));
-    int readBytes, gotBytes, copied;
+    int readBytes, gotBytes;
 
     if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) {
 	return inProc(Tcl_GetChannelInstanceData(cd->parent), buf, toRead,
@@ -3010,12 +3007,20 @@ ZlibTransformInput(
     }
 
     gotBytes = 0;
-    while (toRead > 0) {
+    readBytes = cd->inStream.avail_in; /* how many bytes in buffer now */
+    while (!(cd->flags & STREAM_DONE) && toRead > 0) {
+    	int n, decBytes;
+
+	/* if starting from scratch or continuation after full decompression */
+	if (!cd->inStream.avail_in) {
+	    /* buffer to start, we can read to whole available buffer */
+	    cd->inStream.next_in = (Bytef *) cd->inBuffer;
+	}
 	/* 
-	 * If done - no read (and no genearate) needed anymore, check we have
-	 * to copy decompressed data, otherwise return with size (or 0 for Eof)
+	 * If done - no read needed anymore, check we have to copy rest of
+	 * decompressed data, otherwise return with size (or 0 for Eof)
 	 */
-	if (cd->flags & STREAM_DONE) {
+	if (cd->flags & STREAM_DECOMPRESS) {
 	    goto copyDecompressed;
 	}
 	/*
@@ -3023,11 +3028,21 @@ ZlibTransformInput(
 	 * have to go to the underlying channel, get more bytes and then
 	 * transform them for delivery. We may not get what we want (full EOF
 	 * or temporarily out of data).
-	 *
-	 * Length (cd->decompressed) == 0, toRead > 0 here.
 	 */
 
-	readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, cd->readAheadLimit);
+	/* Check free buffer size and adjust size of next chunk to read. */
+	n = cd->inAllocated - ((char *)cd->inStream.next_in - cd->inBuffer);
+	if (n <= 0) {
+	    /* Normally unreachable: not enough input buffer to uncompress.
+	     * Todo: firstly try to realloc inBuffer upto MAX_BUFFER_SIZE.
+	     */
+	    *errorCodePtr = ENOBUFS;
+	    return -1;
+	}
+	if (n > cd->readAheadLimit) {
+	    n = cd->readAheadLimit;
+	}
+	readBytes = Tcl_ReadRaw(cd->parent, (char *)cd->inStream.next_in, n);
 
 	/*
 	 * Three cases here:
@@ -3050,6 +3065,11 @@ ZlibTransformInput(
 	    return -1;
 	}
 
+	/* more bytes (or Eof if readBytes == 0) */
+	cd->inStream.avail_in += readBytes;
+
+copyDecompressed:
+
 	/*
 	 * Transform the read chunk, if not empty. Anything we get
 	 * back is a transformation result to be put into our buffers, and
@@ -3058,31 +3078,39 @@ ZlibTransformInput(
 	 * partial data waiting is converted and returned.
 	 */
 
-	if (ResultGenerate(cd, readBytes,
+	decBytes = ResultDecompress(cd, buf, toRead,
 		    (readBytes != 0) ? Z_NO_FLUSH : Z_SYNC_FLUSH,
-		    errorCodePtr) != TCL_OK) {
+		    errorCodePtr);
+	if (decBytes == -1) {
 	    return -1;
 	}
+	gotBytes += decBytes;
+	buf += decBytes;
+	toRead -= decBytes;
 
-copyDecompressed:
-
-	if (Tcl_DStringLength(&cd->decompressed) == 0) {
+	if (((decBytes == 0) || (cd->flags & STREAM_DECOMPRESS))) {
 	    /*
-	     * The drain delivered nothing. Time to deliver what we've
-	     * got.
+	     * The drain delivered nothing (or buffer too small to decompress).
+	     * Time to deliver what we've got.
 	     */
+	    if (!gotBytes && !(cd->flags & STREAM_DONE)) {
+		/* if no-data, but not ready - avoid signaling Eof,
+		 * continue in blocking mode, otherwise EAGAIN */
+		if (Tcl_InputBlocked(cd->parent)) {
+		    continue;
+		}
+		*errorCodePtr = EAGAIN;
+		return -1;		
+	    }
 	    break;
 	}
+
 	/*
 	 * Loop until the request is satisfied (or no data available from
 	 * above, possibly EOF).
 	 */
-
-	copied = ResultCopy(cd, buf, toRead);
-	toRead -= copied;
-	buf += copied;
-	gotBytes += copied;
     }
+
     return gotBytes;
 }
 
@@ -3465,7 +3493,7 @@ ZlibTransformWatch(
     watchProc = Tcl_ChannelWatchProc(Tcl_GetChannelType(cd->parent));
     watchProc(Tcl_GetChannelInstanceData(cd->parent), mask);
 
-    if (!(mask & TCL_READABLE) || Tcl_DStringLength(&cd->decompressed) == 0) {
+    if (!(mask & TCL_READABLE) || !(cd->flags & STREAM_DECOMPRESS)) {
 	ZlibTransformEventTimerKill(cd);
     } else if (cd->timer == NULL) {
 	cd->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME,
@@ -3684,8 +3712,6 @@ ZlibStackChannelTransform(
 	}
     }
 
-    Tcl_DStringInit(&cd->decompressed);
-
     chan = Tcl_StackChannel(interp, &zlibChannelType, cd,
 	    Tcl_GetChannelMode(channel), channel);
     if (chan == NULL) {
@@ -3715,96 +3741,37 @@ ZlibStackChannelTransform(
 /*
  *----------------------------------------------------------------------
  *
- * ResultCopy --
- *
- *	Copies the requested number of bytes from the buffer into the
- *	specified array and removes them from the buffer afterward. Copies
- *	less if there is not enough data in the buffer.
- *
- * Side effects:
- *	See above.
- *
- * Result:
- *	The number of actually copied bytes, possibly less than 'toRead'.
- *
- *----------------------------------------------------------------------
- */
-
-static inline int
-ResultCopy(
-    ZlibChannelData *cd,	/* The location of the buffer to read from. */
-    char *buf,			/* The buffer to copy into */
-    int toRead)			/* Number of requested bytes */
-{
-    int have = Tcl_DStringLength(&cd->decompressed);
-
-    if (have == 0) {
-	/*
-	 * Nothing to copy in the case of an empty buffer.
-	 */
-
-	return 0;
-    } else if (have > toRead) {
-	/*
-	 * The internal buffer contains more than requested. Copy the
-	 * requested subset to the caller, shift the remaining bytes down, and
-	 * truncate.
-	 */
-
-	char *src = Tcl_DStringValue(&cd->decompressed);
-
-	memcpy(buf, src, toRead);
-	memmove(src, src + toRead, have - toRead);
-
-	Tcl_DStringSetLength(&cd->decompressed, have - toRead);
-	return toRead;
-    } else /* have <= toRead */ {
-	/*
-	 * There is just or not enough in the buffer to fully satisfy the
-	 * caller, so take everything as best effort.
-	 */
-
-	memcpy(buf, Tcl_DStringValue(&cd->decompressed), have);
-	TclDStringClear(&cd->decompressed);
-	return have;
-    }
-}
-
-/*
- *----------------------------------------------------------------------
- *
- * ResultGenerate --
+ * ResultDecompress --
  *
  *	Extract uncompressed bytes from the compression engine and store them
- *	in our working buffer.
+ *	in our buffer (buf) up to toRead bytes.
  *
  * Result:
- *	TCL_OK/TCL_ERROR (with *errorCodePtr updated with reason).
+ *	Number of bytes decompressed or -1 if error (with *errorCodePtr updated with reason).
  *
  * Side effects:
- *	See above.
+ *	After execution it updates cd->inStream (next_in, avail_in) to reflect
+ *	the data that has been decompressed.
  *
  *----------------------------------------------------------------------
  */
 
 static int
-ResultGenerate(
+ResultDecompress(
     ZlibChannelData *cd,
-    int n,
+    char *buf,
+    int toRead,
     int flush,
     int *errorCodePtr)
 {
-#define MAXBUF	1024
-    unsigned char buf[MAXBUF];
-    int e, written;
+    int e, written, resBytes = 0;
     Tcl_Obj *errObj;
 
-    cd->inStream.next_in = (Bytef *) cd->inBuffer;
-    cd->inStream.avail_in = n;
 
-    while (1) {
-	cd->inStream.next_out = (Bytef *) buf;
-	cd->inStream.avail_out = MAXBUF;
+    cd->flags &= ~STREAM_DECOMPRESS;
+    cd->inStream.next_out = (Bytef *) buf;
+    cd->inStream.avail_out = toRead;
+    while (cd->inStream.avail_out > 0) {
 
 	e = inflate(&cd->inStream, flush);
 	if (e == Z_NEED_DICT && cd->compDictObj) {
@@ -3813,22 +3780,16 @@ ResultGenerate(
 		/*
 		 * A repetition of Z_NEED_DICT is just an error.
 		 */
-
-		cd->inStream.next_out = (Bytef *) buf;
-		cd->inStream.avail_out = MAXBUF;
 		e = inflate(&cd->inStream, flush);
 	    }
 	}
 
 	/*
 	 * avail_out is now the left over space in the output.  Therefore
-	 * "MAXBUF - avail_out" is the amount of bytes generated.
+	 * "toRead - avail_out" is the amount of bytes generated.
 	 */
 
-	written = MAXBUF - cd->inStream.avail_out;
-	if (written) {
-	    Tcl_DStringAppend(&cd->decompressed, (char *) buf, written);
-	}
+	written = toRead - cd->inStream.avail_out;
 
 	/*
 	 * The cases where we're definitely done.
@@ -3836,13 +3797,18 @@ ResultGenerate(
 
 	if (e == Z_STREAM_END) {
 	    cd->flags |= STREAM_DONE;
-	    cd->over = cd->inStream.avail_in;
-	    cd->skip = n - cd->over;
-	    return TCL_OK;
+	    resBytes += written;
+	    break;
 	}
-	if (((flush == Z_SYNC_FLUSH) && (e == Z_BUF_ERROR))
-		|| (e == Z_OK && written == 0)) {
-	    return TCL_OK;
+	if (e == Z_OK) {
+	    if (written == 0) {
+		break;
+	    }
+	    resBytes += written;
+	}
+
+	if ((flush == Z_SYNC_FLUSH) && (e == Z_BUF_ERROR)) {
+	    break;
 	}
 
 	/*
@@ -3863,10 +3829,20 @@ ResultGenerate(
 	 */
 
 	if (cd->inStream.avail_in <= 0 && flush != Z_SYNC_FLUSH) {
-	    return TCL_OK;
+	    break;
 	}
     }
 
+    if (!(cd->flags & STREAM_DONE)) {
+	/* if we have pending input data, but no available output buffer */
+	if (cd->inStream.avail_in && !cd->inStream.avail_out) {
+	    /* next time try to decompress it got readable (new output buffer) */
+	    cd->flags |= STREAM_DECOMPRESS;
+	}
+    }
+
+    return resBytes;
+
   handleError:
     errObj = Tcl_NewListObj(0, NULL);
     Tcl_ListObjAppendElement(NULL, errObj, Tcl_NewStringObj("-errorcode",-1));
@@ -3876,7 +3852,7 @@ ResultGenerate(
 	    Tcl_NewStringObj(cd->inStream.msg, -1));
     Tcl_SetChannelError(cd->parent, errObj);
     *errorCodePtr = EINVAL;
-    return TCL_ERROR;
+    return -1;
 }
 
 /*
-- 
cgit v0.12


From dbc3e640a2807e90bad8ebaef6f4ebb06178a18c Mon Sep 17 00:00:00 2001
From: aku <andreas.kupries@gmail.com>
Date: Tue, 23 Jun 2020 06:37:59 +0000
Subject: Added two tests demonstrating that the tickets [8af92dfb66] (bad
 stream expansion) and [f70ce1fead] (multi-stream decoding without readahead
 limit shenanigans) are fixed.

Verified that the tests fail with commit [4eb66645d6], just before the zlib fixes.

Tweaked the zlib manpage paragraphs about -limit.
Tweaked some code comments.
Fixed typos in description of two unrelated zlib tests.
---
 doc/zlib.n            |  18 ++++++++++-----
 generic/tclZlib.c     |   5 +++--
 tests/assets/zlib.bin | Bin 0 -> 591 bytes
 tests/zlib.test       |  60 ++++++++++++++++++++++++++++++++++++++++++++++++--
 4 files changed, 73 insertions(+), 10 deletions(-)
 create mode 100644 tests/assets/zlib.bin

diff --git a/doc/zlib.n b/doc/zlib.n
index 994a063..3714fc1 100644
--- a/doc/zlib.n
+++ b/doc/zlib.n
@@ -193,12 +193,18 @@ How hard to compress the data. Must be an integer from 0 (uncompressed) to 9
 .TP
 \fB\-limit\fI readaheadLimit\fR
 .
-The maximum number of bytes ahead to read when decompressing. This can be set
-to 1 to find the border for multi-stream inflate or mixed data, to ensure it
-would handle correctly e. g. some operations on following streams after first
-caused to due time.
-Otherwise Tcl would try to automatically return previously read extra data
-to input stream back.
+The maximum number of bytes ahead to read when decompressing.
+.RS
+.PP
+This option has become \fBirrelevant\fR. It was originally introduced
+to prevent Tcl from reading beyond the end of a compressed stream in
+multi-stream channels to ensure that the data after was left alone for
+further reading, at the cost of speed.
+.PP
+Tcl now automatically returns any bytes it has read beyond the end of
+a compressed stream back to the channel, making them appear as unread
+to further readers.
+.RE
 .PP
 Both compressing and decompressing channel transformations add extra
 configuration options that may be accessed through \fBchan configure\fR. The
diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 68bdf92..42d7156 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -2948,8 +2948,9 @@ ZlibTransformClose(
 	(void) deflateEnd(&cd->outStream);
     } else {
 	/*
-	 * If we have rest of read input (overshot by Z_STREAM_END or on possible error),
-	 * unget this part of buffer back to the parent channel.
+	 * If we have unused bytes from the read input (overshot by
+	 * Z_STREAM_END or on possible error), unget them back to the parent
+	 * channel, so that they appear as not being read yet.
 	 */
 	if (cd->inStream.avail_in) {
 	    Tcl_Ungets (cd->parent, (char *)cd->inStream.next_in, cd->inStream.avail_in, 0);
diff --git a/tests/assets/zlib.bin b/tests/assets/zlib.bin
new file mode 100644
index 0000000..dadb639
Binary files /dev/null and b/tests/assets/zlib.bin differ
diff --git a/tests/zlib.test b/tests/zlib.test
index c2f7825..e7ffcfe 100644
--- a/tests/zlib.test
+++ b/tests/zlib.test
@@ -920,7 +920,7 @@ test zlib-10.2 "bug #2818131 (mismatch gets)" -constraints {
     rename zlibRead {}
 } -result {error {invalid block type}}
 
-test zlib-11.1 "Bug #3390073: mis-appled gzip filtering" -setup {
+test zlib-11.1 "Bug #3390073: mis-applied gzip filtering" -setup {
     set file [makeFile {} test.input]
 } -constraints zlib -body {
     set f [open $file wb]
@@ -934,7 +934,7 @@ test zlib-11.1 "Bug #3390073: mis-appled gzip filtering" -setup {
 } -cleanup {
     removeFile $file
 } -result {1000 0}
-test zlib-11.2 "Bug #3390073: mis-appled gzip filtering" -setup {
+test zlib-11.2 "Bug #3390073: mis-applied gzip filtering" -setup {
     set file [makeFile {} test.input]
 } -constraints zlib -body {
     set f [open $file wb]
@@ -1005,6 +1005,62 @@ test zlib-12.2 {Patrick Dunnigan's issue} -constraints zlib -setup {
     removeFile $filesrc
     removeFile $filedst
 } -result 56
+
+test zlib-13.1 {Ticket 8af92dfb66 - zlib stream mis-expansion} -constraints zlib -setup {
+    set pathin  [file join $::tcltest::testsDirectory assets zlib.bin]
+    set chanin  [open $pathin rb]
+    set pathout [file join [pwd] zlib.deflated]
+    set chanout [open $pathout wb]
+    zlib push inflate $chanin
+    fcopy $chanin $chanout
+    close $chanin
+    close $chanout
+} -body {
+    file size $pathout    
+} -cleanup {
+    file delete $pathout
+    unset chanin pathin chanout pathout
+} -result 458752
+
+test zlib-13.2 {Ticket f70ce1fead - zlib multi-stream expansion} -constraints zlib -setup {
+    # Start from the basic asset
+    set pathin  [file join $::tcltest::testsDirectory assets zlib.bin]
+    set chanin  [open $pathin rb]
+    # Create a multi-stream by copying the asset twice into it.
+    set pathout [file join [pwd] zlib.multi]
+    set chanout [open $pathout wb]
+    fcopy $chanin $chanout
+    seek  $chanin 0 start
+    fcopy $chanin $chanout
+    close $chanin
+    close $chanout
+    # The multi-stream file shall be our input
+    set pathin $pathout
+    set chanin [open $pathin rb]
+    # And our destinations
+    set pathout1 [file join [pwd] zlib.1]
+    set pathout2 [file join [pwd] zlib.2]
+} -body {
+    # Decode first stream
+    set chanout [open $pathout1 wb]
+    zlib push inflate $chanin
+    fcopy $chanin $chanout
+    chan pop $chanin
+    close $chanout
+    # Decode second stream
+    set chanout [open $pathout2 wb]
+    zlib push inflate $chanin
+    fcopy $chanin $chanout
+    chan pop $chanin
+    close $chanout
+    #    
+    list [file size $pathout1] [file size $pathout2]
+} -cleanup {
+    close $chanin
+    file delete $pathout $pathout1 $pathout2
+    unset chanin pathin chanout pathout pathout1 pathout2
+} -result {458752 458752}
+
 
 ::tcltest::cleanupTests
 return
-- 
cgit v0.12


From 97499f5ed8cc4ea3177e6fb8544c5cd7aba6e029 Mon Sep 17 00:00:00 2001
From: kjnash <k.j.nash@usa.net>
Date: Tue, 23 Jun 2020 18:10:33 +0000
Subject: Fix for ticket 41c9857bdd

---
 library/init.tcl | 45 +++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/library/init.tcl b/library/init.tcl
index 5cc0fee..e6964e0 100644
--- a/library/init.tcl
+++ b/library/init.tcl
@@ -37,41 +37,46 @@ package require -exact Tcl 8.6.10
 # tcl_pkgPath, which is set by the platform-specific initialization routines
 #	On UNIX it is compiled in
 #       On Windows, it is not used
+#
+# (Ticket 41c9857bdd) In a safe interpreter, this file does not set
+# ::auto_path (other than to {} if it is undefined). The caller, typically
+# a Safe Base command, is responsible for setting ::auto_path.
 
 if {![info exists auto_path]} {
-    if {[info exists env(TCLLIBPATH)]} {
+    if {[info exists env(TCLLIBPATH)] && (![interp issafe])} {
 	set auto_path $env(TCLLIBPATH)
     } else {
 	set auto_path ""
     }
 }
 namespace eval tcl {
-    variable Dir
-    foreach Dir [list $::tcl_library [file dirname $::tcl_library]] {
-	if {$Dir ni $::auto_path} {
-	    lappend ::auto_path $Dir
-	}
-    }
-    set Dir [file join [file dirname [file dirname \
-	    [info nameofexecutable]]] lib]
-    if {$Dir ni $::auto_path} {
-	lappend ::auto_path $Dir
-    }
-    if {[info exists ::tcl_pkgPath]} { catch {
-	foreach Dir $::tcl_pkgPath {
+    if {![interp issafe]} {
+	variable Dir
+	foreach Dir [list $::tcl_library [file dirname $::tcl_library]] {
 	    if {$Dir ni $::auto_path} {
 		lappend ::auto_path $Dir
 	    }
 	}
-    }}
+	set Dir [file join [file dirname [file dirname \
+		[info nameofexecutable]]] lib]
+	if {$Dir ni $::auto_path} {
+	    lappend ::auto_path $Dir
+	}
+	if {[info exists ::tcl_pkgPath]} { catch {
+	    foreach Dir $::tcl_pkgPath {
+		if {$Dir ni $::auto_path} {
+		    lappend ::auto_path $Dir
+		}
+	    }
+	}}
 
-    if {![interp issafe]} {
-        variable Path [encoding dirs]
-        set Dir [file join $::tcl_library encoding]
-        if {$Dir ni $Path} {
+	variable Path [encoding dirs]
+	set Dir [file join $::tcl_library encoding]
+	if {$Dir ni $Path} {
 	    lappend Path $Dir
 	    encoding dirs $Path
-        }
+	}
+	unset Dir Path
     }
 
     # TIP #255 min and max functions
-- 
cgit v0.12


From 094567e57a4212ad576fa972a418938423e8db0d Mon Sep 17 00:00:00 2001
From: sebres <sebres@users.sourceforge.net>
Date: Tue, 23 Jun 2020 20:40:49 +0000
Subject: zlib.test: replaces tests/assets/zlib.bin with inlined version
 (creating test file on demand), use temp files (instead of current directory)
 for interim files

---
 tests/zlib.test | 44 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/tests/zlib.test b/tests/zlib.test
index e7ffcfe..d3a6dff 100644
--- a/tests/zlib.test
+++ b/tests/zlib.test
@@ -1006,10 +1006,26 @@ test zlib-12.2 {Patrick Dunnigan's issue} -constraints zlib -setup {
     removeFile $filedst
 } -result 56
 
-test zlib-13.1 {Ticket 8af92dfb66 - zlib stream mis-expansion} -constraints zlib -setup {
-    set pathin  [file join $::tcltest::testsDirectory assets zlib.bin]
+set zlibbinf ""
+proc _zlibbinf {} {
+  # inlined zlib.bin file creator:
+  variable zlibbinf
+  if {$zlibbinf eq ""} {
+    set zlibbinf [makeFile {} test-zlib-13.bin]
+    set f [open $zlibbinf wb]
+    puts -nonewline $f [zlib decompress [binary decode base64 {
+      eJx7e+6s1+EAgYaLjK3ratptGmOck0vT/y/ZujHAd0qJelDBXfUPJ3tfrtLbpX+wOOFHmtn03/tizm
+      /+tXROXU3d203b79p5X6/0cvUyFzTsqOj4sa9r8SrZI5zT7265e2Xzq595Fb9LbpgffVy7cZaJ/d15
+      4U9L7LLM2vdqut8+aSU/r6q9Ltv6+T9mBhTgIK97bH33m/O1C1eBwf9FDKNgaIDaj9wA+5hToA==
+    }]]
+    close $f
+  }
+  return $zlibbinf
+}
+test zlib-13.1 {Ticket [8af92dfb66] - zlib stream mis-expansion} -constraints zlib -setup {
+    set pathin  [_zlibbinf]
     set chanin  [open $pathin rb]
-    set pathout [file join [pwd] zlib.deflated]
+    set pathout [makeFile {} test-zlib-13.deflated]
     set chanout [open $pathout wb]
     zlib push inflate $chanin
     fcopy $chanin $chanout
@@ -1018,16 +1034,16 @@ test zlib-13.1 {Ticket 8af92dfb66 - zlib stream mis-expansion} -constraints zlib
 } -body {
     file size $pathout    
 } -cleanup {
-    file delete $pathout
+    removeFile $pathout
     unset chanin pathin chanout pathout
 } -result 458752
 
-test zlib-13.2 {Ticket f70ce1fead - zlib multi-stream expansion} -constraints zlib -setup {
+test zlib-13.2 {Ticket [f70ce1fead] - zlib multi-stream expansion} -constraints zlib -setup {
     # Start from the basic asset
-    set pathin  [file join $::tcltest::testsDirectory assets zlib.bin]
+    set pathin  [_zlibbinf]
     set chanin  [open $pathin rb]
     # Create a multi-stream by copying the asset twice into it.
-    set pathout [file join [pwd] zlib.multi]
+    set pathout [makeFile {} test-zlib-13.multi]
     set chanout [open $pathout wb]
     fcopy $chanin $chanout
     seek  $chanin 0 start
@@ -1038,8 +1054,8 @@ test zlib-13.2 {Ticket f70ce1fead - zlib multi-stream expansion} -constraints zl
     set pathin $pathout
     set chanin [open $pathin rb]
     # And our destinations
-    set pathout1 [file join [pwd] zlib.1]
-    set pathout2 [file join [pwd] zlib.2]
+    set pathout1 [makeFile {} test-zlib-13.multi-1]
+    set pathout2 [makeFile {} test-zlib-13.multi-2]
 } -body {
     # Decode first stream
     set chanout [open $pathout1 wb]
@@ -1057,10 +1073,18 @@ test zlib-13.2 {Ticket f70ce1fead - zlib multi-stream expansion} -constraints zl
     list [file size $pathout1] [file size $pathout2]
 } -cleanup {
     close $chanin
-    file delete $pathout $pathout1 $pathout2
+    removeFile $pathout
+    removeFile $pathout1
+    removeFile $pathout2
     unset chanin pathin chanout pathout pathout1 pathout2
 } -result {458752 458752}
 
+if {$zlibbinf ne ""} {
+   removeFile $zlibbinf
+}
+unset zlibbinf
+rename _zlibbinf {}
+
 
 ::tcltest::cleanupTests
 return
-- 
cgit v0.12