merge trunk

1 files changed, 393 insertions, 105 deletions

diff --git a/generic/tclZlib.c b/generic/tclZlib.c
index 356772e..3063b59 100644
--- a/generic/tclZlib.c
+++ b/generic/tclZlib.c
@@ -17,6 +17,7 @@
 #include "tclInt.h"
 #ifdef HAVE_ZLIB
 #include <zlib.h>
+#include "tclIO.h"
 
 /*
  * Magic flags used with wbits fields to indicate that we're handling the gzip
@@ -103,6 +104,7 @@ typedef struct {
     GzipHeader outHeader;	/* Header to write to an output stream, when
 				 * compressing a gzip stream. */
     Tcl_TimerToken timer;	/* Timer used for keeping events fresh. */
+    Tcl_DString decompressed;	/* Buffer for decompression results. */
     Tcl_Obj *compDictObj;	/* Byte-array object containing compression
 				 * dictionary (not dictObj!) to use if
 				 * necessary. */
@@ -128,11 +130,10 @@ typedef struct {
 #define DEFAULT_BUFFER_SIZE	4096
 
 /*
- * Time to wait (in milliseconds) before flushing the channel when reading
- * data through the transform.
+ * Time to wait before delivering a timer event.
  */
 
-#define TRANSFORM_FLUSH_DELAY	5
+#define TRANSFORM_TIMEOUT	0
 
 /*
  * Prototypes for private procedures defined later in this file:
@@ -143,7 +144,7 @@ static Tcl_DriverBlockModeProc	ZlibTransformBlockMode;
 static Tcl_DriverCloseProc	ZlibTransformClose;
 static Tcl_DriverGetHandleProc	ZlibTransformGetHandle;
 static Tcl_DriverGetOptionProc	ZlibTransformGetOption;
-static Tcl_DriverHandlerProc	ZlibTransformHandler;
+static Tcl_DriverHandlerProc	ZlibTransformEventHandler;
 static Tcl_DriverInputProc	ZlibTransformInput;
 static Tcl_DriverOutputProc	ZlibTransformOutput;
 static Tcl_DriverSetOptionProc	ZlibTransformSetOption;
@@ -157,6 +158,10 @@ static int		GenerateHeader(Tcl_Interp *interp, Tcl_Obj *dictObj,
 			    GzipHeader *headerPtr, int *extraSizePtr);
 static int		ZlibPushSubcmd(Tcl_Interp *interp, int objc,
 			    Tcl_Obj *const objv[]);
+static inline int	ResultCopy(ZlibChannelData *cd, char *buf,
+			    int toRead);
+static int		ResultGenerate(ZlibChannelData *cd, int n, int flush,
+			    int *errorCodePtr);
 static Tcl_Channel	ZlibStackChannelTransform(Tcl_Interp *interp,
 			    int mode, int format, int level,
 			    Tcl_Channel channel, Tcl_Obj *gzipHeaderDictPtr,
@@ -164,9 +169,8 @@ static Tcl_Channel	ZlibStackChannelTransform(Tcl_Interp *interp,
 static void		ZlibStreamCleanup(ZlibStreamHandle *zshPtr);
 static int		ZlibStreamSubcmd(Tcl_Interp *interp, int objc,
 			    Tcl_Obj *const objv[]);
-static void		ZlibTransformTimerKill(ZlibChannelData *cd);
+static inline void	ZlibTransformEventTimerKill(ZlibChannelData *cd);
 static void		ZlibTransformTimerRun(ClientData clientData);
-static void		ZlibTransformTimerSetup(ZlibChannelData *cd);
 
 /*
  * Type of zlib-based compressing and decompressing channels.
@@ -186,7 +190,7 @@ static const Tcl_ChannelType zlibChannelType = {
     NULL,			/* close2Proc */
     ZlibTransformBlockMode,
     NULL,			/* flushProc */
-    ZlibTransformHandler,
+    ZlibTransformEventHandler,
     NULL,			/* wideSeekProc */
     NULL,
     NULL
@@ -2595,6 +2599,12 @@ ZlibStreamCmd(
  *----------------------------------------------------------------------
  *	Set of functions to support channel stacking.
  *----------------------------------------------------------------------
+ *
+ * ZlibTransformClose --
+ *
+ *	How to shut down a stacked compressing/decompressing transform.
+ *
+ *----------------------------------------------------------------------
  */
 
 static int
@@ -2609,7 +2619,7 @@ ZlibTransformClose(
      * Delete the support timer.
      */
 
-    ZlibTransformTimerKill(cd);
+    ZlibTransformEventTimerKill(cd);
 
     /*
      * Flush any data waiting to be compressed.
@@ -2654,6 +2664,8 @@ ZlibTransformClose(
      * Release all memory.
      */
 
+    Tcl_DStringFree(&cd->decompressed);
+
     if (cd->inBuffer) {
 	ckfree(cd->inBuffer);
 	cd->inBuffer = NULL;
@@ -2665,6 +2677,16 @@ ZlibTransformClose(
     ckfree(cd);
     return result;
 }
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformInput --
+ *
+ *	Reader filter that does decompression.
+ *
+ *----------------------------------------------------------------------
+ */
 
 static int
 ZlibTransformInput(
@@ -2676,84 +2698,144 @@ ZlibTransformInput(
     ZlibChannelData *cd = instanceData;
     Tcl_DriverInputProc *inProc =
 	    Tcl_ChannelInputProc(Tcl_GetChannelType(cd->parent));
-    int e, readBytes, flush = Z_NO_FLUSH;
+    int readBytes, gotBytes, copied;
 
     if (cd->mode == TCL_ZLIB_STREAM_DEFLATE) {
 	return inProc(Tcl_GetChannelInstanceData(cd->parent), buf, toRead,
 		errorCodePtr);
     }
 
-    cd->inStream.next_out = (Bytef *) buf;
-    cd->inStream.avail_out = toRead;
-    if (cd->inStream.next_in == NULL) {
-	goto doReadFirst;
-    }
-    while (1) {
-	e = inflate(&cd->inStream, flush);
-	if (e == Z_NEED_DICT && cd->compDictObj) {
-	    e = SetInflateDictionary(&cd->inStream, cd->compDictObj);
-	    if (e == Z_OK) {
-		continue;
-	    }
-	}
-	if ((e == Z_STREAM_END) || (e==Z_OK && cd->inStream.avail_out==0)) {
-	    return toRead - cd->inStream.avail_out;
-	}
-
+    gotBytes = 0;
+    while (toRead > 0) {
 	/*
-	 * Z_BUF_ERROR can be ignored as per http://www.zlib.net/zlib_how.html
-	 *
-	 * Just indicates that the zlib couldn't consume input/produce output,
-	 * and is fixed by supplying more input.
+	 * Loop until the request is satisfied (or no data available from
+	 * below, possibly EOF).
 	 */
 
-	if ((e != Z_OK) && (e != Z_BUF_ERROR)) {
-	    Tcl_Obj *errObj = Tcl_NewListObj(0, NULL);
+	copied = ResultCopy(cd, buf, toRead);
+	toRead -= copied;
+	buf += copied;
+	gotBytes += copied;
 
-	    Tcl_ListObjAppendElement(NULL, errObj,
-		    Tcl_NewStringObj(cd->inStream.msg, -1));
-	    Tcl_SetChannelError(cd->parent, errObj);
-	    *errorCodePtr = EINVAL;
-	    return -1;
+	if (toRead == 0) {
+	    return gotBytes;
 	}
 
 	/*
-	 * Check if the inflate stopped early.
+	 * The buffer is exhausted, but the caller wants even more. We now
+	 * have to go to the underlying channel, get more bytes and then
+	 * transform them for delivery. We may not get what we want (full EOF
+	 * or temporarily out of data).
+	 *
+	 * Length (cd->decompressed) == 0, toRead > 0 here.
+	 *
+	 * The zlib transform allows us to read at most one character from the
+	 * underlying channel to properly identify Z_STREAM_END without
+	 * reading over the border.
 	 */
 
-	if (cd->inStream.avail_in > 0) {
-	    continue;
-	}
+	readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, 1);
 
 	/*
-	 * Emptied the buffer of data from the underlying channel. Get some
-	 * more.
+	 * Three cases here:
+	 *  1.	Got some data from the underlying channel (readBytes > 0) so
+	 *	it should be fed through the decompression engine.
+	 *  2.	Got an error (readBytes < 0) which we should report up except
+	 *	for the case where we can convert it to a short read.
+	 *  3.	Got an end-of-data from EOF or blocking (readBytes == 0). If
+	 *	it is EOF, try flushing the data out of the decompressor.
 	 */
 
-    doReadFirst:
-	/*
-	 * Hack for Bug 2762041. Disable pre-reading of lots of input, read
-	 * only one character. This way the Z_END_OF_STREAM can be read
-	 * without triggering an EOF in the base channel. The higher input
-	 * loops in DoReadChars() would react to that by stopping, despite the
-	 * transform still having data which could be read.
-	 *
-	 * This is only a hack because other transforms may not be able to
-	 * work around the general problem in this way.
-	 */
-
-	readBytes = Tcl_ReadRaw(cd->parent, cd->inBuffer, 1);
 	if (readBytes < 0) {
+	    /*
+	     * Report errors to caller. The state of the seek system is
+	     * unchanged!
+	     */
+
+	    if ((Tcl_GetErrno() == EAGAIN) && (gotBytes > 0)) {
+		/*
+		 * EAGAIN is a special situation. If we had some data before
+		 * we report that instead of the request to re-try.
+		 */
+
+		return gotBytes;
+	    }
+
 	    *errorCodePtr = Tcl_GetErrno();
 	    return -1;
 	} else if (readBytes == 0) {
-	    flush = Z_SYNC_FLUSH;
-	}
+	    /*
+	     * Check wether we hit on EOF in 'parent' or not. If not,
+	     * differentiate between blocking and non-blocking modes. In
+	     * non-blocking mode we ran temporarily out of data. Signal this
+	     * to the caller via EWOULDBLOCK and error return (-1). In the
+	     * other cases we simply return what we got and let the caller
+	     * wait for more. On the other hand, if we got an EOF we have to
+	     * convert and flush all waiting partial data.
+	     */
 
-	cd->inStream.next_in = (Bytef *) cd->inBuffer;
-	cd->inStream.avail_in = readBytes;
+	    if (!Tcl_Eof(cd->parent)) {
+		/*
+		 * The state of the seek system is unchanged!
+		 */
+
+		if ((gotBytes == 0) && (cd->flags & ASYNC)) {
+		    *errorCodePtr = EWOULDBLOCK;
+		    return -1;
+		}
+		return gotBytes;
+	    }
+
+	    /*
+	     * (Semi-)Eof in parent.
+	     *
+	     * Now this is a bit different. The partial data waiting is
+	     * converted and returned.
+	     */
+
+	    if (ResultGenerate(cd, 0, Z_SYNC_FLUSH, errorCodePtr) != TCL_OK) {
+		return -1;
+	    }
+
+	    if (Tcl_DStringLength(&cd->decompressed) == 0) {
+		/*
+		 * The drain delivered nothing. Time to deliver what we've
+		 * got.
+		 */
+
+		return gotBytes;
+	    }
+
+	    /*
+	     * Reset eof, force caller to drain result buffer.
+	     */
+
+	    ((Channel *) cd->parent)->state->flags &= ~CHANNEL_EOF;
+	} else /* readBytes > 0 */ {
+	    /*
+	     * Transform the read chunk, which was not empty. Anything we get
+	     * back is a transformation result to be put into our buffers, and
+	     * the next iteration will put it into the result.
+	     */
+
+	    if (ResultGenerate(cd, readBytes, Z_NO_FLUSH,
+		    errorCodePtr) != TCL_OK) {
+		return -1;
+	    }
+	}
     }
+    return gotBytes;
 }
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformOutput --
+ *
+ *	Writer filter that does compression.
+ *
+ *----------------------------------------------------------------------
+ */
 
 static int
 ZlibTransformOutput(
@@ -2798,6 +2880,16 @@ ZlibTransformOutput(
 
     return toWrite - cd->outStream.avail_in;
 }
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformSetOption --
+ *
+ *	Writing side of [fconfigure] on our channel.
+ *
+ *----------------------------------------------------------------------
+ */
 
 static int
 ZlibTransformSetOption(			/* not used */
@@ -2871,7 +2963,7 @@ ZlibTransformSetOption(			/* not used */
 	    }
 
 	    if (Tcl_WriteRaw(cd->parent, cd->outBuffer,
-		    cd->outStream.next_out - (Bytef*)cd->outBuffer) < 0) {
+		    cd->outStream.next_out - (Bytef *) cd->outBuffer) < 0) {
 		Tcl_AppendResult(interp, "problem flushing channel: ",
 			Tcl_PosixError(interp), NULL);
 		return TCL_ERROR;
@@ -2888,9 +2980,24 @@ ZlibTransformSetOption(			/* not used */
 	}
     }
 
+    /*
+     * Pass all unknown options down, to deeper transforms and/or the base
+     * channel.
+     */
+
     return setOptionProc(Tcl_GetChannelInstanceData(cd->parent), interp,
 	    optionName, value);
 }
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformGetOption --
+ *
+ *	Reading side of [fconfigure] on our channel.
+ *
+ *----------------------------------------------------------------------
+ */
 
 static int
 ZlibTransformGetOption(
@@ -2989,6 +3096,17 @@ ZlibTransformGetOption(
     }
     return Tcl_BadChannelOption(interp, optionName, chanOptions);
 }
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformWatch, ZlibTransformEventHandler --
+ *
+ *	If we have data pending, trigger a readable event after a short time
+ *	(in order to allow a real event to catch up).
+ *
+ *----------------------------------------------------------------------
+ */
 
 static void
 ZlibTransformWatch(
@@ -3004,63 +3122,28 @@ ZlibTransformWatch(
 
     watchProc = Tcl_ChannelWatchProc(Tcl_GetChannelType(cd->parent));
     watchProc(Tcl_GetChannelInstanceData(cd->parent), mask);
-    if (!(mask & TCL_READABLE)
-	    || (cd->inStream.avail_in == (uInt) cd->inAllocated)) {
-	ZlibTransformTimerKill(cd);
-    } else {
-	ZlibTransformTimerSetup(cd);
-    }
-}
-
-static int
-ZlibTransformGetHandle(
-    ClientData instanceData,
-    int direction,
-    ClientData *handlePtr)
-{
-    ZlibChannelData *cd = instanceData;
 
-    return Tcl_GetChannelHandle(cd->parent, direction, handlePtr);
-}
-
-static int
-ZlibTransformBlockMode(
-    ClientData instanceData,
-    int mode)
-{
-    ZlibChannelData *cd = instanceData;
-
-    if (mode == TCL_MODE_NONBLOCKING) {
-	cd->flags |= ASYNC;
-    } else {
-	cd->flags &= ~ASYNC;
+    if (!(mask & TCL_READABLE) || Tcl_DStringLength(&cd->decompressed) == 0) {
+	ZlibTransformEventTimerKill(cd);
+    } else if (cd->timer == NULL) {
+	cd->timer = Tcl_CreateTimerHandler(TRANSFORM_TIMEOUT,
+		ZlibTransformTimerRun, cd);
     }
-    return TCL_OK;
 }
 
 static int
-ZlibTransformHandler(
+ZlibTransformEventHandler(
     ClientData instanceData,
     int interestMask)
 {
     ZlibChannelData *cd = instanceData;
 
-    ZlibTransformTimerKill(cd);
+    ZlibTransformEventTimerKill(cd);
     return interestMask;
 }
 
-static void
-ZlibTransformTimerSetup(
-    ZlibChannelData *cd)
-{
-    if (cd->timer == NULL) {
-	cd->timer = Tcl_CreateTimerHandler(TRANSFORM_FLUSH_DELAY,
-		ZlibTransformTimerRun, cd);
-    }
-}
-
-static void
-ZlibTransformTimerKill(
+static inline void
+ZlibTransformEventTimerKill(
     ZlibChannelData *cd)
 {
     if (cd->timer != NULL) {
@@ -3082,6 +3165,53 @@ ZlibTransformTimerRun(
 /*
  *----------------------------------------------------------------------
  *
+ * ZlibTransformGetHandle --
+ *
+ *	Anything that needs the OS handle is told to get it from what we are
+ *	stacked on top of.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+ZlibTransformGetHandle(
+    ClientData instanceData,
+    int direction,
+    ClientData *handlePtr)
+{
+    ZlibChannelData *cd = instanceData;
+
+    return Tcl_GetChannelHandle(cd->parent, direction, handlePtr);
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ZlibTransformBlockMode --
+ *
+ *	We need to keep track of the blocking mode; it changes our behavior.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+ZlibTransformBlockMode(
+    ClientData instanceData,
+    int mode)
+{
+    ZlibChannelData *cd = instanceData;
+
+    if (mode == TCL_MODE_NONBLOCKING) {
+	cd->flags |= ASYNC;
+    } else {
+	cd->flags &= ~ASYNC;
+    }
+    return TCL_OK;
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
  * ZlibStackChannelTransform --
  *
  *	Stacks either compression or decompression onto a channel.
@@ -3207,6 +3337,8 @@ ZlibStackChannelTransform(
 	}
     }
 
+    Tcl_DStringInit(&cd->decompressed);
+
     chan = Tcl_StackChannel(interp, &zlibChannelType, cd,
 	    Tcl_GetChannelMode(channel), channel);
     if (chan == NULL) {
@@ -3235,6 +3367,162 @@ ZlibStackChannelTransform(
 
 /*
  *----------------------------------------------------------------------
+ *
+ * ResultCopy --
+ *
+ *	Copies the requested number of bytes from the buffer into the
+ *	specified array and removes them from the buffer afterward. Copies
+ *	less if there is not enough data in the buffer.
+ *
+ * Side effects:
+ *	See above.
+ *
+ * Result:
+ *	The number of actually copied bytes, possibly less than 'toRead'.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static inline int
+ResultCopy(
+    ZlibChannelData *cd,	/* The location of the buffer to read from. */
+    char *buf,			/* The buffer to copy into */
+    int toRead)			/* Number of requested bytes */
+{
+    int have = Tcl_DStringLength(&cd->decompressed);
+
+    if (have == 0) {
+	/*
+	 * Nothing to copy in the case of an empty buffer.
+	 */
+
+	return 0;
+    } else if (have > toRead) {
+	/*
+	 * The internal buffer contains more than requested. Copy the
+	 * requested subset to the caller, shift the remaining bytes down, and
+	 * truncate.
+	 */
+
+	char *src = Tcl_DStringValue(&cd->decompressed);
+
+	memcpy(buf, src, toRead);
+	memmove(src, src + toRead, have - toRead);
+
+	Tcl_DStringSetLength(&cd->decompressed, have - toRead);
+	return toRead;
+    } else /* have <= toRead */ {
+	/*
+	 * There is just or not enough in the buffer to fully satisfy the
+	 * caller, so take everything as best effort.
+	 */
+
+	memcpy(buf, Tcl_DStringValue(&cd->decompressed), have);
+	Tcl_DStringSetLength(&cd->decompressed, 0);
+	return have;
+    }
+}
+
+/*
+ *----------------------------------------------------------------------
+ *
+ * ResultGenerate --
+ *
+ *	Extract uncompressed bytes from the compression engine and store them
+ *	in our working buffer.
+ *
+ * Result:
+ *	TCL_OK/TCL_ERROR (with *errorCodePtr updated with reason).
+ *
+ * Side effects:
+ *	See above.
+ *
+ *----------------------------------------------------------------------
+ */
+
+static int
+ResultGenerate(
+    ZlibChannelData *cd,
+    int n,
+    int flush,
+    int *errorCodePtr)
+{
+#define MAXBUF	1024
+    unsigned char buf[MAXBUF];
+    int e, written;
+
+    cd->inStream.next_in = (Bytef *) cd->inBuffer;
+    cd->inStream.avail_in = n;
+
+    while (1) {
+	cd->inStream.next_out = (Bytef *) buf;
+	cd->inStream.avail_out = MAXBUF;
+
+	e = inflate(&cd->inStream, flush);
+	if (e == Z_NEED_DICT && cd->compDictObj) {
+	    e = SetInflateDictionary(&cd->inStream, cd->compDictObj);
+	    if (e == Z_OK) {
+		/*
+		 * A repetition of Z_NEED_DICT is just an error.
+		 */
+
+		cd->inStream.next_out = (Bytef *) buf;
+		cd->inStream.avail_out = MAXBUF;
+		e = inflate(&cd->inStream, flush);
+	    }
+	}
+
+	/*
+	 * avail_out is now the left over space in the output.  Therefore
+	 * "MAXBUF - avail_out" is the amount of bytes generated.
+	 */
+
+	written = MAXBUF - cd->inStream.avail_out;
+	if (written) {
+	    Tcl_DStringAppend(&cd->decompressed, (char *) buf, written);
+	}
+
+	/*
+	 * The cases where we're definitely done.
+	 */
+
+	if (((flush == Z_SYNC_FLUSH) && (e == Z_BUF_ERROR))
+		|| (e == Z_STREAM_END)
+		|| (e == Z_OK && cd->inStream.avail_out == 0)) {
+	    return TCL_OK;
+	}
+
+	/*
+	 * Z_BUF_ERROR can be ignored as per http://www.zlib.net/zlib_how.html
+	 *
+	 * Just indicates that the zlib couldn't consume input/produce output,
+	 * and is fixed by supplying more input.
+	 *
+	 * Otherwise, we've got errors and need to report to higher-up.
+	 */
+
+	if ((e != Z_OK) && (e != Z_BUF_ERROR)) {
+	    Tcl_Obj *errObj = Tcl_NewListObj(0, NULL);
+
+	    Tcl_ListObjAppendElement(NULL, errObj,
+		    Tcl_NewStringObj(cd->inStream.msg, -1));
+	    Tcl_SetChannelError(cd->parent, errObj);
+	    *errorCodePtr = EINVAL;
+	    return TCL_ERROR;
+	}
+
+	/*
+	 * Check if the inflate stopped early.
+	 */
+
+	if (cd->inStream.avail_in <= 0 && flush != Z_SYNC_FLUSH) {
+	    return TCL_OK;
+	}
+    }
+}
+
+/*
+ *----------------------------------------------------------------------
  *	Finally, the TclZlibInit function. Used to install the zlib API.
  *----------------------------------------------------------------------
  */