From b555fa4d44d0b13c0c4e61643e22f6c3479baf4e Mon Sep 17 00:00:00 2001 From: dgp Date: Tue, 28 Jan 2014 11:04:57 +0000 Subject: WIP --- generic/tclIO.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/generic/tclIO.c b/generic/tclIO.c index 13494ca..fdb0ddd 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -3779,6 +3779,74 @@ WriteChars( translate = (statePtr->flags & CHANNEL_LINEBUFFERED) || (statePtr->outputTranslation != TCL_TRANSLATE_LF); +#if 0 + consumedSomething = 1; + while (consumedSomething && (srcLen + saved + endEncoding > 0)) { + void *lastNewLine; + int srcLimit; + + /* Get space to write into */ + bufPtr = statePtr->curOutPtr; + if (bufPtr == NULL) { + bufPtr = AllocChannelBuffer(statePtr->bufSize); + statePtr->curOutPtr = bufPtr; + if (saved) { + /* + * Here's some translated bytes left over from the last buffer + * that we need to stick at the beginning of this buffer. + */ + + memcpy(InsertPoint(bufPtr), safe, (size_t) saved); + bufPtr->nextAdded += saved; + saved = 0; + } + } + dst = InsertPoint(bufPtr); + dstLen = SpaceLeft(bufPtr); + + /* + * We have dstLen bytes to write to. The most source bytes + * that could possibly fill that is TCL_UTF_MAX * dstLen. + */ + + srcLimit = TCL_UTF_MAX * dstLen; + if (srcLen < srcLimit) { + srcLimit = srcLen; + } + lastNewLine = memchr(src, '\n', srcLimit); + + if (lastNewLine) { + srcLimit = lastNewLine - src; + } + + result = Tcl_UtfToExternal(NULL, encoding, src, srcLimit, + statePtr->outputEncodingFlags, + &statePtr->outputEncodingState, dst, + dstLen + BUFFER_PADDING, &srcRead, &dstWrote, NULL); + + statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; + + if ((result != 0) && (srcRead + dstWrote == 0)) { + fprintf(stdout, "WDTH?\n"); fflush(stdout); + } + bufPtr->nextAdded += dstWrote; + if (IsBufferOverflowing(bufPtr)) { + /* + * When translating from UTF-8 to external encoding, we + * allowed the translation to produce a character that crossed + * the end of the output buffer, so that we would get a + * completely full buffer before flushing it. The extra bytes + * will be moved to the beginning of the next buffer. + */ + + saved = -SpaceLeft(bufPtr); + memcpy(safe, dst + dstLen, (size_t) saved); + bufPtr->nextAdded = bufPtr->bufLength; + } + + + } +#else /* * Loop over all UTF-8 characters in src, storing them in staging buffer * with proper EOL translation. @@ -3933,6 +4001,7 @@ WriteChars( } } } +#endif /* * If nothing was written and it happened because there was no progress in -- cgit v0.12 From 15c8e0fd00cb9a49203a6de6cd05ec758daf519e Mon Sep 17 00:00:00 2001 From: dgp Date: Tue, 28 Jan 2014 18:51:59 +0000 Subject: Working code with no staging buffer use. --- generic/tclIO.c | 128 ++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 97 insertions(+), 31 deletions(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index fdb0ddd..7203739 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -3756,18 +3756,21 @@ WriteChars( char *dst, *stage; int saved, savedLF, sawLF, total, dstLen, stageMax, dstWrote; int stageLen, toWrite, stageRead, endEncoding, result; - int consumedSomething, translate; + int consumedSomething, translate, flushed, needNlFlush; Tcl_Encoding encoding; char safe[BUFFER_PADDING]; + char *nextNewLine = NULL; if (srcLen) { WillWrite(chanPtr); } + flushed = 0; total = 0; sawLF = 0; savedLF = 0; saved = 0; + needNlFlush = 0; encoding = statePtr->encoding; /* @@ -3779,46 +3782,39 @@ WriteChars( translate = (statePtr->flags & CHANNEL_LINEBUFFERED) || (statePtr->outputTranslation != TCL_TRANSLATE_LF); -#if 0 +#if 1 + if (translate) { + nextNewLine = memchr(src, '\n', srcLen); + } consumedSomething = 1; while (consumedSomething && (srcLen + saved + endEncoding > 0)) { - void *lastNewLine; - int srcLimit; + int srcRead; + int srcLimit = srcLen; + + consumedSomething = 0; + if (nextNewLine) { + srcLimit = nextNewLine - src; + } /* Get space to write into */ bufPtr = statePtr->curOutPtr; if (bufPtr == NULL) { bufPtr = AllocChannelBuffer(statePtr->bufSize); statePtr->curOutPtr = bufPtr; - if (saved) { - /* - * Here's some translated bytes left over from the last buffer - * that we need to stick at the beginning of this buffer. - */ + } + if (saved) { + /* + * Here's some translated bytes left over from the last buffer + * that we need to stick at the beginning of this buffer. + */ - memcpy(InsertPoint(bufPtr), safe, (size_t) saved); - bufPtr->nextAdded += saved; - saved = 0; - } + memcpy(InsertPoint(bufPtr), safe, (size_t) saved); + bufPtr->nextAdded += saved; + saved = 0; } dst = InsertPoint(bufPtr); dstLen = SpaceLeft(bufPtr); - /* - * We have dstLen bytes to write to. The most source bytes - * that could possibly fill that is TCL_UTF_MAX * dstLen. - */ - - srcLimit = TCL_UTF_MAX * dstLen; - if (srcLen < srcLimit) { - srcLimit = srcLen; - } - lastNewLine = memchr(src, '\n', srcLimit); - - if (lastNewLine) { - srcLimit = lastNewLine - src; - } - result = Tcl_UtfToExternal(NULL, encoding, src, srcLimit, statePtr->outputEncodingFlags, &statePtr->outputEncodingState, dst, @@ -3827,9 +3823,59 @@ WriteChars( statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; if ((result != 0) && (srcRead + dstWrote == 0)) { - fprintf(stdout, "WDTH?\n"); fflush(stdout); + /* We're reading from invalid/incomplete UTF-8 */ + break; } + + consumedSomething = 1; bufPtr->nextAdded += dstWrote; + src += srcRead; + srcLen -= srcRead; + total += dstWrote; + dst += dstWrote; + dstLen -= dstWrote; + + if (src == nextNewLine && dstLen > 0) { + static char crln[3] = "\r\n"; + char *nl = NULL; + int nlLen = 0; + + switch (statePtr->outputTranslation) { + case TCL_TRANSLATE_LF: + nl = crln + 1; + nlLen = 1; + break; + case TCL_TRANSLATE_CR: + nl = crln; + nlLen = 1; + break; + case TCL_TRANSLATE_CRLF: + nl = crln; + nlLen = 2; + break; + default: + Tcl_Panic("unknown output translation requested"); + break; + } + + result |= Tcl_UtfToExternal(NULL, encoding, nl, nlLen, + statePtr->outputEncodingFlags, + &statePtr->outputEncodingState, dst, + dstLen + BUFFER_PADDING, &srcRead, &dstWrote, NULL); + + if (srcRead != nlLen) { + Tcl_Panic("Can This Happen?"); + } + + bufPtr->nextAdded += dstWrote; + src++; + srcLen--; + total += dstWrote; + dst += dstWrote; + dstLen -= dstWrote; + nextNewLine = memchr(src, '\n', srcLen); + needNlFlush = 1; + } if (IsBufferOverflowing(bufPtr)) { /* * When translating from UTF-8 to external encoding, we @@ -3843,8 +3889,28 @@ WriteChars( memcpy(safe, dst + dstLen, (size_t) saved); bufPtr->nextAdded = bufPtr->bufLength; } - - + + if ((srcLen + saved == 0) && (result == 0)) { + endEncoding = 0; + } + + /* FLUSH ! */ + if (IsBufferFull(bufPtr)) { + if (FlushChannel(NULL, chanPtr, 0) != 0) { + return -1; + } + flushed += statePtr->bufSize; + if (saved == 0 || src[-1] != '\n') { + needNlFlush = 0; + } + } + } + if ((flushed < total) && (statePtr->flags & CHANNEL_UNBUFFERED || + (needNlFlush && statePtr->flags & CHANNEL_LINEBUFFERED))) { + SetFlag(statePtr, BUFFER_READY); + if (FlushChannel(NULL, chanPtr, 0) != 0) { + return -1; + } } #else /* -- cgit v0.12 From 589975911949b9661152610ca04cf5e8e233a109 Mon Sep 17 00:00:00 2001 From: dgp Date: Tue, 28 Jan 2014 20:19:39 +0000 Subject: tidy things up --- generic/tclIO.c | 211 +++++--------------------------------------------------- 1 file changed, 17 insertions(+), 194 deletions(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index 7203739..c7c11e7 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -3752,46 +3752,30 @@ WriteChars( { ChannelState *statePtr = chanPtr->state; /* State info for channel */ - ChannelBuffer *bufPtr; - char *dst, *stage; - int saved, savedLF, sawLF, total, dstLen, stageMax, dstWrote; - int stageLen, toWrite, stageRead, endEncoding, result; - int consumedSomething, translate, flushed, needNlFlush; - Tcl_Encoding encoding; - char safe[BUFFER_PADDING]; char *nextNewLine = NULL; + int endEncoding, saved = 0, total = 0, flushed = 0, needNlFlush = 0; + Tcl_Encoding encoding = statePtr->encoding; if (srcLen) { WillWrite(chanPtr); } - flushed = 0; - total = 0; - sawLF = 0; - savedLF = 0; - saved = 0; - needNlFlush = 0; - encoding = statePtr->encoding; - /* * Write the terminated escape sequence even if srcLen is 0. */ endEncoding = ((statePtr->outputEncodingFlags & TCL_ENCODING_END) != 0); - translate = (statePtr->flags & CHANNEL_LINEBUFFERED) - || (statePtr->outputTranslation != TCL_TRANSLATE_LF); - -#if 1 - if (translate) { + if ((statePtr->flags & CHANNEL_LINEBUFFERED) + || (statePtr->outputTranslation != TCL_TRANSLATE_LF)) { nextNewLine = memchr(src, '\n', srcLen); } - consumedSomething = 1; - while (consumedSomething && (srcLen + saved + endEncoding > 0)) { - int srcRead; - int srcLimit = srcLen; - consumedSomething = 0; + while (srcLen + saved + endEncoding > 0) { + ChannelBuffer *bufPtr; + char *dst, safe[BUFFER_PADDING]; + int result, srcRead, dstLen, dstWrote, srcLimit = srcLen; + if (nextNewLine) { srcLimit = nextNewLine - src; } @@ -3820,14 +3804,18 @@ WriteChars( &statePtr->outputEncodingState, dst, dstLen + BUFFER_PADDING, &srcRead, &dstWrote, NULL); + /* See chan-io-1.[89]. Tcl Bug 506297. */ statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; - if ((result != 0) && (srcRead + dstWrote == 0)) { + if ((result != TCL_OK) && (srcRead + dstWrote == 0)) { /* We're reading from invalid/incomplete UTF-8 */ + if (total == 0) { + Tcl_SetErrno(EINVAL); + return -1; + } break; } - consumedSomething = 1; bufPtr->nextAdded += dstWrote; src += srcRead; srcLen -= srcRead; @@ -3876,6 +3864,7 @@ WriteChars( nextNewLine = memchr(src, '\n', srcLen); needNlFlush = 1; } + if (IsBufferOverflowing(bufPtr)) { /* * When translating from UTF-8 to external encoding, we @@ -3890,11 +3879,10 @@ WriteChars( bufPtr->nextAdded = bufPtr->bufLength; } - if ((srcLen + saved == 0) && (result == 0)) { + if ((srcLen + saved == 0) && (result == TCL_OK)) { endEncoding = 0; } - /* FLUSH ! */ if (IsBufferFull(bufPtr)) { if (FlushChannel(NULL, chanPtr, 0) != 0) { return -1; @@ -3912,172 +3900,7 @@ WriteChars( return -1; } } -#else - /* - * Loop over all UTF-8 characters in src, storing them in staging buffer - * with proper EOL translation. - */ - - consumedSomething = 1; - while (consumedSomething && (srcLen + savedLF + endEncoding > 0)) { - consumedSomething = 0; - - if (translate) { - stage = statePtr->outputStage; - stageMax = statePtr->bufSize; - stageLen = stageMax; - - toWrite = stageLen; - if (toWrite > srcLen) { - toWrite = srcLen; - } - - if (savedLF) { - /* - * A '\n' was left over from last call to TranslateOutputEOL() - * and we need to store it in the staging buffer. If the channel - * is line-based, we will need to flush the output buffer (after - * translating the staging buffer). - */ - - *stage++ = '\n'; - stageLen--; - sawLF++; - } - if (TranslateOutputEOL(statePtr, stage, src, &stageLen, &toWrite)) { - sawLF++; - } - - stage -= savedLF; - stageLen += savedLF; - savedLF = 0; - - if (stageLen > stageMax) { - savedLF = 1; - stageLen = stageMax; - } - } else { - stage = (char *) src; - stageLen = srcLen; - toWrite = stageLen; - } - src += toWrite; - srcLen -= toWrite; - - /* - * Loop over all UTF-8 characters in staging buffer, converting them - * to external encoding, storing them in output buffer. - */ - - while (stageLen + saved + endEncoding > 0) { - bufPtr = statePtr->curOutPtr; - if (bufPtr == NULL) { - bufPtr = AllocChannelBuffer(statePtr->bufSize); - statePtr->curOutPtr = bufPtr; - } - dst = InsertPoint(bufPtr); - dstLen = SpaceLeft(bufPtr); - if (saved != 0) { - /* - * Here's some translated bytes left over from the last buffer - * that we need to stick at the beginning of this buffer. - */ - - memcpy(dst, safe, (size_t) saved); - bufPtr->nextAdded += saved; - dst += saved; - dstLen -= saved; - saved = 0; - } - - result = Tcl_UtfToExternal(NULL, encoding, stage, stageLen, - statePtr->outputEncodingFlags, - &statePtr->outputEncodingState, dst, - dstLen + BUFFER_PADDING, &stageRead, &dstWrote, NULL); - - /* - * Fix for SF #506297, reported by Martin Forssen - * . - * - * The encoding chosen in the script exposing the bug writes out - * three intro characters when TCL_ENCODING_START is set, but does - * not consume any input as TCL_ENCODING_END is cleared. As some - * output was generated the enclosing loop calls UtfToExternal - * again, again with START set. Three more characters in the out - * and still no use of input ... To break this infinite loop we - * remove TCL_ENCODING_START from the set of flags after the first - * call (no condition is required, the later calls remove an unset - * flag, which is a no-op). This causes the subsequent calls to - * UtfToExternal to consume and convert the actual input. - */ - - statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; - - /* - * The following code must be executed only when result is not 0. - */ - - if ((result != 0) && (stageRead + dstWrote == 0)) { - /* - * We have an incomplete UTF-8 character at the end of the - * staging buffer. It will get moved to the beginning of the - * staging buffer followed by more bytes from src. - */ - - src -= stageLen; - srcLen += stageLen; - stageLen = 0; - savedLF = 0; - break; - } - bufPtr->nextAdded += dstWrote; - if (IsBufferOverflowing(bufPtr)) { - /* - * When translating from UTF-8 to external encoding, we - * allowed the translation to produce a character that crossed - * the end of the output buffer, so that we would get a - * completely full buffer before flushing it. The extra bytes - * will be moved to the beginning of the next buffer. - */ - - saved = -SpaceLeft(bufPtr); - memcpy(safe, dst + dstLen, (size_t) saved); - bufPtr->nextAdded = bufPtr->bufLength; - } - if (CheckFlush(chanPtr, bufPtr, sawLF) != 0) { - return -1; - } - - total += dstWrote; - stage += stageRead; - stageLen -= stageRead; - sawLF = 0; - - consumedSomething = 1; - - /* - * If all translated characters are written to the buffer, - * endEncoding is set to 0 because the escape sequence may be - * output. - */ - - if ((stageLen + saved == 0) && (result == 0)) { - endEncoding = 0; - } - } - } -#endif - - /* - * If nothing was written and it happened because there was no progress in - * the UTF conversion, we throw an error. - */ - - if (!consumedSomething && (total == 0)) { - Tcl_SetErrno(EINVAL); - return -1; - } return total; } -- cgit v0.12 From 9df81fccd3a1cfd90a732c1116cb3bf467bbe802 Mon Sep 17 00:00:00 2001 From: dgp Date: Tue, 28 Jan 2014 20:23:26 +0000 Subject: The outputStage field is now unused, so never allocate it. --- generic/tclIO.c | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index c7c11e7..4ae9ec0 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -1489,12 +1489,7 @@ Tcl_CreateChannel( statePtr->timer = NULL; statePtr->csPtrR = NULL; statePtr->csPtrW = NULL; - statePtr->outputStage = NULL; - if ((statePtr->encoding != NULL) && (statePtr->flags & TCL_WRITABLE)) { - statePtr->outputStage = (char *) - ckalloc((unsigned) (statePtr->bufSize + 2)); - } /* * As we are creating the channel, it is obviously the top for now. @@ -2757,10 +2752,6 @@ CloseChannel( } Tcl_FreeEncoding(statePtr->encoding); - if (statePtr->outputStage != NULL) { - ckfree((char *) statePtr->outputStage); - statePtr->outputStage = NULL; - } } /* @@ -7151,15 +7142,6 @@ Tcl_SetChannelBufferSize( statePtr = ((Channel *) chan)->state; statePtr->bufSize = sz; - - if (statePtr->outputStage != NULL) { - ckfree((char *) statePtr->outputStage); - statePtr->outputStage = NULL; - } - if ((statePtr->encoding != NULL) && (statePtr->flags & TCL_WRITABLE)) { - statePtr->outputStage = (char *) - ckalloc((unsigned) (statePtr->bufSize + 2)); - } } /* @@ -7800,17 +7782,6 @@ Tcl_SetChannelOption( statePtr->inQueueTail = NULL; } - /* - * If encoding or bufsize changes, need to update output staging buffer. - */ - - if (statePtr->outputStage != NULL) { - ckfree(statePtr->outputStage); - statePtr->outputStage = NULL; - } - if ((statePtr->encoding != NULL) && (statePtr->flags & TCL_WRITABLE)) { - statePtr->outputStage = ckalloc((unsigned) (statePtr->bufSize + 2)); - } return TCL_OK; } -- cgit v0.12