diff options
| -rw-r--r-- | generic/tcl.h | 2 | ||||
| -rw-r--r-- | generic/tclEncoding.c | 101 | ||||
| -rw-r--r-- | generic/tclIO.c | 67 | ||||
| -rw-r--r-- | generic/tclTest.c | 16 | ||||
| -rw-r--r-- | tests/encoding.test | 64 | ||||
| -rw-r--r-- | tests/encodingVectors.tcl | 4 | ||||
| -rw-r--r-- | tests/ioTrans.test | 86 | ||||
| -rw-r--r-- | tests/utfext.test | 2 |
8 files changed, 280 insertions, 62 deletions
diff --git a/generic/tcl.h b/generic/tcl.h index fd02ccc..a19acc1 100644 --- a/generic/tcl.h +++ b/generic/tcl.h @@ -1955,10 +1955,8 @@ typedef struct Tcl_EncodingType { #define TCL_ENCODING_START 0x01 #define TCL_ENCODING_END 0x02 #if TCL_MAJOR_VERSION > 8 -# define TCL_ENCODING_STRICT 0x04 # define TCL_ENCODING_STOPONERROR 0x0 /* Not used any more */ #else -# define TCL_ENCODING_STRICT 0x44 # define TCL_ENCODING_STOPONERROR 0x04 #endif #define TCL_ENCODING_NO_TERMINATE 0x08 diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index a170332..36e536b 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -199,11 +199,14 @@ static struct TclEncodingProfiles { {"strict", TCL_ENCODING_PROFILE_STRICT}, {"tcl8", TCL_ENCODING_PROFILE_TCL8}, }; +#define PROFILE_TCL8(flags_) \ + ((TCL_ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8) \ + || (TCL_ENCODING_PROFILE_GET(flags_) == 0 \ + && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_TCL8)) #define PROFILE_STRICT(flags_) \ ((TCL_ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_STRICT) \ || (TCL_ENCODING_PROFILE_GET(flags_) == 0 \ && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_STRICT)) - #define PROFILE_REPLACE(flags_) \ ((TCL_ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE) \ || (TCL_ENCODING_PROFILE_GET(flags_) == 0 \ @@ -1411,6 +1414,9 @@ Tcl_ExternalToUtf( } if (!noTerminate) { + if ((int) dstLen < 1) { + return TCL_CONVERT_NOSPACE; + } /* * If there are any null characters in the middle of the buffer, * they will converted to the UTF-8 null character (\xC0\x80). To get @@ -1419,6 +1425,10 @@ Tcl_ExternalToUtf( */ dstLen--; + } else { + if (dstLen <= 0 && srcLen > 0) { + return TCL_CONVERT_NOSPACE; + } } if (encodingPtr->toUtfProc == UtfToUtfProc) { flags |= ENCODING_INPUT; @@ -1727,10 +1737,17 @@ Tcl_UtfToExternal( dstCharsPtr = &dstChars; } + if (dstLen < encodingPtr->nullSize) { + return TCL_CONVERT_NOSPACE; + } dstLen -= encodingPtr->nullSize; result = encodingPtr->fromUtfProc(encodingPtr->clientData, src, srcLen, flags, statePtr, dst, dstLen, srcReadPtr, dstWrotePtr, dstCharsPtr); + /* + * Buffer is terminated irrespective of result. Not sure this is + * reasonable but keep for historical/compatibility reasons. + */ memset(&dst[*dstWrotePtr], '\0', encodingPtr->nullSize); return result; @@ -2783,18 +2800,23 @@ Utf32ToUtfProc( dst += Tcl_UniCharToUtf(-1, dst); } #endif - if ((unsigned)ch > 0x10FFFF || SURROGATE(ch)) { + + if ((unsigned)ch > 0x10FFFF) { + ch = UNICODE_REPLACE_CHAR; if (PROFILE_STRICT(flags)) { result = TCL_CONVERT_SYNTAX; -#if TCL_UTF_MAX < 4 - ch = 0; -#endif break; } - if (PROFILE_REPLACE(flags)) { - ch = UNICODE_REPLACE_CHAR; - } - } + } else if (PROFILE_STRICT(flags) && SURROGATE(ch)) { + result = TCL_CONVERT_SYNTAX; +#if TCL_UTF_MAX < 4 + ch = 0; +#endif + break; + } else if (PROFILE_REPLACE(flags) && SURROGATE(ch)) { + ch = UNICODE_REPLACE_CHAR; + } + /* * Special case for 1-byte utf chars for speed. Make sure we work with * unsigned short-size data. @@ -2818,6 +2840,7 @@ Utf32ToUtfProc( dst += Tcl_UniCharToUtf(-1, dst); } #endif + if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) { if (dst > dstEnd) { result = TCL_CONVERT_NOSPACE; @@ -3017,7 +3040,7 @@ Utf16ToUtfProc( dstStart = dst; dstEnd = dst + dstLen - TCL_UTF_MAX; - for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) { + for (numChars = 0; src < srcEnd && numChars <= charLimit; src += 2, numChars++) { if (dst > dstEnd) { result = TCL_CONVERT_NOSPACE; break; @@ -3030,8 +3053,31 @@ Utf16ToUtfProc( ch = (src[0] & 0xFF) << 8 | (src[1] & 0xFF); } if (((prev & ~0x3FF) == 0xD800) && ((ch & ~0x3FF) != 0xDC00)) { - /* Bug [10c2c17c32]. If Hi surrogate not followed by Lo surrogate, finish 3-byte UTF-8 */ - dst += Tcl_UniCharToUtf(-1, dst); + if (PROFILE_STRICT(flags)) { + result = TCL_CONVERT_UNKNOWN; + src -= 2; /* Go back to beginning of high surrogate */ + dst--; /* Also undo writing a single byte too much */ + numChars--; + break; + } else if (PROFILE_REPLACE(flags)) { + /* + * Previous loop wrote a single byte to mark the high surrogate. + * Replace it with the replacement character. Further, restart + * current loop iteration since need to recheck destination space + * and reset processing of current character. + */ + ch = UNICODE_REPLACE_CHAR; + dst--; + dst += Tcl_UniCharToUtf(ch, dst); + src -= 2; + numChars--; + continue; + } + else { + /* Bug [10c2c17c32]. If Hi surrogate not followed by Lo + * surrogate, finish 3-byte UTF-8 */ + dst += Tcl_UniCharToUtf(-1, dst); + } } /* @@ -3039,17 +3085,38 @@ Utf16ToUtfProc( * unsigned short-size data. */ - if (ch && ch < 0x80) { + if ((unsigned)ch - 1 < 0x7F) { *dst++ = (ch & 0xFF); - } else { + } else if (((prev & ~0x3FF) == 0xD800) || ((ch & ~0x3FF) == 0xD800)) { dst += Tcl_UniCharToUtf(ch | TCL_COMBINE, dst); + } else if (((ch & ~0x3FF) == 0xDC00) && !PROFILE_TCL8(flags)) { + /* Lo surrogate not preceded by Hi surrogate and not tcl8 profile */ + if (PROFILE_STRICT(flags)) { + result = TCL_CONVERT_UNKNOWN; + break; + } else { + /* PROFILE_REPLACE */ + dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst); + } + } else { + dst += Tcl_UniCharToUtf(ch, dst); } - src += sizeof(unsigned short); } if ((ch & ~0x3FF) == 0xD800) { - /* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */ - dst += Tcl_UniCharToUtf(-1, dst); + if (PROFILE_STRICT(flags)) { + result = TCL_CONVERT_UNKNOWN; + src -= 2; + dst--; + numChars--; + } else if (PROFILE_REPLACE(flags)) { + dst--; + dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst); + } + else { + /* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */ + dst += Tcl_UniCharToUtf(-1, dst); + } } /* diff --git a/generic/tclIO.c b/generic/tclIO.c index e96ac23..cc0af2e 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -8640,6 +8640,7 @@ UpdateInterest( mask &= ~TCL_EXCEPTION; if (!statePtr->timer) { + TclChannelPreserve((Tcl_Channel)chanPtr); statePtr->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME, ChannelTimerProc, chanPtr); } @@ -8650,6 +8651,7 @@ UpdateInterest( && mask & TCL_WRITABLE && GotFlag(statePtr, CHANNEL_NONBLOCKING)) { + TclChannelPreserve((Tcl_Channel)chanPtr); statePtr->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME, ChannelTimerProc,chanPtr); } @@ -8684,44 +8686,55 @@ ChannelTimerProc( /* State info for channel */ ChannelState *statePtr = chanPtr->state; - /* Preserve chanPtr to guard against deallocation in Tcl_NotifyChannel. */ - TclChannelPreserve((Tcl_Channel)chanPtr); - Tcl_Preserve(statePtr); - statePtr->timer = NULL; - if (statePtr->interestMask & TCL_WRITABLE - && GotFlag(statePtr, CHANNEL_NONBLOCKING) - && !GotFlag(statePtr, BG_FLUSH_SCHEDULED) - ) { - /* - * Restart the timer in case a channel handler reenters the event loop - * before UpdateInterest gets called by Tcl_NotifyChannel. - */ - statePtr->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME, - ChannelTimerProc,chanPtr); - Tcl_NotifyChannel((Tcl_Channel) chanPtr, TCL_WRITABLE); - } + /* TclChannelPreserve() must be called before the current function was + * scheduled, is already in effect. In this function it guards against + * deallocation in Tcl_NotifyChannel and also keps the channel preserved + * until ChannelTimerProc is later called again. + */ - /* The channel may have just been closed from within Tcl_NotifyChannel */ - if (!GotFlag(statePtr, CHANNEL_INCLOSE)) { - if (!GotFlag(statePtr, CHANNEL_NEED_MORE_DATA) - && (statePtr->interestMask & TCL_READABLE) - && (statePtr->inQueueHead != NULL) - && IsBufferReady(statePtr->inQueueHead)) { + if (chanPtr->typePtr == NULL) { + TclChannelRelease((Tcl_Channel)chanPtr); + } else { + Tcl_Preserve(statePtr); + statePtr->timer = NULL; + if (statePtr->interestMask & TCL_WRITABLE + && GotFlag(statePtr, CHANNEL_NONBLOCKING) + && !GotFlag(statePtr, BG_FLUSH_SCHEDULED) + ) { /* * Restart the timer in case a channel handler reenters the event loop * before UpdateInterest gets called by Tcl_NotifyChannel. */ - statePtr->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME, ChannelTimerProc,chanPtr); - Tcl_NotifyChannel((Tcl_Channel) chanPtr, TCL_READABLE); + Tcl_NotifyChannel((Tcl_Channel) chanPtr, TCL_WRITABLE); } else { - UpdateInterest(chanPtr); + /* The channel may have just been closed from within Tcl_NotifyChannel */ + if (!GotFlag(statePtr, CHANNEL_INCLOSE)) { + if (!GotFlag(statePtr, CHANNEL_NEED_MORE_DATA) + && (statePtr->interestMask & TCL_READABLE) + && (statePtr->inQueueHead != NULL) + && IsBufferReady(statePtr->inQueueHead)) { + /* + * Restart the timer in case a channel handler reenters the event loop + * before UpdateInterest gets called by Tcl_NotifyChannel. + */ + + statePtr->timer = Tcl_CreateTimerHandler(SYNTHETIC_EVENT_TIME, + ChannelTimerProc,chanPtr); + Tcl_NotifyChannel((Tcl_Channel) chanPtr, TCL_READABLE); + } else { + TclChannelRelease((Tcl_Channel)chanPtr); + UpdateInterest(chanPtr); + } + } else { + TclChannelRelease((Tcl_Channel)chanPtr); + } } + + Tcl_Release(statePtr); } - Tcl_Release(statePtr); - TclChannelRelease((Tcl_Channel)chanPtr); } /* diff --git a/generic/tclTest.c b/generic/tclTest.c index e2d8b3b..eb19d18 100644 --- a/generic/tclTest.c +++ b/generic/tclTest.c @@ -1995,19 +1995,19 @@ static void SpecialFree( * TCL_OK or TCL_ERROR. This any errors running the test, NOT the * result of Tcl_UtfToExternal or Tcl_ExternalToUtf. * - * Side effects: + * Side effects: * * The result in the interpreter is a list of the return code from the * Tcl_UtfToExternal/Tcl_ExternalToUtf functions, the encoding state, and * an encoded binary string of length dstLen. Note the string is the * entire output buffer, not just the part containing the decoded * portion. This allows for additional checks at test script level. - * - * If any of the srcreadvar, dstwrotevar and + * + * If any of the srcreadvar, dstwrotevar and * dstcharsvar are specified and not empty, they are treated as names * of variables where the *srcRead, *dstWrote and *dstChars output * from the functions are stored. - * + * * The function also checks internally whether nuls are correctly * appended as requested but the TCL_ENCODING_NO_TERMINATE flag * and that no buffer overflows occur. @@ -2134,12 +2134,12 @@ static int UtfExtWrapper( } bufLen = dstLen + 4; /* 4 -> overflow detection */ - bufPtr = Tcl_Alloc(bufLen); + bufPtr = (unsigned char *) Tcl_Alloc(bufLen); memset(bufPtr, 0xFF, dstLen); /* Need to check nul terminator */ memmove(bufPtr + dstLen, "\xAB\xCD\xEF\xAB", 4); /* overflow detection */ bytes = Tcl_GetByteArrayFromObj(objv[3], &srcLen); /* Last! to avoid shimmering */ - result = (*transformer)(interp, encoding, bytes, srcLen, flags, - &encState, bufPtr, dstLen, + result = (*transformer)(interp, encoding, (const char *) bytes, srcLen, flags, + &encState, (char *) bufPtr, dstLen, srcReadVar ? &srcRead : NULL, &dstWrote, dstCharsVar ? &dstChars : NULL); @@ -2149,9 +2149,7 @@ static int UtfExtWrapper( TCL_STATIC); result = TCL_ERROR; } else if (result != TCL_ERROR) { - Tcl_Obj *resultObjs[3]; - switch (result) { case TCL_OK: resultObjs[0] = Tcl_NewStringObj("ok", -1); diff --git a/tests/encoding.test b/tests/encoding.test index 8ed7c1d..db5680d 100644 --- a/tests/encoding.test +++ b/tests/encoding.test @@ -564,15 +564,27 @@ test encoding-16.18 { return done } [namespace current]] } -result done -test encoding-16.19 {UnicodeToUtfProc, bug [d19fe0a5b]} -body { +test encoding-16.19 {Utf16ToUtfProc, bug [d19fe0a5b]} -body { encoding convertfrom utf-16 "\x41\x41\x41" } -result \u4141\uFFFD -test encoding-16.20 {UnicodeToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body { +test encoding-16.20 {Utf16ToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body { encoding convertfrom utf-16 "\xD8\xD8" } -result \uD8D8 -test encoding-16.21 {UnicodeToUtfProc, bug [d19fe0a5b]} -body { +test encoding-16.21 {Utf16ToUtfProc, bug [d19fe0a5b]} -body { encoding convertfrom utf-32 "\x00\x00\x00\x00\x41\x41" } -result \x00\uFFFD +test encoding-16.22 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { + encoding convertfrom -profile strict utf-16le \x00\xD8 +} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x00'} +test encoding-16.23 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { + encoding convertfrom -profile strict utf-16le \x00\xDC +} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x00'} +test encoding-16.24 {Utf32ToUtfProc} -body { + encoding convertfrom utf-32 "\xFF\xFF\xFF\xFF" +} -result \uFFFD +test encoding-16.25 {Utf32ToUtfProc} -body { + encoding convertfrom utf-32 "\x01\x00\x00\x01" +} -result \uFFFD test encoding-17.1 {UtfToUtf16Proc} -body { encoding convertto utf-16 "\U460DC" @@ -604,6 +616,12 @@ test encoding-17.9 {Utf32ToUtfProc} -body { test encoding-17.10 {Utf32ToUtfProc} -body { encoding convertfrom -profile tcl8 utf-32 "\xFF\xFF\xFF\xFF" } -result \uFFFD +test encoding-17.11 {Utf32ToUtfProc} -body { + encoding convertfrom -profile strict utf-32le "\x00\xD8\x00\x00" +} -returnCodes error -result {unexpected byte sequence starting at index 0: '\x00'} +test encoding-17.12 {Utf32ToUtfProc} -body { + encoding convertfrom -profile strict utf-32le "\x00\xDC\x00\x00" +} -returnCodes error -result {unexpected byte sequence starting at index 0: '\x00'} test encoding-18.1 {TableToUtfProc on invalid input} -body { list [catch {encoding convertto jis0208 \\} res] $res @@ -1049,6 +1067,46 @@ test encoding-28.0 {all encodings load} -body { runtests +test encoding-bug-183a1adcc0-1 {Bug [183a1adcc0] Buffer overflow Tcl_UtfToExternal} -constraints { + testencoding +} -body { + # Note - buffers are initialized to \xff + list [catch {testencoding Tcl_UtfToExternal utf-16 A {start end} {} 1} result] $result +} -result [list 0 [list nospace {} \xff]] + +test encoding-bug-183a1adcc0-2 {Bug [183a1adcc0] Buffer overflow Tcl_UtfToExternal} -constraints { + testencoding +} -body { + # Note - buffers are initialized to \xff + list [catch {testencoding Tcl_UtfToExternal utf-16 A {start end} {} 0} result] $result +} -result [list 0 [list nospace {} {}]] + +test encoding-bug-183a1adcc0-3 {Bug [183a1adcc0] Buffer overflow Tcl_UtfToExternal} -constraints { + testencoding +} -body { + # Note - buffers are initialized to \xff + list [catch {testencoding Tcl_UtfToExternal utf-16 A {start end} {} 2} result] $result +} -result [list 0 [list nospace {} \x00\x00]] + +test encoding-bug-183a1adcc0-4 {Bug [183a1adcc0] Buffer overflow Tcl_UtfToExternal} -constraints { + testencoding +} -body { + # Note - buffers are initialized to \xff + list [catch {testencoding Tcl_UtfToExternal utf-16 A {start end} {} 3} result] $result +} -result [list 0 [list nospace {} \x00\x00\xff]] + +test encoding-bug-183a1adcc0-5 {Bug [183a1adcc0] Buffer overflow Tcl_UtfToExternal} -constraints { + testencoding +} -constraints { + knownBug +} -body { + # The knownBug constraint is because test depends on TCL_UTF_MAX and + # also UtfToUtf16 assumes space required in destination buffer is + # sizeof(Tcl_UniChar) which is incorrect when TCL_UTF_MAX==4 + # Note - buffers are initialized to \xff + list [catch {testencoding Tcl_UtfToExternal utf-16 A {start end} {} 4} result] $result +} -result [list 0 [list ok {} [expr {$::tcl_platform(byteOrder) eq "littleEndian" ? "\x41\x00" : "\x00\x41"}]\x00\x00]] + } test encoding-29.0 {get encoding nul terminator lengths} -constraints { diff --git a/tests/encodingVectors.tcl b/tests/encodingVectors.tcl index b3f3efa..1b569a1 100644 --- a/tests/encodingVectors.tcl +++ b/tests/encodingVectors.tcl @@ -553,10 +553,10 @@ lappend encInvalidBytes {*}{ utf-16le 41 replace \uFFFD -1 {solo tail} {Truncated} utf-16le 41 strict {} 0 {solo tail} {Truncated} utf-16le 00D8 tcl8 \uD800 -1 {} {Missing low surrogate} - utf-16le 00D8 replace \uFFFD -1 {knownBug} {Missing low surrogate} + utf-16le 00D8 replace \uFFFD -1 {} {Missing low surrogate} utf-16le 00D8 strict {} 0 {knownBug} {Missing low surrogate} utf-16le 00DC tcl8 \uDC00 -1 {} {Missing high surrogate} - utf-16le 00DC replace \uFFFD -1 {knownBug} {Missing high surrogate} + utf-16le 00DC replace \uFFFD -1 {} {Missing high surrogate} utf-16le 00DC strict {} 0 {knownBug} {Missing high surrogate} utf-16be 41 tcl8 \uFFFD -1 {solo tail} {Truncated} diff --git a/tests/ioTrans.test b/tests/ioTrans.test index 79493e0..9639576 100644 --- a/tests/ioTrans.test +++ b/tests/ioTrans.test @@ -634,6 +634,58 @@ test iortrans-4.9 {chan read, gets, bug 2921116} -setup { } } + + +namespace eval reflector { + proc initialize {_ chan mode} { + return {initialize finalize watch read} + } + + + proc finalize {_ chan} { + namespace delete $_ + } + + + proc read {_ chan count} { + namespace upvar $_ source source + set res [string range $source 0 $count-1] + set source [string range $source $count end] + return $res + } + + + proc watch {_ chan events} { + after 0 [list chan postevent $chan read] + return read + } + + namespace ensemble create -parameters _ + namespace export * +} + + + + +namespace eval inputfilter { + proc initialize {chan mode} { + return {initialize finalize read} + } + + proc read {chan buffer} { + return $buffer + } + + proc finalize chan { + namespace delete $chan + } + + namespace ensemble create + namespace export * +} + + + # Channel read transform that is just the identity - pass all through proc idxform {cmd handle args} { switch -- $cmd { @@ -2089,7 +2141,39 @@ test iortrans.tf-11.1 {origin thread of moved transform destroyed during access} thread::release $tidb } -result {Owner lost} -# ### ### ### ######### ######### ######### + +test iortrans-ea69b0258a9833cb { + Crash when using a channel transformation on TCP client socket + + "line two" does not make it into result. This issue should probably be + addressed, but it is outside the scope of this test. +} -setup { + set res {} + set read 0 +} -body { + namespace eval reflector1 { + variable source "line one\nline two" + interp alias {} [namespace current]::dispatch {} [ + namespace parent]::reflector [namespace current] + } + set chan [chan create read [namespace which reflector1::dispatch]] + chan configure $chan -blocking 0 + chan push $chan inputfilter + chan event $chan read [list ::apply [list chan { + variable res + variable read + set gets [gets $chan] + append res $gets + incr read + } [namespace current]] $chan] + vwait [namespace current]::read + chan pop $chan + vwait [namespace current]::read + return $res +} -cleanup { + catch {unset read} + close $chan +} -result {line one} cleanupTests return diff --git a/tests/utfext.test b/tests/utfext.test index 61e36b8..4ceb72f 100644 --- a/tests/utfext.test +++ b/tests/utfext.test @@ -86,7 +86,7 @@ foreach {enc utfhex hex} $utfExtMap { # Test for insufficient space test xx-bufferoverflow {buffer overflow Tcl_ExternalToUtf} -body { testencoding Tcl_UtfToExternal unicode A {start end} {} 1 -} -result {nospace {} {}} +} -result [list nospace {} \xFF] ::tcltest::cleanupTests return |