From f24bab84df2f1940c0a8004c4ee69c3c48da3126 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 3 Jul 2023 14:16:37 +0000 Subject: Branch meant for bug-fixing [5be203d6ca]: io-7.3 and chanio-7.3 fail in debug mode --- tests/chanio.test | 3 +-- tests/io.test | 3 +-- tests/tcltests.tcl | 1 - 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/chanio.test b/tests/chanio.test index fc8620f..c5d3aca 100644 --- a/tests/chanio.test +++ b/tests/chanio.test @@ -1090,10 +1090,9 @@ test chan-io-7.2 {FilterInputBytes: split up character in middle of buffer} -bod } -cleanup { chan close $f } -result {10 1234567890 0} -# This testcase fails in "debug" builds. See: [5be203d6ca] test chan-io-7.3 {FilterInputBytes: split up character at EOF} -setup { set x "" -} -constraints {testchannel ndebug} -body { +} -constraints {testchannel} -body { set f [open $path(test1) w] chan configure $f -encoding binary chan puts -nonewline $f "1234567890123\x82\x4F\x82\x50\x82" diff --git a/tests/io.test b/tests/io.test index ad68bf1..ca636ce 100644 --- a/tests/io.test +++ b/tests/io.test @@ -1124,8 +1124,7 @@ test io-7.2 {FilterInputBytes: split up character in middle of buffer} { close $f set x } [list 10 "1234567890" 0] -# This testcase fails in "debug" builds. See: [5be203d6ca] -test io-7.3 {FilterInputBytes: split up character at EOF} {testchannel ndebug} { +test io-7.3 {FilterInputBytes: split up character at EOF} {testchannel} { set f [open $path(test1) w] fconfigure $f -encoding binary puts -nonewline $f "1234567890123\x82\x4F\x82\x50\x82" diff --git a/tests/tcltests.tcl b/tests/tcltests.tcl index 0cabaaa..61366a4 100644 --- a/tests/tcltests.tcl +++ b/tests/tcltests.tcl @@ -8,7 +8,6 @@ namespace import ::tcltest::* testConstraint exec [llength [info commands exec]] testConstraint deprecated [expr {![tcl::build-info no-deprecate]}] testConstraint debug [tcl::build-info debug] -testConstraint ndebug [expr {![tcl::build-info debug]}] testConstraint purify [tcl::build-info purify] testConstraint debugpurify [ expr { -- cgit v0.12 From 3d2de6ed070b76c6a985c1cfe0be7a50fb12b650 Mon Sep 17 00:00:00 2001 From: apnadkarni Date: Tue, 4 Jul 2023 06:47:13 +0000 Subject: Correctly compute consumed bytes in case of truncated byte sequence at end. --- generic/tclEncoding.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index d0f04a7..42fcb7f 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -3448,6 +3448,14 @@ TableToUtfProc( src++; } + if (src > srcEnd) { + /* + * This can happen in the case of a trailing truncated byte sequence + * where src is incremented past the prefix byte which is the last. + * See bug [5be203d6ca] + */ + src--; + } *srcReadPtr = src - srcStart; *dstWrotePtr = dst - dstStart; *dstCharsPtr = numChars; -- cgit v0.12 From b49afc700b71491f3ae17396d4ce473d4b067712 Mon Sep 17 00:00:00 2001 From: apnadkarni Date: Wed, 5 Jul 2023 03:23:43 +0000 Subject: Bug [5be203d6ca] - better fix and add test --- generic/tclEncoding.c | 20 ++++++-------------- tests/utfext.test | 8 ++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index 42fcb7f..405c179 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -10,6 +10,7 @@ */ #include "tclInt.h" +#include typedef size_t (LengthProc)(const char *src); @@ -3397,16 +3398,13 @@ TableToUtfProc( } byte = *((unsigned char *) src); if (prefixBytes[byte]) { - src++; - if (src >= srcEnd) { + if (src >= srcEnd-1) { + /* Prefix byte but nothing after it */ if (!(flags & TCL_ENCODING_END)) { - /* Suffix bytes expected, don't consume prefix */ - src--; + /* More data to come */ result = TCL_CONVERT_MULTIBYTE; break; } else if (PROFILE_STRICT(flags)) { - /* Truncation. Do not consume so error location correct */ - src--; result = TCL_CONVERT_SYNTAX; break; } else if (PROFILE_REPLACE(flags)) { @@ -3415,6 +3413,7 @@ TableToUtfProc( ch = (unsigned) byte; } } else { + ++src; ch = toUnicode[byte][*((unsigned char *)src)]; } } else { @@ -3448,14 +3447,7 @@ TableToUtfProc( src++; } - if (src > srcEnd) { - /* - * This can happen in the case of a trailing truncated byte sequence - * where src is incremented past the prefix byte which is the last. - * See bug [5be203d6ca] - */ - src--; - } + assert(src <= srcEnd); *srcReadPtr = src - srcStart; *dstWrotePtr = dst - dstStart; *dstCharsPtr = numChars; diff --git a/tests/utfext.test b/tests/utfext.test index bc996c9..a6c9dd8 100644 --- a/tests/utfext.test +++ b/tests/utfext.test @@ -93,6 +93,14 @@ test xx-bufferoverflow {buffer overflow Tcl_ExternalToUtf} -body { # % testencoding Tcl_ExternalToUtf utf-8 abcdefgh {start end noterminate charlimit} {} 20 rv wv cv # nospace {} abcÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ +test TableToUtf-bug-5be203d6ca {Bug 5be203d6ca - truncated prefix in table encoding} -body { + set src \x82\x4f\x82\x50\x82 + lassign [testencoding Tcl_ExternalToUtf shiftjis $src {start} 0 16 srcRead dstWritten charsWritten] buf + set result [list [testencoding Tcl_ExternalToUtf shiftjis $src {start} 0 16 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] + lappend result {*}[list [testencoding Tcl_ExternalToUtf shiftjis [string range $src $srcRead end] {end} 0 10 srcRead dstWritten charsWritten] $srcRead $dstWritten $charsWritten] +} -result [list [list multibyte 0 \xEF\xBC\x90\xEF\xBC\x91\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF] 4 6 2 [list ok 0 \xC2\x82\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF] 1 2 1] + + ::tcltest::cleanupTests return -- cgit v0.12