Fix [ed29806baf]: Tcl_UtfToUniChar reads more than TCL_UTF_MAX bytes

4 files changed, 94 insertions, 165 deletions

diff --git a/generic/tclUtf.c b/generic/tclUtf.c
index c7f8a94..4454558 100644
--- a/generic/tclUtf.c
+++ b/generic/tclUtf.c
@@ -64,16 +64,11 @@ static const unsigned char totalBytes[256] = {
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-#if TCL_UTF_MAX < 4
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-#else /* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */
-    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-    3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-#endif
     2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
     3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-#if TCL_UTF_MAX > 4
+#if TCL_UTF_MAX > 3
     4,4,4,4,4,
 #else
     1,1,1,1,1,
@@ -86,19 +81,14 @@ static const unsigned char complete[256] = {
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-#if TCL_UTF_MAX < 4
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-#else /* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */
+/* Tcl_UtfCharComplete() might point to 2nd byte of valid 4-byte sequence */
     3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
     3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
-#endif
+/* End of "continuation byte section" */
     2,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
     3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
 #if TCL_UTF_MAX > 4
     4,4,4,4,4,
-#elif TCL_UTF_MAX < 4
-    1,1,1,1,1,
 #else
     3,3,3,3,3,
 #endif
@@ -407,7 +397,7 @@ Tcl_UtfToUniChar(
 	 * characters representing themselves.
 	 */
 
-#if TCL_UTF_MAX == 4
+#if TCL_UTF_MAX <= 4
 	/* If *chPtr contains a high surrogate (produced by a previous
 	 * Tcl_UtfToUniChar() call) and the next 3 bytes are UTF-8 continuation
 	 * bytes, then we must produce a follow-up low surrogate. We only
@@ -463,7 +453,7 @@ Tcl_UtfToUniChar(
 	     * Four-byte-character lead byte followed by at least two trail bytes.
 	     * We don't test the validity of 3th trail byte, see [ed29806ba]
 	     */
-#if TCL_UTF_MAX == 4
+#if TCL_UTF_MAX <= 4
 	    Tcl_UniChar high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2)
 		    | ((src[2] & 0x3F) >> 4)) - 0x40;
 	    if (high < 0x400) {
@@ -472,7 +462,7 @@ Tcl_UtfToUniChar(
 		return 1;
 	    }
 	    /* out of range, < 0x10000 or > 0x10FFFF */
-#elif TCL_UTF_MAX > 4
+#else
 	    if ((src[3] & 0xC0) == 0x80) {
 		*chPtr = (((byte & 0x07) << 18) | ((src[1] & 0x3F) << 12)
 			| ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
@@ -857,7 +847,7 @@ Tcl_UtfPrev(
 		 * it (the fallback) is correct.
 		 */
 
-		    || (trailBytesSeen >= complete[byte])) {
+		    || (trailBytesSeen >= totalBytes[byte])) {
 		/*
 		 * That is, (1 + trailBytesSeen > needed).
 		 * We've examined more bytes than needed to complete
@@ -898,19 +888,15 @@ Tcl_UtfPrev(
 
 	/* Continue the search backwards... */
 	look--;
-    } while (trailBytesSeen < ((TCL_UTF_MAX > 4) ? 4 : 3));
+    } while (trailBytesSeen < 3);
 
     /*
-     * We've seen 3 (or 4) trail bytes, so we know there will not be a
+     * We've seen 3 trail bytes, so we know there will not be a
      * properly formed byte sequence to find, and we can stop looking,
-     * accepting the fallback (for TCL_UTF_MAX > 4) or just go back as
-     * far as we can.
+     * accepting the fallback.
      */
-#if TCL_UTF_MAX > 4
+
     return fallback;
-#else
-    return src - 3;
-#endif
 }
 
 /*
diff --git a/tests/encoding.test b/tests/encoding.test
index a2bfe5d..935bef8 100644
--- a/tests/encoding.test
+++ b/tests/encoding.test
@@ -335,7 +335,7 @@ test encoding-15.4 {UtfToUtfProc emoji character input} -body {
     set y [encoding convertfrom utf-8 \xED\xA0\xBD\xED\xB8\x82]
     list [string length $x] $y
 } -result "6 \uD83D\uDE02"
-test encoding-15.5 {UtfToUtfProc emoji character input} knownBug {
+test encoding-15.5 {UtfToUtfProc emoji character input} {
     set x \xF0\x9F\x98\x82
     set y [encoding convertfrom utf-8 \xF0\x9F\x98\x82]
     list [string length $x] $y
diff --git a/tests/utf.test b/tests/utf.test
index 52b979c..69ef1f4 100644
--- a/tests/utf.test
+++ b/tests/utf.test
@@ -105,20 +105,14 @@ test utf-2.7 {Tcl_UtfToUniChar: lead (3-byte) followed by 2 trail} testbytestrin
 } 1
 test utf-2.8.0 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {testbytestring ucs2} {
     string length [testbytestring \xF0\x90\x80\x80]
-} 4
-test utf-2.8.1 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {testbytestring utf16} {
-    string length [testbytestring \xF0\x90\x80\x80]
 } 2
-test utf-2.8.2 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {testbytestring ucs4} {
+test utf-2.8.1 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {testbytestring ucs4} {
     string length [testbytestring \xF0\x90\x80\x80]
 } 1
 test utf-2.9.0 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {testbytestring ucs2} {
     string length [testbytestring \xF4\x8F\xBF\xBF]
-} 4
-test utf-2.9.1 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} utf16 {
-    string length \uDBFF\uDFFF
 } 2
-test utf-2.9.2 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {Uesc ucs4} {
+test utf-2.9.1 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail} {Uesc ucs4} {
     string length \U10FFFF
 } 1
 test utf-2.10 {Tcl_UtfToUniChar: lead (4-byte) followed by 3 trail, underflow} testbytestring {
@@ -171,11 +165,8 @@ test utf-4.11 {Tcl_NumUtfChars: 3 bytes of 4-byte UTF-8 characater} {testnumutfc
 } 3
 test utf-4.12.0 {Tcl_NumUtfChars: #4-byte UTF-8 character} {testnumutfchars testbytestring ucs2} {
     testnumutfchars [testbytestring \xF0\x9F\x92\xA9] end
-} 4
-test utf-4.12.1 {Tcl_NumUtfChars: #4-byte UTF-8 character} {testnumutfchars testbytestring utf16} {
-    testnumutfchars [testbytestring \xF0\x9F\x92\xA9] end
 } 2
-test utf-4.12.2 {Tcl_NumUtfChars: #4-byte UTF-8 character} {testnumutfchars testbytestring ucs4} {
+test utf-4.12.1 {Tcl_NumUtfChars: #4-byte UTF-8 character} {testnumutfchars testbytestring ucs4} {
     testnumutfchars [testbytestring \xF0\x9F\x92\xA9] end
 } 1
 test utf-4.13 {Tcl_NumUtfChars: end of string} {testnumutfchars testbytestring} {
@@ -218,12 +209,9 @@ test utf-6.7 {Tcl_UtfNext} {testutfnext testbytestring} {
 test utf-6.8 {Tcl_UtfNext} {testutfnext testbytestring} {
     testutfnext A[testbytestring \xF8]
 } 1
-test utf-6.9.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
+test utf-6.9 {Tcl_UtfNext} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0]
 } 1
-test utf-6.9.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
-    testutfnext [testbytestring \xA0]
-} -1
 test utf-6.10 {Tcl_UtfNext} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0]G
 } 1
@@ -287,7 +275,7 @@ test utf-6.29 {Tcl_UtfNext} {testutfnext testbytestring} {
 test utf-6.30.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2]
 } 1
-test utf-6.30.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.30.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2]
 } -1
 test utf-6.31 {Tcl_UtfNext} {testutfnext testbytestring} {
@@ -296,7 +284,7 @@ test utf-6.31 {Tcl_UtfNext} {testutfnext testbytestring} {
 test utf-6.32.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0]
 } 1
-test utf-6.32.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.32.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0]
 } -1
 test utf-6.33 {Tcl_UtfNext} {testutfnext testbytestring} {
@@ -410,7 +398,7 @@ test utf-6.68 {Tcl_UtfNext} {testutfnext testbytestring} {
 test utf-6.69.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]
 } 1
-test utf-6.69.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.69.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]
 } 4
 test utf-6.70 {Tcl_UtfNext} {testutfnext testbytestring} {
@@ -428,37 +416,37 @@ test utf-6.73 {Tcl_UtfNext} {testutfnext testbytestring} {
 test utf-6.74.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G
 } 1
-test utf-6.74.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.74.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G
 } 4
 test utf-6.75.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0]
 } 1
-test utf-6.75.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.75.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0]
 } 4
 test utf-6.76.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xD0]
 } 1
-test utf-6.76.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.76.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xD0]
 } 4
 test utf-6.77.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xE8]
 } 1
-test utf-6.77.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.77.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xE8]
 } 4
 test utf-6.78.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xF2]
 } 1
-test utf-6.78.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.78.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xF2]
 } 4
 test utf-6.79.0 {Tcl_UtfNext} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0G\xF8]
 } 1
-test utf-6.79.1 {Tcl_UtfNext} {testutfnext testbytestring ucs4} {
+test utf-6.79.1 {Tcl_UtfNext} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0G\xF8]
 } 4
 test utf-6.80 {Tcl_UtfNext - overlong sequences} testutfnext {
@@ -485,7 +473,7 @@ test utf-6.86 {Tcl_UtfNext - overlong sequences} {testutfnext testbytestring} {
 test utf-6.87.0 {Tcl_UtfNext - overlong sequences} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF0\x90\x80\x80]
 } 1
-test utf-6.87.1 {Tcl_UtfNext - overlong sequences} {testutfnext testbytestring ucs4} {
+test utf-6.87.1 {Tcl_UtfNext - overlong sequences} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF0\x90\x80\x80]
 } 4
 test utf-6.88 {Tcl_UtfNext, pointing to 2th byte of 3-byte valid sequence} {testutfnext testbytestring} {
@@ -497,22 +485,22 @@ test utf-6.89 {Tcl_UtfNext, pointing to 2th byte of 3-byte invalid sequence} {te
 test utf-6.90.0 {Tcl_UtfNext, validity check [493dccc2de]} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF4\x8F\xBF\xBF]
 } 1
-test utf-6.90.1 {Tcl_UtfNext, validity check [493dccc2de]} {testutfnext testbytestring ucs4} {
+test utf-6.90.1 {Tcl_UtfNext, validity check [493dccc2de]} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF4\x8F\xBF\xBF]
 } 4
 test utf-6.91 {Tcl_UtfNext, validity check [493dccc2de]} {testutfnext testbytestring} {
     testutfnext [testbytestring \xF4\x90\x80\x80]
 } 1
-test utf-6.92 {Tcl_UtfNext, pointing to 2th byte of 4-byte valid sequence} {testutfnext testbytestring ucs2} {
+test utf-6.92 {Tcl_UtfNext, pointing to 2th byte of 4-byte valid sequence} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0]
 } 1
-test utf-6.93 {Tcl_UtfNext, pointing to 2th byte of 4-byte invalid sequence} {testutfnext testbytestring ucs2} {
+test utf-6.93 {Tcl_UtfNext, pointing to 2th byte of 4-byte invalid sequence} {testutfnext testbytestring} {
     testutfnext [testbytestring \x80\x80\x80]
 } 1
-test utf-6.94 {Tcl_UtfNext, pointing to 2th byte of 5-byte invalid sequence} {testutfnext testbytestring ucs2} {
+test utf-6.94 {Tcl_UtfNext, pointing to 2th byte of 5-byte invalid sequence} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0\xA0]
 } 1
-test utf-6.95 {Tcl_UtfNext, pointing to 2th byte of 5-byte invalid sequence} {testutfnext testbytestring ucs2} {
+test utf-6.95 {Tcl_UtfNext, pointing to 2th byte of 5-byte invalid sequence} {testutfnext testbytestring} {
     testutfnext [testbytestring \x80\x80\x80\x80]
 } 1
 test utf-6.96 {Tcl_UtfNext, read limits} testutfnext {
@@ -557,79 +545,73 @@ test utf-6.108 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
 test utf-6.109 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext \u8820[testbytestring \xA0] 3
 } 3
-test utf-6.110.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
-    testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 1
-} 1
-test utf-6.110.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.110 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 1
 } 0
-test utf-6.111.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
-    testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 2
-} 1
-test utf-6.111.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.111 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 2
 } 0
 test utf-6.112.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 3
 } 1
-test utf-6.112.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.112.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring utf16} {
+    testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 3
+} 4
+test utf-6.112.3 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 3
 } 0
 test utf-6.113.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 4
 } 1
-test utf-6.113.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.113.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0]G 4
 } 4
-test utf-6.114.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
-    testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 1
-} 1
-test utf-6.114.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.114 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 1
 } 0
-test utf-6.115.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
-    testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 2
-} 1
-test utf-6.115.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.115 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 2
 } 0
 test utf-6.116.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 3
 } 1
-test utf-6.116.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.116.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring utf16} {
+    testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 3
+} 4
+test utf-6.116.2 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 3
 } 0
 test utf-6.117.0 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 4
 } 1
-test utf-6.117.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs4} {
+test utf-6.117.1 {Tcl_UtfNext, read limits} {testutfnext testbytestring fullutf} {
     testutfnext [testbytestring \xF2\xA0\xA0\xA0\xA0] 4
 } 4
 test utf-6.118 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0]G 0
 } 0
-test utf-6.119 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+test utf-6.119 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0]G 1
-} 1
-test utf-6.120 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+} 0
+test utf-6.120 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0] 1
-} 1
-test utf-6.121 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+} 0
+test utf-6.121 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0]G 2
-} 1
-test utf-6.122 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+} 0
+test utf-6.122 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0] 2
-} 1
-test utf-6.123 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+} 0
+test utf-6.123 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0]G 3
 } 1
-test utf-6.124 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+test utf-6.124 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0\xA0] 3
 } 1
-test utf-6.125 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+test utf-6.125 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0\xA0]G 4
 } 1
-test utf-6.126 {Tcl_UtfNext, read limits} {testutfnext testbytestring ucs2} {
+test utf-6.126 {Tcl_UtfNext, read limits} {testutfnext testbytestring} {
     testutfnext [testbytestring \xA0\xA0\xA0\xA0\xA0] 4
 } 1
 
@@ -699,19 +681,19 @@ test utf-7.9.2 {Tcl_UtfPrev} {testutfprev testbytestring} {
 test utf-7.10.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0]
 } 2
-test utf-7.10.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.10.1 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0]
 } 1
 test utf-7.10.2 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xA0] 3
 } 2
-test utf-7.10.3 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.10.3 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xA0] 3
 } 1
 test utf-7.10.4 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0\xF8\xA0] 3
 } 2
-test utf-7.10.5 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.10.5 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0\xF8\xA0] 3
 } 1
 test utf-7.11 {Tcl_UtfPrev} {testutfprev testbytestring} {
@@ -756,19 +738,19 @@ test utf-7.14.2 {Tcl_UtfPrev} {testutfprev testbytestring} {
 test utf-7.15.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0\xA0]
 } 3
-test utf-7.15.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.15.1 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0\xA0]
 } 1
 test utf-7.15.2 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xA0] 4
 } 3
-test utf-7.15.3 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.15.3 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xA0] 4
 } 1
 test utf-7.15.4 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xF8] 4
 } 3
-test utf-7.15.5 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.15.5 {Tcl_UtfPrev} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xF8] 4
 } 1
 test utf-7.16 {Tcl_UtfPrev} testutfprev {
@@ -789,52 +771,28 @@ test utf-7.17.1 {Tcl_UtfPrev} {testutfprev testbytestring} {
 test utf-7.17.2 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xD0\xA0\xA0\xF8] 4
 } 3
-test utf-7.18.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xA0\xA0\xA0]
-} 1
-test utf-7.18.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.18.0 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xA0\xA0\xA0]
 } 3
-test utf-7.18.2 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xA0\xA0\xA0\xA0] 4
-} 1
-test utf-7.18.3 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.18.1 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xA0\xA0\xA0\xA0] 4
 } 3
-test utf-7.18.4 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xA0\xA0\xA0\xF8] 4
-} 1
-test utf-7.18.5 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.18.2 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xA0\xA0\xA0\xF8] 4
 } 3
-test utf-7.19.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xF8\xA0\xA0\xA0]
-} 2
-test utf-7.19.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.19 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF8\xA0\xA0\xA0]
 } 4
-test utf-7.20.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xF2\xA0\xA0\xA0]
-} 2
-test utf-7.20.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.20 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF2\xA0\xA0\xA0]
-} 1
-test utf-7.21.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A\u8820[testbytestring \xA0]
-} 2
-test utf-7.21.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+} 4
+test utf-7.21 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A\u8820[testbytestring \xA0]
 } 4
-test utf-7.22.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xD0\xA0\xA0\xA0]
-} 2
-test utf-7.22.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.22 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xD0\xA0\xA0\xA0]
 } 4
-test utf-7.23.0 {Tcl_UtfPrev} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xA0\xA0\xA0\xA0]
-} 2
-test utf-7.23.1 {Tcl_UtfPrev} {testutfprev testbytestring ucs4} {
+test utf-7.23 {Tcl_UtfPrev} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xA0\xA0\xA0\xA0]
 } 4
 test utf-7.24 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
@@ -858,10 +816,7 @@ test utf-7.28 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
 test utf-7.28.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
     testutfprev A[testbytestring \xE0\x80\x80] 2
 } 1
-test utf-7.29.0 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xF0\x80\x80\x80]
-} 2
-test utf-7.29.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs4} {
+test utf-7.29 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF0\x80\x80\x80]
 } 4
 test utf-7.30 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
@@ -891,22 +846,19 @@ test utf-7.37 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
 test utf-7.38 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
     testutfprev A[testbytestring \xE0\xA0\x80] 2
 } 1
-test utf-7.39.0 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs2} {
+test utf-7.39 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF0\x90\x80\x80]
-} 2
-test utf-7.39.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs4} {
-    testutfprev A[testbytestring \xF0\x90\x80\x80]
-} 1
+} 4
 test utf-7.40.0 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF0\x90\x80\x80] 4
 } 3
-test utf-7.40.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs4} {
+test utf-7.40.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF0\x90\x80\x80] 4
 } 1
 test utf-7.41.0 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF0\x90\x80\x80] 3
 } 2
-test utf-7.41.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring ucs4} {
+test utf-7.41.1 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF0\x90\x80\x80] 3
 } 1
 test utf-7.42 {Tcl_UtfPrev -- overlong sequence}  {testutfprev testbytestring} {
@@ -921,10 +873,7 @@ test utf-7.44 {Tcl_UtfPrev -- no lead byte at start}  {testutfprev testbytestrin
 test utf-7.45 {Tcl_UtfPrev -- no lead byte at start}  {testutfprev testbytestring} {
     testutfprev [testbytestring \xA0\xA0\xA0]
 } 2
-test utf-7.46.0 {Tcl_UtfPrev -- no lead byte at start}  {testutfprev testbytestring ucs2} {
-    testutfprev [testbytestring \xA0\xA0\xA0\xA0]
-} 1
-test utf-7.46.1 {Tcl_UtfPrev -- no lead byte at start}  {testutfprev testbytestring ucs4} {
+test utf-7.46 {Tcl_UtfPrev -- no lead byte at start}  {testutfprev testbytestring} {
     testutfprev [testbytestring \xA0\xA0\xA0\xA0]
 } 3
 test utf-7.47 {Tcl_UtfPrev, pointing to 3th byte of 3-byte valid sequence} {testutfprev testbytestring} {
@@ -936,40 +885,34 @@ test utf-7.47.1 {Tcl_UtfPrev, pointing to 3th byte of 3-byte valid sequence} tes
 test utf-7.47.2 {Tcl_UtfPrev, pointing to 3th byte of 3-byte invalid sequence} {testutfprev testbytestring} {
     testutfprev [testbytestring \xE8\xA0\x00] 2
 } 0
-test utf-7.48.0 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xF4\x8F\xBF\xBF]
-} 2
-test utf-7.48.1 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs4} {
+test utf-7.48.0 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF]
-} 1
-test utf-7.48.2 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
+} 4
+test utf-7.48.1 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF] 4
 } 3
-test utf-7.48.3 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs4} {
+test utf-7.48.2 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF] 4
 } 1
-test utf-7.48.4 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
+test utf-7.48.3 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF] 3
 } 2
-test utf-7.48.5 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs4} {
+test utf-7.48.4 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring fullutf} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF] 3
 } 1
-test utf-7.48.6 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
+test utf-7.48.5 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x8F\xBF\xBF] 2
 } 1
-test utf-7.49.0 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs2} {
-    testutfprev A[testbytestring \xF4\x90\x80\x80]
-} 2
-test utf-7.49.1 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring ucs4} {
+test utf-7.49.0 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x90\x80\x80]
 } 4
-test utf-7.49.2 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
+test utf-7.49.1 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x90\x80\x80] 4
 } 3
-test utf-7.49.3 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
+test utf-7.49.2 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x90\x80\x80] 3
 } 2
-test utf-7.49.4 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
+test utf-7.49.3 {Tcl_UtfPrev, validity check [493dccc2de]} {testutfprev testbytestring} {
     testutfprev A[testbytestring \xF4\x90\x80\x80] 2
 } 1
 
diff --git a/win/tclWinSerial.c b/win/tclWinSerial.c
index be7ec2c..aeebb56 100644
--- a/win/tclWinSerial.c
+++ b/win/tclWinSerial.c
@@ -1752,16 +1752,16 @@ SerialSetOptionProc(
 	dcb.XonChar = argv[0][0];
 	dcb.XoffChar = argv[1][0];
 	if (argv[0][0] & 0x80 || argv[1][0] & 0x80) {
-	    Tcl_UniChar character;
+	    int character;
 	    int charLen;
 
-	    charLen = Tcl_UtfToUniChar(argv[0], &character);
-	    if (argv[0][charLen]) {
+	    charLen = TclUtfToUCS4(argv[0], &character);
+	    if ((character & ~0xFF) || argv[0][charLen]) {
 		goto badXchar;
 	    }
 	    dcb.XonChar = (char) character;
-	    charLen = Tcl_UtfToUniChar(argv[1], &character);
-	    if (argv[1][charLen]) {
+	    charLen = TclUtfToUCS4(argv[1], &character);
+	    if ((character & ~0xFF) || argv[1][charLen]) {
 		goto badXchar;
 	    }
 	    dcb.XoffChar = (char) character;