Complete fix for [53fdb87e49]: "Glyph indexing is still unperfect". Tk 8.7 now uses UTF-32 indices everywhere, just as Tk 9.0.

17 files changed, 191 insertions, 128 deletions

diff --git a/generic/tkCanvText.c b/generic/tkCanvText.c
index 8e85946..3f02158 100644
--- a/generic/tkCanvText.c
+++ b/generic/tkCanvText.c
@@ -594,7 +594,7 @@ ConfigureText(
      */
 
     textPtr->numBytes = strlen(textPtr->text);
-    textPtr->numChars = Tcl_NumUtfChars(textPtr->text, textPtr->numBytes);
+    textPtr->numChars = TkNumUtfChars(textPtr->text, textPtr->numBytes);
     if (textInfoPtr->selItemPtr == itemPtr) {
 
 	if (textInfoPtr->selectFirst >= textPtr->numChars) {
@@ -1119,7 +1119,7 @@ TextInsert(
 
     ckfree(text);
     textPtr->text = newStr;
-    charsAdded = Tcl_NumUtfChars(string, byteCount);
+    charsAdded = TkNumUtfChars(string, byteCount);
     textPtr->numChars += charsAdded;
     textPtr->numBytes += byteCount;
 
@@ -1189,8 +1189,8 @@ TextDeleteChars(
     }
     charsRemoved = last + 1 - first;
 
-    byteIndex = Tcl_UtfAtIndex(text, first) - text;
-    byteCount = Tcl_UtfAtIndex(text + byteIndex, charsRemoved)
+    byteIndex = TkUtfAtIndex(text, first) - text;
+    byteCount = TkUtfAtIndex(text + byteIndex, charsRemoved)
 	- (text + byteIndex);
 
     newStr = (char *)ckalloc(textPtr->numBytes + 1 - byteCount);
@@ -1612,8 +1612,8 @@ GetSelText(
 	return 0;
     }
     text = textPtr->text;
-    selStart = Tcl_UtfAtIndex(text, textInfoPtr->selectFirst);
-    selEnd = Tcl_UtfAtIndex(selStart,
+    selStart = TkUtfAtIndex(text, textInfoPtr->selectFirst);
+    selEnd = TkUtfAtIndex(selStart,
 	    textInfoPtr->selectLast + 1 - textInfoPtr->selectFirst);
     if (selEnd  <= selStart + offset) {
 	return 0;
diff --git a/generic/tkEntry.c b/generic/tkEntry.c
index dfb2ce2..404836c 100644
--- a/generic/tkEntry.c
+++ b/generic/tkEntry.c
@@ -2164,7 +2164,7 @@ InsertChars(
     char *newStr;
 
     string = entryPtr->string;
-    byteIndex = Tcl_UtfAtIndex(string, index) - string;
+    byteIndex = TkUtfAtIndex(string, index) - string;
     byteCount = strlen(value);
     if (byteCount == 0) {
 	return TCL_OK;
@@ -2197,7 +2197,7 @@ InsertChars(
      */
 
     oldChars = entryPtr->numChars;
-    entryPtr->numChars = Tcl_NumUtfChars(newStr, TCL_INDEX_NONE);
+    entryPtr->numChars = TkNumUtfChars(newStr, TCL_INDEX_NONE);
     charsAdded = entryPtr->numChars - oldChars;
     entryPtr->numBytes += byteCount;
 
@@ -2268,8 +2268,8 @@ DeleteChars(
     }
 
     string = entryPtr->string;
-    byteIndex = Tcl_UtfAtIndex(string, index) - string;
-    byteCount = Tcl_UtfAtIndex(string + byteIndex, count) - (string+byteIndex);
+    byteIndex = TkUtfAtIndex(string, index) - string;
+    byteCount = TkUtfAtIndex(string + byteIndex, count) - (string+byteIndex);
 
     newByteCount = entryPtr->numBytes + 1 - byteCount;
     newStr = (char *)ckalloc(newByteCount);
@@ -2497,7 +2497,7 @@ EntrySetValue(
 	entryPtr->string = tmp;
     }
     entryPtr->numBytes = valueLen;
-    entryPtr->numChars = Tcl_NumUtfChars(value, valueLen);
+    entryPtr->numChars = TkNumUtfChars(value, valueLen);
 
     if (entryPtr->displayString == oldSource) {
 	entryPtr->displayString = entryPtr->string;
@@ -2927,8 +2927,8 @@ EntryFetchSelection(
 	return -1;
     }
     string = entryPtr->displayString;
-    selStart = Tcl_UtfAtIndex(string, entryPtr->selectFirst);
-    selEnd = Tcl_UtfAtIndex(selStart,
+    selStart = TkUtfAtIndex(string, entryPtr->selectFirst);
+    selEnd = TkUtfAtIndex(selStart,
 	    entryPtr->selectLast - entryPtr->selectFirst);
     if (selEnd <= selStart + offset) {
 	return 0;
diff --git a/generic/tkFont.c b/generic/tkFont.c
index 0f77a91..840d9b6 100644
--- a/generic/tkFont.c
+++ b/generic/tkFont.c
@@ -2013,7 +2013,7 @@ Tk_ComputeTextLayout(
     height = fmPtr->ascent + fmPtr->descent;
 
     if (numChars < 0) {
-	numChars = Tcl_NumUtfChars(string, TCL_INDEX_NONE);
+	numChars = TkNumUtfChars(string, TCL_INDEX_NONE);
     }
     if (wrapLength == 0) {
 	wrapLength = -1;
@@ -2036,7 +2036,7 @@ Tk_ComputeTextLayout(
 
     curX = 0;
 
-    endp = Tcl_UtfAtIndex(string, numChars);
+    endp = TkUtfAtIndex(string, numChars);
     special = string;
 
     flags &= TK_IGNORE_TABS | TK_IGNORE_NEWLINES;
@@ -2153,7 +2153,7 @@ Tk_ComputeTextLayout(
 		bytesThisChunk = Tk_MeasureChars(tkfont, end, bytesThisChunk,
 			-1, 0, &chunkPtr->totalWidth);
 		chunkPtr->numBytes += bytesThisChunk;
-		chunkPtr->numChars += Tcl_NumUtfChars(end, bytesThisChunk);
+		chunkPtr->numChars += TkNumUtfChars(end, bytesThisChunk);
 		chunkPtr->totalWidth += curX;
 	    }
 	}
@@ -2345,14 +2345,14 @@ Tk_DrawTextLayout(
 		firstChar = 0;
 		firstByte = chunkPtr->start;
 	    } else {
-		firstByte = Tcl_UtfAtIndex(chunkPtr->start, firstChar);
+		firstByte = TkUtfAtIndex(chunkPtr->start, firstChar);
 		Tk_MeasureChars(layoutPtr->tkfont, chunkPtr->start,
 			firstByte - chunkPtr->start, -1, 0, &drawX);
 	    }
 	    if (lastChar < numDisplayChars) {
 		numDisplayChars = lastChar;
 	    }
-	    lastByte = Tcl_UtfAtIndex(chunkPtr->start, numDisplayChars);
+	    lastByte = TkUtfAtIndex(chunkPtr->start, numDisplayChars);
 #ifdef TK_DRAW_IN_CONTEXT
 	    TkpDrawCharsInContext(display, drawable, gc, layoutPtr->tkfont,
 		    chunkPtr->start, chunkPtr->numBytes,
@@ -2415,14 +2415,14 @@ TkDrawAngledTextLayout(
 		firstChar = 0;
 		firstByte = chunkPtr->start;
 	    } else {
-		firstByte = Tcl_UtfAtIndex(chunkPtr->start, firstChar);
+		firstByte = TkUtfAtIndex(chunkPtr->start, firstChar);
 		Tk_MeasureChars(layoutPtr->tkfont, chunkPtr->start,
 			firstByte - chunkPtr->start, -1, 0, &drawX);
 	    }
 	    if (lastChar < numDisplayChars) {
 		numDisplayChars = lastChar;
 	    }
-	    lastByte = Tcl_UtfAtIndex(chunkPtr->start, numDisplayChars);
+	    lastByte = TkUtfAtIndex(chunkPtr->start, numDisplayChars);
 #ifdef TK_DRAW_IN_CONTEXT
 	    dx = cosA * (chunkPtr->x) + sinA * (chunkPtr->y);
 	    dy = -sinA * (chunkPtr->x) + cosA * (chunkPtr->y);
@@ -2675,7 +2675,7 @@ Tk_PointToChar(
 		    }
 		    n = Tk_MeasureChars((Tk_Font) fontPtr, chunkPtr->start,
 			    chunkPtr->numBytes, x - chunkPtr->x, 0, &dummy);
-		    return numChars + Tcl_NumUtfChars(chunkPtr->start, n);
+		    return numChars + TkNumUtfChars(chunkPtr->start, n);
 		}
 		numChars += chunkPtr->numChars;
 		lastPtr = chunkPtr;
@@ -2784,7 +2784,7 @@ Tk_CharBbox(
 		goto check;
 	    }
 	} else if (index < chunkPtr->numChars) {
-	    end = Tcl_UtfAtIndex(chunkPtr->start, index);
+	    end = TkUtfAtIndex(chunkPtr->start, index);
 	    if (xPtr != NULL) {
 		Tk_MeasureChars(tkfont, chunkPtr->start,
 			end - chunkPtr->start, -1, 0, &x);
@@ -3859,7 +3859,7 @@ NewChunk(
 	*layoutPtrPtr = layoutPtr;
 	*maxPtr = maxChunks;
     }
-    numChars = Tcl_NumUtfChars(start, numBytes);
+    numChars = TkNumUtfChars(start, numBytes);
     chunkPtr = &layoutPtr->chunks[layoutPtr->numChunks];
     chunkPtr->start		= start;
     chunkPtr->numBytes		= numBytes;
diff --git a/generic/tkIcu.c b/generic/tkIcu.c
index 9696868..257003c 100644
--- a/generic/tkIcu.c
+++ b/generic/tkIcu.c
@@ -92,7 +92,8 @@ startEndOfCmd(
     str = Tcl_GetStringFromObj(objv[1], &len);
     Tcl_UtfToChar16DString(str, len, &ds);
     len = Tcl_DStringLength(&ds)/2;
-    if (TkGetIntForIndex(objv[2], len-1, 0, &idx) != TCL_OK) {
+    Tcl_Size ulen = TkGetCharLength(objv[1]);
+    if (TkGetIntForIndex(objv[2], ulen-1, 0, &idx) != TCL_OK) {
 	Tcl_DStringFree(&ds);
 	Tcl_SetObjResult(interp, Tcl_ObjPrintf("bad index \"%s\": must be integer?[+-]integer?, end?[+-]integer?, or \"\"", Tcl_GetString(objv[2])));
 	Tcl_SetErrorCode(interp, "TK", "ICU", "INDEX", NULL);
@@ -111,6 +112,14 @@ startEndOfCmd(
     	Tcl_SetErrorCode(interp, "TK", "ICU", "CANNOTOPEN", NULL);
     	return TCL_ERROR;
     }
+    if (idx > 0 && len != ulen) {
+	/* The string contains codepoints > \uFFFF. Determine UTF-16 index */
+	Tcl_Size newIdx = 0;
+	for (Tcl_Size i = 0; i < idx; i++) {
+	    newIdx += 1 + (((newIdx < (Tcl_Size)len-1) && (ustr[newIdx]&0xFC00) == 0xD800) && ((ustr[newIdx+1]&0xFC00) == 0xDC00));
+	}
+	idx = newIdx;
+    }
     if (flags & FLAG_FOLLOWING) {
 	if ((idx < 0) && (flags & FLAG_WORD)) {
 	    idx = 0;
@@ -145,6 +154,14 @@ startEndOfCmd(
     icu_close(it);
     Tcl_DStringFree(&ds);
     if (idx != TCL_INDEX_NONE) {
+	if (idx > 0 && len != ulen) {
+	    /* The string contains codepoints > \uFFFF. Determine UTF-32 index */
+	    Tcl_Size newIdx = 1;
+	    for (Tcl_Size i = 1; i < idx; i++) {
+    	if (((ustr[i-1]&0xFC00) != 0xD800) || ((ustr[i]&0xFC00) != 0xDC00)) newIdx++;
+	    }
+	    idx = newIdx;
+	}
 	Tcl_SetObjResult(interp, TkNewIndexObj(idx));
     }
     return TCL_OK;
diff --git a/generic/tkInt.h b/generic/tkInt.h
index 2b89d2a..cfc311a 100644
--- a/generic/tkInt.h
+++ b/generic/tkInt.h
@@ -76,13 +76,40 @@
 #endif
 
 #if (TCL_MAJOR_VERSION == 8) && (TCL_MINOR_VERSION < 7)
-# define Tcl_WCharToUtfDString ((char * (*)(const WCHAR *, int len, Tcl_DString *))Tcl_UniCharToUtfDString)
-# define Tcl_UtfToWCharDString ((WCHAR * (*)(const char *, int len, Tcl_DString *))Tcl_UtfToUniCharDString)
+# define Tcl_WCharToUtfDString ((char * (*)(const WCHAR *, int, Tcl_DString *))Tcl_UniCharToUtfDString)
+# define Tcl_UtfToWCharDString ((WCHAR * (*)(const char *, int, Tcl_DString *))Tcl_UtfToUniCharDString)
 # define Tcl_Char16ToUtfDString Tcl_UniCharToUtfDString
 # define Tcl_UtfToChar16DString Tcl_UtfToUniCharDString
 # define TCL_COMBINE		0
 #endif
 
+/* Make available UTF-32 versions of the API, even though we compile with TCL_UTF_MAX=3 */
+#if TCL_MAJOR_VERSION > 8
+#   define TkUtfToUniChar (tclStubsPtr->tcl_UtfToUniChar) /* 646 */
+#   define TkUniCharToUtf (tclStubsPtr->tcl_UniCharToUtf) /* 324 (without TCL_COMBINE) */
+#   define TkNumUtfChars (tclStubsPtr->tcl_NumUtfChars) /* 669 */
+#   define TkGetCharLength (tclStubsPtr->tcl_GetCharLength) /* 670 */
+#   define TkUtfAtIndex (tclStubsPtr->tcl_UtfAtIndex) /* 671 */
+#else
+    MODULE_SCOPE Tcl_Size TkUtfToUniChar(const char *, int *);
+    MODULE_SCOPE Tcl_Size TkUniCharToUtf(int, char *);
+#   ifdef USE_TCL_STUBS
+#	define TkNumUtfChars (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((Tcl_Size (*)(const char *, Tcl_Size))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[669])) \
+		: (tclStubsPtr->tcl_NumUtfChars) /* 312 */)
+#	define TkGetCharLength (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((Tcl_Size (*)(Tcl_Obj *))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[670])) \
+		: (tclStubsPtr->tcl_GetCharLength) /* 380 */)
+#	define TkUtfAtIndex (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((const char *(*)(const char *, Tcl_Size))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[671])) \
+		: (tclStubsPtr->tcl_UtfAtIndex) /* 325 */)
+#   else
+#	define TkNumUtfChars TclNumUtfChars
+#	define TkGetCharLength TclGetCharLength
+#	define TkUtfAtIndex TclUtfAtIndex
+#   endif
+#endif
+
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
 #   define TKFLEXARRAY
 #elif defined(__GNUC__) && (__GNUC__ > 2)
@@ -1377,15 +1404,6 @@ MODULE_SCOPE void	TkpCopyRegion(TkRegion dst, TkRegion src);
 # define c_class class
 #endif
 
-/* Tcl 8.6 has a different definition of Tcl_UniChar than other Tcl versions for TCL_UTF_MAX > 3 */
-#if TCL_UTF_MAX > (3 + (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION == 6))
-#   define TkUtfToUniChar(src, ch) (size_t)(((int (*)(const char *, int *))Tcl_UtfToUniChar)(src, ch))
-#   define TkUniCharToUtf(ch, src) (size_t)(((int (*)(int, char *))Tcl_UniCharToUtf)(ch, src))
-#else
-    MODULE_SCOPE size_t TkUtfToUniChar(const char *, int *);
-    MODULE_SCOPE size_t TkUniCharToUtf(int, char *);
-#endif
-
 #if defined(_WIN32) && !defined(STATIC_BUILD) && TCL_MAJOR_VERSION < 9
 #   define tcl_CreateFileHandler reserved9
 #endif
diff --git a/generic/tkMessage.c b/generic/tkMessage.c
index 0f70b41..fd36e62 100644
--- a/generic/tkMessage.c
+++ b/generic/tkMessage.c
@@ -501,7 +501,7 @@ ConfigureMessage(
      * be specified to Tk_ConfigureWidget.
      */
 
-    msgPtr->numChars = Tcl_NumUtfChars(msgPtr->string, TCL_INDEX_NONE);
+    msgPtr->numChars = TkNumUtfChars(msgPtr->string, TCL_INDEX_NONE);
 
     if (msgPtr->highlightWidth < 0) {
 	msgPtr->highlightWidth = 0;
@@ -902,7 +902,7 @@ MessageTextVarProc(
     if (msgPtr->string != NULL) {
 	ckfree(msgPtr->string);
     }
-    msgPtr->numChars = Tcl_NumUtfChars(value, TCL_INDEX_NONE);
+    msgPtr->numChars = TkNumUtfChars(value, TCL_INDEX_NONE);
     msgPtr->string = (char *)ckalloc(strlen(value) + 1);
     strcpy(msgPtr->string, value);
     ComputeMessageGeometry(msgPtr);
diff --git a/generic/tkSelect.c b/generic/tkSelect.c
index ef15a7e..9a36b5b 100644
--- a/generic/tkSelect.c
+++ b/generic/tkSelect.c
@@ -1401,15 +1401,15 @@ HandleTclCommand(
 
 	if (cmdInfoPtr->interp != NULL) {
 	    if (length <= maxBytes) {
-		cmdInfoPtr->charOffset += Tcl_NumUtfChars(string, TCL_INDEX_NONE);
+		cmdInfoPtr->charOffset += TkNumUtfChars(string, TCL_INDEX_NONE);
 		cmdInfoPtr->buffer[0] = '\0';
 	    } else {
-		Tcl_UniChar ch = 0;
+		int ch = 0;
 		p = string;
 		string += count;
 		numChars = 0;
 		while (p < string) {
-		    p += Tcl_UtfToUniChar(p, &ch);
+		    p += TkUtfToUniChar(p, &ch);
 		    numChars++;
 		}
 		cmdInfoPtr->charOffset += numChars;
diff --git a/generic/tkText.c b/generic/tkText.c
index 8e6c9c1..b186e18 100644
--- a/generic/tkText.c
+++ b/generic/tkText.c
@@ -1461,7 +1461,7 @@ TextWidgetObjCmd(
 
 		insertLength = 0;
 		for (j = 4; j < objc; j += 2) {
-		    insertLength += Tcl_GetCharLength(objv[j]);
+		    insertLength += TkGetCharLength(objv[j]);
 		}
 
 		/*
@@ -4092,12 +4092,12 @@ TextSearchIndexInLine(
 		if (searchSpecPtr->exact) {
 		    index += leftToScan;
 		} else {
-		    index += Tcl_NumUtfChars(segPtr->body.chars, leftToScan);
+		    index += TkNumUtfChars(segPtr->body.chars, leftToScan);
 		}
 	    } else if (searchSpecPtr->exact) {
 		index += segPtr->size;
 	    } else {
-		index += Tcl_NumUtfChars(segPtr->body.chars, -1);
+		index += TkNumUtfChars(segPtr->body.chars, -1);
 	    }
 	}
 	leftToScan -= segPtr->size;
@@ -4222,7 +4222,7 @@ TextSearchAddNextLine(
 	    Tcl_GetString(theLine);
 	    *lenPtr = theLine->length;
 	} else {
-	    *lenPtr = Tcl_GetCharLength(theLine);
+	    *lenPtr = TkGetCharLength(theLine);
 	}
     }
     return linePtr;
@@ -4292,7 +4292,7 @@ TextSearchFoundMatch(
     if (searchSpecPtr->exact) {
 	const char *startOfLine = Tcl_GetString(theLine);
 
-	numChars = Tcl_NumUtfChars(startOfLine + matchOffset, matchLength);
+	numChars = TkNumUtfChars(startOfLine + matchOffset, matchLength);
     } else {
 	numChars = matchLength;
     }
@@ -4351,13 +4351,13 @@ TextSearchFoundMatch(
 		if (searchSpecPtr->exact) {
 		    matchOffset += segPtr->size;
 		} else {
-		    matchOffset += Tcl_NumUtfChars(segPtr->body.chars, -1);
+		    matchOffset += TkNumUtfChars(segPtr->body.chars, -1);
 		}
 	    } else {
 		if (searchSpecPtr->exact) {
 		    leftToScan -= (int)segPtr->size;
 		} else {
-		    leftToScan -= Tcl_NumUtfChars(segPtr->body.chars, -1);
+		    leftToScan -= TkNumUtfChars(segPtr->body.chars, -1);
 		}
 	    }
 	    curIndex.byteIndex += segPtr->size;
@@ -4442,13 +4442,13 @@ TextSearchFoundMatch(
 	    continue;
 	} else if (!searchSpecPtr->searchElide
 		&& TkTextIsElided(textPtr, &curIndex, NULL)) {
-	    numChars += Tcl_NumUtfChars(segPtr->body.chars, -1);
+	    numChars += TkNumUtfChars(segPtr->body.chars, -1);
 	    continue;
 	}
 	if (searchSpecPtr->exact) {
 	    leftToScan -= segPtr->size;
 	} else {
-	    leftToScan -= Tcl_NumUtfChars(segPtr->body.chars, -1);
+	    leftToScan -= TkNumUtfChars(segPtr->body.chars, -1);
 	}
     }
 
diff --git a/generic/tkTextIndex.c b/generic/tkTextIndex.c
index 31a95a7..c62a198 100644
--- a/generic/tkTextIndex.c
+++ b/generic/tkTextIndex.c
@@ -494,7 +494,7 @@ TkTextMakeCharIndex(
     TkTextSegment *segPtr;
     char *p, *start, *end;
     int index, offset;
-    Tcl_UniChar ch = 0;
+    int ch = 0;
 
     indexPtr->tree = tree;
     if (lineIndex < 0) {
@@ -541,7 +541,7 @@ TkTextMakeCharIndex(
 		    return indexPtr;
 		}
 		charIndex--;
-		offset = Tcl_UtfToUniChar(p, &ch);
+		offset = TkUtfToUniChar(p, &ch);
 		index += offset;
 	    }
 	} else {
@@ -1129,7 +1129,7 @@ TkTextPrintIndex(
 	    break;
 	}
 	if (segPtr->typePtr == &tkTextCharType) {
-	    charIndex += Tcl_NumUtfChars(segPtr->body.chars, segPtr->size);
+	    charIndex += TkNumUtfChars(segPtr->body.chars, segPtr->size);
 	} else {
 	    charIndex += segPtr->size;
 	}
@@ -1137,7 +1137,7 @@ TkTextPrintIndex(
     }
 
     if (segPtr->typePtr == &tkTextCharType) {
-	charIndex += Tcl_NumUtfChars(segPtr->body.chars, numBytes);
+	charIndex += TkNumUtfChars(segPtr->body.chars, numBytes);
     } else {
 	charIndex += numBytes;
     }
@@ -1948,7 +1948,7 @@ TkTextIndexCount(
 		}
 		count += byteLen - i;
 		if (i) {
-		    count += Tcl_NumUtfChars(segPtr->body.chars + byteOffset
+		    count += TkNumUtfChars(segPtr->body.chars + byteOffset
 			    + (byteLen - i), i);
 		}
 	    } else {
@@ -2210,9 +2210,6 @@ TkTextIndexBackChars(
 		    if (p == start) {
 			break;
 		    }
-		    if ((sizeof(Tcl_UniChar) == 2) &&  (unsigned)(UCHAR(*p) - 0xF0) <= 5) {
-			charCount--; /* Characters > U+FFFF count as 2 here */
-		    }
 		    if (charCount != 0) {
 			charCount--;
 		    }
diff --git a/generic/tkUtil.c b/generic/tkUtil.c
index 13b91d3..ef65880 100644
--- a/generic/tkUtil.c
+++ b/generic/tkUtil.c
@@ -1246,8 +1246,7 @@ Tk_SendVirtualEvent(
     Tk_QueueWindowEvent(&event.general, TCL_QUEUE_TAIL);
 }
 
-/* Tcl 8.6 has a different definition of Tcl_UniChar than other Tcl versions for TCL_UTF_MAX > 3 */
-#if TCL_UTF_MAX <= (3 + (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION == 6))
+#if TCL_MAJOR_VERSION < 9
 /*
  *---------------------------------------------------------------------------
  *
@@ -1267,7 +1266,7 @@ Tk_SendVirtualEvent(
  *---------------------------------------------------------------------------
  */
 
-size_t
+Tcl_Size
 TkUtfToUniChar(
     const char *src,	/* The UTF-8 string. */
     int *chPtr)		/* Filled with the Unicode value represented by
@@ -1275,12 +1274,11 @@ TkUtfToUniChar(
 {
     Tcl_UniChar uniChar = 0;
 
-    size_t len = Tcl_UtfToUniChar(src, &uniChar);
+    Tcl_Size len = Tcl_UtfToUniChar(src, &uniChar);
     if ((uniChar & 0xFC00) == 0xD800) {
 	Tcl_UniChar low = uniChar;
-	/* This can only happen if sizeof(Tcl_UniChar)== 2 and src points
-	 * to a character > U+FFFF  */
-	size_t len2 = Tcl_UtfToUniChar(src+len, &low);
+	/* This can only happen if src points to a character > U+FFFF  */
+	Tcl_Size len2 = Tcl_UtfToUniChar(src+len, &low);
 	if ((low & 0xFC00) == 0xDC00) {
 	    *chPtr = (((uniChar & 0x3FF) << 10) | (low & 0x3FF)) + 0x10000;
 	    return len + len2;
@@ -1309,12 +1307,12 @@ TkUtfToUniChar(
  *---------------------------------------------------------------------------
  */
 
-size_t TkUniCharToUtf(int ch, char *buf)
+Tcl_Size TkUniCharToUtf(int ch, char *buf)
 {
     if ((unsigned)(ch - 0x10000) <= 0xFFFFF) {
-	/* Spit out a 4-byte UTF-8 character or 2 x 3-byte UTF-8 characters, depending on Tcl
-	 * version and/or TCL_UTF_MAX build value */
-	int len = Tcl_UniCharToUtf(0xD800 | ((ch - 0x10000) >> 10), buf);
+	/* Spit out a 4-byte UTF-8 character (Tcl 8.7+) or
+	 * 2 x 3-byte UTF-8 characters (Tcl 8.6) */
+	Tcl_Size len = Tcl_UniCharToUtf(0xD800 | ((ch - 0x10000) >> 10), buf);
 	return len + Tcl_UniCharToUtf(0xDC00 | (ch & 0x7FF), buf + len);
     }
     return Tcl_UniCharToUtf(ch, buf);
diff --git a/generic/tkWindow.c b/generic/tkWindow.c
index 37cda60..53f6936 100644
--- a/generic/tkWindow.c
+++ b/generic/tkWindow.c
@@ -1028,9 +1028,6 @@ TkCreateMainWindow(
 #ifdef STATIC_BUILD
 		".static"
 #endif
-#if TCL_UTF_MAX <= (3 + (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION == 6))
-		".utf-16"
-#endif
 #if defined(_WIN32)
 		".win32"
 #endif
diff --git a/generic/ttk/ttkEntry.c b/generic/ttk/ttkEntry.c
index 756e41c..25add27 100644
--- a/generic/ttk/ttkEntry.c
+++ b/generic/ttk/ttkEntry.c
@@ -369,8 +369,8 @@ EntryFetchSelection(
     }
     string = entryPtr->entry.displayString;
 
-    selStart = Tcl_UtfAtIndex(string, entryPtr->entry.selectFirst);
-    selEnd = Tcl_UtfAtIndex(selStart,
+    selStart = TkUtfAtIndex(string, entryPtr->entry.selectFirst);
+    selEnd = TkUtfAtIndex(selStart,
 	    entryPtr->entry.selectLast - entryPtr->entry.selectFirst);
     if (selEnd  <= selStart + offset) {
 	return 0;
@@ -488,11 +488,11 @@ ExpandPercents(
 		break;
 	    case 'S': /* string to be inserted/deleted, if any */
 		if (reason == VALIDATE_INSERT) {
-		    string = Tcl_UtfAtIndex(newValue, index);
-		    stringLength = Tcl_UtfAtIndex(string, count) - string;
+		    string = TkUtfAtIndex(newValue, index);
+		    stringLength = TkUtfAtIndex(string, count) - string;
 		} else if (reason == VALIDATE_DELETE) {
-		    string = Tcl_UtfAtIndex(entryPtr->entry.string, index);
-		    stringLength = Tcl_UtfAtIndex(string, count) - string;
+		    string = TkUtfAtIndex(entryPtr->entry.string, index);
+		    stringLength = TkUtfAtIndex(string, count) - string;
 		} else {
 		    string = "";
 		    stringLength = 0;
@@ -740,7 +740,7 @@ static void
 EntryStoreValue(Entry *entryPtr, const char *value)
 {
     size_t numBytes = strlen(value);
-    Tcl_Size numChars = Tcl_NumUtfChars(value, numBytes);
+    Tcl_Size numChars = TkNumUtfChars(value, numBytes);
 
     if (entryPtr->core.flags & VALIDATING)
 	entryPtr->core.flags |= VALIDATION_SET_VALUE;
@@ -845,9 +845,9 @@ InsertChars(
     const char *value)		/* New characters to add */
 {
     char *string = entryPtr->entry.string;
-    size_t byteIndex = Tcl_UtfAtIndex(string, index) - string;
+    size_t byteIndex = TkUtfAtIndex(string, index) - string;
     size_t byteCount = strlen(value);
-    int charsAdded = Tcl_NumUtfChars(value, byteCount);
+    int charsAdded = TkNumUtfChars(value, byteCount);
     size_t newByteCount = entryPtr->entry.numBytes + byteCount + 1;
     char *newBytes;
     int code;
@@ -899,8 +899,8 @@ DeleteChars(
 	return TCL_OK;
     }
 
-    byteIndex = Tcl_UtfAtIndex(string, index) - string;
-    byteCount = Tcl_UtfAtIndex(string+byteIndex, count) - (string+byteIndex);
+    byteIndex = TkUtfAtIndex(string, index) - string;
+    byteCount = TkUtfAtIndex(string+byteIndex, count) - (string+byteIndex);
 
     newByteCount = entryPtr->entry.numBytes + 1 - byteCount;
     newBytes =  (char *)ckalloc(newByteCount);
@@ -1314,7 +1314,7 @@ static void EntryDisplay(void *clientData, Drawable d)
     if ((*(entryPtr->entry.displayString) == '\0')
 		&& (entryPtr->entry.placeholderObj != NULL)) {
 	/* No text displayed, but -placeholder is given */
-	if (Tcl_GetCharLength(es.placeholderForegroundObj) > 0) {
+	if (TkGetCharLength(es.placeholderForegroundObj) > 0) {
 	    foregroundObj = es.placeholderForegroundObj;
 	} else {
 	    foregroundObj = es.foregroundObj;
diff --git a/macosx/tkMacOSXFont.c b/macosx/tkMacOSXFont.c
index fb1e403..e39d7c0 100644
--- a/macosx/tkMacOSXFont.c
+++ b/macosx/tkMacOSXFont.c
@@ -451,31 +451,49 @@ startOfClusterObjCmd(
 {
     TKNSString *S;
     const char *stringArg;
-    Tcl_Size numBytes, index;
+    Tcl_Size len, idx;
     if ((unsigned)(objc - 3) > 1) {
 	Tcl_WrongNumArgs(interp, 1 , objv, "str start ?locale?");
 	return TCL_ERROR;
     }
-    stringArg = Tcl_GetStringFromObj(objv[1], &numBytes);
+    stringArg = Tcl_GetStringFromObj(objv[1], &len);
     if (stringArg == NULL) {
 	return TCL_ERROR;
     }
-    S = [[TKNSString alloc] initWithTclUtfBytes:stringArg length:numBytes];
-    if (TkGetIntForIndex(objv[2], [S length] - 1, 0, &index) != TCL_OK) {
+    Tcl_Size ulen = TkGetCharLength(objv[1]);
+    S = [[TKNSString alloc] initWithTclUtfBytes:stringArg length:len];
+    len = [S length];
+    if (TkGetIntForIndex(objv[2], ulen - 1, 0, &idx) != TCL_OK) {
 	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
 		"bad index \"%s\": must be integer?[+-]integer?, end?[+-]integer?, or \"\"",
 		Tcl_GetString(objv[2])));
 	Tcl_SetErrorCode(interp, "TK", "VALUE", "INDEX", NULL);
 	return TCL_ERROR;
     }
-    if (index >= 0) {
-	if ((size_t)index >= [S length]) {
-	    index = (Tcl_Size)[S length];
+    if (idx > 0 && len != ulen) {
+	/* The string contains codepoints > \uFFFF. Determine UTF-16 index */
+	Tcl_Size newIdx = 0;
+	for (Tcl_Size i = 0; i < idx; i++) {
+	    newIdx += 1 + (((newIdx < (Tcl_Size)len-1) && ([S characterAtIndex:newIdx]&0xFC00) == 0xD800) && (([S characterAtIndex:newIdx+1]&0xFC00) == 0xDC00));
+	}
+	idx = newIdx;
+    }
+    if (idx >= 0) {
+	if (idx >= len) {
+	    idx = len;
 	} else {
-	    NSRange range = [S rangeOfComposedCharacterSequenceAtIndex:index];
-	    index = range.location;
+	    NSRange range = [S rangeOfComposedCharacterSequenceAtIndex:idx];
+	    idx = range.location;
+	}
+	if (idx > 0 && len != ulen) {
+	    /* The string contains codepoints > \uFFFF. Determine UTF-32 index */
+	    Tcl_Size newIdx = 1;
+	    for (Tcl_Size i = 1; i < idx; i++) {
+		if ((([S characterAtIndex:i-1]&0xFC00) != 0xD800) || (([S characterAtIndex:i]&0xFC00) != 0xDC00)) newIdx++;
+	    }
+	    idx = newIdx;
 	}
-	Tcl_SetObjResult(interp, TkNewIndexObj(index));
+	Tcl_SetObjResult(interp, TkNewIndexObj(idx));
     }
     return TCL_OK;
 }
@@ -489,32 +507,50 @@ endOfClusterObjCmd(
 {
     TKNSString *S;
     char *stringArg;
-    Tcl_Size index, numBytes;
+    Tcl_Size idx, len;
 
     if ((unsigned)(objc - 3) > 1) {
 	Tcl_WrongNumArgs(interp, 1 , objv, "str start ?locale?");
 	return TCL_ERROR;
     }
-    stringArg = Tcl_GetStringFromObj(objv[1], &numBytes);
+    stringArg = Tcl_GetStringFromObj(objv[1], &len);
     if (stringArg == NULL) {
 	return TCL_ERROR;
     }
-    S = [[TKNSString alloc] initWithTclUtfBytes:stringArg length:numBytes];
-    if (TkGetIntForIndex(objv[2], [S length] - 1, 0, &index) != TCL_OK) {
+    Tcl_Size ulen = TkGetCharLength(objv[1]);
+    S = [[TKNSString alloc] initWithTclUtfBytes:stringArg length:len];
+    len = [S length];
+    if (TkGetIntForIndex(objv[2], ulen - 1, 0, &idx) != TCL_OK) {
 	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
 		"bad index \"%s\": must be integer?[+-]integer?, end?[+-]integer?, or \"\"",
 		Tcl_GetString(objv[2])));
 	Tcl_SetErrorCode(interp, "TK", "VALUE", "INDEX", NULL);
 	return TCL_ERROR;
     }
-    if ((size_t)index + 1 <= [S length]) {
-	if (index < 0) {
-	    index = 0;
+    if (idx > 0 && len != ulen) {
+	/* The string contains codepoints > \uFFFF. Determine UTF-16 index */
+	Tcl_Size newIdx = 0;
+	for (Tcl_Size i = 0; i < idx; i++) {
+	    newIdx += 1 + (((newIdx < len-1) && ([S characterAtIndex:newIdx]&0xFC00) == 0xD800) && (([S characterAtIndex:newIdx+1]&0xFC00) == 0xDC00));
+	}
+	idx = newIdx;
+    }
+    if (idx + 1 <= len) {
+	if (idx < 0) {
+	    idx = 0;
 	} else {
-	    NSRange range = [S rangeOfComposedCharacterSequenceAtIndex:index];
-	    index = range.location + range.length;
+	    NSRange range = [S rangeOfComposedCharacterSequenceAtIndex:idx];
+	    idx = range.location + range.length;
+	    if (idx > 0 && len != ulen) {
+		/* The string contains codepoints > \uFFFF. Determine UTF-32 index */
+		Tcl_Size newIdx = 1;
+		for (Tcl_Size i = 1; i < idx; i++) {
+		if ((([S characterAtIndex:i-1]&0xFC00) != 0xD800) || (([S characterAtIndex:i]&0xFC00) != 0xDC00)) newIdx++;
+		}
+		idx = newIdx;
+	    }
 	}
-	Tcl_SetObjResult(interp, TkNewIndexObj(index));
+	Tcl_SetObjResult(interp, TkNewIndexObj(idx));
     }
     return TCL_OK;
 }
diff --git a/macosx/tkMacOSXPrivate.h b/macosx/tkMacOSXPrivate.h
index 51901c9..de230da 100644
--- a/macosx/tkMacOSXPrivate.h
+++ b/macosx/tkMacOSXPrivate.h
@@ -532,8 +532,8 @@ VISIBILITY_HIDDEN
  * NSString class does not provide a constructor which accepts a CESU-8 encoded
  * byte sequence as initial data.  So we add a new class which does provide
  * such a constructor.  It also has a DString property which is a DString whose
- * string pointer is a byte sequence encoding the NSString with the current Tk
- * encoding, namely UTF-8 if TCL_UTF_MAX >= 4 or CESU-8 if TCL_UTF_MAX = 3.
+ * string pointer is a byte sequence encoding the NSString with the current Tcl
+ * internal encoding, namely UTF-8 for Tcl8.7+ or CESU-8 otherwise.
  *
  *---------------------------------------------------------------------------
  */
diff --git a/tests/cluster.test b/tests/cluster.test
index 998759e..be889c3 100644
--- a/tests/cluster.test
+++ b/tests/cluster.test
@@ -15,53 +15,53 @@ testConstraint needsICU [expr {[catch {info body ::tk::startOfCluster}]}]
 
 
 test cluster-1.0 {::tk::startOfCluster} -body {
-    ::tk::startOfCluster 🤡 -1
+    ::tk::startOfCluster é -1
 } -result {}
 test cluster-1.1 {::tk::startOfCluster} -body {
-    ::tk::startOfCluster 🤡 0
+    ::tk::startOfCluster é 0
 } -result 0
 test cluster-1.2 {::tk::startOfCluster} -constraints needsICU -body {
-    ::tk::startOfCluster 🤡 1
+    ::tk::startOfCluster é 1
 } -result 0
 test cluster-1.3 {::tk::startOfCluster} -constraints needsICU -body {
-    ::tk::startOfCluster 🤡 2
+    ::tk::startOfCluster é 2
 } -result 2
 test cluster-1.4 {::tk::startOfCluster} -constraints needsICU -body {
-    ::tk::startOfCluster 🤡 3
+    ::tk::startOfCluster é 3
 } -result 2
 test cluster-1.5 {::tk::startOfCluster} -constraints needsICU -body {
-    ::tk::startOfCluster 🤡 end
+    ::tk::startOfCluster é end
 } -result 0
 test cluster-1.6 {::tk::startOfCluster} -body {
-    ::tk::startOfCluster 🤡 {}
+    ::tk::startOfCluster é {}
 } -result {}
 test cluster-1.7 {::tk::startOfCluster} -constraints needsICU -body {
-    ::tk::startOfCluster 🤡 end-1
+    ::tk::startOfCluster é end-1
 } -result 0
 
 test cluster-2.0 {::tk::endOfCluster} -body {
-    ::tk::endOfCluster 🤡 -1
+    ::tk::endOfCluster é -1
 } -result 0
 test cluster-2.1 {::tk::endOfCluster} -constraints needsICU -body {
-    ::tk::endOfCluster 🤡 0
+    ::tk::endOfCluster é 0
 } -result 2
 test cluster-2.2 {::tk::endOfCluster} -constraints needsICU -body {
-    ::tk::endOfCluster 🤡 1
+    ::tk::endOfCluster é 1
 } -result 2
 test cluster-2.3 {::tk::endOfCluster} -body {
-    ::tk::endOfCluster 🤡 2
+    ::tk::endOfCluster é 2
 } -result {}
 test cluster-2.4 {::tk::endOfCluster} -body {
-    ::tk::endOfCluster 🤡 3
+    ::tk::endOfCluster é 3
 } -result {}
 test cluster-2.5 {::tk::endOfCluster} -constraints needsICU -body {
-    ::tk::endOfCluster 🤡 end
+    ::tk::endOfCluster é end
 } -result 2
 test cluster-2.6 {::tk::endOfCluster} -body {
-    ::tk::endOfCluster 🤡 {}
+    ::tk::endOfCluster é {}
 } -result 0
 test cluster-2.7 {::tk::endOfCluster} -constraints needsICU -body {
-    ::tk::endOfCluster 🤡 end-1
+    ::tk::endOfCluster é end-1
 } -result 2
 
 test cluster-3.0 {::tk::endOfWord} -body {
diff --git a/unix/tkUnixMenu.c b/unix/tkUnixMenu.c
index 4ae343c..08d45b0 100644
--- a/unix/tkUnixMenu.c
+++ b/unix/tkUnixMenu.c
@@ -860,13 +860,13 @@ DrawMenuUnderline(
     if (mePtr->labelPtr != NULL) {
 	int len;
 
-	len = Tcl_GetCharLength(mePtr->labelPtr);
+	len = TkGetCharLength(mePtr->labelPtr);
 	if (mePtr->underline < len && mePtr->underline >= -len) {
 	    int activeBorderWidth, leftEdge, ch;
 	    const char *label, *start, *end;
 
 	    label = Tcl_GetString(mePtr->labelPtr);
-	    start = Tcl_UtfAtIndex(label, (mePtr->underline < 0) ? mePtr->underline + len : mePtr->underline);
+	    start = TkUtfAtIndex(label, (mePtr->underline < 0) ? mePtr->underline + len : mePtr->underline);
 	    end = start + TkUtfToUniChar(start, &ch);
 
 	    Tk_GetPixelsFromObj(NULL, menuPtr->tkwin,
diff --git a/win/tkWinMenu.c b/win/tkWinMenu.c
index c102e14..0175514 100644
--- a/win/tkWinMenu.c
+++ b/win/tkWinMenu.c
@@ -509,7 +509,7 @@ GetEntryText(
 		: Tcl_GetString(mePtr->accelPtr);
 	const char *p, *next;
 	Tcl_DString itemString;
-	Tcl_UniChar ch = 0;
+	int ch = 0;
 
 	/*
 	 * We have to construct the string with an ampersand preceeding the
@@ -526,7 +526,7 @@ GetEntryText(
 	    if (*p == '&') {
 		Tcl_DStringAppend(&itemString, "&", 1);
 	    }
-	    next = p + Tcl_UtfToUniChar(p, &ch);
+	    next = p + TkUtfToUniChar(p, &ch);
 	    Tcl_DStringAppend(&itemString, p, (int) (next - p));
 	}
 	ch = 0;
@@ -536,7 +536,7 @@ GetEntryText(
 		if (*p == '&') {
 		    Tcl_DStringAppend(&itemString, "&", 1);
 		}
-		next = p + Tcl_UtfToUniChar(p, &ch);
+		next = p + TkUtfToUniChar(p, &ch);
 		Tcl_DStringAppend(&itemString, p, (int) (next - p));
 	    }
 	}
@@ -2102,13 +2102,13 @@ DrawMenuUnderline(
     if ((mePtr->underline >= 0) && (mePtr->labelPtr != NULL)) {
 	int len;
 
-	len = Tcl_GetCharLength(mePtr->labelPtr);
+	len = TkGetCharLength(mePtr->labelPtr);
 	if (mePtr->underline < len) {
 	    const char *label, *start, *end;
 	    int ch;
 
 	    label = Tcl_GetString(mePtr->labelPtr);
-	    start = Tcl_UtfAtIndex(label, mePtr->underline);
+	    start = TkUtfAtIndex(label, mePtr->underline);
 	    end = start + TkUtfToUniChar(start, &ch);
 	    Tk_UnderlineChars(menuPtr->display, d,
 		    gc, tkfont, label, x + mePtr->indicatorSpace,