Experimental new functions TkUtfPrev/TkUtfNext

7 files changed, 54 insertions, 19 deletions

diff --git a/generic/tkFont.c b/generic/tkFont.c
index bec8807..cfa41e3 100644
--- a/generic/tkFont.c
+++ b/generic/tkFont.c
@@ -2730,7 +2730,7 @@ Tk_CharBbox(
 		x += chunkPtr->x;
 	    }
 	    if (widthPtr != NULL) {
-		Tk_MeasureChars(tkfont, end, Tcl_UtfNext(end) - end,
+		Tk_MeasureChars(tkfont, end, TkUtfNext(end) - end,
 			-1, 0, &w);
 	    }
 	    goto check;
diff --git a/generic/tkInt.h b/generic/tkInt.h
index 8c64c9a..88bef01 100644
--- a/generic/tkInt.h
+++ b/generic/tkInt.h
@@ -1242,10 +1242,14 @@ MODULE_SCOPE void	TkUnixSetXftClipRegion(TkRegion clipRegion);
 #   define TkUtfToUniChar Tcl_UtfToUniChar
 #   define TkUniCharToUtf Tcl_UniCharToUtf
 #   define TkUtfCharComplete Tcl_UtfCharComplete
+#   define TkUtfPrev Tcl_UtfPrev
+#   define TkUtfNext Tcl_UtfNext
 #else
     MODULE_SCOPE int TkUtfToUniChar(const char *, int *);
     MODULE_SCOPE int TkUniCharToUtf(int, char *);
     MODULE_SCOPE int TkUtfCharComplete(const char *, int);
+    MODULE_SCOPE const char *TkUtfPrev(const char *, const char *);
+    MODULE_SCOPE const char *TkUtfNext(const char *);
 #endif
 
 /*
diff --git a/generic/tkTextIndex.c b/generic/tkTextIndex.c
index faa1afd..a1695c0 100644
--- a/generic/tkTextIndex.c
+++ b/generic/tkTextIndex.c
@@ -387,7 +387,7 @@ TkTextMakeByteIndex(
     TkTextSegment *segPtr;
     int index;
     const char *p, *start;
-    Tcl_UniChar ch;
+    int ch;
 
     indexPtr->tree = tree;
     if (lineIndex < 0) {
@@ -433,11 +433,12 @@ TkTextMakeByteIndex(
 		 * that byteIndex falls on a character boundary. If the index
 		 * falls in the middle of a UTF-8 character, it will be
 		 * adjusted to the end of that UTF-8 character.
+		 * Surrogate pairs are handled as well, the same way.
 		 */
 
 		start = segPtr->body.chars + (byteIndex - index);
-		p = Tcl_UtfPrev(start, segPtr->body.chars);
-		p += Tcl_UtfToUniChar(p, &ch);
+		p = TkUtfPrev(start, segPtr->body.chars);
+		p += TkUtfToUniChar(p, &ch);
 		indexPtr->byteIndex += p - start;
 	    }
 	    break;
@@ -480,7 +481,7 @@ TkTextMakeCharIndex(
     register TkTextSegment *segPtr;
     char *p, *start, *end;
     int index, offset;
-    Tcl_UniChar ch;
+    int ch;
 
     indexPtr->tree = tree;
     if (lineIndex < 0) {
@@ -527,7 +528,7 @@ TkTextMakeCharIndex(
 		    return indexPtr;
 		}
 		charIndex--;
-		offset = Tcl_UtfToUniChar(p, &ch);
+		offset = TkUtfToUniChar(p, &ch);
 		index += offset;
 	    }
 	} else {
@@ -1475,7 +1476,7 @@ TkTextIndexForwChars(
     TkTextElideInfo *infoPtr = NULL;
     int byteOffset;
     char *start, *end, *p;
-    Tcl_UniChar ch;
+    int ch;
     int elide = 0;
     int checkElided = (type & COUNT_DISPLAY);
 
@@ -1574,7 +1575,7 @@ TkTextIndexForwChars(
 		if (segPtr->typePtr == &tkTextCharType) {
 		    start = segPtr->body.chars + byteOffset;
 		    end = segPtr->body.chars + segPtr->size;
-		    for (p = start; p < end; p += Tcl_UtfToUniChar(p, &ch)) {
+		    for (p = start; p < end; p += TkUtfToUniChar(p, &ch)) {
 			if (charCount == 0) {
 			    dstPtr->byteIndex += (p - start);
 			    goto forwardCharDone;
diff --git a/generic/tkUtil.c b/generic/tkUtil.c
index 0fa7ad9..b8bd473 100644
--- a/generic/tkUtil.c
+++ b/generic/tkUtil.c
@@ -1284,6 +1284,36 @@ int TkUtfCharComplete(const char *source, int numBytes) {
     return 1;
 }
 
+const char *TkUtfPrev(const char *start, const char *source) {
+    const char *p = Tcl_UtfPrev(start, source);
+    if (((source[0]&0xFF) == 0xEB) && ((source[1]&0xF0) == 0xB0)
+	    && ((source[2]&0xC0) == 0x80)) {
+	/* We are pointing to a low surrogate. If the previous
+	 * codepoint is a high surrogate, we want that in stead. */
+	const char *q = Tcl_UtfPrev(start, p);
+	if (((q[0]&0xFF) == 0xEB) && ((q[1]&0xF0) == 0xA0)
+		&& ((q[2]&0xC0) == 0x80)) {
+	    p = q;
+	}
+    }
+    return p;
+}
+
+const char *TkUtfNext(const char *source) {
+    const char *p = Tcl_UtfNext(source);
+    if (((source[0]&0xFF) == 0xEB) && ((source[1]&0xF0) == 0xA0)
+	    && ((source[2]&0xC0) == 0x80)) {
+	const char *q = Tcl_UtfNext(p);
+	/* We are pointing to a high surrogate. If the next
+	 * codepoint is a low surrogate, we want that in stead. */
+	if (((q[0]&0xFF) == 0xEB) && ((q[1]&0xF0) == 0xB0)
+		&& ((q[2]&0xC0) == 0x80)) {
+	    p = q;
+	}
+    }
+    return p;
+}
+
 #endif
 /*
  * Local Variables:
diff --git a/unix/tkUnixFont.c b/unix/tkUnixFont.c
index 0c663a3..f77affc 100644
--- a/unix/tkUnixFont.c
+++ b/unix/tkUnixFont.c
@@ -1090,7 +1090,7 @@ Tk_MeasureChars(
     } else {
 	const char *p, *end, *next, *term;
 	int newX, termX, sawNonSpace, dstWrote;
-	Tcl_UniChar ch;
+	int ch;
 	FontFamily *familyPtr;
 	XChar2b buf[8];
 
@@ -1100,7 +1100,7 @@ Tk_MeasureChars(
 	 * individually.
 	 */
 
-	next = source + Tcl_UtfToUniChar(source, &ch);
+	next = source + TkUtfToUniChar(source, &ch);
 	newX = curX = termX = 0;
 
 	term = source;
@@ -1135,7 +1135,7 @@ Tk_MeasureChars(
 		break;
 	    }
 
-	    next += Tcl_UtfToUniChar(next, &ch);
+	    next += TkUtfToUniChar(next, &ch);
 	    if ((ch < 256) && isspace(ch)) {
 		if (sawNonSpace) {
 		    term = p;
@@ -1160,13 +1160,13 @@ Tk_MeasureChars(
 	     */
 
 	    curX = newX;
-	    p += Tcl_UtfToUniChar(p, &ch);
+	    p += TkUtfToUniChar(p, &ch);
 	}
 	if ((flags & TK_AT_LEAST_ONE) && (term == source) && (p < end)) {
 	    term = p;
 	    termX = curX;
 	    if (term == source) {
-		term += Tcl_UtfToUniChar(term, &ch);
+		term += TkUtfToUniChar(term, &ch);
 		termX = newX;
 	    }
 	} else if ((p >= end) || !(flags & TK_WHOLE_WORDS)) {
@@ -1279,7 +1279,7 @@ Tk_DrawChars(
     Tcl_DString runString;
     const char *p, *end, *next;
     int xStart, needWidth, window_width, do_width;
-    Tcl_UniChar ch;
+    int ch;
     FontFamily *familyPtr;
 #ifdef TK_DRAW_CHAR_XWINDOW_CHECK
     int rx, ry;
@@ -1314,7 +1314,7 @@ Tk_DrawChars(
     needWidth = fontPtr->font.fa.underline + fontPtr->font.fa.overstrike;
     for (p = source; p <= end; ) {
 	if (p < end) {
-	    next = p + Tcl_UtfToUniChar(p, &ch);
+	    next = p + TkUtfToUniChar(p, &ch);
 	    thisSubFontPtr = FindSubFontForChar(fontPtr, ch, &lastSubFontPtr);
 	} else {
 	    next = p + 1;
diff --git a/unix/tkUnixMenu.c b/unix/tkUnixMenu.c
index bc1bd2e..c11858e 100644
--- a/unix/tkUnixMenu.c
+++ b/unix/tkUnixMenu.c
@@ -866,7 +866,7 @@ DrawMenuUnderline(
 
 	    label = Tcl_GetString(mePtr->labelPtr);
 	    start = Tcl_UtfAtIndex(label, mePtr->underline);
-	    end = Tcl_UtfNext(start);
+	    end = TkUtfNext(start);
 
 	    Tk_GetPixelsFromObj(NULL, menuPtr->tkwin,
 		    menuPtr->activeBorderWidthPtr, &activeBorderWidth);
diff --git a/win/tkWinMenu.c b/win/tkWinMenu.c
index 3cf7c10..d6c011e 100644
--- a/win/tkWinMenu.c
+++ b/win/tkWinMenu.c
@@ -527,7 +527,7 @@ GetEntryText(
 	    if (*p == '&') {
 		Tcl_DStringAppend(&itemString, "&", 1);
 	    }
-	    next = Tcl_UtfNext(p);
+	    next = TkUtfNext(p);
 	    Tcl_DStringAppend(&itemString, p, (int) (next - p));
 	}
 	if (mePtr->accelLength > 0) {
@@ -536,7 +536,7 @@ GetEntryText(
 		if (*p == '&') {
 		    Tcl_DStringAppend(&itemString, "&", 1);
 		}
-		next = Tcl_UtfNext(p);
+		next = TkUtfNext(p);
 		Tcl_DStringAppend(&itemString, p, (int) (next - p));
 	    }
 	}
@@ -1965,7 +1965,7 @@ DrawMenuUnderline(
 
 	    label = Tcl_GetString(mePtr->labelPtr);
 	    start = Tcl_UtfAtIndex(label, mePtr->underline);
-	    end = Tcl_UtfNext(start);
+	    end = TkUtfNext(start);
 	    Tk_UnderlineChars(menuPtr->display, d,
 		    gc, tkfont, label, x + mePtr->indicatorSpace,
 		    y + (height + fmPtr->ascent - fmPtr->descent) / 2,