Merge 8.6. Introduce the new Unicode-handling functions here too: Tk 8.7 could be loaded in Tcl 8.6 .... Further improvements are still possible

5 files changed, 172 insertions, 60 deletions

diff --git a/macosx/tkMacOSXClipboard.c b/macosx/tkMacOSXClipboard.c
index 6717afa..88a0d93 100644
--- a/macosx/tkMacOSXClipboard.c
+++ b/macosx/tkMacOSXClipboard.c
@@ -35,10 +35,7 @@ static Tk_Window clipboardOwner = NULL;
 		    targetPtr->type == dispPtr->utf8Atom) {
 		for (TkClipboardBuffer *cbPtr = targetPtr->firstBufferPtr;
 			cbPtr; cbPtr = cbPtr->nextPtr) {
-		    NSString *s = [[NSString alloc] initWithBytesNoCopy:
-			    cbPtr->buffer length:cbPtr->length
-			    encoding:NSUTF8StringEncoding freeWhenDone:NO];
-
+		    NSString *s = TkUtfToNSString(cbPtr->buffer, cbPtr->length);
 		    [string appendString:s];
 		    [s release];
 		}
@@ -130,7 +127,6 @@ TkSelGetSelection(
 	        && selection == dispPtr->clipboardAtom
 	        && (target == XA_STRING || target == dispPtr->utf8Atom)) {
 	NSString *string = nil;
-	NSString *clean;
 	NSPasteboard *pb = [NSPasteboard generalPasteboard];
 	NSString *type = [pb availableTypeFromArray:[NSArray arrayWithObject:
 		NSStringPboardType]];
@@ -139,25 +135,20 @@ TkSelGetSelection(
 	    string = [pb stringForType:type];
 	}
 	if (string) {
+
 	    /*
-	     * Replace all non-BMP characters by the replacement character 0xfffd.
-	     * This is a workaround until Tcl supports TCL_UTF_MAX > 3.
+	     * Encode the string using the encoding which is used in Tcl
+	     * when TCL_UTF_MAX = 3.  This replaces each UTF-16 surrogate with
+	     * a 3-byte sequence generated using the UTF-8 algorithm. (Even
+	     * though UTF-8 does not allow encoding surrogates, the algorithm
+	     * does produce a 3-byte sequence.)
 	     */
-	    int i, j, len = [string length];
-	    CFRange all = CFRangeMake(0, len);
-	    UniChar *buffer = ckalloc(len*sizeof(UniChar));
-	    CFStringGetCharacters((CFStringRef) string, all, buffer);
-	    for (i = 0, j = 0 ; j < len ; i++, j++) {
-		if (CFStringIsSurrogateHighCharacter(buffer[j])) {
-		    buffer[i] = 0xfffd;
-		    j++;
-		} else {
-		    buffer[i] = buffer[j];
-		}
+
+	    char *bytes = TkNSStringToUtf(string, NULL);
+	    result = proc(clientData, interp, bytes);
+	    if (bytes) {
+		ckfree(bytes);
 	    }
-	    clean = (NSString *)CFStringCreateWithCharacters(NULL, buffer, i);
-	    ckfree(buffer);
-	    result = proc(clientData, interp, [clean UTF8String]);
 	}
     } else {
 	Tcl_SetObjResult(interp, Tcl_ObjPrintf(
diff --git a/macosx/tkMacOSXFont.c b/macosx/tkMacOSXFont.c
index 41da390..a402345 100644
--- a/macosx/tkMacOSXFont.c
+++ b/macosx/tkMacOSXFont.c
@@ -101,6 +101,141 @@ static void		DrawCharsInContext(Display *display, Drawable drawable,
 #pragma mark -
 #pragma mark Font Helpers:
 
+/*
+ *---------------------------------------------------------------------------
+ *
+ * TkUtfToNSString --
+ *
+ * When Tcl is compiled with TCL_UTF_MAX = 3 (the default for 8.6) it cannot
+ * deal directly with UTF-8 encoded non-BMP characters, since their UTF-8
+ * encoding requires 4 bytes.
+ *
+ * As a workaround, these versions of Tcl encode non-BMP characters as a string
+ * of length 6 in which the high and low UTF-16 surrogates have been encoded
+ * using the UTF-8 algorithm.  The UTF-8 encoding does not allow encoding
+ * surrogates, so these 6-byte strings are not valid UTF-8, and hence Apple's
+ * NString class will refuse to instantiate an NSString from the 6-byte
+ * encoding.  This function allows creating an NSString from a C-string which
+ * has been encoded using this scheme.
+ *
+ * Results:
+ *	An NSString, which may be nil.
+ *
+ * Side effects:
+ *	None.
+ *---------------------------------------------------------------------------
+ */
+
+MODULE_SCOPE NSString*
+TkUtfToNSString(
+   const char *source,
+   size_t numBytes)
+{
+    NSString *string;
+    Tcl_DString ds;
+
+    Tcl_DStringInit(&ds);
+    Tcl_UtfToUniCharDString(source, numBytes, &ds);
+    string = [[NSString alloc] initWithCharacters:(const unichar *)Tcl_DStringValue(&ds)
+	    length:(Tcl_DStringLength(&ds)>>1)];
+    Tcl_DStringFree(&ds);
+    return string;
+}
+
+/*
+ *---------------------------------------------------------------------------
+ *
+ * TkUtfAtIndex --
+ *
+ *  Write a sequence of bytes up to length 6 which is an encoding of a UTF-16
+ *  character in an NSString.  Also record the unicode code point of the character.
+ *  this may be a non-BMP character constructed by reading two surrogates from
+ *  the NSString.
+ *
+ * Results:
+ *	Returns the number of bytes written.
+ *
+ * Side effects:
+ *	Bytes are written to the char array referenced by the pointer uni and
+ *      the unicode code point is written to the integer referenced by the
+ *      pointer code.
+ *
+ */
+
+MODULE_SCOPE int
+TkUtfAtIndex(
+    NSString *string,
+    int index,
+    char *uni,
+    unsigned int *code)
+{
+    char *ptr = uni;
+    UniChar uniChar = [string characterAtIndex: index];
+    if (CFStringIsSurrogateHighCharacter(uniChar)) {
+	UniChar lowChar = [string characterAtIndex: ++index];
+	*code = CFStringGetLongCharacterForSurrogatePair(
+	    uniChar, lowChar);
+	ptr += Tcl_UniCharToUtf(uniChar, ptr);
+        ptr += Tcl_UniCharToUtf(lowChar, ptr);
+	return ptr - uni;
+    } else {
+	*code = (int) uniChar;
+	[[string substringWithRange: NSMakeRange(index, 1)]
+     	        getCString: uni
+		 maxLength: XMaxTransChars
+		  encoding: NSUTF8StringEncoding];
+	return strlen(uni);
+    }
+}
+
+/*
+ *---------------------------------------------------------------------------
+ *
+ * TkNSStringToUtf --
+ *
+ * Encodes the unicode string represented by an NSString object with the
+ * internal encoding that Tcl uses when TCL_UTF_MAX = 3.  This encoding
+ * is similar to UTF-8 except that non-BMP characters are encoded as two
+ * successive 3-byte sequences which are constructed from UTF-16 surrogates
+ * by applying the UTF-8 algorithm.  Even though the UTF-8 encoding does not
+ * allow encoding surrogates, the algorithm does produce a well-defined
+ * 3-byte sequence.
+ *
+ * Results:
+ *	Returns a pointer to a null-terminated byte array which encodes the
+ *	NSString.
+ *
+ * Side effects:
+ *      Memory is allocated to hold the byte array, which must be freed with
+ *      ckalloc.  If the pointer numBytes is not NULL the number of non-null
+ *      bytes written to the array is stored in the integer it references.
+ */
+
+MODULE_SCOPE char*
+TkNSStringToUtf(
+   NSString *string,
+   int *numBytes)
+{
+    unsigned int code;
+    size_t i;
+    char *ptr, *bytes = ckalloc(6*[string length] + 1);
+
+    ptr = bytes;
+    if (ptr) {
+	for (i = 0; i < [string length]; i++) {
+	    ptr += TkUtfAtIndex(string, i, ptr, &code);
+	    if (code > 0xffff){
+		i++;
+	    }
+	}
+	*ptr = '\0';
+    }
+    if (numBytes) {
+	*numBytes = ptr - bytes;
+    }
+    return bytes;
+}
+
 #define GetNSFontTraitsFromTkFontAttributes(faPtr) \
 	((faPtr)->weight == TK_FW_BOLD ? NSBoldFontMask : NSUnboldFontMask) | \
 	((faPtr)->slant == TK_FS_ITALIC ? NSItalicFontMask : NSUnitalicFontMask)
@@ -844,8 +979,7 @@ TkpMeasureCharsInContext(
     if (maxLength > 32767) {
 	maxLength = 32767;
     }
-    string = [[NSString alloc] initWithBytesNoCopy:(void*)source
-		length:numBytes encoding:NSUTF8StringEncoding freeWhenDone:NO];
+    string = TkUtfToNSString((const char *)source, numBytes);
     if (!string) {
 	length = 0;
 	fit = rangeLength;
@@ -1124,8 +1258,7 @@ DrawCharsInContext(
 	    !TkMacOSXSetupDrawingContext(drawable, gc, 1, &drawingContext)) {
 	return;
     }
-    string = [[NSString alloc] initWithBytesNoCopy:(void*)source
-		length:numBytes encoding:NSUTF8StringEncoding freeWhenDone:NO];
+    string = TkUtfToNSString((const char *)source, numBytes);
     if (!string) {
 	return;
     }
diff --git a/macosx/tkMacOSXKeyEvent.c b/macosx/tkMacOSXKeyEvent.c
index e3aed98..d3704d2 100644
--- a/macosx/tkMacOSXKeyEvent.c
+++ b/macosx/tkMacOSXKeyEvent.c
@@ -331,42 +331,30 @@ static unsigned	isFunctionKey(unsigned int code);
     }
 
     /*
-     * NSString represents a non-BMP character as a string of length 2 where
-     * the first character is the high surrogate and the second character is
-     * the low surrogate.  We could record this in the XEvent by setting the
-     * keycode to the unicode code point and setting the trans_chars to the
-     * 4-byte UTF-8 string.  However, that will not help as long as TCL_UTF_MAX
-     * is set to 3.  Until that changes, we just replace non-BMP characters by
-     * the "replacement character" U+FFFD.
+     * Next we generate an XEvent for each unicode character in our string.
+     *
+     * NSString uses UTF-16 internally, which means that a non-BMP character is
+     * represented by a sequence of two 16-bit "surrogates".  In principle we
+     * could record this in the XEvent by setting the keycode to the 32-bit
+     * unicode code point and setting the trans_chars string to the 4-byte
+     * UTF-8 string for the non-BMP character.  However, that will not work
+     * when TCL_UTF_MAX is set to 3, as is the case for Tcl 8.6.  A workaround
+     * used internally by Tcl 8.6 is to encode each surrogate as a 3-byte
+     * sequence using the UTF-8 algorithm (ignoring the fact that the UTF-8
+     * encoding specification does not allow encoding UTF-16 surrogates).
+     * This gives a 6-byte encoding of the non-BMP character which we write into
+     * the trans_chars field of the XEvent.
      */
 
     for (i = 0; i < len; i++) {
-	UniChar nextChar = [str characterAtIndex: i];
-	if (CFStringIsSurrogateHighCharacter(nextChar)) {
-#if 0
-	    UniChar lowChar = [str characterAtIndex: ++i];
-	    xEvent.xkey.keycode = CFStringGetLongCharacterForSurrogatePair(
-		nextChar, lowChar);
-	    xEvent.xkey.nbytes = TkUniCharToUtf(xEvent.xkey.keycode,
-						&xEvent.xkey.trans_chars);
-#else
+	xEvent.xkey.nbytes = TkUtfAtIndex(str, i, xEvent.xkey.trans_chars,
+					   &xEvent.xkey.keycode);
+	if (xEvent.xkey.keycode > 0xffff){
 	    i++;
-	    xEvent.xkey.keycode = 0xfffd;
-	    strcpy(xEvent.xkey.trans_chars, "\xef\xbf\xbd");
-	    xEvent.xkey.nbytes = strlen(xEvent.xkey.trans_chars);
-#endif
-	} else {
-	    xEvent.xkey.keycode = (int) nextChar;
-	    [[str substringWithRange: NSMakeRange(i,1)]
-	        getCString: xEvent.xkey.trans_chars
-		maxLength: XMaxTransChars encoding: NSUTF8StringEncoding];
-	    xEvent.xkey.nbytes = strlen(xEvent.xkey.trans_chars);
 	}
-	xEvent.xany.type = KeyPress;
-	releaseCode = (UInt16) nextChar;
-	Tk_QueueWindowEvent(&xEvent, TCL_QUEUE_TAIL);
+    	xEvent.xany.type = KeyPress;
+    	Tk_QueueWindowEvent(&xEvent, TCL_QUEUE_TAIL);
     }
-
     releaseCode = (UInt16) [str characterAtIndex: 0];
 }
 
diff --git a/macosx/tkMacOSXPrivate.h b/macosx/tkMacOSXPrivate.h
index 7aa6840..7f291f1 100644
--- a/macosx/tkMacOSXPrivate.h
+++ b/macosx/tkMacOSXPrivate.h
@@ -144,10 +144,6 @@ MODULE_SCOPE long tkMacOSXMacOSXVersion;
  * Prototypes for TkMacOSXRegion.c.
  */
 
-#if 0
-MODULE_SCOPE void	TkMacOSXEmtpyRegion(Region r);
-MODULE_SCOPE int	TkMacOSXIsEmptyRegion(Region r);
-#endif
 MODULE_SCOPE HIShapeRef	TkMacOSXGetNativeRegion(Region r);
 MODULE_SCOPE void	TkMacOSXSetWithNativeRegion(Region r,
 			    HIShapeRef rgn);
@@ -239,6 +235,10 @@ MODULE_SCOPE int 	TkMacOSXServices_Init(Tcl_Interp *interp);
 MODULE_SCOPE int	TkMacOSXRegisterServiceWidgetObjCmd(ClientData clientData,
 			    Tcl_Interp *interp, int objc,
 			    Tcl_Obj *const objv[]);
+MODULE_SCOPE NSString*  TkUtfToNSString(const char *source, size_t numBytes);
+MODULE_SCOPE int        TkUtfAtIndex(NSString *string, int index, char *uni,
+				      unsigned int *code);
+MODULE_SCOPE char*      TkNSStringToUtf(NSString *string, int *numBytes);
 
 #pragma mark Private Objective-C Classes
 
diff --git a/macosx/tkMacOSXRegion.c b/macosx/tkMacOSXRegion.c
index 0afc752..0a5ff6d 100644
--- a/macosx/tkMacOSXRegion.c
+++ b/macosx/tkMacOSXRegion.c
@@ -175,7 +175,7 @@ XUnionRectWithRegion(
  *----------------------------------------------------------------------
  */
 
-int
+static int
 TkMacOSXIsEmptyRegion(
     Region r)
 {