TclNeedSpace is now UTF8-aware. (Bug #411825 from <arobert3434@users.sf.net>)

3 files changed, 33 insertions, 19 deletions

diff --git a/ChangeLog b/ChangeLog
index 5bb6036..0e27b4b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2001-09-19  Donal K. Fellows  <fellowsd@cs.man.ac.uk>
+
+	* tests/util.test (util-8.1): Test derived from code to detect the
+	problem, but the test always works in the C locale, so beware if
+	you are maintaining the code.
+	* generic/tclUtil.c (TclNeedSpace): Rewrote to be UTF-8 aware.
+	[Bug 411825, but not that patch which would have added extra
+	spaces if there was a real non-ASCII space involved. ]
+
 2001-09-18 Andreas Kupries  <andreas_kupries@users.sourceforge.net> 
 
 	* generic/tclIOCmd.c (Tcl_PutsObjCmd): Rewritten to have saner and
diff --git a/generic/tclUtil.c b/generic/tclUtil.c
index 96deda9..0afef5c 100644
--- a/generic/tclUtil.c
+++ b/generic/tclUtil.c
@@ -10,7 +10,7 @@
  * See the file "license.terms" for information on usage and redistribution
  * of this file, and for a DISCLAIMER OF ALL WARRANTIES.
  *
- *  RCS: @(#) $Id: tclUtil.c,v 1.22 2001/08/31 17:53:57 hobbs Exp $
+ *  RCS: @(#) $Id: tclUtil.c,v 1.23 2001/09/19 08:52:46 dkf Exp $
  */
 
 #include "tclInt.h"
@@ -1931,6 +1931,8 @@ TclNeedSpace(start, end)
     char *end;			/* End of string (place where space will
 				 * be added, if appropriate). */
 {
+    Tcl_UniChar ch;
+
     /*
      * A space is needed unless either
      * (a) we're at the start of the string, or
@@ -1944,10 +1946,14 @@ TclNeedSpace(start, end)
     if (end == start) {
 	return 0;
     }
-    end--;
+    end = Tcl_UtfPrev(end, start);
     if (*end != '{') {
-	if (isspace(UCHAR(*end)) /* INTL: ISO space. */
-		&& ((end == start) || (end[-1] != '\\'))) {
+	Tcl_UtfToUniChar(end, &ch);
+	/*
+	 * Direct char comparison on next line is safe as it is with
+	 * a character in the ASCII subset, and so single-byte in UTF8.
+	 */
+	if (Tcl_UniCharIsSpace(ch) && ((end == start) || (end[-1] != '\\'))) {
 	    return 0;
 	}
 	return 1;
@@ -1956,9 +1962,10 @@ TclNeedSpace(start, end)
 	if (end == start) {
 	    return 0;
 	}
-	end--;
+	end = Tcl_UtfPrev(end, start);
     } while (*end == '{');
-    if (isspace(UCHAR(*end))) {	/* INTL: ISO space. */
+    Tcl_UtfToUniChar(end, &ch);
+    if (Tcl_UniCharIsSpace(ch)) {
 	return 0;
     }
     return 1;
diff --git a/tests/util.test b/tests/util.test
index d783eff..d39a6da 100644
--- a/tests/util.test
+++ b/tests/util.test
@@ -7,7 +7,7 @@
 # See the file "license.terms" for information on usage and redistribution
 # of this file, and for a DISCLAIMER OF ALL WARRANTIES.
 #
-# RCS: @(#) $Id: util.test,v 1.8 2001/07/03 03:33:42 hobbs Exp $
+# RCS: @(#) $Id: util.test,v 1.9 2001/09/19 08:52:46 dkf Exp $
 
 if {[lsearch [namespace children] ::tcltest] == -1} {
     package require tcltest
@@ -294,18 +294,16 @@ test util-7.4 {TclPrecTraceProc - write traces, bogus values} {
 
 set tcl_precision 12
 
+# This test always succeeded in the C locale anyway...
+test util-8.1 {TclNeedSpace - correct UTF8 handling} {
+    interp create \u5420
+    interp create [list \u5420 foo]
+    interp alias {} fooset [list \u5420 foo] set
+    set result [interp target {} fooset]
+    interp delete \u5420
+    set result
+} "\u5420 foo"
+
 # cleanup
 ::tcltest::cleanupTests
 return
-
-
-
-
-
-
-
-
-
-
-
-