[Bug 3393714] overflow in toupper delta

4 files changed, 48 insertions, 35 deletions

diff --git a/ChangeLog b/ChangeLog
index 28418c6..df5aef0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2011-08-18  Jan Nijtmans  <nijtmans@users.sf.net>
+
+	* generic/tclUniData.c: [Bug 3393714] overflow in toupper delta
+	* tools/uniParse.tcl
+	* tests/utf.test
+
 2011-08-17  Don Porter  <dgp@users.sourceforge.net>
 
 	* generic/tclGet.c: [Bug 3393150] Overlooked free of intreps.
diff --git a/generic/tclUniData.c b/generic/tclUniData.c
index 83b3058..6cff83a 100644
--- a/generic/tclUniData.c
+++ b/generic/tclUniData.c
@@ -739,38 +739,35 @@ static const unsigned char groupMap[] = {
  *				 101 = sub delta for upper, sub 1 for title
  *				 110 = sub delta for upper, add delta for lower
  *
- * Bits 8-21	Reserved for future use.
+ * Bits 8-14	Reserved for future use.
  *
- * Bits 22-31	Case delta: delta for case conversions.  This should be the
+ * Bits 15-31	Case delta: delta for case conversions.  This should be the
  *			    highest field so we can easily sign extend.
  */
 
 static const int groups[] = {
-    0, 15, 12, 25, 27, 21, 22, 26, 20, 9, 134217793, 28, 19, 134217858,
-    29, 2, 23, 16, 11, 1178599554, 24, -507510654, 4194369, 4194434,
-    -834666431, 973078658, -507510719, 1258291330, -817889150, 880803905,
-    864026689, 859832385, 331350081, 847249473, 851443777, 868220993,
-    -406847358, 884998209, 876609601, -683671422, 893386817, -545259390,
-    897581121, 914358337, 910164033, 918552641, 5, -234880894, 8388705,
-    4194467, 8388770, 331350146, -406847423, -234880959, -545259455,
-    -1967128511, -683671487, -1979711423, 1883242626, -817889215,
-    289407041, 297795649, 2017460354, 2030043266, 2021654658, 880803970,
-    864026754, 859832450, 847249538, 851443842, 868221058, -1241513854,
-    876609666, 884998274, -2109734782, -2134900606, 893386882, 897581186,
-    -2042625918, 914358402, 289407106, 910164098, 297795714, 918552706,
-    4, 6, -352321402, 159383617, 155189313, 268435521, 264241217,
-    159383682, 155189378, 130023554, 268435586, 264241282, 33554497,
-    260046978, 239075458, 1, 197132418, 226492546, 33554562, 360710274,
-    335544450, -29359998, -251658175, 402653314, -29360063, 335544385,
-    7, 62914625, 62914690, 201326657, 201326722, 8, 402653249, 10,
-    2130706562, 1182793858, 247464066, -1874853823, -33554302, -33554367,
-    -310378366, -360710014, -419430270, -536870782, -469761918, -528482174,
-    -33554365, -37748606, -310378431, -37748669, 155189378, -360710079,
-    -419430335, -469761983, -536870847, -528482239, 13, 14, -1463812031,
-    -801111999, -293601215, 117440577, 117440642, 67108938, 67109002,
-    109051997, 109052061, -2109734847, 1182793793, -2042625983, -1967128446,
-    -1979711358, 2030043201, -2134900671, 2017460289, 2021654593,
-    1883242561, 402653314, 2130706497, -1241513919, 18, 17
+    0, 15, 12, 25, 27, 21, 22, 26, 20, 9, 1048641, 28, 19, 1048706,
+    29, 2, 23, 16, 11, -24346494, 24, -3964798, 32833, 32898, -6520767,
+    7602306, -3964863, 9830530, -6389630, 6881345, 6750273, 6717505,
+    2588737, 6619201, 6651969, 6783041, -3178366, 6914113, 6848577,
+    -5341054, 6979649, -4259710, 7012417, 7143489, 7110721, 7176257,
+    5, -1834878, 65633, 32931, 65698, 2588802, -3178431, -1834943,
+    -4259775, 353730625, -5341119, 353632321, -354385790, -6389695,
+    2261057, 2326593, -353337214, -353238910, -353304446, 6881410,
+    6750338, 6717570, 6619266, 6652034, 6783106, -1385430910, 6848642,
+    6914178, -352026494, -352223102, 6979714, 7012482, -351502206,
+    7143554, 2261122, 7110786, 2326658, 7176322, 4, 6, -2752378, 1245249,
+    1212481, 2097217, 2064449, 1245314, 1212546, 1015938, 2097282,
+    2064514, 262209, 2031746, 1867906, 1, 1540226, 1769602, 262274,
+    2818178, 2621570, -229246, -1966015, 3145858, -229311, 2621505,
+    7, 491585, 491650, 1572929, 1572994, 8, 238026817, 10, -1157758846,
+    -124977022, 1933442, -249528255, -262014, -262079, -2424702, -2817918,
+    -3276670, -4194174, -3669886, -4128638, -262077, -294782, -2424767,
+    -294845, 236093570, -2817983, -3276735, -3669951, -4194239, -4128703,
+    13, 14, -246316991, -274694079, -270729151, 917569, 917634, 524362,
+    524426, 852061, 852125, -352026559, -124977087, -351502271, 353730690,
+    353632386, -353238975, -352223167, -353337279, -353304511, -354385855,
+    238026882, -1157758911, -1385430975, 18, 17
 };
 
 /*
@@ -821,7 +818,7 @@ enum {
 
 #define GetCaseType(info) (((info) & 0xE0) >> 5)
 #define GetCategory(info) ((info) & 0x1F)
-#define GetDelta(info) (((info) > 0) ? ((info) >> 22) : (~(~((info)) >> 22)))
+#define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15)))
 
 /*
  * This macro extracts the information about a character from the
diff --git a/tests/utf.test b/tests/utf.test
index d319f6e..a9485a8 100644
--- a/tests/utf.test
+++ b/tests/utf.test
@@ -244,8 +244,9 @@ test utf-16.1 {Tcl_UniCharToLower, negative delta} {
     string tolower aA
 } aa
 test utf-16.2 {Tcl_UniCharToLower, positive delta} {
-    string tolower \u0178\u00ff
-} \u00ff\u00ff
+    string tolower \u0178\u00ff\uA78D
+} \u00ff\u00ff\u0265
+
 test utf-17.1 {Tcl_UniCharToLower, no delta} {
     string tolower !
 } !
diff --git a/tools/uniParse.tcl b/tools/uniParse.tcl
index 99ecf40..3b0f965 100644
--- a/tools/uniParse.tcl
+++ b/tools/uniParse.tcl
@@ -179,7 +179,7 @@ proc uni::main {} {
     buildTables $data
     puts "X = [llength $pMap]  Y= [llength $pages]  A= [llength $groups]"
     set size [expr {[llength $pMap] + [llength $pages]*(1<<$shift)}]
-    puts "shift = 6, space = $size"
+    puts "shift = $shift, space = $size"
     puts "title case count = $titleCount"
 
     set f [open [file join [lindex $argv 1] tclUniData.c] w]
@@ -214,6 +214,9 @@ static const unsigned short pageMap\[\] = {"
     set last [expr {[llength $pMap] - 1}]
     for {set i 0} {$i <= $last} {incr i} {
 	append line [lindex $pMap $i]
+	if {[lindex $pMap $i] == 17} {
+	    puts stdout [list ZZZZ: $i]
+	}
 	if {$i != $last} {
 	    append line ", "
 	}
@@ -239,6 +242,9 @@ static const unsigned char groupMap\[\] = {"
 	set lastj [expr {[llength $page] - 1}]
 	for {set j 0} {$j <= $lastj} {incr j} {
 	    append line [lindex $page $j]
+	    if {[lindex $page $j] == 71} {
+		puts stdout [list YYYY: $i $j]
+	    }
 	    if {$j != $lastj || $i != $lasti} {
 		append line ", "
 	    }
@@ -264,9 +270,9 @@ static const unsigned char groupMap\[\] = {"
  *				 101 = sub delta for upper, sub 1 for title
  *				 110 = sub delta for upper, add delta for lower
  *
- * Bits 8-21	Reserved for future use.
+ * Bits 8-14	Reserved for future use.
  *
- * Bits 22-31	Case delta: delta for case conversions.  This should be the
+ * Bits 15-31	Case delta: delta for case conversions.  This should be the
  *			    highest field so we can easily sign extend.
  */
 
@@ -306,7 +312,10 @@ static const int groups\[\] = {"
 	    set delta 0
 	}
 
-	set val [expr {($delta << 22) | ($case << 5) | $type}]
+	set val [expr {($delta << 15) | ($case << 5) | $type}]
+	if {($val > 0x3fffffff) || ($val < -0x3fffffff)} {
+		puts stdout [list "XXXXXXXXXXX:" $i $delta $val]
+	}
 
 	append line [format "%d" $val]
 	if {$i != $last} {
@@ -368,7 +377,7 @@ enum {
 
 #define GetCaseType(info) (((info) & 0xE0) >> 5)
 #define GetCategory(info) ((info) & 0x1F)
-#define GetDelta(info) (((info) > 0) ? ((info) >> 22) : (~(~((info)) >> 22)))
+#define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15)))
 
 /*
  * This macro extracts the information about a character from the