summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--ChangeLog9
-rw-r--r--generic/regc_locale.c222
-rw-r--r--generic/tclUniData.c6
-rw-r--r--generic/tclUtf.c2
-rw-r--r--tests/utf.test17
-rw-r--r--tools/uniParse.tcl36
6 files changed, 125 insertions, 167 deletions
diff --git a/ChangeLog b/ChangeLog
index 410f689..a700258 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+2011-12-23 Jan Nijtmans <nijtmans@users.sf.net>
+
+ * generic/tclUtf.c: [Bug 3464428] string is graph \u0120 is wrong
+ * generic/tclUniData.c:
+ * generic/regc_locale.c:
+ * tests/utf.test:
+ * tools/uniParse.tcl: clean up some unused stuff, and be more robust
+ against changes in UnicodeData.txt syntax
+
2011-12-11 Jan Nijtmans <nijtmans@users.sf.net>
* generic/regc_locale.c: [Bug 3457031]: Some Unicode 6.0 chars not
diff --git a/generic/regc_locale.c b/generic/regc_locale.c
index f393820..f652f0e 100644
--- a/generic/regc_locale.c
+++ b/generic/regc_locale.c
@@ -218,7 +218,7 @@ static const chr alphaCharTable[] = {
0x2107, 0x2115, 0x2124, 0x2126, 0x2128, 0x214e, 0x2183, 0x2184, 0x2d6f,
0x2e2f, 0x3005, 0x3006, 0x303b, 0x303c, 0xa62a, 0xa62b, 0xa790, 0xa791,
0xa8fb, 0xa9cf, 0xaa7a, 0xaab1, 0xaab5, 0xaab6, 0xaac0, 0xaac2, 0xfb1d,
- 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfffe, 0xffff
+ 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44
};
#define NUM_ALPHA_CHAR (sizeof(alphaCharTable)/sizeof(chr))
@@ -478,153 +478,99 @@ static const chr upperCharTable[] = {
*/
static const crange graphRangeTable[] = {
- {0x0021, 0x007e}, {0x00a0, 0x00ac}, {0x00ae, 0x011f}, {0x0121, 0x021f},
- {0x0221, 0x031f}, {0x0321, 0x0377}, {0x037a, 0x037e}, {0x0384, 0x038a},
- {0x038e, 0x03a1}, {0x03a3, 0x041f}, {0x0421, 0x051f}, {0x0521, 0x0527},
- {0x0531, 0x0556}, {0x0559, 0x055f}, {0x0561, 0x0587}, {0x0591, 0x05c7},
- {0x05d0, 0x05ea}, {0x05f0, 0x05f4}, {0x0606, 0x061b}, {0x0621, 0x06dc},
- {0x06de, 0x070d}, {0x0710, 0x071f}, {0x0721, 0x074a}, {0x074d, 0x07b1},
- {0x07c0, 0x07fa}, {0x0800, 0x081f}, {0x0821, 0x082d}, {0x0830, 0x083e},
- {0x0840, 0x085b}, {0x0900, 0x091f}, {0x0921, 0x0977}, {0x0979, 0x097f},
+ {0x0021, 0x007e}, {0x00a0, 0x00ac}, {0x00ae, 0x0377}, {0x037a, 0x037e},
+ {0x0384, 0x038a}, {0x038e, 0x03a1}, {0x03a3, 0x0527}, {0x0531, 0x0556},
+ {0x0559, 0x055f}, {0x0561, 0x0587}, {0x0591, 0x05c7}, {0x05d0, 0x05ea},
+ {0x05f0, 0x05f4}, {0x0606, 0x061b}, {0x061e, 0x06dc}, {0x06de, 0x070d},
+ {0x0710, 0x074a}, {0x074d, 0x07b1}, {0x07c0, 0x07fa}, {0x0800, 0x082d},
+ {0x0830, 0x083e}, {0x0840, 0x085b}, {0x0900, 0x0977}, {0x0979, 0x097f},
{0x0981, 0x0983}, {0x0985, 0x098c}, {0x0993, 0x09a8}, {0x09aa, 0x09b0},
{0x09b6, 0x09b9}, {0x09bc, 0x09c4}, {0x09cb, 0x09ce}, {0x09df, 0x09e3},
- {0x09e6, 0x09fb}, {0x0a01, 0x0a03}, {0x0a05, 0x0a0a}, {0x0a13, 0x0a1f},
- {0x0a21, 0x0a28}, {0x0a2a, 0x0a30}, {0x0a3e, 0x0a42}, {0x0a4b, 0x0a4d},
- {0x0a59, 0x0a5c}, {0x0a66, 0x0a75}, {0x0a81, 0x0a83}, {0x0a85, 0x0a8d},
- {0x0a8f, 0x0a91}, {0x0a93, 0x0aa8}, {0x0aaa, 0x0ab0}, {0x0ab5, 0x0ab9},
- {0x0abc, 0x0ac5}, {0x0ac7, 0x0ac9}, {0x0acb, 0x0acd}, {0x0ae0, 0x0ae3},
- {0x0ae6, 0x0aef}, {0x0b01, 0x0b03}, {0x0b05, 0x0b0c}, {0x0b13, 0x0b1f},
- {0x0b21, 0x0b28}, {0x0b2a, 0x0b30}, {0x0b35, 0x0b39}, {0x0b3c, 0x0b44},
- {0x0b4b, 0x0b4d}, {0x0b5f, 0x0b63}, {0x0b66, 0x0b77}, {0x0b85, 0x0b8a},
- {0x0b8e, 0x0b90}, {0x0b92, 0x0b95}, {0x0ba8, 0x0baa}, {0x0bae, 0x0bb9},
- {0x0bbe, 0x0bc2}, {0x0bc6, 0x0bc8}, {0x0bca, 0x0bcd}, {0x0be6, 0x0bfa},
- {0x0c01, 0x0c03}, {0x0c05, 0x0c0c}, {0x0c0e, 0x0c10}, {0x0c12, 0x0c1f},
- {0x0c21, 0x0c28}, {0x0c2a, 0x0c33}, {0x0c35, 0x0c39}, {0x0c3d, 0x0c44},
- {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, {0x0c60, 0x0c63}, {0x0c66, 0x0c6f},
- {0x0c78, 0x0c7f}, {0x0c85, 0x0c8c}, {0x0c8e, 0x0c90}, {0x0c92, 0x0ca8},
- {0x0caa, 0x0cb3}, {0x0cb5, 0x0cb9}, {0x0cbc, 0x0cc4}, {0x0cc6, 0x0cc8},
- {0x0cca, 0x0ccd}, {0x0ce0, 0x0ce3}, {0x0ce6, 0x0cef}, {0x0d05, 0x0d0c},
- {0x0d0e, 0x0d10}, {0x0d12, 0x0d1f}, {0x0d21, 0x0d3a}, {0x0d3d, 0x0d44},
+ {0x09e6, 0x09fb}, {0x0a01, 0x0a03}, {0x0a05, 0x0a0a}, {0x0a13, 0x0a28},
+ {0x0a2a, 0x0a30}, {0x0a3e, 0x0a42}, {0x0a4b, 0x0a4d}, {0x0a59, 0x0a5c},
+ {0x0a66, 0x0a75}, {0x0a81, 0x0a83}, {0x0a85, 0x0a8d}, {0x0a8f, 0x0a91},
+ {0x0a93, 0x0aa8}, {0x0aaa, 0x0ab0}, {0x0ab5, 0x0ab9}, {0x0abc, 0x0ac5},
+ {0x0ac7, 0x0ac9}, {0x0acb, 0x0acd}, {0x0ae0, 0x0ae3}, {0x0ae6, 0x0aef},
+ {0x0b01, 0x0b03}, {0x0b05, 0x0b0c}, {0x0b13, 0x0b28}, {0x0b2a, 0x0b30},
+ {0x0b35, 0x0b39}, {0x0b3c, 0x0b44}, {0x0b4b, 0x0b4d}, {0x0b5f, 0x0b63},
+ {0x0b66, 0x0b77}, {0x0b85, 0x0b8a}, {0x0b8e, 0x0b90}, {0x0b92, 0x0b95},
+ {0x0ba8, 0x0baa}, {0x0bae, 0x0bb9}, {0x0bbe, 0x0bc2}, {0x0bc6, 0x0bc8},
+ {0x0bca, 0x0bcd}, {0x0be6, 0x0bfa}, {0x0c01, 0x0c03}, {0x0c05, 0x0c0c},
+ {0x0c0e, 0x0c10}, {0x0c12, 0x0c28}, {0x0c2a, 0x0c33}, {0x0c35, 0x0c39},
+ {0x0c3d, 0x0c44}, {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, {0x0c60, 0x0c63},
+ {0x0c66, 0x0c6f}, {0x0c78, 0x0c7f}, {0x0c85, 0x0c8c}, {0x0c8e, 0x0c90},
+ {0x0c92, 0x0ca8}, {0x0caa, 0x0cb3}, {0x0cb5, 0x0cb9}, {0x0cbc, 0x0cc4},
+ {0x0cc6, 0x0cc8}, {0x0cca, 0x0ccd}, {0x0ce0, 0x0ce3}, {0x0ce6, 0x0cef},
+ {0x0d05, 0x0d0c}, {0x0d0e, 0x0d10}, {0x0d12, 0x0d3a}, {0x0d3d, 0x0d44},
{0x0d46, 0x0d48}, {0x0d4a, 0x0d4e}, {0x0d60, 0x0d63}, {0x0d66, 0x0d75},
{0x0d79, 0x0d7f}, {0x0d85, 0x0d96}, {0x0d9a, 0x0db1}, {0x0db3, 0x0dbb},
{0x0dc0, 0x0dc6}, {0x0dcf, 0x0dd4}, {0x0dd8, 0x0ddf}, {0x0df2, 0x0df4},
- {0x0e01, 0x0e1f}, {0x0e21, 0x0e3a}, {0x0e3f, 0x0e5b}, {0x0e94, 0x0e97},
- {0x0e99, 0x0e9f}, {0x0ea1, 0x0ea3}, {0x0ead, 0x0eb9}, {0x0ebb, 0x0ebd},
- {0x0ec0, 0x0ec4}, {0x0ec8, 0x0ecd}, {0x0ed0, 0x0ed9}, {0x0f00, 0x0f1f},
- {0x0f21, 0x0f47}, {0x0f49, 0x0f6c}, {0x0f71, 0x0f97}, {0x0f99, 0x0fbc},
- {0x0fbe, 0x0fcc}, {0x0fce, 0x0fda}, {0x1000, 0x101f}, {0x1021, 0x10c5},
- {0x10d0, 0x10fc}, {0x1100, 0x111f}, {0x1121, 0x121f}, {0x1221, 0x1248},
- {0x124a, 0x124d}, {0x1250, 0x1256}, {0x125a, 0x125d}, {0x1260, 0x1288},
- {0x128a, 0x128d}, {0x1290, 0x12b0}, {0x12b2, 0x12b5}, {0x12b8, 0x12be},
- {0x12c2, 0x12c5}, {0x12c8, 0x12d6}, {0x12d8, 0x1310}, {0x1312, 0x1315},
- {0x1318, 0x131f}, {0x1321, 0x135a}, {0x135d, 0x137c}, {0x1380, 0x1399},
- {0x13a0, 0x13f4}, {0x1400, 0x141f}, {0x1421, 0x151f}, {0x1521, 0x161f},
- {0x1621, 0x169c}, {0x16a0, 0x16f0}, {0x1700, 0x170c}, {0x170e, 0x1714},
- {0x1721, 0x1736}, {0x1740, 0x1753}, {0x1760, 0x176c}, {0x176e, 0x1770},
- {0x1780, 0x17b3}, {0x17b6, 0x17dd}, {0x17e0, 0x17e9}, {0x17f0, 0x17f9},
- {0x1800, 0x180e}, {0x1810, 0x1819}, {0x1821, 0x1877}, {0x1880, 0x18aa},
- {0x18b0, 0x18f5}, {0x1900, 0x191c}, {0x1921, 0x192b}, {0x1930, 0x193b},
- {0x1944, 0x196d}, {0x1970, 0x1974}, {0x1980, 0x19ab}, {0x19b0, 0x19c9},
- {0x19d0, 0x19da}, {0x19de, 0x1a1b}, {0x1a21, 0x1a5e}, {0x1a60, 0x1a7c},
- {0x1a7f, 0x1a89}, {0x1a90, 0x1a99}, {0x1aa0, 0x1aad}, {0x1b00, 0x1b1f},
- {0x1b21, 0x1b4b}, {0x1b50, 0x1b7c}, {0x1b80, 0x1baa}, {0x1bae, 0x1bb9},
- {0x1bc0, 0x1bf3}, {0x1bfc, 0x1c1f}, {0x1c21, 0x1c37}, {0x1c3b, 0x1c49},
- {0x1c4d, 0x1c7f}, {0x1cd0, 0x1cf2}, {0x1d00, 0x1d1f}, {0x1d21, 0x1de6},
- {0x1dfc, 0x1e1f}, {0x1e21, 0x1f15}, {0x1f18, 0x1f1d}, {0x1f21, 0x1f45},
- {0x1f48, 0x1f4d}, {0x1f50, 0x1f57}, {0x1f5f, 0x1f7d}, {0x1f80, 0x1fb4},
- {0x1fb6, 0x1fc4}, {0x1fc6, 0x1fd3}, {0x1fd6, 0x1fdb}, {0x1fdd, 0x1fef},
- {0x1ff2, 0x1ff4}, {0x1ff6, 0x1ffe}, {0x2000, 0x200a}, {0x2010, 0x201f},
- {0x2021, 0x2029}, {0x202f, 0x205f}, {0x2074, 0x208e}, {0x2090, 0x209c},
- {0x20a0, 0x20b9}, {0x20d0, 0x20f0}, {0x2100, 0x211f}, {0x2121, 0x2189},
- {0x2190, 0x221f}, {0x2221, 0x231f}, {0x2321, 0x23f3}, {0x2400, 0x241f},
- {0x2421, 0x2426}, {0x2440, 0x244a}, {0x2460, 0x251f}, {0x2521, 0x261f},
- {0x2621, 0x26ff}, {0x2701, 0x271f}, {0x2721, 0x27ca}, {0x27ce, 0x281f},
- {0x2821, 0x291f}, {0x2921, 0x2a1f}, {0x2a21, 0x2b1f}, {0x2b21, 0x2b4c},
- {0x2b50, 0x2b59}, {0x2c00, 0x2c1f}, {0x2c21, 0x2c2e}, {0x2c30, 0x2c5e},
- {0x2c60, 0x2cf1}, {0x2cf9, 0x2d1f}, {0x2d21, 0x2d25}, {0x2d30, 0x2d65},
- {0x2d7f, 0x2d96}, {0x2da0, 0x2da6}, {0x2da8, 0x2dae}, {0x2db0, 0x2db6},
- {0x2db8, 0x2dbe}, {0x2dc0, 0x2dc6}, {0x2dc8, 0x2dce}, {0x2dd0, 0x2dd6},
- {0x2dd8, 0x2dde}, {0x2de0, 0x2e1f}, {0x2e21, 0x2e31}, {0x2e80, 0x2e99},
- {0x2e9b, 0x2ef3}, {0x2f00, 0x2f1f}, {0x2f21, 0x2fd5}, {0x2ff0, 0x2ffb},
- {0x3000, 0x301f}, {0x3021, 0x303f}, {0x3041, 0x3096}, {0x3099, 0x30ff},
- {0x3105, 0x311f}, {0x3121, 0x312d}, {0x3131, 0x318e}, {0x3190, 0x31ba},
- {0x31c0, 0x31e3}, {0x31f0, 0x321e}, {0x3221, 0x32fe}, {0x3300, 0x331f},
- {0x3321, 0x341f}, {0x3421, 0x351f}, {0x3521, 0x361f}, {0x3621, 0x371f},
- {0x3721, 0x381f}, {0x3821, 0x391f}, {0x3921, 0x3a1f}, {0x3a21, 0x3b1f},
- {0x3b21, 0x3c1f}, {0x3c21, 0x3d1f}, {0x3d21, 0x3e1f}, {0x3e21, 0x3f1f},
- {0x3f21, 0x401f}, {0x4021, 0x411f}, {0x4121, 0x421f}, {0x4221, 0x431f},
- {0x4321, 0x441f}, {0x4421, 0x451f}, {0x4521, 0x461f}, {0x4621, 0x471f},
- {0x4721, 0x481f}, {0x4821, 0x491f}, {0x4921, 0x4a1f}, {0x4a21, 0x4b1f},
- {0x4b21, 0x4c1f}, {0x4c21, 0x4d1f}, {0x4d21, 0x4db5}, {0x4dc0, 0x4e1f},
- {0x4e21, 0x4f1f}, {0x4f21, 0x501f}, {0x5021, 0x511f}, {0x5121, 0x521f},
- {0x5221, 0x531f}, {0x5321, 0x541f}, {0x5421, 0x551f}, {0x5521, 0x561f},
- {0x5621, 0x571f}, {0x5721, 0x581f}, {0x5821, 0x591f}, {0x5921, 0x5a1f},
- {0x5a21, 0x5b1f}, {0x5b21, 0x5c1f}, {0x5c21, 0x5d1f}, {0x5d21, 0x5e1f},
- {0x5e21, 0x5f1f}, {0x5f21, 0x601f}, {0x6021, 0x611f}, {0x6121, 0x621f},
- {0x6221, 0x631f}, {0x6321, 0x641f}, {0x6421, 0x651f}, {0x6521, 0x661f},
- {0x6621, 0x671f}, {0x6721, 0x681f}, {0x6821, 0x691f}, {0x6921, 0x6a1f},
- {0x6a21, 0x6b1f}, {0x6b21, 0x6c1f}, {0x6c21, 0x6d1f}, {0x6d21, 0x6e1f},
- {0x6e21, 0x6f1f}, {0x6f21, 0x701f}, {0x7021, 0x711f}, {0x7121, 0x721f},
- {0x7221, 0x731f}, {0x7321, 0x741f}, {0x7421, 0x751f}, {0x7521, 0x761f},
- {0x7621, 0x771f}, {0x7721, 0x781f}, {0x7821, 0x791f}, {0x7921, 0x7a1f},
- {0x7a21, 0x7b1f}, {0x7b21, 0x7c1f}, {0x7c21, 0x7d1f}, {0x7d21, 0x7e1f},
- {0x7e21, 0x7f1f}, {0x7f21, 0x801f}, {0x8021, 0x811f}, {0x8121, 0x821f},
- {0x8221, 0x831f}, {0x8321, 0x841f}, {0x8421, 0x851f}, {0x8521, 0x861f},
- {0x8621, 0x871f}, {0x8721, 0x881f}, {0x8821, 0x891f}, {0x8921, 0x8a1f},
- {0x8a21, 0x8b1f}, {0x8b21, 0x8c1f}, {0x8c21, 0x8d1f}, {0x8d21, 0x8e1f},
- {0x8e21, 0x8f1f}, {0x8f21, 0x901f}, {0x9021, 0x911f}, {0x9121, 0x921f},
- {0x9221, 0x931f}, {0x9321, 0x941f}, {0x9421, 0x951f}, {0x9521, 0x961f},
- {0x9621, 0x971f}, {0x9721, 0x981f}, {0x9821, 0x991f}, {0x9921, 0x9a1f},
- {0x9a21, 0x9b1f}, {0x9b21, 0x9c1f}, {0x9c21, 0x9d1f}, {0x9d21, 0x9e1f},
- {0x9e21, 0x9f1f}, {0x9f21, 0x9fcb}, {0xa000, 0xa01f}, {0xa021, 0xa11f},
- {0xa121, 0xa21f}, {0xa221, 0xa31f}, {0xa321, 0xa41f}, {0xa421, 0xa48c},
- {0xa490, 0xa4c6}, {0xa4d0, 0xa51f}, {0xa521, 0xa61f}, {0xa621, 0xa62b},
- {0xa640, 0xa673}, {0xa67c, 0xa697}, {0xa6a0, 0xa6f7}, {0xa700, 0xa71f},
- {0xa721, 0xa78e}, {0xa7a0, 0xa7a9}, {0xa7fa, 0xa81f}, {0xa821, 0xa82b},
- {0xa830, 0xa839}, {0xa840, 0xa877}, {0xa880, 0xa8c4}, {0xa8ce, 0xa8d9},
- {0xa8e0, 0xa8fb}, {0xa900, 0xa91f}, {0xa921, 0xa953}, {0xa95f, 0xa97c},
- {0xa980, 0xa9cd}, {0xa9cf, 0xa9d9}, {0xaa00, 0xaa1f}, {0xaa21, 0xaa36},
- {0xaa40, 0xaa4d}, {0xaa50, 0xaa59}, {0xaa5c, 0xaa7b}, {0xaa80, 0xaac2},
- {0xaadb, 0xaadf}, {0xab01, 0xab06}, {0xab09, 0xab0e}, {0xab11, 0xab16},
- {0xab21, 0xab26}, {0xab28, 0xab2e}, {0xabc0, 0xabed}, {0xabf0, 0xabf9},
- {0xac00, 0xac1f}, {0xac21, 0xad1f}, {0xad21, 0xae1f}, {0xae21, 0xaf1f},
- {0xaf21, 0xb01f}, {0xb021, 0xb11f}, {0xb121, 0xb21f}, {0xb221, 0xb31f},
- {0xb321, 0xb41f}, {0xb421, 0xb51f}, {0xb521, 0xb61f}, {0xb621, 0xb71f},
- {0xb721, 0xb81f}, {0xb821, 0xb91f}, {0xb921, 0xba1f}, {0xba21, 0xbb1f},
- {0xbb21, 0xbc1f}, {0xbc21, 0xbd1f}, {0xbd21, 0xbe1f}, {0xbe21, 0xbf1f},
- {0xbf21, 0xc01f}, {0xc021, 0xc11f}, {0xc121, 0xc21f}, {0xc221, 0xc31f},
- {0xc321, 0xc41f}, {0xc421, 0xc51f}, {0xc521, 0xc61f}, {0xc621, 0xc71f},
- {0xc721, 0xc81f}, {0xc821, 0xc91f}, {0xc921, 0xca1f}, {0xca21, 0xcb1f},
- {0xcb21, 0xcc1f}, {0xcc21, 0xcd1f}, {0xcd21, 0xce1f}, {0xce21, 0xcf1f},
- {0xcf21, 0xd01f}, {0xd021, 0xd11f}, {0xd121, 0xd21f}, {0xd221, 0xd31f},
- {0xd321, 0xd41f}, {0xd421, 0xd51f}, {0xd521, 0xd61f}, {0xd621, 0xd71f},
- {0xd721, 0xd7a3}, {0xd7b0, 0xd7c6}, {0xd7cb, 0xd7fb}, {0xf900, 0xf91f},
- {0xf921, 0xfa1f}, {0xfa21, 0xfa2d}, {0xfa30, 0xfa6d}, {0xfa70, 0xfad9},
- {0xfb00, 0xfb06}, {0xfb13, 0xfb17}, {0xfb1d, 0xfb1f}, {0xfb21, 0xfb36},
- {0xfb38, 0xfb3c}, {0xfb46, 0xfbc1}, {0xfbd3, 0xfc1f}, {0xfc21, 0xfd1f},
- {0xfd21, 0xfd3f}, {0xfd50, 0xfd8f}, {0xfd92, 0xfdc7}, {0xfdf0, 0xfdfd},
- {0xfe00, 0xfe19}, {0xfe21, 0xfe26}, {0xfe30, 0xfe52}, {0xfe54, 0xfe66},
- {0xfe68, 0xfe6b}, {0xfe70, 0xfe74}, {0xfe76, 0xfefc}, {0xff01, 0xff1f},
- {0xff21, 0xffbe}, {0xffc2, 0xffc7}, {0xffca, 0xffcf}, {0xffd2, 0xffd7},
- {0xffda, 0xffdc}, {0xffe0, 0xffe6}, {0xffe8, 0xffee}, {0xfffc, 0xffff}
+ {0x0e01, 0x0e3a}, {0x0e3f, 0x0e5b}, {0x0e94, 0x0e97}, {0x0e99, 0x0e9f},
+ {0x0ea1, 0x0ea3}, {0x0ead, 0x0eb9}, {0x0ebb, 0x0ebd}, {0x0ec0, 0x0ec4},
+ {0x0ec8, 0x0ecd}, {0x0ed0, 0x0ed9}, {0x0f00, 0x0f47}, {0x0f49, 0x0f6c},
+ {0x0f71, 0x0f97}, {0x0f99, 0x0fbc}, {0x0fbe, 0x0fcc}, {0x0fce, 0x0fda},
+ {0x1000, 0x10c5}, {0x10d0, 0x10fc}, {0x1100, 0x1248}, {0x124a, 0x124d},
+ {0x1250, 0x1256}, {0x125a, 0x125d}, {0x1260, 0x1288}, {0x128a, 0x128d},
+ {0x1290, 0x12b0}, {0x12b2, 0x12b5}, {0x12b8, 0x12be}, {0x12c2, 0x12c5},
+ {0x12c8, 0x12d6}, {0x12d8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135a},
+ {0x135d, 0x137c}, {0x1380, 0x1399}, {0x13a0, 0x13f4}, {0x1400, 0x169c},
+ {0x16a0, 0x16f0}, {0x1700, 0x170c}, {0x170e, 0x1714}, {0x1720, 0x1736},
+ {0x1740, 0x1753}, {0x1760, 0x176c}, {0x176e, 0x1770}, {0x1780, 0x17b3},
+ {0x17b6, 0x17dd}, {0x17e0, 0x17e9}, {0x17f0, 0x17f9}, {0x1800, 0x180e},
+ {0x1810, 0x1819}, {0x1820, 0x1877}, {0x1880, 0x18aa}, {0x18b0, 0x18f5},
+ {0x1900, 0x191c}, {0x1920, 0x192b}, {0x1930, 0x193b}, {0x1944, 0x196d},
+ {0x1970, 0x1974}, {0x1980, 0x19ab}, {0x19b0, 0x19c9}, {0x19d0, 0x19da},
+ {0x19de, 0x1a1b}, {0x1a1e, 0x1a5e}, {0x1a60, 0x1a7c}, {0x1a7f, 0x1a89},
+ {0x1a90, 0x1a99}, {0x1aa0, 0x1aad}, {0x1b00, 0x1b4b}, {0x1b50, 0x1b7c},
+ {0x1b80, 0x1baa}, {0x1bae, 0x1bb9}, {0x1bc0, 0x1bf3}, {0x1bfc, 0x1c37},
+ {0x1c3b, 0x1c49}, {0x1c4d, 0x1c7f}, {0x1cd0, 0x1cf2}, {0x1d00, 0x1de6},
+ {0x1dfc, 0x1f15}, {0x1f18, 0x1f1d}, {0x1f20, 0x1f45}, {0x1f48, 0x1f4d},
+ {0x1f50, 0x1f57}, {0x1f5f, 0x1f7d}, {0x1f80, 0x1fb4}, {0x1fb6, 0x1fc4},
+ {0x1fc6, 0x1fd3}, {0x1fd6, 0x1fdb}, {0x1fdd, 0x1fef}, {0x1ff2, 0x1ff4},
+ {0x1ff6, 0x1ffe}, {0x2000, 0x200a}, {0x2010, 0x2029}, {0x202f, 0x205f},
+ {0x2074, 0x208e}, {0x2090, 0x209c}, {0x20a0, 0x20b9}, {0x20d0, 0x20f0},
+ {0x2100, 0x2189}, {0x2190, 0x23f3}, {0x2400, 0x2426}, {0x2440, 0x244a},
+ {0x2460, 0x26ff}, {0x2701, 0x27ca}, {0x27ce, 0x2b4c}, {0x2b50, 0x2b59},
+ {0x2c00, 0x2c2e}, {0x2c30, 0x2c5e}, {0x2c60, 0x2cf1}, {0x2cf9, 0x2d25},
+ {0x2d30, 0x2d65}, {0x2d7f, 0x2d96}, {0x2da0, 0x2da6}, {0x2da8, 0x2dae},
+ {0x2db0, 0x2db6}, {0x2db8, 0x2dbe}, {0x2dc0, 0x2dc6}, {0x2dc8, 0x2dce},
+ {0x2dd0, 0x2dd6}, {0x2dd8, 0x2dde}, {0x2de0, 0x2e31}, {0x2e80, 0x2e99},
+ {0x2e9b, 0x2ef3}, {0x2f00, 0x2fd5}, {0x2ff0, 0x2ffb}, {0x3000, 0x303f},
+ {0x3041, 0x3096}, {0x3099, 0x30ff}, {0x3105, 0x312d}, {0x3131, 0x318e},
+ {0x3190, 0x31ba}, {0x31c0, 0x31e3}, {0x31f0, 0x321e}, {0x3220, 0x32fe},
+ {0x3300, 0x4db5}, {0x4dc0, 0x9fcb}, {0xa000, 0xa48c}, {0xa490, 0xa4c6},
+ {0xa4d0, 0xa62b}, {0xa640, 0xa673}, {0xa67c, 0xa697}, {0xa6a0, 0xa6f7},
+ {0xa700, 0xa78e}, {0xa7a0, 0xa7a9}, {0xa7fa, 0xa82b}, {0xa830, 0xa839},
+ {0xa840, 0xa877}, {0xa880, 0xa8c4}, {0xa8ce, 0xa8d9}, {0xa8e0, 0xa8fb},
+ {0xa900, 0xa953}, {0xa95f, 0xa97c}, {0xa980, 0xa9cd}, {0xa9cf, 0xa9d9},
+ {0xaa00, 0xaa36}, {0xaa40, 0xaa4d}, {0xaa50, 0xaa59}, {0xaa5c, 0xaa7b},
+ {0xaa80, 0xaac2}, {0xaadb, 0xaadf}, {0xab01, 0xab06}, {0xab09, 0xab0e},
+ {0xab11, 0xab16}, {0xab20, 0xab26}, {0xab28, 0xab2e}, {0xabc0, 0xabed},
+ {0xabf0, 0xabf9}, {0xac00, 0xd7a3}, {0xd7b0, 0xd7c6}, {0xd7cb, 0xd7fb},
+ {0xf900, 0xfa2d}, {0xfa30, 0xfa6d}, {0xfa70, 0xfad9}, {0xfb00, 0xfb06},
+ {0xfb13, 0xfb17}, {0xfb1d, 0xfb36}, {0xfb38, 0xfb3c}, {0xfb46, 0xfbc1},
+ {0xfbd3, 0xfd3f}, {0xfd50, 0xfd8f}, {0xfd92, 0xfdc7}, {0xfdf0, 0xfdfd},
+ {0xfe00, 0xfe19}, {0xfe20, 0xfe26}, {0xfe30, 0xfe52}, {0xfe54, 0xfe66},
+ {0xfe68, 0xfe6b}, {0xfe70, 0xfe74}, {0xfe76, 0xfefc}, {0xff01, 0xffbe},
+ {0xffc2, 0xffc7}, {0xffca, 0xffcf}, {0xffd2, 0xffd7}, {0xffda, 0xffdc},
+ {0xffe0, 0xffe6}, {0xffe8, 0xffee}
};
#define NUM_GRAPH_RANGE (sizeof(graphRangeTable)/sizeof(crange))
static const chr graphCharTable[] = {
- 0x038c, 0x0589, 0x058a, 0x061e, 0x061f, 0x085e, 0x098f, 0x0990, 0x09b2,
- 0x09c7, 0x09c8, 0x09d7, 0x09dc, 0x09dd, 0x0a0f, 0x0a10, 0x0a32, 0x0a33,
- 0x0a35, 0x0a36, 0x0a38, 0x0a39, 0x0a3c, 0x0a47, 0x0a48, 0x0a51, 0x0a5e,
- 0x0ab2, 0x0ab3, 0x0ad0, 0x0af1, 0x0b0f, 0x0b10, 0x0b32, 0x0b33, 0x0b47,
- 0x0b48, 0x0b56, 0x0b57, 0x0b5c, 0x0b5d, 0x0b82, 0x0b83, 0x0b99, 0x0b9a,
- 0x0b9c, 0x0b9e, 0x0b9f, 0x0ba3, 0x0ba4, 0x0bd0, 0x0bd7, 0x0c55, 0x0c56,
- 0x0c58, 0x0c59, 0x0c82, 0x0c83, 0x0cd5, 0x0cd6, 0x0cde, 0x0cf1, 0x0cf2,
- 0x0d02, 0x0d03, 0x0d57, 0x0d82, 0x0d83, 0x0dbd, 0x0dca, 0x0dd6, 0x0e81,
- 0x0e82, 0x0e84, 0x0e87, 0x0e88, 0x0e8a, 0x0e8d, 0x0ea5, 0x0ea7, 0x0eaa,
- 0x0eab, 0x0ec6, 0x0edc, 0x0edd, 0x1258, 0x12c0, 0x1772, 0x1773, 0x1940,
- 0x1a1e, 0x1a1f, 0x1f59, 0x1f5b, 0x1f5d, 0x2070, 0x2071, 0x27cc, 0x2d6f,
- 0x2d70, 0xa790, 0xa791, 0xa9de, 0xa9df, 0xfb3e, 0xfb40, 0xfb41, 0xfb43,
- 0xfb44
+ 0x038c, 0x0589, 0x058a, 0x085e, 0x098f, 0x0990, 0x09b2, 0x09c7, 0x09c8,
+ 0x09d7, 0x09dc, 0x09dd, 0x0a0f, 0x0a10, 0x0a32, 0x0a33, 0x0a35, 0x0a36,
+ 0x0a38, 0x0a39, 0x0a3c, 0x0a47, 0x0a48, 0x0a51, 0x0a5e, 0x0ab2, 0x0ab3,
+ 0x0ad0, 0x0af1, 0x0b0f, 0x0b10, 0x0b32, 0x0b33, 0x0b47, 0x0b48, 0x0b56,
+ 0x0b57, 0x0b5c, 0x0b5d, 0x0b82, 0x0b83, 0x0b99, 0x0b9a, 0x0b9c, 0x0b9e,
+ 0x0b9f, 0x0ba3, 0x0ba4, 0x0bd0, 0x0bd7, 0x0c55, 0x0c56, 0x0c58, 0x0c59,
+ 0x0c82, 0x0c83, 0x0cd5, 0x0cd6, 0x0cde, 0x0cf1, 0x0cf2, 0x0d02, 0x0d03,
+ 0x0d57, 0x0d82, 0x0d83, 0x0dbd, 0x0dca, 0x0dd6, 0x0e81, 0x0e82, 0x0e84,
+ 0x0e87, 0x0e88, 0x0e8a, 0x0e8d, 0x0ea5, 0x0ea7, 0x0eaa, 0x0eab, 0x0ec6,
+ 0x0edc, 0x0edd, 0x1258, 0x12c0, 0x1772, 0x1773, 0x1940, 0x1f59, 0x1f5b,
+ 0x1f5d, 0x2070, 0x2071, 0x27cc, 0x2d6f, 0x2d70, 0xa790, 0xa791, 0xa9de,
+ 0xa9df, 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfffc, 0xfffd
};
#define NUM_GRAPH_CHAR (sizeof(graphCharTable)/sizeof(chr))
diff --git a/generic/tclUniData.c b/generic/tclUniData.c
index a06e84d..896580c 100644
--- a/generic/tclUniData.c
+++ b/generic/tclUniData.c
@@ -775,7 +775,7 @@ static const int groups[] = {
* Unicode character.
*/
-#define UNICODE_CATEGORY_MASK 0x1F
+#define UNICODE_CATEGORY_MASK 0x1f
#define UNICODE_OUT_OF_RANGE 0x10000u
enum {
@@ -817,8 +817,8 @@ enum {
* to do sign extension on right shifts.
*/
-#define GetCaseType(info) (((info) & 0xE0) >> 5)
-#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1F)
+#define GetCaseType(info) (((info) & 0xe0) >> 5)
+#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1f)
#define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15)))
/*
diff --git a/generic/tclUtf.c b/generic/tclUtf.c
index 00b652e..40f6272 100644
--- a/generic/tclUtf.c
+++ b/generic/tclUtf.c
@@ -1424,7 +1424,7 @@ Tcl_UniCharIsGraph(
int ch) /* Unicode character to test. */
{
register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK);
- return (((PRINT_BITS >> category) & 1) && ((unsigned char) ch != ' '));
+ return (((PRINT_BITS >> category) & 1) && (ch != ' '));
}
/*
diff --git a/tests/utf.test b/tests/utf.test
index 60e3d52..273773b 100644
--- a/tests/utf.test
+++ b/tests/utf.test
@@ -27,9 +27,12 @@ test utf-1.3 {Tcl_UniCharToUtf: 2 byte sequences} {
test utf-1.4 {Tcl_UniCharToUtf: 3 byte sequences} {
set x "\u4e4e"
} [bytestring "\xe4\xb9\x8e"]
-test utf-1.5 {Tcl_UniCharToUtf: negative Tcl_UniChar} {
- string length [format %c -1]
-} 1
+test utf-1.5 {Tcl_UniCharToUtf: overflowed Tcl_UniChar} {
+ format %c 0x110000
+} [bytestring "\xef\xbf\xbd"]
+test utf-1.6 {Tcl_UniCharToUtf: negative Tcl_UniChar} {
+ format %c -1
+} [bytestring "\xef\xbf\xbd"]
test utf-2.1 {Tcl_UtfToUniChar: low ascii} {
string length "abc"
@@ -286,6 +289,14 @@ test utf-21.3 {unicode print char in regc_locale.c} {
# this returns 1 with Unicode 6 compliance
regexp {^[[:print:]]+$} \ufbc1
} 1
+test utf-21.4 {TclUniCharIsGraph} {
+ # [Bug 3464428]
+ string is graph \u0120
+} {1}
+test utf-21.5 {unicode graph char in regc_locale.c} {
+ # [Bug 3464428]
+ regexp {^[[:graph:]]+$} \u0120
+} {1}
test utf-22.1 {TclUniCharIsWordChar} {
string wordend "xyz123_bar fg" 0
diff --git a/tools/uniParse.tcl b/tools/uniParse.tcl
index 6850897..a13bae6 100644
--- a/tools/uniParse.tcl
+++ b/tools/uniParse.tcl
@@ -30,45 +30,39 @@ namespace eval uni {
Cc Cf Co Cs Pc Pd Ps Pe Pi Pf Po Sm Sc Sk So
}; # Ordered list of character categories, must
# match the enumeration in the header file.
-
- variable titleCount 0; # Count of the number of title case
- # characters. This value is used in the
- # regular expression code to allocate enough
- # space for the title case variants.
}
proc uni::getValue {items index} {
variable categories
- variable titleCount
# Extract character info
set category [lindex $items 2]
- if {[scan [lindex $items 12] %6x toupper] == 1} {
+ if {[scan [lindex $items 12] %x toupper] == 1} {
set toupper [expr {$index - $toupper}]
} else {
set toupper 0
}
- if {[scan [lindex $items 13] %6x tolower] == 1} {
+ if {[scan [lindex $items 13] %x tolower] == 1} {
set tolower [expr {$tolower - $index}]
} else {
set tolower 0
}
- if {[scan [lindex $items 14] %6x totitle] == 1} {
+ if {[scan [lindex $items 14] %x totitle] == 1} {
set totitle [expr {$index - $totitle}]
- } else {
+ } elseif {$tolower} {
set totitle 0
+ } else {
+ set totitle $toupper
}
set categoryIndex [lsearch -exact $categories $category]
if {$categoryIndex < 0} {
puts "Unexpected character category: $index($category)"
set categoryIndex 0
- } elseif {$category eq "Lt"} {
- incr titleCount
}
- return "$categoryIndex,$toupper,$tolower,$totitle"
+ return [list $categoryIndex $toupper $tolower $totitle]
}
proc uni::getGroup {value} {
@@ -100,7 +94,7 @@ proc uni::buildTables {data} {
variable pMap {}
variable pages {}
- variable groups {{0,0,0,0}}
+ variable groups {{0 0 0 0}}
variable next 0
set info {} ;# temporary page info
@@ -120,7 +114,7 @@ proc uni::buildTables {data} {
set items [split $line \;]
scan [lindex $items 0] %x index
- if {$index > 0xFFFF} then {
+ if {$index > 0xffff} then {
# Ignore non-BMP characters, as long as Tcl doesn't support them
continue
}
@@ -171,7 +165,6 @@ proc uni::main {} {
variable pages
variable groups
variable shift
- variable titleCount
variable next
if {$argc != 2} {
@@ -186,7 +179,6 @@ proc uni::main {} {
puts "X = [llength $pMap] Y= [llength $pages] A= [llength $groups]"
set size [expr {[llength $pMap]*2 + [llength $pages]*(1<<$shift)}]
puts "shift = $shift, space = $size"
- puts "title case count = $titleCount"
set f [open [file join [lindex $argv 1] tclUniData.c] w]
fconfigure $f -translation lf
@@ -280,7 +272,7 @@ static const int groups\[\] = {"
set line " "
set last [expr {[llength $groups] - 1}]
for {set i 0} {$i <= $last} {incr i} {
- foreach {type toupper tolower totitle} [split [lindex $groups $i] ,] {}
+ foreach {type toupper tolower totitle} [lindex $groups $i] {}
# Compute the case conversion type and delta
@@ -329,9 +321,9 @@ static const int groups\[\] = {"
* Unicode character.
*/
-#define UNICODE_CATEGORY_MASK 0x1F
+#define UNICODE_CATEGORY_MASK 0x1f
#define UNICODE_OUT_OF_RANGE "
- puts $f [format 0x%Xu $next]
+ puts $f [format 0x%xu $next]
puts $f "
enum {
UNASSIGNED,
@@ -372,8 +364,8 @@ enum {
* to do sign extension on right shifts.
*/
-#define GetCaseType(info) (((info) & 0xE0) >> 5)
-#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1F)
+#define GetCaseType(info) (((info) & 0xe0) >> 5)
+#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1f)
#define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15)))
/*