diff options
| -rw-r--r-- | ChangeLog | 9 | ||||
| -rw-r--r-- | generic/regc_locale.c | 222 | ||||
| -rw-r--r-- | generic/tclUniData.c | 6 | ||||
| -rw-r--r-- | generic/tclUtf.c | 2 | ||||
| -rw-r--r-- | tests/utf.test | 17 | ||||
| -rw-r--r-- | tools/uniParse.tcl | 36 |
6 files changed, 125 insertions, 167 deletions
@@ -1,3 +1,12 @@ +2011-12-23 Jan Nijtmans <nijtmans@users.sf.net> + + * generic/tclUtf.c: [Bug 3464428] string is graph \u0120 is wrong + * generic/tclUniData.c: + * generic/regc_locale.c: + * tests/utf.test: + * tools/uniParse.tcl: clean up some unused stuff, and be more robust + against changes in UnicodeData.txt syntax + 2011-12-11 Jan Nijtmans <nijtmans@users.sf.net> * generic/regc_locale.c: [Bug 3457031]: Some Unicode 6.0 chars not diff --git a/generic/regc_locale.c b/generic/regc_locale.c index f393820..f652f0e 100644 --- a/generic/regc_locale.c +++ b/generic/regc_locale.c @@ -218,7 +218,7 @@ static const chr alphaCharTable[] = { 0x2107, 0x2115, 0x2124, 0x2126, 0x2128, 0x214e, 0x2183, 0x2184, 0x2d6f, 0x2e2f, 0x3005, 0x3006, 0x303b, 0x303c, 0xa62a, 0xa62b, 0xa790, 0xa791, 0xa8fb, 0xa9cf, 0xaa7a, 0xaab1, 0xaab5, 0xaab6, 0xaac0, 0xaac2, 0xfb1d, - 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfffe, 0xffff + 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44 }; #define NUM_ALPHA_CHAR (sizeof(alphaCharTable)/sizeof(chr)) @@ -478,153 +478,99 @@ static const chr upperCharTable[] = { */ static const crange graphRangeTable[] = { - {0x0021, 0x007e}, {0x00a0, 0x00ac}, {0x00ae, 0x011f}, {0x0121, 0x021f}, - {0x0221, 0x031f}, {0x0321, 0x0377}, {0x037a, 0x037e}, {0x0384, 0x038a}, - {0x038e, 0x03a1}, {0x03a3, 0x041f}, {0x0421, 0x051f}, {0x0521, 0x0527}, - {0x0531, 0x0556}, {0x0559, 0x055f}, {0x0561, 0x0587}, {0x0591, 0x05c7}, - {0x05d0, 0x05ea}, {0x05f0, 0x05f4}, {0x0606, 0x061b}, {0x0621, 0x06dc}, - {0x06de, 0x070d}, {0x0710, 0x071f}, {0x0721, 0x074a}, {0x074d, 0x07b1}, - {0x07c0, 0x07fa}, {0x0800, 0x081f}, {0x0821, 0x082d}, {0x0830, 0x083e}, - {0x0840, 0x085b}, {0x0900, 0x091f}, {0x0921, 0x0977}, {0x0979, 0x097f}, + {0x0021, 0x007e}, {0x00a0, 0x00ac}, {0x00ae, 0x0377}, {0x037a, 0x037e}, + {0x0384, 0x038a}, {0x038e, 0x03a1}, {0x03a3, 0x0527}, {0x0531, 0x0556}, + {0x0559, 0x055f}, {0x0561, 0x0587}, {0x0591, 0x05c7}, {0x05d0, 0x05ea}, + {0x05f0, 0x05f4}, {0x0606, 0x061b}, {0x061e, 0x06dc}, {0x06de, 0x070d}, + {0x0710, 0x074a}, {0x074d, 0x07b1}, {0x07c0, 0x07fa}, {0x0800, 0x082d}, + {0x0830, 0x083e}, {0x0840, 0x085b}, {0x0900, 0x0977}, {0x0979, 0x097f}, {0x0981, 0x0983}, {0x0985, 0x098c}, {0x0993, 0x09a8}, {0x09aa, 0x09b0}, {0x09b6, 0x09b9}, {0x09bc, 0x09c4}, {0x09cb, 0x09ce}, {0x09df, 0x09e3}, - {0x09e6, 0x09fb}, {0x0a01, 0x0a03}, {0x0a05, 0x0a0a}, {0x0a13, 0x0a1f}, - {0x0a21, 0x0a28}, {0x0a2a, 0x0a30}, {0x0a3e, 0x0a42}, {0x0a4b, 0x0a4d}, - {0x0a59, 0x0a5c}, {0x0a66, 0x0a75}, {0x0a81, 0x0a83}, {0x0a85, 0x0a8d}, - {0x0a8f, 0x0a91}, {0x0a93, 0x0aa8}, {0x0aaa, 0x0ab0}, {0x0ab5, 0x0ab9}, - {0x0abc, 0x0ac5}, {0x0ac7, 0x0ac9}, {0x0acb, 0x0acd}, {0x0ae0, 0x0ae3}, - {0x0ae6, 0x0aef}, {0x0b01, 0x0b03}, {0x0b05, 0x0b0c}, {0x0b13, 0x0b1f}, - {0x0b21, 0x0b28}, {0x0b2a, 0x0b30}, {0x0b35, 0x0b39}, {0x0b3c, 0x0b44}, - {0x0b4b, 0x0b4d}, {0x0b5f, 0x0b63}, {0x0b66, 0x0b77}, {0x0b85, 0x0b8a}, - {0x0b8e, 0x0b90}, {0x0b92, 0x0b95}, {0x0ba8, 0x0baa}, {0x0bae, 0x0bb9}, - {0x0bbe, 0x0bc2}, {0x0bc6, 0x0bc8}, {0x0bca, 0x0bcd}, {0x0be6, 0x0bfa}, - {0x0c01, 0x0c03}, {0x0c05, 0x0c0c}, {0x0c0e, 0x0c10}, {0x0c12, 0x0c1f}, - {0x0c21, 0x0c28}, {0x0c2a, 0x0c33}, {0x0c35, 0x0c39}, {0x0c3d, 0x0c44}, - {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, {0x0c60, 0x0c63}, {0x0c66, 0x0c6f}, - {0x0c78, 0x0c7f}, {0x0c85, 0x0c8c}, {0x0c8e, 0x0c90}, {0x0c92, 0x0ca8}, - {0x0caa, 0x0cb3}, {0x0cb5, 0x0cb9}, {0x0cbc, 0x0cc4}, {0x0cc6, 0x0cc8}, - {0x0cca, 0x0ccd}, {0x0ce0, 0x0ce3}, {0x0ce6, 0x0cef}, {0x0d05, 0x0d0c}, - {0x0d0e, 0x0d10}, {0x0d12, 0x0d1f}, {0x0d21, 0x0d3a}, {0x0d3d, 0x0d44}, + {0x09e6, 0x09fb}, {0x0a01, 0x0a03}, {0x0a05, 0x0a0a}, {0x0a13, 0x0a28}, + {0x0a2a, 0x0a30}, {0x0a3e, 0x0a42}, {0x0a4b, 0x0a4d}, {0x0a59, 0x0a5c}, + {0x0a66, 0x0a75}, {0x0a81, 0x0a83}, {0x0a85, 0x0a8d}, {0x0a8f, 0x0a91}, + {0x0a93, 0x0aa8}, {0x0aaa, 0x0ab0}, {0x0ab5, 0x0ab9}, {0x0abc, 0x0ac5}, + {0x0ac7, 0x0ac9}, {0x0acb, 0x0acd}, {0x0ae0, 0x0ae3}, {0x0ae6, 0x0aef}, + {0x0b01, 0x0b03}, {0x0b05, 0x0b0c}, {0x0b13, 0x0b28}, {0x0b2a, 0x0b30}, + {0x0b35, 0x0b39}, {0x0b3c, 0x0b44}, {0x0b4b, 0x0b4d}, {0x0b5f, 0x0b63}, + {0x0b66, 0x0b77}, {0x0b85, 0x0b8a}, {0x0b8e, 0x0b90}, {0x0b92, 0x0b95}, + {0x0ba8, 0x0baa}, {0x0bae, 0x0bb9}, {0x0bbe, 0x0bc2}, {0x0bc6, 0x0bc8}, + {0x0bca, 0x0bcd}, {0x0be6, 0x0bfa}, {0x0c01, 0x0c03}, {0x0c05, 0x0c0c}, + {0x0c0e, 0x0c10}, {0x0c12, 0x0c28}, {0x0c2a, 0x0c33}, {0x0c35, 0x0c39}, + {0x0c3d, 0x0c44}, {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, {0x0c60, 0x0c63}, + {0x0c66, 0x0c6f}, {0x0c78, 0x0c7f}, {0x0c85, 0x0c8c}, {0x0c8e, 0x0c90}, + {0x0c92, 0x0ca8}, {0x0caa, 0x0cb3}, {0x0cb5, 0x0cb9}, {0x0cbc, 0x0cc4}, + {0x0cc6, 0x0cc8}, {0x0cca, 0x0ccd}, {0x0ce0, 0x0ce3}, {0x0ce6, 0x0cef}, + {0x0d05, 0x0d0c}, {0x0d0e, 0x0d10}, {0x0d12, 0x0d3a}, {0x0d3d, 0x0d44}, {0x0d46, 0x0d48}, {0x0d4a, 0x0d4e}, {0x0d60, 0x0d63}, {0x0d66, 0x0d75}, {0x0d79, 0x0d7f}, {0x0d85, 0x0d96}, {0x0d9a, 0x0db1}, {0x0db3, 0x0dbb}, {0x0dc0, 0x0dc6}, {0x0dcf, 0x0dd4}, {0x0dd8, 0x0ddf}, {0x0df2, 0x0df4}, - {0x0e01, 0x0e1f}, {0x0e21, 0x0e3a}, {0x0e3f, 0x0e5b}, {0x0e94, 0x0e97}, - {0x0e99, 0x0e9f}, {0x0ea1, 0x0ea3}, {0x0ead, 0x0eb9}, {0x0ebb, 0x0ebd}, - {0x0ec0, 0x0ec4}, {0x0ec8, 0x0ecd}, {0x0ed0, 0x0ed9}, {0x0f00, 0x0f1f}, - {0x0f21, 0x0f47}, {0x0f49, 0x0f6c}, {0x0f71, 0x0f97}, {0x0f99, 0x0fbc}, - {0x0fbe, 0x0fcc}, {0x0fce, 0x0fda}, {0x1000, 0x101f}, {0x1021, 0x10c5}, - {0x10d0, 0x10fc}, {0x1100, 0x111f}, {0x1121, 0x121f}, {0x1221, 0x1248}, - {0x124a, 0x124d}, {0x1250, 0x1256}, {0x125a, 0x125d}, {0x1260, 0x1288}, - {0x128a, 0x128d}, {0x1290, 0x12b0}, {0x12b2, 0x12b5}, {0x12b8, 0x12be}, - {0x12c2, 0x12c5}, {0x12c8, 0x12d6}, {0x12d8, 0x1310}, {0x1312, 0x1315}, - {0x1318, 0x131f}, {0x1321, 0x135a}, {0x135d, 0x137c}, {0x1380, 0x1399}, - {0x13a0, 0x13f4}, {0x1400, 0x141f}, {0x1421, 0x151f}, {0x1521, 0x161f}, - {0x1621, 0x169c}, {0x16a0, 0x16f0}, {0x1700, 0x170c}, {0x170e, 0x1714}, - {0x1721, 0x1736}, {0x1740, 0x1753}, {0x1760, 0x176c}, {0x176e, 0x1770}, - {0x1780, 0x17b3}, {0x17b6, 0x17dd}, {0x17e0, 0x17e9}, {0x17f0, 0x17f9}, - {0x1800, 0x180e}, {0x1810, 0x1819}, {0x1821, 0x1877}, {0x1880, 0x18aa}, - {0x18b0, 0x18f5}, {0x1900, 0x191c}, {0x1921, 0x192b}, {0x1930, 0x193b}, - {0x1944, 0x196d}, {0x1970, 0x1974}, {0x1980, 0x19ab}, {0x19b0, 0x19c9}, - {0x19d0, 0x19da}, {0x19de, 0x1a1b}, {0x1a21, 0x1a5e}, {0x1a60, 0x1a7c}, - {0x1a7f, 0x1a89}, {0x1a90, 0x1a99}, {0x1aa0, 0x1aad}, {0x1b00, 0x1b1f}, - {0x1b21, 0x1b4b}, {0x1b50, 0x1b7c}, {0x1b80, 0x1baa}, {0x1bae, 0x1bb9}, - {0x1bc0, 0x1bf3}, {0x1bfc, 0x1c1f}, {0x1c21, 0x1c37}, {0x1c3b, 0x1c49}, - {0x1c4d, 0x1c7f}, {0x1cd0, 0x1cf2}, {0x1d00, 0x1d1f}, {0x1d21, 0x1de6}, - {0x1dfc, 0x1e1f}, {0x1e21, 0x1f15}, {0x1f18, 0x1f1d}, {0x1f21, 0x1f45}, - {0x1f48, 0x1f4d}, {0x1f50, 0x1f57}, {0x1f5f, 0x1f7d}, {0x1f80, 0x1fb4}, - {0x1fb6, 0x1fc4}, {0x1fc6, 0x1fd3}, {0x1fd6, 0x1fdb}, {0x1fdd, 0x1fef}, - {0x1ff2, 0x1ff4}, {0x1ff6, 0x1ffe}, {0x2000, 0x200a}, {0x2010, 0x201f}, - {0x2021, 0x2029}, {0x202f, 0x205f}, {0x2074, 0x208e}, {0x2090, 0x209c}, - {0x20a0, 0x20b9}, {0x20d0, 0x20f0}, {0x2100, 0x211f}, {0x2121, 0x2189}, - {0x2190, 0x221f}, {0x2221, 0x231f}, {0x2321, 0x23f3}, {0x2400, 0x241f}, - {0x2421, 0x2426}, {0x2440, 0x244a}, {0x2460, 0x251f}, {0x2521, 0x261f}, - {0x2621, 0x26ff}, {0x2701, 0x271f}, {0x2721, 0x27ca}, {0x27ce, 0x281f}, - {0x2821, 0x291f}, {0x2921, 0x2a1f}, {0x2a21, 0x2b1f}, {0x2b21, 0x2b4c}, - {0x2b50, 0x2b59}, {0x2c00, 0x2c1f}, {0x2c21, 0x2c2e}, {0x2c30, 0x2c5e}, - {0x2c60, 0x2cf1}, {0x2cf9, 0x2d1f}, {0x2d21, 0x2d25}, {0x2d30, 0x2d65}, - {0x2d7f, 0x2d96}, {0x2da0, 0x2da6}, {0x2da8, 0x2dae}, {0x2db0, 0x2db6}, - {0x2db8, 0x2dbe}, {0x2dc0, 0x2dc6}, {0x2dc8, 0x2dce}, {0x2dd0, 0x2dd6}, - {0x2dd8, 0x2dde}, {0x2de0, 0x2e1f}, {0x2e21, 0x2e31}, {0x2e80, 0x2e99}, - {0x2e9b, 0x2ef3}, {0x2f00, 0x2f1f}, {0x2f21, 0x2fd5}, {0x2ff0, 0x2ffb}, - {0x3000, 0x301f}, {0x3021, 0x303f}, {0x3041, 0x3096}, {0x3099, 0x30ff}, - {0x3105, 0x311f}, {0x3121, 0x312d}, {0x3131, 0x318e}, {0x3190, 0x31ba}, - {0x31c0, 0x31e3}, {0x31f0, 0x321e}, {0x3221, 0x32fe}, {0x3300, 0x331f}, - {0x3321, 0x341f}, {0x3421, 0x351f}, {0x3521, 0x361f}, {0x3621, 0x371f}, - {0x3721, 0x381f}, {0x3821, 0x391f}, {0x3921, 0x3a1f}, {0x3a21, 0x3b1f}, - {0x3b21, 0x3c1f}, {0x3c21, 0x3d1f}, {0x3d21, 0x3e1f}, {0x3e21, 0x3f1f}, - {0x3f21, 0x401f}, {0x4021, 0x411f}, {0x4121, 0x421f}, {0x4221, 0x431f}, - {0x4321, 0x441f}, {0x4421, 0x451f}, {0x4521, 0x461f}, {0x4621, 0x471f}, - {0x4721, 0x481f}, {0x4821, 0x491f}, {0x4921, 0x4a1f}, {0x4a21, 0x4b1f}, - {0x4b21, 0x4c1f}, {0x4c21, 0x4d1f}, {0x4d21, 0x4db5}, {0x4dc0, 0x4e1f}, - {0x4e21, 0x4f1f}, {0x4f21, 0x501f}, {0x5021, 0x511f}, {0x5121, 0x521f}, - {0x5221, 0x531f}, {0x5321, 0x541f}, {0x5421, 0x551f}, {0x5521, 0x561f}, - {0x5621, 0x571f}, {0x5721, 0x581f}, {0x5821, 0x591f}, {0x5921, 0x5a1f}, - {0x5a21, 0x5b1f}, {0x5b21, 0x5c1f}, {0x5c21, 0x5d1f}, {0x5d21, 0x5e1f}, - {0x5e21, 0x5f1f}, {0x5f21, 0x601f}, {0x6021, 0x611f}, {0x6121, 0x621f}, - {0x6221, 0x631f}, {0x6321, 0x641f}, {0x6421, 0x651f}, {0x6521, 0x661f}, - {0x6621, 0x671f}, {0x6721, 0x681f}, {0x6821, 0x691f}, {0x6921, 0x6a1f}, - {0x6a21, 0x6b1f}, {0x6b21, 0x6c1f}, {0x6c21, 0x6d1f}, {0x6d21, 0x6e1f}, - {0x6e21, 0x6f1f}, {0x6f21, 0x701f}, {0x7021, 0x711f}, {0x7121, 0x721f}, - {0x7221, 0x731f}, {0x7321, 0x741f}, {0x7421, 0x751f}, {0x7521, 0x761f}, - {0x7621, 0x771f}, {0x7721, 0x781f}, {0x7821, 0x791f}, {0x7921, 0x7a1f}, - {0x7a21, 0x7b1f}, {0x7b21, 0x7c1f}, {0x7c21, 0x7d1f}, {0x7d21, 0x7e1f}, - {0x7e21, 0x7f1f}, {0x7f21, 0x801f}, {0x8021, 0x811f}, {0x8121, 0x821f}, - {0x8221, 0x831f}, {0x8321, 0x841f}, {0x8421, 0x851f}, {0x8521, 0x861f}, - {0x8621, 0x871f}, {0x8721, 0x881f}, {0x8821, 0x891f}, {0x8921, 0x8a1f}, - {0x8a21, 0x8b1f}, {0x8b21, 0x8c1f}, {0x8c21, 0x8d1f}, {0x8d21, 0x8e1f}, - {0x8e21, 0x8f1f}, {0x8f21, 0x901f}, {0x9021, 0x911f}, {0x9121, 0x921f}, - {0x9221, 0x931f}, {0x9321, 0x941f}, {0x9421, 0x951f}, {0x9521, 0x961f}, - {0x9621, 0x971f}, {0x9721, 0x981f}, {0x9821, 0x991f}, {0x9921, 0x9a1f}, - {0x9a21, 0x9b1f}, {0x9b21, 0x9c1f}, {0x9c21, 0x9d1f}, {0x9d21, 0x9e1f}, - {0x9e21, 0x9f1f}, {0x9f21, 0x9fcb}, {0xa000, 0xa01f}, {0xa021, 0xa11f}, - {0xa121, 0xa21f}, {0xa221, 0xa31f}, {0xa321, 0xa41f}, {0xa421, 0xa48c}, - {0xa490, 0xa4c6}, {0xa4d0, 0xa51f}, {0xa521, 0xa61f}, {0xa621, 0xa62b}, - {0xa640, 0xa673}, {0xa67c, 0xa697}, {0xa6a0, 0xa6f7}, {0xa700, 0xa71f}, - {0xa721, 0xa78e}, {0xa7a0, 0xa7a9}, {0xa7fa, 0xa81f}, {0xa821, 0xa82b}, - {0xa830, 0xa839}, {0xa840, 0xa877}, {0xa880, 0xa8c4}, {0xa8ce, 0xa8d9}, - {0xa8e0, 0xa8fb}, {0xa900, 0xa91f}, {0xa921, 0xa953}, {0xa95f, 0xa97c}, - {0xa980, 0xa9cd}, {0xa9cf, 0xa9d9}, {0xaa00, 0xaa1f}, {0xaa21, 0xaa36}, - {0xaa40, 0xaa4d}, {0xaa50, 0xaa59}, {0xaa5c, 0xaa7b}, {0xaa80, 0xaac2}, - {0xaadb, 0xaadf}, {0xab01, 0xab06}, {0xab09, 0xab0e}, {0xab11, 0xab16}, - {0xab21, 0xab26}, {0xab28, 0xab2e}, {0xabc0, 0xabed}, {0xabf0, 0xabf9}, - {0xac00, 0xac1f}, {0xac21, 0xad1f}, {0xad21, 0xae1f}, {0xae21, 0xaf1f}, - {0xaf21, 0xb01f}, {0xb021, 0xb11f}, {0xb121, 0xb21f}, {0xb221, 0xb31f}, - {0xb321, 0xb41f}, {0xb421, 0xb51f}, {0xb521, 0xb61f}, {0xb621, 0xb71f}, - {0xb721, 0xb81f}, {0xb821, 0xb91f}, {0xb921, 0xba1f}, {0xba21, 0xbb1f}, - {0xbb21, 0xbc1f}, {0xbc21, 0xbd1f}, {0xbd21, 0xbe1f}, {0xbe21, 0xbf1f}, - {0xbf21, 0xc01f}, {0xc021, 0xc11f}, {0xc121, 0xc21f}, {0xc221, 0xc31f}, - {0xc321, 0xc41f}, {0xc421, 0xc51f}, {0xc521, 0xc61f}, {0xc621, 0xc71f}, - {0xc721, 0xc81f}, {0xc821, 0xc91f}, {0xc921, 0xca1f}, {0xca21, 0xcb1f}, - {0xcb21, 0xcc1f}, {0xcc21, 0xcd1f}, {0xcd21, 0xce1f}, {0xce21, 0xcf1f}, - {0xcf21, 0xd01f}, {0xd021, 0xd11f}, {0xd121, 0xd21f}, {0xd221, 0xd31f}, - {0xd321, 0xd41f}, {0xd421, 0xd51f}, {0xd521, 0xd61f}, {0xd621, 0xd71f}, - {0xd721, 0xd7a3}, {0xd7b0, 0xd7c6}, {0xd7cb, 0xd7fb}, {0xf900, 0xf91f}, - {0xf921, 0xfa1f}, {0xfa21, 0xfa2d}, {0xfa30, 0xfa6d}, {0xfa70, 0xfad9}, - {0xfb00, 0xfb06}, {0xfb13, 0xfb17}, {0xfb1d, 0xfb1f}, {0xfb21, 0xfb36}, - {0xfb38, 0xfb3c}, {0xfb46, 0xfbc1}, {0xfbd3, 0xfc1f}, {0xfc21, 0xfd1f}, - {0xfd21, 0xfd3f}, {0xfd50, 0xfd8f}, {0xfd92, 0xfdc7}, {0xfdf0, 0xfdfd}, - {0xfe00, 0xfe19}, {0xfe21, 0xfe26}, {0xfe30, 0xfe52}, {0xfe54, 0xfe66}, - {0xfe68, 0xfe6b}, {0xfe70, 0xfe74}, {0xfe76, 0xfefc}, {0xff01, 0xff1f}, - {0xff21, 0xffbe}, {0xffc2, 0xffc7}, {0xffca, 0xffcf}, {0xffd2, 0xffd7}, - {0xffda, 0xffdc}, {0xffe0, 0xffe6}, {0xffe8, 0xffee}, {0xfffc, 0xffff} + {0x0e01, 0x0e3a}, {0x0e3f, 0x0e5b}, {0x0e94, 0x0e97}, {0x0e99, 0x0e9f}, + {0x0ea1, 0x0ea3}, {0x0ead, 0x0eb9}, {0x0ebb, 0x0ebd}, {0x0ec0, 0x0ec4}, + {0x0ec8, 0x0ecd}, {0x0ed0, 0x0ed9}, {0x0f00, 0x0f47}, {0x0f49, 0x0f6c}, + {0x0f71, 0x0f97}, {0x0f99, 0x0fbc}, {0x0fbe, 0x0fcc}, {0x0fce, 0x0fda}, + {0x1000, 0x10c5}, {0x10d0, 0x10fc}, {0x1100, 0x1248}, {0x124a, 0x124d}, + {0x1250, 0x1256}, {0x125a, 0x125d}, {0x1260, 0x1288}, {0x128a, 0x128d}, + {0x1290, 0x12b0}, {0x12b2, 0x12b5}, {0x12b8, 0x12be}, {0x12c2, 0x12c5}, + {0x12c8, 0x12d6}, {0x12d8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135a}, + {0x135d, 0x137c}, {0x1380, 0x1399}, {0x13a0, 0x13f4}, {0x1400, 0x169c}, + {0x16a0, 0x16f0}, {0x1700, 0x170c}, {0x170e, 0x1714}, {0x1720, 0x1736}, + {0x1740, 0x1753}, {0x1760, 0x176c}, {0x176e, 0x1770}, {0x1780, 0x17b3}, + {0x17b6, 0x17dd}, {0x17e0, 0x17e9}, {0x17f0, 0x17f9}, {0x1800, 0x180e}, + {0x1810, 0x1819}, {0x1820, 0x1877}, {0x1880, 0x18aa}, {0x18b0, 0x18f5}, + {0x1900, 0x191c}, {0x1920, 0x192b}, {0x1930, 0x193b}, {0x1944, 0x196d}, + {0x1970, 0x1974}, {0x1980, 0x19ab}, {0x19b0, 0x19c9}, {0x19d0, 0x19da}, + {0x19de, 0x1a1b}, {0x1a1e, 0x1a5e}, {0x1a60, 0x1a7c}, {0x1a7f, 0x1a89}, + {0x1a90, 0x1a99}, {0x1aa0, 0x1aad}, {0x1b00, 0x1b4b}, {0x1b50, 0x1b7c}, + {0x1b80, 0x1baa}, {0x1bae, 0x1bb9}, {0x1bc0, 0x1bf3}, {0x1bfc, 0x1c37}, + {0x1c3b, 0x1c49}, {0x1c4d, 0x1c7f}, {0x1cd0, 0x1cf2}, {0x1d00, 0x1de6}, + {0x1dfc, 0x1f15}, {0x1f18, 0x1f1d}, {0x1f20, 0x1f45}, {0x1f48, 0x1f4d}, + {0x1f50, 0x1f57}, {0x1f5f, 0x1f7d}, {0x1f80, 0x1fb4}, {0x1fb6, 0x1fc4}, + {0x1fc6, 0x1fd3}, {0x1fd6, 0x1fdb}, {0x1fdd, 0x1fef}, {0x1ff2, 0x1ff4}, + {0x1ff6, 0x1ffe}, {0x2000, 0x200a}, {0x2010, 0x2029}, {0x202f, 0x205f}, + {0x2074, 0x208e}, {0x2090, 0x209c}, {0x20a0, 0x20b9}, {0x20d0, 0x20f0}, + {0x2100, 0x2189}, {0x2190, 0x23f3}, {0x2400, 0x2426}, {0x2440, 0x244a}, + {0x2460, 0x26ff}, {0x2701, 0x27ca}, {0x27ce, 0x2b4c}, {0x2b50, 0x2b59}, + {0x2c00, 0x2c2e}, {0x2c30, 0x2c5e}, {0x2c60, 0x2cf1}, {0x2cf9, 0x2d25}, + {0x2d30, 0x2d65}, {0x2d7f, 0x2d96}, {0x2da0, 0x2da6}, {0x2da8, 0x2dae}, + {0x2db0, 0x2db6}, {0x2db8, 0x2dbe}, {0x2dc0, 0x2dc6}, {0x2dc8, 0x2dce}, + {0x2dd0, 0x2dd6}, {0x2dd8, 0x2dde}, {0x2de0, 0x2e31}, {0x2e80, 0x2e99}, + {0x2e9b, 0x2ef3}, {0x2f00, 0x2fd5}, {0x2ff0, 0x2ffb}, {0x3000, 0x303f}, + {0x3041, 0x3096}, {0x3099, 0x30ff}, {0x3105, 0x312d}, {0x3131, 0x318e}, + {0x3190, 0x31ba}, {0x31c0, 0x31e3}, {0x31f0, 0x321e}, {0x3220, 0x32fe}, + {0x3300, 0x4db5}, {0x4dc0, 0x9fcb}, {0xa000, 0xa48c}, {0xa490, 0xa4c6}, + {0xa4d0, 0xa62b}, {0xa640, 0xa673}, {0xa67c, 0xa697}, {0xa6a0, 0xa6f7}, + {0xa700, 0xa78e}, {0xa7a0, 0xa7a9}, {0xa7fa, 0xa82b}, {0xa830, 0xa839}, + {0xa840, 0xa877}, {0xa880, 0xa8c4}, {0xa8ce, 0xa8d9}, {0xa8e0, 0xa8fb}, + {0xa900, 0xa953}, {0xa95f, 0xa97c}, {0xa980, 0xa9cd}, {0xa9cf, 0xa9d9}, + {0xaa00, 0xaa36}, {0xaa40, 0xaa4d}, {0xaa50, 0xaa59}, {0xaa5c, 0xaa7b}, + {0xaa80, 0xaac2}, {0xaadb, 0xaadf}, {0xab01, 0xab06}, {0xab09, 0xab0e}, + {0xab11, 0xab16}, {0xab20, 0xab26}, {0xab28, 0xab2e}, {0xabc0, 0xabed}, + {0xabf0, 0xabf9}, {0xac00, 0xd7a3}, {0xd7b0, 0xd7c6}, {0xd7cb, 0xd7fb}, + {0xf900, 0xfa2d}, {0xfa30, 0xfa6d}, {0xfa70, 0xfad9}, {0xfb00, 0xfb06}, + {0xfb13, 0xfb17}, {0xfb1d, 0xfb36}, {0xfb38, 0xfb3c}, {0xfb46, 0xfbc1}, + {0xfbd3, 0xfd3f}, {0xfd50, 0xfd8f}, {0xfd92, 0xfdc7}, {0xfdf0, 0xfdfd}, + {0xfe00, 0xfe19}, {0xfe20, 0xfe26}, {0xfe30, 0xfe52}, {0xfe54, 0xfe66}, + {0xfe68, 0xfe6b}, {0xfe70, 0xfe74}, {0xfe76, 0xfefc}, {0xff01, 0xffbe}, + {0xffc2, 0xffc7}, {0xffca, 0xffcf}, {0xffd2, 0xffd7}, {0xffda, 0xffdc}, + {0xffe0, 0xffe6}, {0xffe8, 0xffee} }; #define NUM_GRAPH_RANGE (sizeof(graphRangeTable)/sizeof(crange)) static const chr graphCharTable[] = { - 0x038c, 0x0589, 0x058a, 0x061e, 0x061f, 0x085e, 0x098f, 0x0990, 0x09b2, - 0x09c7, 0x09c8, 0x09d7, 0x09dc, 0x09dd, 0x0a0f, 0x0a10, 0x0a32, 0x0a33, - 0x0a35, 0x0a36, 0x0a38, 0x0a39, 0x0a3c, 0x0a47, 0x0a48, 0x0a51, 0x0a5e, - 0x0ab2, 0x0ab3, 0x0ad0, 0x0af1, 0x0b0f, 0x0b10, 0x0b32, 0x0b33, 0x0b47, - 0x0b48, 0x0b56, 0x0b57, 0x0b5c, 0x0b5d, 0x0b82, 0x0b83, 0x0b99, 0x0b9a, - 0x0b9c, 0x0b9e, 0x0b9f, 0x0ba3, 0x0ba4, 0x0bd0, 0x0bd7, 0x0c55, 0x0c56, - 0x0c58, 0x0c59, 0x0c82, 0x0c83, 0x0cd5, 0x0cd6, 0x0cde, 0x0cf1, 0x0cf2, - 0x0d02, 0x0d03, 0x0d57, 0x0d82, 0x0d83, 0x0dbd, 0x0dca, 0x0dd6, 0x0e81, - 0x0e82, 0x0e84, 0x0e87, 0x0e88, 0x0e8a, 0x0e8d, 0x0ea5, 0x0ea7, 0x0eaa, - 0x0eab, 0x0ec6, 0x0edc, 0x0edd, 0x1258, 0x12c0, 0x1772, 0x1773, 0x1940, - 0x1a1e, 0x1a1f, 0x1f59, 0x1f5b, 0x1f5d, 0x2070, 0x2071, 0x27cc, 0x2d6f, - 0x2d70, 0xa790, 0xa791, 0xa9de, 0xa9df, 0xfb3e, 0xfb40, 0xfb41, 0xfb43, - 0xfb44 + 0x038c, 0x0589, 0x058a, 0x085e, 0x098f, 0x0990, 0x09b2, 0x09c7, 0x09c8, + 0x09d7, 0x09dc, 0x09dd, 0x0a0f, 0x0a10, 0x0a32, 0x0a33, 0x0a35, 0x0a36, + 0x0a38, 0x0a39, 0x0a3c, 0x0a47, 0x0a48, 0x0a51, 0x0a5e, 0x0ab2, 0x0ab3, + 0x0ad0, 0x0af1, 0x0b0f, 0x0b10, 0x0b32, 0x0b33, 0x0b47, 0x0b48, 0x0b56, + 0x0b57, 0x0b5c, 0x0b5d, 0x0b82, 0x0b83, 0x0b99, 0x0b9a, 0x0b9c, 0x0b9e, + 0x0b9f, 0x0ba3, 0x0ba4, 0x0bd0, 0x0bd7, 0x0c55, 0x0c56, 0x0c58, 0x0c59, + 0x0c82, 0x0c83, 0x0cd5, 0x0cd6, 0x0cde, 0x0cf1, 0x0cf2, 0x0d02, 0x0d03, + 0x0d57, 0x0d82, 0x0d83, 0x0dbd, 0x0dca, 0x0dd6, 0x0e81, 0x0e82, 0x0e84, + 0x0e87, 0x0e88, 0x0e8a, 0x0e8d, 0x0ea5, 0x0ea7, 0x0eaa, 0x0eab, 0x0ec6, + 0x0edc, 0x0edd, 0x1258, 0x12c0, 0x1772, 0x1773, 0x1940, 0x1f59, 0x1f5b, + 0x1f5d, 0x2070, 0x2071, 0x27cc, 0x2d6f, 0x2d70, 0xa790, 0xa791, 0xa9de, + 0xa9df, 0xfb3e, 0xfb40, 0xfb41, 0xfb43, 0xfb44, 0xfffc, 0xfffd }; #define NUM_GRAPH_CHAR (sizeof(graphCharTable)/sizeof(chr)) diff --git a/generic/tclUniData.c b/generic/tclUniData.c index a06e84d..896580c 100644 --- a/generic/tclUniData.c +++ b/generic/tclUniData.c @@ -775,7 +775,7 @@ static const int groups[] = { * Unicode character. */ -#define UNICODE_CATEGORY_MASK 0x1F +#define UNICODE_CATEGORY_MASK 0x1f #define UNICODE_OUT_OF_RANGE 0x10000u enum { @@ -817,8 +817,8 @@ enum { * to do sign extension on right shifts. */ -#define GetCaseType(info) (((info) & 0xE0) >> 5) -#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1F) +#define GetCaseType(info) (((info) & 0xe0) >> 5) +#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1f) #define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15))) /* diff --git a/generic/tclUtf.c b/generic/tclUtf.c index 00b652e..40f6272 100644 --- a/generic/tclUtf.c +++ b/generic/tclUtf.c @@ -1424,7 +1424,7 @@ Tcl_UniCharIsGraph( int ch) /* Unicode character to test. */ { register int category = (GetUniCharInfo(ch) & UNICODE_CATEGORY_MASK); - return (((PRINT_BITS >> category) & 1) && ((unsigned char) ch != ' ')); + return (((PRINT_BITS >> category) & 1) && (ch != ' ')); } /* diff --git a/tests/utf.test b/tests/utf.test index 60e3d52..273773b 100644 --- a/tests/utf.test +++ b/tests/utf.test @@ -27,9 +27,12 @@ test utf-1.3 {Tcl_UniCharToUtf: 2 byte sequences} { test utf-1.4 {Tcl_UniCharToUtf: 3 byte sequences} { set x "\u4e4e" } [bytestring "\xe4\xb9\x8e"] -test utf-1.5 {Tcl_UniCharToUtf: negative Tcl_UniChar} { - string length [format %c -1] -} 1 +test utf-1.5 {Tcl_UniCharToUtf: overflowed Tcl_UniChar} { + format %c 0x110000 +} [bytestring "\xef\xbf\xbd"] +test utf-1.6 {Tcl_UniCharToUtf: negative Tcl_UniChar} { + format %c -1 +} [bytestring "\xef\xbf\xbd"] test utf-2.1 {Tcl_UtfToUniChar: low ascii} { string length "abc" @@ -286,6 +289,14 @@ test utf-21.3 {unicode print char in regc_locale.c} { # this returns 1 with Unicode 6 compliance regexp {^[[:print:]]+$} \ufbc1 } 1 +test utf-21.4 {TclUniCharIsGraph} { + # [Bug 3464428] + string is graph \u0120 +} {1} +test utf-21.5 {unicode graph char in regc_locale.c} { + # [Bug 3464428] + regexp {^[[:graph:]]+$} \u0120 +} {1} test utf-22.1 {TclUniCharIsWordChar} { string wordend "xyz123_bar fg" 0 diff --git a/tools/uniParse.tcl b/tools/uniParse.tcl index 6850897..a13bae6 100644 --- a/tools/uniParse.tcl +++ b/tools/uniParse.tcl @@ -30,45 +30,39 @@ namespace eval uni { Cc Cf Co Cs Pc Pd Ps Pe Pi Pf Po Sm Sc Sk So }; # Ordered list of character categories, must # match the enumeration in the header file. - - variable titleCount 0; # Count of the number of title case - # characters. This value is used in the - # regular expression code to allocate enough - # space for the title case variants. } proc uni::getValue {items index} { variable categories - variable titleCount # Extract character info set category [lindex $items 2] - if {[scan [lindex $items 12] %6x toupper] == 1} { + if {[scan [lindex $items 12] %x toupper] == 1} { set toupper [expr {$index - $toupper}] } else { set toupper 0 } - if {[scan [lindex $items 13] %6x tolower] == 1} { + if {[scan [lindex $items 13] %x tolower] == 1} { set tolower [expr {$tolower - $index}] } else { set tolower 0 } - if {[scan [lindex $items 14] %6x totitle] == 1} { + if {[scan [lindex $items 14] %x totitle] == 1} { set totitle [expr {$index - $totitle}] - } else { + } elseif {$tolower} { set totitle 0 + } else { + set totitle $toupper } set categoryIndex [lsearch -exact $categories $category] if {$categoryIndex < 0} { puts "Unexpected character category: $index($category)" set categoryIndex 0 - } elseif {$category eq "Lt"} { - incr titleCount } - return "$categoryIndex,$toupper,$tolower,$totitle" + return [list $categoryIndex $toupper $tolower $totitle] } proc uni::getGroup {value} { @@ -100,7 +94,7 @@ proc uni::buildTables {data} { variable pMap {} variable pages {} - variable groups {{0,0,0,0}} + variable groups {{0 0 0 0}} variable next 0 set info {} ;# temporary page info @@ -120,7 +114,7 @@ proc uni::buildTables {data} { set items [split $line \;] scan [lindex $items 0] %x index - if {$index > 0xFFFF} then { + if {$index > 0xffff} then { # Ignore non-BMP characters, as long as Tcl doesn't support them continue } @@ -171,7 +165,6 @@ proc uni::main {} { variable pages variable groups variable shift - variable titleCount variable next if {$argc != 2} { @@ -186,7 +179,6 @@ proc uni::main {} { puts "X = [llength $pMap] Y= [llength $pages] A= [llength $groups]" set size [expr {[llength $pMap]*2 + [llength $pages]*(1<<$shift)}] puts "shift = $shift, space = $size" - puts "title case count = $titleCount" set f [open [file join [lindex $argv 1] tclUniData.c] w] fconfigure $f -translation lf @@ -280,7 +272,7 @@ static const int groups\[\] = {" set line " " set last [expr {[llength $groups] - 1}] for {set i 0} {$i <= $last} {incr i} { - foreach {type toupper tolower totitle} [split [lindex $groups $i] ,] {} + foreach {type toupper tolower totitle} [lindex $groups $i] {} # Compute the case conversion type and delta @@ -329,9 +321,9 @@ static const int groups\[\] = {" * Unicode character. */ -#define UNICODE_CATEGORY_MASK 0x1F +#define UNICODE_CATEGORY_MASK 0x1f #define UNICODE_OUT_OF_RANGE " - puts $f [format 0x%Xu $next] + puts $f [format 0x%xu $next] puts $f " enum { UNASSIGNED, @@ -372,8 +364,8 @@ enum { * to do sign extension on right shifts. */ -#define GetCaseType(info) (((info) & 0xE0) >> 5) -#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1F) +#define GetCaseType(info) (((info) & 0xe0) >> 5) +#define GetCategory(ch) (GetUniCharInfo(ch) & 0x1f) #define GetDelta(info) (((info) > 0) ? ((info) >> 15) : (~(~((info)) >> 15))) /* |