1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
#
# genmap_tchinese.py: Traditional Chinese Codecs Map Generator
#
# Original Author: Hye-Shik Chang <perky@FreeBSD.org>
#
import os
from genmap_support import *
# ranges for (lead byte, follower byte)
BIG5_C1 = (0xa1, 0xfe)
BIG5_C2 = (0x40, 0xfe)
BIG5HKSCS_C1 = (0x87, 0xfe)
BIG5HKSCS_C2 = (0x40, 0xfe)
MAPPINGS_BIG5 = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT'
MAPPINGS_CP950 = 'https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT'
HKSCS_VERSION = '2004'
# The files for HKSCS mappings are available under a restrictive license.
# Users of the script need to download the files from the HKSARG CCLI website:
MAPPINGS_HKSCS = f'https://www.ccli.gov.hk/en/archive/terms_hkscs-{HKSCS_VERSION}-big5-iso.html'
def bh2s(code):
return ((code >> 8) - 0x87) * (0xfe - 0x40 + 1) + ((code & 0xff) - 0x40)
def split_bytes(code):
"""Split 0xABCD into 0xAB, 0xCD"""
return code >> 8, code & 0xff
def parse_hkscs_map(fo):
fo.seek(0, 0)
table = []
for line in fo:
line = line.split('#', 1)[0].strip()
# We expect 4 columns in supported HKSCS files:
# [1999]: unsupported
# [2001]: unsupported
# [2004]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd1
# [2008]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd6
# [2016]: not supported here--uses a json file instead
#
# In both supported cases, we only need the first and last column:
# * Big-5 is a hex string (always 4 digits)
# * iso10646:2003 is either a hex string (4 or 5 digits) or a sequence
# of hex strings like: `<code_point1,code_point2>`
try:
hkscs_col, _, _, uni_col = line.split()
hkscs = int(hkscs_col, 16)
seq = tuple(int(cp, 16) for cp in uni_col.strip('<>').split(','))
except ValueError:
continue
table.append((hkscs, seq))
return table
def make_hkscs_map(table):
decode_map = {}
encode_map_bmp, encode_map_notbmp = {}, {}
is_bmp_map = {}
sequences = []
beginnings = {}
single_cp_table = []
# Determine multi-codepoint sequences, and sequence beginnings that encode
# multiple multibyte (i.e. Big-5) codes.
for mbcode, cp_seq in table:
cp, *_ = cp_seq
if len(cp_seq) == 1:
single_cp_table.append((mbcode, cp))
else:
sequences.append((mbcode, cp_seq))
beginnings.setdefault(cp, []).append(mbcode)
# Decode table only cares about single code points (no sequences) currently
for mbcode, cp in single_cp_table:
b1, b2 = split_bytes(mbcode)
decode_map.setdefault(b1, {})
decode_map[b1][b2] = cp & 0xffff
# Encode table needs to mark code points beginning a sequence as tuples.
for cp, mbcodes in beginnings.items():
plane = cp >> 16
if plane == 0:
encode_map = encode_map_bmp
elif plane == 2:
encode_map = encode_map_notbmp
is_bmp_map[bh2s(mbcodes[0])] = 1
else:
assert False, 'only plane 0 (BMP) and plane 2 (SIP) allowed'
if len(mbcodes) == 1:
encode_value = mbcodes[0]
else:
encode_value = tuple(mbcodes)
uni_b1, uni_b2 = split_bytes(cp & 0xffff)
encode_map.setdefault(uni_b1, {})
encode_map[uni_b1][uni_b2] = encode_value
return decode_map, encode_map_bmp, encode_map_notbmp, is_bmp_map
def load_big5_map():
mapfile = open_mapping_file('python-mappings/BIG5.txt', MAPPINGS_BIG5)
with mapfile:
big5decmap = loadmap(mapfile)
# big5 mapping fix: use the cp950 mapping for these characters as the file
# provided by unicode.org doesn't define a mapping. See notes in BIG5.txt.
# Since U+5341, U+5345, U+FF0F, U+FF3C already have a big5 mapping, no
# roundtrip compatibility is guaranteed for those.
for m in """\
0xA15A 0x2574
0xA1C3 0xFFE3
0xA1C5 0x02CD
0xA1FE 0xFF0F
0xA240 0xFF3C
0xA2CC 0x5341
0xA2CE 0x5345""".splitlines():
bcode, ucode = list(map(eval, m.split()))
big5decmap[bcode >> 8][bcode & 0xff] = ucode
# encoding map
big5encmap = {}
for c1, m in list(big5decmap.items()):
for c2, code in list(m.items()):
big5encmap.setdefault(code >> 8, {})
if code & 0xff not in big5encmap[code >> 8]:
big5encmap[code >> 8][code & 0xff] = c1 << 8 | c2
# fix unicode->big5 priority for the above-mentioned duplicate characters
big5encmap[0xFF][0x0F] = 0xA241
big5encmap[0xFF][0x3C] = 0xA242
big5encmap[0x53][0x41] = 0xA451
big5encmap[0x53][0x45] = 0xA4CA
return big5decmap, big5encmap
def load_cp950_map():
mapfile = open_mapping_file('python-mappings/CP950.TXT', MAPPINGS_CP950)
with mapfile:
cp950decmap = loadmap(mapfile)
cp950encmap = {}
for c1, m in list(cp950decmap.items()):
for c2, code in list(m.items()):
cp950encmap.setdefault(code >> 8, {})
if code & 0xff not in cp950encmap[code >> 8]:
cp950encmap[code >> 8][code & 0xff] = c1 << 8 | c2
# fix unicode->big5 duplicated mapping priority
cp950encmap[0x53][0x41] = 0xA451
cp950encmap[0x53][0x45] = 0xA4CA
return cp950decmap, cp950encmap
def main_tw():
big5decmap, big5encmap = load_big5_map()
cp950decmap, cp950encmap = load_cp950_map()
# CP950 extends Big5, and the codec can use the Big5 lookup tables
# for most entries. So the CP950 tables should only include entries
# that are not in Big5:
for c1, m in list(cp950encmap.items()):
for c2, code in list(m.items()):
if (c1 in big5encmap and c2 in big5encmap[c1]
and big5encmap[c1][c2] == code):
del cp950encmap[c1][c2]
for c1, m in list(cp950decmap.items()):
for c2, code in list(m.items()):
if (c1 in big5decmap and c2 in big5decmap[c1]
and big5decmap[c1][c2] == code):
del cp950decmap[c1][c2]
with open('mappings_tw.h', 'w') as fp:
print_autogen(fp, os.path.basename(__file__))
write_big5_maps(fp, 'BIG5', 'big5', big5decmap, big5encmap)
write_big5_maps(fp, 'CP950', 'cp950ext', cp950decmap, cp950encmap)
def write_big5_maps(fp, display_name, table_name, decode_map, encode_map):
print(f'Generating {display_name} decode map...')
writer = DecodeMapWriter(fp, table_name, decode_map)
writer.update_decode_map(BIG5_C1, BIG5_C2)
writer.generate()
print(f'Generating {display_name} encode map...')
writer = EncodeMapWriter(fp, table_name, encode_map)
writer.generate()
class HintsWriter:
filler_class = BufferedFiller
def __init__(self, fp, prefix, isbmpmap):
self.fp = fp
self.prefix = prefix
self.isbmpmap = isbmpmap
self.filler = self.filler_class()
def fillhints(self, hintfrom, hintto):
name = f'{self.prefix}_phint_{hintfrom}'
self.fp.write(f'static const unsigned char {name}[] = {{\n')
for msbcode in range(hintfrom, hintto+1, 8):
v = 0
for c in range(msbcode, msbcode+8):
v |= self.isbmpmap.get(c, 0) << (c - msbcode)
self.filler.write('%d,' % v)
self.filler.printout(self.fp)
self.fp.write('};\n\n')
def main_hkscs():
filename = f'python-mappings/hkscs-{HKSCS_VERSION}-big5-iso.txt'
with open_mapping_file(filename, MAPPINGS_HKSCS) as f:
table = parse_hkscs_map(f)
hkscsdecmap, hkscsencmap_bmp, hkscsencmap_nonbmp, isbmpmap = (
make_hkscs_map(table)
)
with open('mappings_hk.h', 'w') as fp:
print('Generating BIG5HKSCS decode map...')
print_autogen(fp, os.path.basename(__file__))
writer = DecodeMapWriter(fp, 'big5hkscs', hkscsdecmap)
writer.update_decode_map(BIG5HKSCS_C1, BIG5HKSCS_C2)
writer.generate()
print('Generating BIG5HKSCS decode map Unicode plane hints...')
writer = HintsWriter(fp, 'big5hkscs', isbmpmap)
writer.fillhints(bh2s(0x8740), bh2s(0xa0fe))
writer.fillhints(bh2s(0xc6a1), bh2s(0xc8fe))
writer.fillhints(bh2s(0xf9d6), bh2s(0xfefe))
print('Generating BIG5HKSCS encode map (BMP)...')
writer = EncodeMapWriter(fp, 'big5hkscs_bmp', hkscsencmap_bmp)
writer.generate()
print('Generating BIG5HKSCS encode map (non-BMP)...')
writer = EncodeMapWriter(fp, 'big5hkscs_nonbmp', hkscsencmap_nonbmp)
writer.generate()
if __name__ == '__main__':
main_tw()
main_hkscs()
|
