summaryrefslogtreecommitdiffstats
path: root/Lib/test/test_normalization.py
blob: a263fc5464fe7099e9ccced39ad8cf787ec1e003 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
from test.test_support import verbose, TestFailed, TestSkipped, verify
import sys
from unicodedata import normalize
try:
    data = open("NormalizationTest.txt","r").readlines()
except IOError:
    raise TestSkipped("NormalizationTest.txt not found, download from http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt")

class RangeError:
    pass

def NFC(str):
    return normalize("NFC", str)

def NFKC(str):
    return normalize("NFKC", str)

def NFD(str):
    return normalize("NFD", str)

def NFKD(str):
    return normalize("NFKD", str)

def unistr(data):
    data = [int(x, 16) for x in data.split(" ")]
    for x in data:
        if x > sys.maxunicode:
            raise RangeError
    return u"".join([unichr(x) for x in data])

part1_data = {}
for line in data:
    if '#' in line:
        line = line.split('#')[0]
    line = line.strip()
    if not line:
        continue
    if line.startswith("@Part"):
        part = line
        continue
    try:
        c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]]
    except RangeError:
        # Skip unsupported characters
        continue

    if verbose:
        print line
    
    # Perform tests
    verify(c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3), line)
    verify(c4 ==  NFC(c4) ==  NFC(c5), line)
    verify(c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3), line)
    verify(c5 ==  NFD(c4) ==  NFD(c5), line)
    verify(c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5), line)
    verify(c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5), line)

    # Record part 1 data
    if part == "@Part1":
        part1_data[c1] = 1

# Perform tests for all other data
for c in range(sys.maxunicode+1):
    X = unichr(c)
    if X in part1_data:
        continue
    assert X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c