Issue #3811: The Unicode database was updated to 5.1.

Reviewed by Fredrik Lundh and Marc-Andre Lemburg.

1 files changed, 30 insertions, 10 deletions

diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py
index 2dfaa29..8ede83c 100644
--- a/Tools/unicode/makeunicodedata.py
+++ b/Tools/unicode/makeunicodedata.py
@@ -27,10 +27,10 @@
 import sys
 
 SCRIPT = sys.argv[0]
-VERSION = "2.5"
+VERSION = "2.6"
 
 # The Unicode Database
-UNIDATA_VERSION = "4.1.0"
+UNIDATA_VERSION = "5.1.0"
 UNICODE_DATA = "UnicodeData%s.txt"
 COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt"
 EASTASIAN_WIDTH = "EastAsianWidth%s.txt"
@@ -57,6 +57,7 @@ LINEBREAK_MASK = 0x10
 SPACE_MASK = 0x20
 TITLE_MASK = 0x40
 UPPER_MASK = 0x80
+NODELTA_MASK = 0x100
 
 def maketables(trace=0):
 
@@ -355,6 +356,7 @@ def makeunicodetype(unicode, trace):
             category = record[2]
             bidirectional = record[4]
             flags = 0
+            delta = True
             if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
                 flags |= ALPHA_MASK
             if category == "Ll":
@@ -367,25 +369,36 @@ def makeunicodetype(unicode, trace):
                 flags |= TITLE_MASK
             if category == "Lu":
                 flags |= UPPER_MASK
-            # use delta predictor for upper/lower/title
+            # use delta predictor for upper/lower/title if it fits
             if record[12]:
                 upper = int(record[12], 16) - char
-                assert -32768 <= upper <= 32767
-                upper = upper & 0xffff
+                if  -32768 <= upper <= 32767 and delta:
+                    upper = upper & 0xffff
+                else:
+                    upper += char
+                    delta = False
             else:
                 upper = 0
             if record[13]:
                 lower = int(record[13], 16) - char
-                assert -32768 <= lower <= 32767
-                lower = lower & 0xffff
+                if -32768 <= lower <= 32767 and delta:
+                    lower = lower & 0xffff
+                else:
+                    lower += char
+                    delta = False
             else:
                 lower = 0
             if record[14]:
                 title = int(record[14], 16) - char
-                assert -32768 <= lower <= 32767
-                title = title & 0xffff
+                if -32768 <= lower <= 32767 and delta:
+                    title = title & 0xffff
+                else:
+                    title += char
+                    delta = False
             else:
                 title = 0
+            if not delta:
+                flags |= NODELTA_MASK
             # decimal digit, integer digit
             decimal = 0
             if record[6]:
@@ -603,6 +616,7 @@ def merge_old_version(version, new, old):
     bidir_changes = [0xFF]*0x110000
     category_changes = [0xFF]*0x110000
     decimal_changes = [0xFF]*0x110000
+    mirrored_changes = [0xFF]*0x110000
     # In numeric data, 0 means "no change",
     # -1 means "did not have a numeric value
     numeric_changes = [0] * 0x110000
@@ -649,6 +663,11 @@ def merge_old_version(version, new, old):
                         else:
                             assert re.match("^[0-9]+$", value)
                             numeric_changes[i] = int(value)
+                    elif k == 9:
+                        if value == 'Y':
+                            mirrored_changes[i] = '1'
+                        else:
+                            mirrored_changes[i] = '0'
                     elif k == 11:
                         # change to ISO comment, ignore
                         pass
@@ -665,7 +684,8 @@ def merge_old_version(version, new, old):
                         class Difference(Exception):pass
                         raise Difference, (hex(i), k, old.table[i], new.table[i])
     new.changed.append((version, zip(bidir_changes, category_changes,
-                                     decimal_changes, numeric_changes),
+                                     decimal_changes, mirrored_changes,
+                                     numeric_changes),
                         normalization_changes))