Issue #12728: Different Unicode characters having the same uppercase but

different lowercase are now matched in case-insensitive regular expressions.

3 files changed, 134 insertions, 10 deletions

diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
index 7d8f775..5ecd33a 100644
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -26,6 +26,46 @@ _REPEATING_CODES = {REPEAT, MIN_REPEAT, MAX_REPEAT}
 _SUCCESS_CODES = {SUCCESS, FAILURE}
 _ASSERT_CODES = {ASSERT, ASSERT_NOT}
 
+# Sets of lowercase characters which have the same uppercase.
+_equivalences = (
+    # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
+    (0x69, 0x131), # iı
+    # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
+    (0x73, 0x17f), # sſ
+    # MICRO SIGN, GREEK SMALL LETTER MU
+    (0xb5, 0x3bc), # µμ
+    # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
+    (0x345, 0x3b9, 0x1fbe), # \u0345ιι
+    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+    (0x390, 0x1fd3), # ΐΐ
+    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+    (0x3b0, 0x1fe3), # ΰΰ
+    # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
+    (0x3b2, 0x3d0), # βϐ
+    # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
+    (0x3b5, 0x3f5), # εϵ
+    # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
+    (0x3b8, 0x3d1), # θϑ
+    # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
+    (0x3ba, 0x3f0), # κϰ
+    # GREEK SMALL LETTER PI, GREEK PI SYMBOL
+    (0x3c0, 0x3d6), # πϖ
+    # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
+    (0x3c1, 0x3f1), # ρϱ
+    # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
+    (0x3c2, 0x3c3), # ςσ
+    # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
+    (0x3c6, 0x3d5), # φϕ
+    # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
+    (0x1e61, 0x1e9b), # ṡẛ
+    # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST
+    (0xfb05, 0xfb06), # ſtst
+)
+
+# Maps the lowercase code to lowercase codes which have the same uppercase.
+_ignorecase_fixes = {i: tuple(j for j in t if i != j)
+                     for t in _equivalences for i in t}
+
 def _compile(code, pattern, flags):
     # internal: compile a (sub)pattern
     emit = code.append
@@ -34,11 +74,29 @@ def _compile(code, pattern, flags):
     REPEATING_CODES = _REPEATING_CODES
     SUCCESS_CODES = _SUCCESS_CODES
     ASSERT_CODES = _ASSERT_CODES
+    if (flags & SRE_FLAG_IGNORECASE and
+            not (flags & SRE_FLAG_LOCALE) and
+            flags & SRE_FLAG_UNICODE):
+        fixes = _ignorecase_fixes
+    else:
+        fixes = None
     for op, av in pattern:
         if op in LITERAL_CODES:
             if flags & SRE_FLAG_IGNORECASE:
-                emit(OP_IGNORE[op])
-                emit(_sre.getlower(av, flags))
+                lo = _sre.getlower(av, flags)
+                if fixes and lo in fixes:
+                    emit(IN_IGNORE)
+                    skip = _len(code); emit(0)
+                    if op is NOT_LITERAL:
+                        emit(NEGATE)
+                    for k in (lo,) + fixes[lo]:
+                        emit(LITERAL)
+                        emit(k)
+                    emit(FAILURE)
+                    code[skip] = _len(code) - skip
+                else:
+                    emit(OP_IGNORE[op])
+                    emit(lo)
             else:
                 emit(op)
                 emit(av)
@@ -51,7 +109,7 @@ def _compile(code, pattern, flags):
                 emit(op)
                 fixup = None
             skip = _len(code); emit(0)
-            _compile_charset(av, flags, code, fixup)
+            _compile_charset(av, flags, code, fixup, fixes)
             code[skip] = _len(code) - skip
         elif op is ANY:
             if flags & SRE_FLAG_DOTALL:
@@ -165,10 +223,10 @@ def _compile(code, pattern, flags):
         else:
             raise ValueError("unsupported operand type", op)
 
-def _compile_charset(charset, flags, code, fixup=None):
+def _compile_charset(charset, flags, code, fixup=None, fixes=None):
     # compile charset subprogram
     emit = code.append
-    for op, av in _optimize_charset(charset, fixup):
+    for op, av in _optimize_charset(charset, fixup, fixes):
         emit(op)
         if op is NEGATE:
             pass
@@ -192,7 +250,7 @@ def _compile_charset(charset, flags, code, fixup=None):
             raise error("internal: unsupported set operator")
     emit(FAILURE)
 
-def _optimize_charset(charset, fixup):
+def _optimize_charset(charset, fixup, fixes):
     # internal: optimize character set
     out = []
     tail = []
@@ -202,14 +260,26 @@ def _optimize_charset(charset, fixup):
             try:
                 if op is LITERAL:
                     if fixup:
-                        av = fixup(av)
-                    charmap[av] = 1
+                        lo = fixup(av)
+                        charmap[lo] = 1
+                        if fixes and lo in fixes:
+                            for k in fixes[lo]:
+                                charmap[k] = 1
+                    else:
+                        charmap[av] = 1
                 elif op is RANGE:
                     r = range(av[0], av[1]+1)
                     if fixup:
                         r = map(fixup, r)
-                    for i in r:
-                        charmap[i] = 1
+                    if fixup and fixes:
+                        for i in r:
+                            charmap[i] = 1
+                            if i in fixes:
+                                for k in fixes[i]:
+                                    charmap[k] = 1
+                    else:
+                        for i in r:
+                            charmap[i] = 1
                 elif op is NEGATE:
                     out.append((op, av))
                 else:
diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py
index 8baf3b4..3bd716d 100644
--- a/Lib/test/test_re.py
+++ b/Lib/test/test_re.py
@@ -637,6 +637,43 @@ class ReTests(unittest.TestCase):
         self.assertEqual(re.match(r"((a)\s(abc|a))", "a a", re.I).group(1), "a a")
         self.assertEqual(re.match(r"((a)\s(abc|a)*)", "a aa", re.I).group(1), "a aa")
 
+        assert '\u212a'.lower() == 'k' # 'K'
+        self.assertTrue(re.match(r'K', '\u212a', re.I))
+        self.assertTrue(re.match(r'k', '\u212a', re.I))
+        self.assertTrue(re.match(r'\u212a', 'K', re.I))
+        self.assertTrue(re.match(r'\u212a', 'k', re.I))
+        assert '\u017f'.upper() == 'S' # 'ſ'
+        self.assertTrue(re.match(r'S', '\u017f', re.I))
+        self.assertTrue(re.match(r's', '\u017f', re.I))
+        self.assertTrue(re.match(r'\u017f', 'S', re.I))
+        self.assertTrue(re.match(r'\u017f', 's', re.I))
+        assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
+        self.assertTrue(re.match(r'\ufb05', '\ufb06', re.I))
+        self.assertTrue(re.match(r'\ufb06', '\ufb05', re.I))
+
+    def test_ignore_case_set(self):
+        self.assertTrue(re.match(r'[19A]', 'A', re.I))
+        self.assertTrue(re.match(r'[19a]', 'a', re.I))
+        self.assertTrue(re.match(r'[19a]', 'A', re.I))
+        self.assertTrue(re.match(r'[19A]', 'a', re.I))
+        self.assertTrue(re.match(br'[19A]', b'A', re.I))
+        self.assertTrue(re.match(br'[19a]', b'a', re.I))
+        self.assertTrue(re.match(br'[19a]', b'A', re.I))
+        self.assertTrue(re.match(br'[19A]', b'a', re.I))
+        assert '\u212a'.lower() == 'k' # 'K'
+        self.assertTrue(re.match(r'[19K]', '\u212a', re.I))
+        self.assertTrue(re.match(r'[19k]', '\u212a', re.I))
+        self.assertTrue(re.match(r'[19\u212a]', 'K', re.I))
+        self.assertTrue(re.match(r'[19\u212a]', 'k', re.I))
+        assert '\u017f'.upper() == 'S' # 'ſ'
+        self.assertTrue(re.match(r'[19S]', '\u017f', re.I))
+        self.assertTrue(re.match(r'[19s]', '\u017f', re.I))
+        self.assertTrue(re.match(r'[19\u017f]', 'S', re.I))
+        self.assertTrue(re.match(r'[19\u017f]', 's', re.I))
+        assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
+        self.assertTrue(re.match(r'[19\ufb05]', '\ufb06', re.I))
+        self.assertTrue(re.match(r'[19\ufb06]', '\ufb05', re.I))
+
     def test_ignore_case_range(self):
         # Issues #3511, #17381.
         self.assertTrue(re.match(r'[9-a]', '_', re.I))
@@ -656,6 +693,20 @@ class ReTests(unittest.TestCase):
         self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010428', re.I))
         self.assertTrue(re.match(r'[\U00010400-\U00010427]', '\U00010400', re.I))
 
+        assert '\u212a'.lower() == 'k' # 'K'
+        self.assertTrue(re.match(r'[J-M]', '\u212a', re.I))
+        self.assertTrue(re.match(r'[j-m]', '\u212a', re.I))
+        self.assertTrue(re.match(r'[\u2129-\u212b]', 'K', re.I))
+        self.assertTrue(re.match(r'[\u2129-\u212b]', 'k', re.I))
+        assert '\u017f'.upper() == 'S' # 'ſ'
+        self.assertTrue(re.match(r'[R-T]', '\u017f', re.I))
+        self.assertTrue(re.match(r'[r-t]', '\u017f', re.I))
+        self.assertTrue(re.match(r'[\u017e-\u0180]', 'S', re.I))
+        self.assertTrue(re.match(r'[\u017e-\u0180]', 's', re.I))
+        assert '\ufb05'.upper() == '\ufb06'.upper() == 'ST' # 'ſt', 'st'
+        self.assertTrue(re.match(r'[\ufb04-\ufb05]', '\ufb06', re.I))
+        self.assertTrue(re.match(r'[\ufb06-\ufb07]', '\ufb05', re.I))
+
     def test_category(self):
         self.assertEqual(re.match(r"(\s)", " ").group(1), " ")
 
diff --git a/Misc/NEWS b/Misc/NEWS
index 4b495d7..955cee6 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -183,6 +183,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #12728: Different Unicode characters having the same uppercase but
+  different lowercase are now matched in case-insensitive regular expressions.
+
 - Issue #22821: Fixed fcntl() with integer argument on 64-bit big-endian
   platforms.