gh-91575: Add a script for generating data for case-insensitive matching in re (GH-91660)

Also test that all extra cases are in BMP.

2 files changed, 108 insertions, 57 deletions

diff --git a/Lib/re/_casefix.py b/Lib/re/_casefix.py
new file mode 100644
index 0000000..06507d0
--- /dev/null
+++ b/Lib/re/_casefix.py
@@ -0,0 +1,106 @@
+# Auto-generated by Tools/scripts/generate_re_casefix.py.
+
+# Maps the code of lowercased character to codes of different lowercased
+# characters which have the same uppercase.
+_EXTRA_CASES = {
+    # LATIN SMALL LETTER I: LATIN SMALL LETTER DOTLESS I
+    0x0069: (0x0131,), # 'i': 'ı'
+    # LATIN SMALL LETTER S: LATIN SMALL LETTER LONG S
+    0x0073: (0x017f,), # 's': 'ſ'
+    # MICRO SIGN: GREEK SMALL LETTER MU
+    0x00b5: (0x03bc,), # 'µ': 'μ'
+    # LATIN SMALL LETTER DOTLESS I: LATIN SMALL LETTER I
+    0x0131: (0x0069,), # 'ı': 'i'
+    # LATIN SMALL LETTER LONG S: LATIN SMALL LETTER S
+    0x017f: (0x0073,), # 'ſ': 's'
+    # COMBINING GREEK YPOGEGRAMMENI: GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
+    0x0345: (0x03b9, 0x1fbe), # '\u0345': 'ιι'
+    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+    0x0390: (0x1fd3,), # 'ΐ': 'ΐ'
+    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+    0x03b0: (0x1fe3,), # 'ΰ': 'ΰ'
+    # GREEK SMALL LETTER BETA: GREEK BETA SYMBOL
+    0x03b2: (0x03d0,), # 'β': 'ϐ'
+    # GREEK SMALL LETTER EPSILON: GREEK LUNATE EPSILON SYMBOL
+    0x03b5: (0x03f5,), # 'ε': 'ϵ'
+    # GREEK SMALL LETTER THETA: GREEK THETA SYMBOL
+    0x03b8: (0x03d1,), # 'θ': 'ϑ'
+    # GREEK SMALL LETTER IOTA: COMBINING GREEK YPOGEGRAMMENI, GREEK PROSGEGRAMMENI
+    0x03b9: (0x0345, 0x1fbe), # 'ι': '\u0345ι'
+    # GREEK SMALL LETTER KAPPA: GREEK KAPPA SYMBOL
+    0x03ba: (0x03f0,), # 'κ': 'ϰ'
+    # GREEK SMALL LETTER MU: MICRO SIGN
+    0x03bc: (0x00b5,), # 'μ': 'µ'
+    # GREEK SMALL LETTER PI: GREEK PI SYMBOL
+    0x03c0: (0x03d6,), # 'π': 'ϖ'
+    # GREEK SMALL LETTER RHO: GREEK RHO SYMBOL
+    0x03c1: (0x03f1,), # 'ρ': 'ϱ'
+    # GREEK SMALL LETTER FINAL SIGMA: GREEK SMALL LETTER SIGMA
+    0x03c2: (0x03c3,), # 'ς': 'σ'
+    # GREEK SMALL LETTER SIGMA: GREEK SMALL LETTER FINAL SIGMA
+    0x03c3: (0x03c2,), # 'σ': 'ς'
+    # GREEK SMALL LETTER PHI: GREEK PHI SYMBOL
+    0x03c6: (0x03d5,), # 'φ': 'ϕ'
+    # GREEK BETA SYMBOL: GREEK SMALL LETTER BETA
+    0x03d0: (0x03b2,), # 'ϐ': 'β'
+    # GREEK THETA SYMBOL: GREEK SMALL LETTER THETA
+    0x03d1: (0x03b8,), # 'ϑ': 'θ'
+    # GREEK PHI SYMBOL: GREEK SMALL LETTER PHI
+    0x03d5: (0x03c6,), # 'ϕ': 'φ'
+    # GREEK PI SYMBOL: GREEK SMALL LETTER PI
+    0x03d6: (0x03c0,), # 'ϖ': 'π'
+    # GREEK KAPPA SYMBOL: GREEK SMALL LETTER KAPPA
+    0x03f0: (0x03ba,), # 'ϰ': 'κ'
+    # GREEK RHO SYMBOL: GREEK SMALL LETTER RHO
+    0x03f1: (0x03c1,), # 'ϱ': 'ρ'
+    # GREEK LUNATE EPSILON SYMBOL: GREEK SMALL LETTER EPSILON
+    0x03f5: (0x03b5,), # 'ϵ': 'ε'
+    # CYRILLIC SMALL LETTER VE: CYRILLIC SMALL LETTER ROUNDED VE
+    0x0432: (0x1c80,), # 'в': 'ᲀ'
+    # CYRILLIC SMALL LETTER DE: CYRILLIC SMALL LETTER LONG-LEGGED DE
+    0x0434: (0x1c81,), # 'д': 'ᲁ'
+    # CYRILLIC SMALL LETTER O: CYRILLIC SMALL LETTER NARROW O
+    0x043e: (0x1c82,), # 'о': 'ᲂ'
+    # CYRILLIC SMALL LETTER ES: CYRILLIC SMALL LETTER WIDE ES
+    0x0441: (0x1c83,), # 'с': 'ᲃ'
+    # CYRILLIC SMALL LETTER TE: CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
+    0x0442: (0x1c84, 0x1c85), # 'т': 'ᲄᲅ'
+    # CYRILLIC SMALL LETTER HARD SIGN: CYRILLIC SMALL LETTER TALL HARD SIGN
+    0x044a: (0x1c86,), # 'ъ': 'ᲆ'
+    # CYRILLIC SMALL LETTER YAT: CYRILLIC SMALL LETTER TALL YAT
+    0x0463: (0x1c87,), # 'ѣ': 'ᲇ'
+    # CYRILLIC SMALL LETTER ROUNDED VE: CYRILLIC SMALL LETTER VE
+    0x1c80: (0x0432,), # 'ᲀ': 'в'
+    # CYRILLIC SMALL LETTER LONG-LEGGED DE: CYRILLIC SMALL LETTER DE
+    0x1c81: (0x0434,), # 'ᲁ': 'д'
+    # CYRILLIC SMALL LETTER NARROW O: CYRILLIC SMALL LETTER O
+    0x1c82: (0x043e,), # 'ᲂ': 'о'
+    # CYRILLIC SMALL LETTER WIDE ES: CYRILLIC SMALL LETTER ES
+    0x1c83: (0x0441,), # 'ᲃ': 'с'
+    # CYRILLIC SMALL LETTER TALL TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
+    0x1c84: (0x0442, 0x1c85), # 'ᲄ': 'тᲅ'
+    # CYRILLIC SMALL LETTER THREE-LEGGED TE: CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE
+    0x1c85: (0x0442, 0x1c84), # 'ᲅ': 'тᲄ'
+    # CYRILLIC SMALL LETTER TALL HARD SIGN: CYRILLIC SMALL LETTER HARD SIGN
+    0x1c86: (0x044a,), # 'ᲆ': 'ъ'
+    # CYRILLIC SMALL LETTER TALL YAT: CYRILLIC SMALL LETTER YAT
+    0x1c87: (0x0463,), # 'ᲇ': 'ѣ'
+    # CYRILLIC SMALL LETTER UNBLENDED UK: CYRILLIC SMALL LETTER MONOGRAPH UK
+    0x1c88: (0xa64b,), # 'ᲈ': 'ꙋ'
+    # LATIN SMALL LETTER S WITH DOT ABOVE: LATIN SMALL LETTER LONG S WITH DOT ABOVE
+    0x1e61: (0x1e9b,), # 'ṡ': 'ẛ'
+    # LATIN SMALL LETTER LONG S WITH DOT ABOVE: LATIN SMALL LETTER S WITH DOT ABOVE
+    0x1e9b: (0x1e61,), # 'ẛ': 'ṡ'
+    # GREEK PROSGEGRAMMENI: COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA
+    0x1fbe: (0x0345, 0x03b9), # 'ι': '\u0345ι'
+    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+    0x1fd3: (0x0390,), # 'ΐ': 'ΐ'
+    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA: GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+    0x1fe3: (0x03b0,), # 'ΰ': 'ΰ'
+    # CYRILLIC SMALL LETTER MONOGRAPH UK: CYRILLIC SMALL LETTER UNBLENDED UK
+    0xa64b: (0x1c88,), # 'ꙋ': 'ᲈ'
+    # LATIN SMALL LIGATURE LONG S T: LATIN SMALL LIGATURE ST
+    0xfb05: (0xfb06,), # 'ſt': 'st'
+    # LATIN SMALL LIGATURE ST: LATIN SMALL LIGATURE LONG S T
+    0xfb06: (0xfb05,), # 'st': 'ſt'
+}
diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py
index 065f6fb..f621d04 100644
--- a/Lib/re/_compiler.py
+++ b/Lib/re/_compiler.py
@@ -13,6 +13,7 @@
 import _sre
 from . import _parser
 from ._constants import *
+from ._casefix import _EXTRA_CASES
 
 assert _sre.MAGIC == MAGIC, "SRE module mismatch"
 
@@ -27,62 +28,6 @@ _REPEATING_CODES = {
     POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE),
 }
 
-# Sets of lowercase characters which have the same uppercase.
-_equivalences = (
-    # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
-    (0x69, 0x131), # iı
-    # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
-    (0x73, 0x17f), # sſ
-    # MICRO SIGN, GREEK SMALL LETTER MU
-    (0xb5, 0x3bc), # µμ
-    # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
-    (0x345, 0x3b9, 0x1fbe), # \u0345ιι
-    # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
-    (0x390, 0x1fd3), # ΐΐ
-    # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS, GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
-    (0x3b0, 0x1fe3), # ΰΰ
-    # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL
-    (0x3b2, 0x3d0), # βϐ
-    # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL
-    (0x3b5, 0x3f5), # εϵ
-    # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL
-    (0x3b8, 0x3d1), # θϑ
-    # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL
-    (0x3ba, 0x3f0), # κϰ
-    # GREEK SMALL LETTER PI, GREEK PI SYMBOL
-    (0x3c0, 0x3d6), # πϖ
-    # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL
-    (0x3c1, 0x3f1), # ρϱ
-    # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA
-    (0x3c2, 0x3c3), # ςσ
-    # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL
-    (0x3c6, 0x3d5), # φϕ
-    # CYRILLIC SMALL LETTER VE, CYRILLIC SMALL LETTER ROUNDED VE
-    (0x432, 0x1c80), # вᲀ
-    # CYRILLIC SMALL LETTER DE, CYRILLIC SMALL LETTER LONG-LEGGED DE
-    (0x434, 0x1c81), # дᲁ
-    # CYRILLIC SMALL LETTER O, CYRILLIC SMALL LETTER NARROW O
-    (0x43e, 0x1c82), # оᲂ
-    # CYRILLIC SMALL LETTER ES, CYRILLIC SMALL LETTER WIDE ES
-    (0x441, 0x1c83), # сᲃ
-    # CYRILLIC SMALL LETTER TE, CYRILLIC SMALL LETTER TALL TE, CYRILLIC SMALL LETTER THREE-LEGGED TE
-    (0x442, 0x1c84, 0x1c85), # тᲄᲅ
-    # CYRILLIC SMALL LETTER HARD SIGN, CYRILLIC SMALL LETTER TALL HARD SIGN
-    (0x44a, 0x1c86), # ъᲆ
-    # CYRILLIC SMALL LETTER YAT, CYRILLIC SMALL LETTER TALL YAT
-    (0x463, 0x1c87), # ѣᲇ
-    # CYRILLIC SMALL LETTER UNBLENDED UK, CYRILLIC SMALL LETTER MONOGRAPH UK
-    (0x1c88, 0xa64b), # ᲈꙋ
-    # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE
-    (0x1e61, 0x1e9b), # ṡẛ
-    # LATIN SMALL LIGATURE LONG S T, LATIN SMALL LIGATURE ST
-    (0xfb05, 0xfb06), # ſtst
-)
-
-# Maps the lowercase code to lowercase codes which have the same uppercase.
-_ignorecase_fixes = {i: tuple(j for j in t if i != j)
-                     for t in _equivalences for i in t}
-
 class _CompileData:
     __slots__ = ('code', 'repeat_count')
     def __init__(self):
@@ -111,7 +56,7 @@ def _compile(data, pattern, flags):
         if flags & SRE_FLAG_UNICODE:
             iscased = _sre.unicode_iscased
             tolower = _sre.unicode_tolower
-            fixes = _ignorecase_fixes
+            fixes = _EXTRA_CASES
         else:
             iscased = _sre.ascii_iscased
             tolower = _sre.ascii_tolower