diff options
Diffstat (limited to 'Lib/sre_compile.py')
| -rw-r--r-- | Lib/sre_compile.py | 87 |
1 files changed, 76 insertions, 11 deletions
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index febbde1..c5a7e89 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # # Secret Labs' Regular Expression Engine # @@ -26,6 +27,40 @@ _REPEATING_CODES = set([REPEAT, MIN_REPEAT, MAX_REPEAT]) _SUCCESS_CODES = set([SUCCESS, FAILURE]) _ASSERT_CODES = set([ASSERT, ASSERT_NOT]) +# Sets of lowercase characters which have the same uppercase. +_equivalences = ( + # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I + (0x69, 0x131), # iı + # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S + (0x73, 0x17f), # sſ + # MICRO SIGN, GREEK SMALL LETTER MU + (0xb5, 0x3bc), # µμ + # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI + (0x345, 0x3b9, 0x1fbe), # \u0345ιι + # GREEK SMALL LETTER BETA, GREEK BETA SYMBOL + (0x3b2, 0x3d0), # βϐ + # GREEK SMALL LETTER EPSILON, GREEK LUNATE EPSILON SYMBOL + (0x3b5, 0x3f5), # εϵ + # GREEK SMALL LETTER THETA, GREEK THETA SYMBOL + (0x3b8, 0x3d1), # θϑ + # GREEK SMALL LETTER KAPPA, GREEK KAPPA SYMBOL + (0x3ba, 0x3f0), # κϰ + # GREEK SMALL LETTER PI, GREEK PI SYMBOL + (0x3c0, 0x3d6), # πϖ + # GREEK SMALL LETTER RHO, GREEK RHO SYMBOL + (0x3c1, 0x3f1), # ρϱ + # GREEK SMALL LETTER FINAL SIGMA, GREEK SMALL LETTER SIGMA + (0x3c2, 0x3c3), # ςσ + # GREEK SMALL LETTER PHI, GREEK PHI SYMBOL + (0x3c6, 0x3d5), # φϕ + # LATIN SMALL LETTER S WITH DOT ABOVE, LATIN SMALL LETTER LONG S WITH DOT ABOVE + (0x1e61, 0x1e9b), # ṡẛ +) + +# Maps the lowercase code to lowercase codes which have the same uppercase. +_ignorecase_fixes = {i: tuple(j for j in t if i != j) + for t in _equivalences for i in t} + def _compile(code, pattern, flags): # internal: compile a (sub)pattern emit = code.append @@ -34,11 +69,29 @@ def _compile(code, pattern, flags): REPEATING_CODES = _REPEATING_CODES SUCCESS_CODES = _SUCCESS_CODES ASSERT_CODES = _ASSERT_CODES + if (flags & SRE_FLAG_IGNORECASE and + not (flags & SRE_FLAG_LOCALE) and + flags & SRE_FLAG_UNICODE): + fixes = _ignorecase_fixes + else: + fixes = None for op, av in pattern: if op in LITERAL_CODES: if flags & SRE_FLAG_IGNORECASE: - emit(OPCODES[OP_IGNORE[op]]) - emit(_sre.getlower(av, flags)) + lo = _sre.getlower(av, flags) + if fixes and lo in fixes: + emit(OPCODES[IN_IGNORE]) + skip = _len(code); emit(0) + if op is NOT_LITERAL: + emit(OPCODES[NEGATE]) + for k in (lo,) + fixes[lo]: + emit(OPCODES[LITERAL]) + emit(k) + emit(OPCODES[FAILURE]) + code[skip] = _len(code) - skip + else: + emit(OPCODES[OP_IGNORE[op]]) + emit(lo) else: emit(OPCODES[op]) emit(av) @@ -51,7 +104,7 @@ def _compile(code, pattern, flags): emit(OPCODES[op]) fixup = None skip = _len(code); emit(0) - _compile_charset(av, flags, code, fixup) + _compile_charset(av, flags, code, fixup, fixes) code[skip] = _len(code) - skip elif op is ANY: if flags & SRE_FLAG_DOTALL: @@ -172,10 +225,11 @@ def _compile(code, pattern, flags): else: raise ValueError, ("unsupported operand type", op) -def _compile_charset(charset, flags, code, fixup=None): +def _compile_charset(charset, flags, code, fixup=None, fixes=None): # compile charset subprogram emit = code.append - for op, av in _optimize_charset(charset, fixup, flags & SRE_FLAG_UNICODE): + for op, av in _optimize_charset(charset, fixup, fixes, + flags & SRE_FLAG_UNICODE): emit(OPCODES[op]) if op is NEGATE: pass @@ -199,7 +253,7 @@ def _compile_charset(charset, flags, code, fixup=None): raise error, "internal: unsupported set operator" emit(OPCODES[FAILURE]) -def _optimize_charset(charset, fixup, isunicode): +def _optimize_charset(charset, fixup, fixes, isunicode): # internal: optimize character set out = [] tail = [] @@ -208,16 +262,27 @@ def _optimize_charset(charset, fixup, isunicode): while True: try: if op is LITERAL: - i = av if fixup: - i = fixup(i) - charmap[i] = 1 + i = fixup(av) + charmap[i] = 1 + if fixes and i in fixes: + for k in fixes[i]: + charmap[k] = 1 + else: + charmap[av] = 1 elif op is RANGE: r = range(av[0], av[1]+1) if fixup: r = map(fixup, r) - for i in r: - charmap[i] = 1 + if fixup and fixes: + for i in r: + charmap[i] = 1 + if i in fixes: + for k in fixes[i]: + charmap[k] = 1 + else: + for i in r: + charmap[i] = 1 elif op is NEGATE: out.append((op, av)) else: |