diff options
| -rw-r--r-- | Lib/re.py | 132 | ||||
| -rwxr-xr-x | Lib/test/re_tests.py | 77 | ||||
| -rw-r--r-- | Lib/test/test_re.py | 50 |
3 files changed, 201 insertions, 58 deletions
@@ -1,7 +1,5 @@ #!/usr/bin/env python # -*- mode: python -*- -# $Id$ - import sys import string @@ -12,11 +10,12 @@ from pcre import * # # pcre.error and re.error should be the same, since exceptions can be -# raised from either module. +# raised from either module. # compilation flags I = IGNORECASE +L = LOCALE M = MULTILINE S = DOTALL X = VERBOSE @@ -61,8 +60,25 @@ def split(pattern, string, maxsplit=0): pattern = _cachecompile(pattern) return pattern.split(string, maxsplit) +def escape(pattern): + "Escape all non-alphanumeric characters in pattern." + result = [] + alphanum=string.letters+'_'+string.digits + for char in pattern: + if char not in alphanum: + result.append('\\') + result.append(char) + return string.join(result, '') + +def compile(pattern, flags=0): + "Compile a regular expression pattern, returning a RegexObject." + groupindex={} + code=pcre_compile(pattern, flags, groupindex) + return RegexObject(pattern, flags, code, groupindex) + + # -# +# Class definitions # class RegexObject: @@ -71,31 +87,54 @@ class RegexObject: self.flags = flags self.pattern = pattern self.groupindex = groupindex - def search(self, string, pos=0): - regs = self.code.match(string, pos, 0) + + def search(self, string, pos=0, endpos=None): + """Scan through string looking for a match to the pattern, returning + a MatchObject instance, or None if no match was found.""" + + if endpos is None or endpos>len(string): + endpos=len(string) + if endpos<pos: endpos=pos + regs = self.code.match(string, pos, endpos, 0) if regs is None: return None - self.num_regs=len(regs) + self._num_regs=len(regs) return MatchObject(self, string, - pos, + pos, endpos, regs) - def match(self, string, pos=0): - regs = self.code.match(string, pos, ANCHORED) + def match(self, string, pos=0, endpos=None): + """Try to apply the pattern at the start of the string, returning + a MatchObject instance, or None if no match was found.""" + + if endpos is None or endpos>len(string): + endpos=len(string) + if endpos<pos: endpos=pos + regs = self.code.match(string, pos, endpos, ANCHORED) if regs is None: return None - self.num_regs=len(regs) + self._num_regs=len(regs) return MatchObject(self, string, - pos, + pos, endpos, regs) def sub(self, repl, string, count=0): + """Return the string obtained by replacing the leftmost + non-overlapping occurrences of the pattern in string by the + replacement repl""" + return self.subn(repl, string, count)[0] - def subn(self, repl, source, count=0): + def subn(self, repl, source, count=0): + """Return a 2-tuple containing (new_string, number). + new_string is the string obtained by replacing the leftmost + non-overlapping occurrences of the pattern in string by the + replacement repl. number is the number of substitutions that + were made.""" + if count < 0: raise error, "negative substitution count" if count == 0: @@ -134,6 +173,9 @@ class RegexObject: return (string.join(results, ''), n) def split(self, source, maxsplit=0): + """Split \var{string} by the occurrences of the pattern, + returning a list containing the resulting substrings.""" + if maxsplit < 0: raise error, "negative split count" if maxsplit == 0: @@ -156,21 +198,39 @@ class RegexObject: pos = pos+1 continue results.append(source[lastmatch:i]) - g = m.group() + g = m.groups() if g: + if type(g)==type( "" ): g = [g] results[len(results):] = list(g) pos = lastmatch = j results.append(source[lastmatch:]) return results + # The following 3 functions were contributed by Mike Fletcher, and + # allow pickling and unpickling of RegexObject instances. + def __getinitargs__(self): + return (None,None,None,None) # any 4 elements, to work around + # problems with the + # pickle/cPickle modules not yet + # ignoring the __init__ function + def __getstate__(self): + return self.pattern, self.flags, self.groupindex + def __setstate__(self, statetuple): + self.pattern = statetuple[0] + self.flags = statetuple[1] + self.groupindex = statetuple[2] + self.code = apply(pcre_compile, statetuple) + class MatchObject: - def __init__(self, re, string, pos, regs): + def __init__(self, re, string, pos, endpos, regs): self.re = re self.string = string - self.pos = pos + self.pos = pos + self.endpos = endpos self.regs = regs - def start(self, g): + def start(self, g = 0): + "Return the start of the substring matched by group g" if type(g) == type(''): try: g = self.re.groupindex[g] @@ -178,7 +238,8 @@ class MatchObject: raise IndexError, ('group "' + g + '" is undefined') return self.regs[g][0] - def end(self, g): + def end(self, g = 0): + "Return the end of the substring matched by group g" if type(g) == type(''): try: g = self.re.groupindex[g] @@ -186,7 +247,9 @@ class MatchObject: raise IndexError, ('group "' + g + '" is undefined') return self.regs[g][1] - def span(self, g): + def span(self, g = 0): + """Return a tuple containing the start,end of the substring + matched by group g""" if type(g) == type(''): try: g = self.re.groupindex[g] @@ -194,12 +257,18 @@ class MatchObject: raise IndexError, ('group "' + g + '" is undefined') return self.regs[g] + def groups(self): + "Return a tuple containing all subgroups of the match object" + + # If _num_regs==1, we don't want to call self.group with an + # empty tuple. + if self.re._num_regs == 1: return () + return apply(self.group, tuple(range(1, self.re._num_regs) ) ) + def group(self, *groups): + "Return one or more groups of the match." if len(groups) == 0: - groups = range(1, self.re.num_regs) - use_all = 1 - else: - use_all = 0 + groups = (0,) result = [] for g in groups: if type(g) == type(''): @@ -212,25 +281,10 @@ class MatchObject: result.append(None) else: result.append(self.string[self.regs[g][0]:self.regs[g][1]]) - if use_all or len(result) > 1: + if len(result) > 1: return tuple(result) elif len(result) == 1: return result[0] else: return () -def escape(pattern): - result = [] - alphanum=string.letters+'_'+string.digits - for char in pattern: - if char not in alphanum: - result.append('\\') - result.append(char) - return string.join(result, '') - -def compile(pattern, flags=0): - groupindex={} - code=pcre_compile(pattern, flags, groupindex) - return RegexObject(pattern, flags, code, groupindex) - - diff --git a/Lib/test/re_tests.py b/Lib/test/re_tests.py index 9447dd2..9bbfa54 100755 --- a/Lib/test/re_tests.py +++ b/Lib/test/re_tests.py @@ -2,7 +2,7 @@ # -*- mode: python -*- # $Id$ -# Re test suite and benchmark suite v1.5a2 +# Re test suite and benchmark suite v1.5b2 # The 3 possible outcomes for each pattern [SUCCEED, FAIL, SYNTAX_ERROR] = range(3) @@ -47,7 +47,62 @@ benchmarks = [ # # If the regex isn't expected to work, the latter two elements can be omitted. -tests = [ +tests = [ + # Test ?P< and ?P= extensions + ('(?P<foo_123', '', SYNTAX_ERROR), # Unterminated group identifier + ('(?P<1>a)', '', SYNTAX_ERROR), # Begins with a digit + ('(?P<!>a)', '', SYNTAX_ERROR), # Begins with an illegal char + ('(?P<foo!>a)', '', SYNTAX_ERROR), # Begins with an illegal char + + # Same tests, for the ?P= form + ('(?P<foo_123>a)(?P=foo_123', 'aa', SYNTAX_ERROR), + ('(?P<foo_123>a)(?P=1)', 'aa', SYNTAX_ERROR), + ('(?P<foo_123>a)(?P=!)', 'aa', SYNTAX_ERROR), + ('(?P<foo_123>a)(?P=foo_124', 'aa', SYNTAX_ERROR), # Backref to undefined group + + ('(?P<foo_123>a)', 'a', SUCCEED, 'g1', 'a'), + ('(?P<foo_123>a)(?P=foo_123)', 'aa', SUCCEED, 'g1', 'a'), + + # Test octal escapes + ('\\1', 'a', SYNTAX_ERROR), + ('\\09', chr(0) + '9', SUCCEED, 'found', chr(0) + '9'), + ('\\141', 'a', SUCCEED, 'found', 'a'), + ('(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)\\119', 'abcdefghijklk9', SUCCEED, 'found+"-"+g11', 'abcdefghijklk9-k'), + + # Test that a literal \0 is handled everywhere + ('\0', '\0', SUCCEED, 'found', '\0'), + (r'\0', '\0', SUCCEED, 'found', '\0'), + ('[\0a]', '\0', SUCCEED, 'found', '\0'), + ('[a\0]', '\0', SUCCEED, 'found', '\0'), + ('[^a\0]', '\0', FAIL), + (r'[\0a]', '\0', SUCCEED, 'found', '\0'), + (r'[a\0]', '\0', SUCCEED, 'found', '\0'), + (r'[^a\0]', '\0', FAIL), + + # Test various letter escapes + (r'\a[\b]\f\n\r\t\v', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'), + (r'[\a][\b][\f][\n][\r][\t][\v]', '\a\b\f\n\r\t\v', SUCCEED, 'found', '\a\b\f\n\r\t\v'), + (r'\u', '', SYNTAX_ERROR), # A Perl escape + (r'\c\e\g\h\i\j\k\m\o\p\q\y\z', 'ceghijkmopqyz', SUCCEED, 'found', 'ceghijkmopqyz'), + (r'\xff', '\377', SUCCEED, 'found', chr(255)), + (r'\x00ffffffffffffff', '\377', SUCCEED, 'found', chr(255)), + (r'\x00f', '\017', SUCCEED, 'found', chr(15)), + (r'\x00fe', '\376', SUCCEED, 'found', chr(254)), + + (r"^\w+=(\\[\000-\277]|[^\n\\])*", "SRC=eval.c g.c blah blah blah \\\\\n\tapes.c", + SUCCEED, 'found', "SRC=eval.c g.c blah blah blah \\\\"), + + # Test that . only matches \n in DOTALL mode + ('a.b', 'acb', SUCCEED, 'found', 'acb'), + ('a.b', 'a\nb', FAIL), + ('a.*b', 'acc\nccb', FAIL), + ('a.{4,5}b', 'acc\nccb', FAIL), + ('a.b', 'a\rb', SUCCEED, 'found', 'a\rb'), + ('a.b(?s)', 'a\nb', SUCCEED, 'found', 'a\nb'), + ('a.*(?s)b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), + ('(?s)a.{4,5}b', 'acc\nccb', SUCCEED, 'found', 'acc\nccb'), + ('(?s)a.b', 'a\nb', SUCCEED, 'found', 'a\nb'), + ('abc', 'abc', SUCCEED, 'found', 'abc'), ('abc', 'xbc', FAIL), ('abc', 'axc', FAIL), @@ -338,8 +393,9 @@ tests = [ ('(.*)c(.*)', 'abcde', SUCCEED, 'found+"-"+g1+"-"+g2', 'abcde-ab-de'), ('\\((.*), (.*)\\)', '(a, b)', SUCCEED, 'g2+"-"+g1', 'b-a'), ('[k]', 'ab', FAIL), - ##('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'), - ##('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'), +# XXX +# ('abcd', 'abcd', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'abcd-$&-\\abcd'), +# ('a(bc)d', 'abcd', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'bc-$1-\\bc'), ('a[-]?c', 'ac', SUCCEED, 'found', 'ac'), ('(abc)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), ('([a-c]*)\\1', 'abcabc', SUCCEED, 'g1', 'abc'), @@ -470,15 +526,14 @@ tests = [ ('(?i)(.*)c(.*)', 'ABCDE', SUCCEED, 'found+"-"+g1+"-"+g2', 'ABCDE-AB-DE'), ('(?i)\\((.*), (.*)\\)', '(A, B)', SUCCEED, 'g2+"-"+g1', 'B-A'), ('(?i)[k]', 'AB', FAIL), - ##('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'), - ##('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'), +# ('(?i)abcd', 'ABCD', SUCCEED, 'found+"-"+\\found+"-"+\\\\found', 'ABCD-$&-\\ABCD'), +# ('(?i)a(bc)d', 'ABCD', SUCCEED, 'g1+"-"+\\g1+"-"+\\\\g1', 'BC-$1-\\BC'), ('(?i)a[-]?c', 'AC', SUCCEED, 'found', 'AC'), ('(?i)(abc)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'), ('(?i)([a-c]*)\\1', 'ABCABC', SUCCEED, 'g1', 'ABC'), - # these zero-width assertions are not supported - #('a(?!b).', 'abad', SUCCEED, 'found', 'ad'), - #('a(?=d).', 'abad', SUCCEED, 'found', 'ad'), - #('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'), + ('a(?!b).', 'abad', SUCCEED, 'found', 'ad'), + ('a(?=d).', 'abad', SUCCEED, 'found', 'ad'), + ('a(?=c|d).', 'abad', SUCCEED, 'found', 'ad'), ('a(?:b|c|d)(.)', 'ace', SUCCEED, 'g1', 'e'), ('a(?:b|c|d)*(.)', 'ace', SUCCEED, 'g1', 'e'), ('a(?:b|c|d)+?(.)', 'ace', SUCCEED, 'g1', 'e'), @@ -535,5 +590,5 @@ xyzabc (r'\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'), ('\t\n\v\r\f\a\g', '\t\n\v\r\f\ag', SUCCEED, 'found', '\t\n\v\r\f\ag'), (r'\t\n\v\r\f\a', '\t\n\v\r\f\a', SUCCEED, 'found', chr(9)+chr(10)+chr(11)+chr(13)+chr(12)+chr(7)), - (r'[\t][\n][\v][\r][\f][\a][\A][\b][\B][\Z][\g]', '\t\n\v\r\f\aA\bBZg', SUCCEED, 'found', '\t\n\v\r\f\aA\bBZg'), + (r'[\t][\n][\v][\r][\f][\b]', '\t\n\v\r\f\b', SUCCEED, 'found', '\t\n\v\r\f\b'), ] diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index d90e2a8..83eab9f 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -4,7 +4,6 @@ from test_support import verbose, TestFailed import re -import reop import sys, os, string, traceback # Misc tests from Tim Peters' re.doc @@ -23,12 +22,13 @@ try: assert re.sub('.', lambda m: r"\n", 'x') == '\\n' assert re.sub('.', r"\n", 'x') == '\n' - + s = r"\1\1" assert re.sub('(.)', s, 'x') == 'xx' assert re.sub('(.)', re.escape(s), 'x') == s assert re.sub('(.)', lambda m: s, 'x') == s + assert re.sub('(?P<a>x)', '\g<a>\g<a>', 'xx') == 'xxxx' assert re.sub('(?P<unk>x)', '\g<unk>\g<unk>', 'xx') == 'xxxx' assert re.sub('a', r'\t\n\v\r\f\a\b\B\Z\a\A\w\W\s\S\d\D', 'a') == '\t\n\v\r\f\a\bBZ\aAwWsSdD' @@ -98,13 +98,9 @@ try: assert re.subn("b+", "x", "bbbb BBBB") == ('x BBBB', 1) assert re.subn("b+", "x", "xyz") == ('xyz', 0) assert re.subn("b*", "x", "xyz") == ('xxxyxzx', 4) - except AssertionError: raise TestFailed, "re.subn" -if verbose: - print 'Running tests on re.split' - try: assert re.split(":", ":a:b::c") == ['', 'a', 'b', '', 'c'] assert re.split(":*", ":a:b::c") == ['', 'a', 'b', 'c'] @@ -119,12 +115,37 @@ try: except AssertionError: raise TestFailed, "re.split" +if verbose: + print 'Pickling a RegexObject instance' + import pickle + pat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)') + s = pickle.dumps(pat) + pat = pickle.loads(s) + +if verbose: + print 'Running tests on re.split' + +try: + assert re.I == re.IGNORECASE + assert re.L == re.LOCALE + assert re.M == re.MULTILINE + assert re.S == re.DOTALL + assert re.X == re.VERBOSE +except AssertionError: + raise TestFailed, 're module constants' + +for flags in [re.I, re.M, re.X, re.S]: + try: + r = re.compile('^pattern$', flags) + except: + print 'Exception raised on flag', flags + from re_tests import * if verbose: print 'Running re_tests test suite' else: # To save time, only run the first and last 10 tests - tests = tests[:10] + tests[-10:] + pass #tests = tests[:10] + tests[-10:] for t in tests: sys.stdout.flush() @@ -150,7 +171,7 @@ for t in tests: else: try: result=obj.search(s) - except (re.error, reop.error), msg: + except (re.error), msg: print '=== Unexpected exception', t, repr(msg) if outcome==SYNTAX_ERROR: # This should have been a syntax error; forget it. @@ -190,9 +211,22 @@ for t in tests: else: print '=== Failed incorrectly', t + # Try the match with the search area limited to the extent + # of the match and see if it still succeeds. \B will + # break (because it won't match at the end or start of a + # string), so we'll ignore patterns that feature it. + + if pattern[:2]!='\\B' and pattern[-2:]!='\\B': + obj=re.compile(pattern) + result=obj.search(s, pos=result.start(0), endpos=result.end(0)+1) + if result==None: + print '=== Failed on range-limited match', t + # Try the match with IGNORECASE enabled, and check that it # still succeeds. obj=re.compile(pattern, re.IGNORECASE) result=obj.search(s) if result==None: print '=== Fails on case-insensitive match', t + + |