- fixed split

  (test_sre still complains about split, but that's caused by
  the group reset bug, not split itself)

- added more mark slots
  (should be dynamically allocated, but 100 is better than 32.
  and checking for the upper limit is better than overwriting
  the memory ;-)

- internal: renamed the cursor helper class

- internal: removed some bloat from sre_compile

2 files changed, 78 insertions, 101 deletions

diff --git a/Lib/sre.py b/Lib/sre.py
index 32b3e8f..e0a51e3 100644
--- a/Lib/sre.py
+++ b/Lib/sre.py
@@ -26,7 +26,7 @@ T = TEMPLATE = sre_compile.SRE_FLAG_TEMPLATE
 U = UNICODE = sre_compile.SRE_FLAG_UNICODE
 
 # sre exception
-error = sre_parse.error
+error = sre_compile.error
 
 # --------------------------------------------------------------------
 # public interface
@@ -105,7 +105,7 @@ def _subn(pattern, template, string, count=0):
     n = i = 0
     s = []
     append = s.append
-    c = pattern.cursor(string)
+    c = pattern.scanner(string)
     while not count or n < count:
         m = c.search()
         if not m:
@@ -127,16 +127,20 @@ def _split(pattern, string, maxsplit=0):
     n = i = 0
     s = []
     append = s.append
-    c = pattern.cursor(string)
+    extend = s.extend
+    c = pattern.scanner(string)
+    g = c.groups
     while not maxsplit or n < maxsplit:
         m = c.search()
         if not m:
             break
-        j = m.start()
-        append(string[i:j])
-        i = m.end()
-	if i <= j:
-	    break
+	b, e = m.span()
+	if e == i:
+	    continue
+        append(string[i:b])
+	if g and b != e:
+	    extend(m.groups())
+	i = e
         n = n + 1
     if i < len(string):
         append(string[i:])
diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py
index 2d7c021..a51531b 100644
--- a/Lib/sre_compile.py
+++ b/Lib/sre_compile.py
@@ -11,8 +11,7 @@
 # other compatibility work.
 #
 
-import array, string, sys
-
+import array
 import _sre
 
 from sre_constants import *
@@ -24,123 +23,101 @@ for WORDSIZE in "BHil":
 else:
     raise RuntimeError, "cannot find a useable array type"
 
-# FIXME: <fl> should move some optimizations from the parser to here!
-
-class Code:
-    def __init__(self):
-	self.data = []
-    def __len__(self):
-	return len(self.data)
-    def __getitem__(self, index):
-	return self.data[index]
-    def __setitem__(self, index, code):
-	self.data[index] = code
-    def append(self, code):
-	self.data.append(code)
-    def todata(self):
-	# print self.data
-	try:
-	    return array.array(WORDSIZE, self.data).tostring()
-	except OverflowError:
-	    print self.data
-	    raise
-
 def _compile(code, pattern, flags):
-    append = code.append
+    emit = code.append
     for op, av in pattern:
 	if op is ANY:
 	    if flags & SRE_FLAG_DOTALL:
-		append(OPCODES[op]) # any character at all!
+		emit(OPCODES[op])
 	    else:
-		append(OPCODES[CATEGORY])
-		append(CHCODES[CATEGORY_NOT_LINEBREAK])
+		emit(OPCODES[CATEGORY])
+		emit(CHCODES[CATEGORY_NOT_LINEBREAK])
 	elif op in (SUCCESS, FAILURE):
-	    append(OPCODES[op])
+	    emit(OPCODES[op])
 	elif op is AT:
-	    append(OPCODES[op])
+	    emit(OPCODES[op])
 	    if flags & SRE_FLAG_MULTILINE:
-		append(ATCODES[AT_MULTILINE[av]])
+		emit(ATCODES[AT_MULTILINE[av]])
 	    else:
-		append(ATCODES[av])
+		emit(ATCODES[av])
 	elif op is BRANCH:
-	    append(OPCODES[op])
+	    emit(OPCODES[op])
 	    tail = []
 	    for av in av[1]:
-		skip = len(code); append(0)
+		skip = len(code); emit(0)
 		_compile(code, av, flags)
-##		append(OPCODES[SUCCESS])
- 		append(OPCODES[JUMP])
- 		tail.append(len(code)); append(0)
+ 		emit(OPCODES[JUMP])
+ 		tail.append(len(code)); emit(0)
 		code[skip] = len(code) - skip
-	    append(0) # end of branch
+	    emit(0) # end of branch
  	    for tail in tail:
 		code[tail] = len(code) - tail
 	elif op is CALL:
-	    append(OPCODES[op])
-	    skip = len(code); append(0)
+	    emit(OPCODES[op])
+	    skip = len(code); emit(0)
 	    _compile(code, av, flags)
-	    append(OPCODES[SUCCESS])
+	    emit(OPCODES[SUCCESS])
 	    code[skip] = len(code) - skip
 	elif op is CATEGORY:
-	    append(OPCODES[op])
+	    emit(OPCODES[op])
 	    if flags & SRE_FLAG_LOCALE:
-		append(CH_LOCALE[CHCODES[av]])
+		emit(CH_LOCALE[CHCODES[av]])
 	    elif flags & SRE_FLAG_UNICODE:
-		append(CH_UNICODE[CHCODES[av]])
+		emit(CH_UNICODE[CHCODES[av]])
 	    else:
-		append(CHCODES[av])
+		emit(CHCODES[av])
 	elif op is GROUP:
 	    if flags & SRE_FLAG_IGNORECASE:
-		append(OPCODES[OP_IGNORE[op]])
+		emit(OPCODES[OP_IGNORE[op]])
 	    else:
-		append(OPCODES[op])
-	    append(av-1)
+		emit(OPCODES[op])
+	    emit(av-1)
 	elif op is IN:
 	    if flags & SRE_FLAG_IGNORECASE:
-		append(OPCODES[OP_IGNORE[op]])
+		emit(OPCODES[OP_IGNORE[op]])
 		def fixup(literal, flags=flags):
 		    return _sre.getlower(ord(literal), flags)
 	    else:
-		append(OPCODES[op])
+		emit(OPCODES[op])
 		fixup = ord
-	    skip = len(code); append(0)
+	    skip = len(code); emit(0)
 	    for op, av in av:
-		append(OPCODES[op])
+		emit(OPCODES[op])
 		if op is NEGATE:
 		    pass
 		elif op is LITERAL:
-		    append(fixup(av))
+		    emit(fixup(av))
 		elif op is RANGE:
-		    append(fixup(av[0]))
-		    append(fixup(av[1]))
+		    emit(fixup(av[0]))
+		    emit(fixup(av[1]))
 		elif op is CATEGORY:
 		    if flags & SRE_FLAG_LOCALE:
-			append(CH_LOCALE[CHCODES[av]])
+			emit(CH_LOCALE[CHCODES[av]])
 		    elif flags & SRE_FLAG_UNICODE:
-			append(CH_UNICODE[CHCODES[av]])
+			emit(CH_UNICODE[CHCODES[av]])
 		    else:
-			append(CHCODES[av])
+			emit(CHCODES[av])
 		else:
-		    raise ValueError, "unsupported set operator"
-	    append(OPCODES[FAILURE])
+		    raise error, "internal: unsupported set operator"
+	    emit(OPCODES[FAILURE])
 	    code[skip] = len(code) - skip
 	elif op in (LITERAL, NOT_LITERAL):
 	    if flags & SRE_FLAG_IGNORECASE:
-		append(OPCODES[OP_IGNORE[op]])
+		emit(OPCODES[OP_IGNORE[op]])
 	    else:
-		append(OPCODES[op])
-	    append(ord(av))
+		emit(OPCODES[op])
+	    emit(ord(av))
 	elif op is MARK:
-	    append(OPCODES[op])
-	    append(av)
+	    emit(OPCODES[op])
+	    emit(av)
  	elif op in (REPEAT, MIN_REPEAT, MAX_REPEAT):
 	    if flags & SRE_FLAG_TEMPLATE:
-		append(OPCODES[REPEAT])
-		skip = len(code); append(0)
-		append(av[0])
-		append(av[1])
+		emit(OPCODES[REPEAT])
+		skip = len(code); emit(0)
+		emit(av[0])
+		emit(av[1])
 		_compile(code, av[2], flags)
-		append(OPCODES[SUCCESS])
+		emit(OPCODES[SUCCESS])
 		code[skip] = len(code) - skip
 	    else:
 		lo, hi = av[2].getwidth()
@@ -149,35 +126,35 @@ def _compile(code, pattern, flags):
 		if 0 and lo == hi == 1 and op is MAX_REPEAT:
 		    # FIXME: <fl> need a better way to figure out when
 		    # it's safe to use this one (in the parser, probably)
-		    append(OPCODES[MAX_REPEAT_ONE])
-		    skip = len(code); append(0)
-		    append(av[0])
-		    append(av[1])
+		    emit(OPCODES[MAX_REPEAT_ONE])
+		    skip = len(code); emit(0)
+		    emit(av[0])
+		    emit(av[1])
 		    _compile(code, av[2], flags)
-		    append(OPCODES[SUCCESS])
+		    emit(OPCODES[SUCCESS])
 		    code[skip] = len(code) - skip
 		else:
-		    append(OPCODES[op])
-		    skip = len(code); append(0)
-		    append(av[0])
-		    append(av[1])
+		    emit(OPCODES[op])
+		    skip = len(code); emit(0)
+		    emit(av[0])
+		    emit(av[1])
 		    _compile(code, av[2], flags)
-		    append(OPCODES[SUCCESS])
+		    emit(OPCODES[SUCCESS])
 		    code[skip] = len(code) - skip
 	elif op is SUBPATTERN:
  	    group = av[0]
  	    if group:
- 		append(OPCODES[MARK])
- 		append((group-1)*2)
+ 		emit(OPCODES[MARK])
+ 		emit((group-1)*2)
 	    _compile(code, av[1], flags)
  	    if group:
- 		append(OPCODES[MARK])
- 		append((group-1)*2+1)
+ 		emit(OPCODES[MARK])
+ 		emit((group-1)*2+1)
 	else:
 	    raise ValueError, ("unsupported operand type", op)
 
 def compile(p, flags=0):
-    # convert pattern list to internal format
+    # internal: convert pattern list to internal format
     if type(p) in (type(""), type(u"")):
 	import sre_parse
 	pattern = p
@@ -185,18 +162,14 @@ def compile(p, flags=0):
     else:
 	pattern = None
     flags = p.pattern.flags | flags
-    code = Code()
+    code = []
     _compile(code, p.data, flags)
     code.append(OPCODES[SUCCESS])
-    data = code.todata()
-    if 0: # debugging
-	print
-	print "-" * 68
-	import sre_disasm
-	sre_disasm.disasm(data)
-	print "-" * 68
+    # FIXME: <fl> get rid of this limitation
+    assert p.pattern.groups <= 100,\
+	   "sorry, but this version only supports 100 named groups"
     return _sre.compile(
 	pattern, flags,
-	data,
+	array.array(WORDSIZE, code).tostring(),
 	p.pattern.groups-1, p.pattern.groupdict
 	)