Reformatted with 4-space tab stops.

Allow '=' and '~' in unquoted attribute values.

Added overridable methods handle_starttag(tag, method, attrs) and
handle_endtag(tag, method) so subclasses can decide whether they
really want to call the method (e.g. when suppressing some portion of
the document).

Added support for a number of SGML shortcuts:

        shorthand               full notation
        <tag>...<>...           <tag>...<tag>...
        <tag>...</>             <tag>...</tag>
        <tag/.../               <tag>...</tag>
        <tag1<tag2>             <tag1><tag2>
        </tag1</tag2>           </tag1></tag2>
        </tag1<tag2>            </tag1><tag2>

This required factoring out some common actions and rationalizing the
interface to parse_endtag(), so as to make the code more readable.

Fixed syntax for &entity and &#char references so the trailing
semicolon is optional; removed explicit support for trailing period
(which was a TBL mistake in HTML 0.0).

Generalized the test program.

Tried to speed things up a little.  (More to come after the profile
results are in.)

Fix error recovery: call the end methods popped from the stack instead
of the one that triggers.  (Plus some complications because of the way
HTML extensions are handled in Grail.)

1 files changed, 406 insertions, 286 deletions

diff --git a/Lib/sgmllib.py b/Lib/sgmllib.py
index b46f829..304bbdb 100644
--- a/Lib/sgmllib.py
+++ b/Lib/sgmllib.py
@@ -14,16 +14,28 @@ import string
 
 # Regular expressions used for parsing
 
-incomplete = regex.compile(
-	  '<!-?\|</[a-zA-Z][a-zA-Z0-9]*[ \t\n]*\|</?\|' +
-	  '&#[a-zA-Z0-9]*\|&[a-zA-Z][a-zA-Z0-9]*\|&')
-entityref = regex.compile('&[a-zA-Z][a-zA-Z0-9]*[;.]')
-charref = regex.compile('&#[a-zA-Z0-9]+;')
-starttagopen = regex.compile('<[a-zA-Z]')
-endtag = regex.compile('</[a-zA-Z][a-zA-Z0-9]*[ \t\n]*>')
+interesting = regex.compile('[&<]')
+incomplete = regex.compile('&\([a-zA-Z][a-zA-Z0-9]*\|#[0-9]*\)?\|'
+			   '<\([a-zA-Z][^<>]*\|'
+			      '/\([a-zA-Z][^<>]*\)?\|'
+			      '![^<>]*\)?')
+
+entityref = regex.compile('&\([a-zA-Z][a-zA-Z0-9]*\)[^a-zA-Z0-9]')
+charref = regex.compile('&#\([0-9]+\)[^0-9]')
+
+starttagopen = regex.compile('<[>a-zA-Z]')
+shorttagopen = regex.compile('<[a-zA-Z][a-zA-Z0-9]*/')
+shorttag = regex.compile('<\([a-zA-Z][a-zA-Z0-9]*\)/\([^/]*\)/')
+endtagopen = regex.compile('</[<>a-zA-Z]')
+endbracket = regex.compile('[<>]')
 special = regex.compile('<![^<>]*>')
 commentopen = regex.compile('<!--')
 commentclose = regex.compile('--[ \t\n]*>')
+tagfind = regex.compile('[a-zA-Z][a-zA-Z0-9]*')
+attrfind = regex.compile(
+    '[ \t\n]+\([a-zA-Z_][a-zA-Z_0-9]*\)'
+    '\([ \t\n]*=[ \t\n]*'
+    '\(\'[^\']*\'\|"[^"]*"\|[-a-zA-Z0-9./:+*%?!()_#=~]*\)\)?')
 
 
 # SGML parser base class -- find tags and call handler functions.
@@ -39,288 +51,396 @@ commentclose = regex.compile('--[ \t\n]*>')
 
 class SGMLParser:
 
-	# Interface -- initialize and reset this instance
-	def __init__(self, verbose=0):
-		self.verbose = verbose
-		self.reset()
-
-	# Interface -- reset this instance.  Loses all unprocessed data
-	def reset(self):
-		self.rawdata = ''
-		self.stack = []
-		self.nomoretags = 0
-		self.literal = 0
-
-	# For derived classes only -- enter literal mode (CDATA) till EOF
-	def setnomoretags(self):
-		self.nomoretags = self.literal = 1
-
-	# For derived classes only -- enter literal mode (CDATA)
-	def setliteral(self, *args):
-		self.literal = 1
-
-	# Interface -- feed some data to the parser.  Call this as
-	# often as you want, with as little or as much text as you
-	# want (may include '\n').  (This just saves the text, all the
-	# processing is done by goahead().)
-	def feed(self, data):
-		self.rawdata = self.rawdata + data
-		self.goahead(0)
-
-	# Interface -- handle the remaining data
-	def close(self):
-		self.goahead(1)
-
-	# Internal -- handle data as far as reasonable.  May leave state
-	# and data to be processed by a subsequent call.  If 'end' is
-	# true, force handling all data as if followed by EOF marker.
-	def goahead(self, end):
-		rawdata = self.rawdata
-		i = 0
-		n = len(rawdata)
-		while i < n:
-			if self.nomoretags:
-				self.handle_data(rawdata[i:n])
-				i = n
-				break
-			j = incomplete.search(rawdata, i)
-			if j < 0: j = n
-			if i < j: self.handle_data(rawdata[i:j])
-			i = j
-			if i == n: break
-			if rawdata[i] == '<':
-				if starttagopen.match(rawdata, i) >= 0:
-					if self.literal:
-						self.handle_data(rawdata[i])
-						i = i+1
-						continue
-					k = self.parse_starttag(i)
-					if k < 0: break
-					i = i + k
-					continue
-				k = endtag.match(rawdata, i)
-				if k >= 0:
-					j = i+k
-					self.parse_endtag(rawdata[i:j])
-					i = j
-					self.literal = 0
-					continue
-				if commentopen.match(rawdata, i) >= 0:
-					if self.literal:
-						self.handle_data(rawdata[i])
-						i = i+1
-						continue
-					k = self.parse_comment(i)
-					if k < 0: break
-					i = i+k
-					continue
-				k = special.match(rawdata, i)
-				if k >= 0:
-					if self.literal:
-						self.handle_data(rawdata[i])
-						i = i+1
-						continue
-					i = i+k
-					continue
-			elif rawdata[i] == '&':
-				k = charref.match(rawdata, i)
-				if k >= 0:
-					j = i+k
-					self.handle_charref(rawdata[i+2:j-1])
-					i = j
-					continue
-				k = entityref.match(rawdata, i)
-				if k >= 0:
-					j = i+k
-					self.handle_entityref(rawdata[i+1:j-1])
-					i = j
-					continue
-			else:
-				raise RuntimeError, 'neither < nor & ??'
-			# We get here only if incomplete matches but
-			# nothing else
-			k = incomplete.match(rawdata, i)
-			if k < 0: raise RuntimeError, 'no incomplete match ??'
-			j = i+k
-			if j == n or rawdata[i:i+2] == '<!':
-				break # Really incomplete
-			self.handle_data(rawdata[i:j])
-			i = j
-		# end while
-		if end and i < n:
-			self.handle_data(rawdata[i:n])
-			i = n
-		self.rawdata = rawdata[i:]
-		# XXX if end: check for empty stack
-
-	# Internal -- parse comment, return length or -1 if not terminated
-	def parse_comment(self, i):
-		rawdata = self.rawdata
-		if rawdata[i:i+4] <> '<!--':
-			raise RuntimeError, 'unexpected call to handle_comment'
-		j = commentclose.search(rawdata, i+4)
-		if j < 0:
-			return -1
-		self.handle_comment(rawdata[i+4: j])
-		j = j+commentclose.match(rawdata, j)
-		return j-i
-
-	# Internal -- handle starttag, return length or -1 if not terminated
-	def parse_starttag(self, i):
-		rawdata = self.rawdata
+    # Interface -- initialize and reset this instance
+    def __init__(self, verbose=0):
+	self.verbose = verbose
+	self.reset()
+
+    # Interface -- reset this instance.  Loses all unprocessed data
+    def reset(self):
+	self.rawdata = ''
+	self.stack = []
+	self.lasttag = '???'
+	self.nomoretags = 0
+	self.literal = 0
+
+    # For derived classes only -- enter literal mode (CDATA) till EOF
+    def setnomoretags(self):
+	self.nomoretags = self.literal = 1
+
+    # For derived classes only -- enter literal mode (CDATA)
+    def setliteral(self, *args):
+	self.literal = 1
+
+    # Interface -- feed some data to the parser.  Call this as
+    # often as you want, with as little or as much text as you
+    # want (may include '\n').  (This just saves the text, all the
+    # processing is done by goahead().)
+    def feed(self, data):
+	self.rawdata = self.rawdata + data
+	self.goahead(0)
+
+    # Interface -- handle the remaining data
+    def close(self):
+	self.goahead(1)
+
+    # Internal -- handle data as far as reasonable.  May leave state
+    # and data to be processed by a subsequent call.  If 'end' is
+    # true, force handling all data as if followed by EOF marker.
+    def goahead(self, end):
+	rawdata = self.rawdata
+	i = 0
+	n = len(rawdata)
+	while i < n:
+	    if self.nomoretags:
+		self.handle_data(rawdata[i:n])
+		i = n
+		break
+	    j = interesting.search(rawdata, i)
+	    if j < 0: j = n
+	    if i < j: self.handle_data(rawdata[i:j])
+	    i = j
+	    if i == n: break
+	    if rawdata[i] == '<':
+		if starttagopen.match(rawdata, i) >= 0:
+		    if self.literal:
+			self.handle_data(rawdata[i])
+			i = i+1
+			continue
+		    k = self.parse_starttag(i)
+		    if k < 0: break
+		    i = k
+		    continue
+		if endtagopen.match(rawdata, i) >= 0:
+		    k = self.parse_endtag(i)
+		    if k < 0: break
+		    i =  k
+		    self.literal = 0
+		    continue
+		if commentopen.match(rawdata, i) >= 0:
+		    if self.literal:
+			self.handle_data(rawdata[i])
+			i = i+1
+			continue
+		    k = self.parse_comment(i)
+		    if k < 0: break
+		    i = i+k
+		    continue
+		k = special.match(rawdata, i)
+		if k >= 0:
+		    if self.literal:
+			self.handle_data(rawdata[i])
+			i = i+1
+			continue
+		    i = i+k
+		    continue
+	    elif rawdata[i] == '&':
+		k = charref.match(rawdata, i)
+		if k >= 0:
+		    k = i+k
+		    if rawdata[k-1] != ';': k = k-1
+		    name = charref.group(1)
+		    self.handle_charref(name)
+		    i = k
+		    continue
+		k = entityref.match(rawdata, i)
+		if k >= 0:
+		    k = i+k
+		    if rawdata[k-1] != ';': k = k-1
+		    name = entityref.group(1)
+		    self.handle_entityref(name)
+		    i = k
+		    continue
+	    else:
+		raise RuntimeError, 'neither < nor & ??'
+	    # We get here only if incomplete matches but
+	    # nothing else
+	    k = incomplete.match(rawdata, i)
+	    if k < 0:
+		self.handle_data(rawdata[i])
+		i = i+1
+		continue
+	    j = i+k
+	    if j == n:
+		break # Really incomplete
+	    self.handle_data(rawdata[i:j])
+	    i = j
+	# end while
+	if end and i < n:
+	    self.handle_data(rawdata[i:n])
+	    i = n
+	self.rawdata = rawdata[i:]
+	# XXX if end: check for empty stack
+
+    # Internal -- parse comment, return length or -1 if not terminated
+    def parse_comment(self, i):
+	rawdata = self.rawdata
+	if rawdata[i:i+4] <> '<!--':
+	    raise RuntimeError, 'unexpected call to handle_comment'
+	j = commentclose.search(rawdata, i+4)
+	if j < 0:
+	    return -1
+	self.handle_comment(rawdata[i+4: j])
+	j = j+commentclose.match(rawdata, j)
+	return j-i
+
+    # Internal -- handle starttag, return length or -1 if not terminated
+    def parse_starttag(self, i):
+	rawdata = self.rawdata
+	if shorttagopen.match(rawdata, i) >= 0:
+	    # SGML shorthand: <tag/data/ == <tag>data</tag>
+	    # XXX Can data contain &... (entity or char refs)?
+	    # XXX Can data contain < or > (tag characters)?
+	    # XXX Can there be whitespace before the first /?
+	    j = shorttag.match(rawdata, i)
+	    if j < 0:
+		return -1
+	    tag, data = shorttag.group(1, 2)
+	    tag = string.lower(tag)
+	    self.finish_shorttag(tag, data)
+	    k = i+j
+	    if rawdata[k-1] == '<':
+		k = k-1
+	    return k
+	# XXX The following should skip matching quotes (' or ")
+	j = endbracket.search(rawdata, i+1)
+	if j < 0:
+	    return -1
+	# Now parse the data between i+1 and j into a tag and attrs
+	attrs = []
+	if rawdata[i:i+2] == '<>':
+	    # SGML shorthand: <> == <last open tag seen>
+	    k = j
+	    tag = self.lasttag
+	else:
+	    k = tagfind.match(rawdata, i+1)
+	    if k < 0:
+		raise RuntimeError, 'unexpected call to parse_starttag'
+	    k = i+1+k
+	    tag = string.lower(rawdata[i+1:k])
+	    self.lasttag = tag
+	while k < j:
+	    l = attrfind.match(rawdata, k)
+	    if l < 0: break
+	    attrname, rest, attrvalue = attrfind.group(1, 2, 3)
+	    if not rest:
+		attrvalue = attrname
+	    elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
+		 attrvalue[:1] == '"' == attrvalue[-1:]:
+		attrvalue = attrvalue[1:-1]
+	    attrs.append((string.lower(attrname), attrvalue))
+	    k = k + l
+	if rawdata[j] == '>':
+	    j = j+1
+	self.finish_starttag(tag, attrs)
+	return j
+
+    # Internal -- parse endtag
+    def parse_endtag(self, i):
+	rawdata = self.rawdata
+	j = endbracket.search(rawdata, i+1)
+	if j < 0:
+	    return -1
+	tag = string.lower(string.strip(rawdata[i+2:j]))
+	if rawdata[j] == '>':
+	    j = j+1
+	self.finish_endtag(tag)
+	return j
+
+    # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
+    def finish_shorttag(self, tag, data):
+	self.finish_starttag(tag, [])
+	self.handle_data(data)
+	self.finish_endtag(tag)
+
+    # Internal -- finish processing of start tag
+    # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
+    def finish_starttag(self, tag, attrs):
+	try:
+	    method = getattr(self, 'start_' + tag)
+	except AttributeError:
+	    try:
+		method = getattr(self, 'do_' + tag)
+	    except AttributeError:
+		self.unknown_starttag(tag, attrs)
+		return -1
+	    else:
+		self.handle_starttag(tag, method, attrs)
+		return 0
+	else:
+	    self.stack.append(tag)
+	    self.handle_starttag(tag, method, attrs)
+	    return 1
+
+    # Internal -- finish processing of end tag
+    def finish_endtag(self, tag):
+	if not tag:
+	    found = len(self.stack) - 1
+	    if found < 0:
+		self.unknown_endtag(tag)
+		return
+	else:
+	    if tag not in self.stack:
 		try:
-			j = string.index(rawdata, '>', i)
-		except string.index_error:
-			return -1
-		# Now parse the data between i+1 and j into a tag and attrs
-		attrs = []
-		tagfind = regex.compile('[a-zA-Z][a-zA-Z0-9]*')
-		attrfind = regex.compile(
-		  '[ \t\n]+\([a-zA-Z_][a-zA-Z_0-9]*\)' +
-		  '\([ \t\n]*=[ \t\n]*' +
-		     '\(\'[^\']*\'\|"[^"]*"\|[-a-zA-Z0-9./:+*%?!()_#]*\)\)?')
-		k = tagfind.match(rawdata, i+1)
-		if k < 0:
-			raise RuntimeError, 'unexpected call to parse_starttag'
-		k = i+1+k
-		tag = string.lower(rawdata[i+1:k])
-		while k < j:
-			l = attrfind.match(rawdata, k)
-			if l < 0: break
-			attrname, rest, attrvalue = attrfind.group(1, 2, 3)
-			if not rest:
-				attrvalue = attrname
-			elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
-			     attrvalue[:1] == '"' == attrvalue[-1:]:
-				attrvalue = attrvalue[1:-1]
-			attrs.append((string.lower(attrname), attrvalue))
-			k = k + l
-		j = j+1
-		try:
-			method = getattr(self, 'start_' + tag)
-		except AttributeError:
-			try:
-				method = getattr(self, 'do_' + tag)
-			except AttributeError:
-				self.unknown_starttag(tag, attrs)
-				return j-i
-			method(attrs)
-			return j-i
-		self.stack.append(tag)
-		method(attrs)
-		return j-i
-
-	# Internal -- parse endtag
-	def parse_endtag(self, data):
-		if data[:2] <> '</' or data[-1:] <> '>':
-			raise RuntimeError, 'unexpected call to parse_endtag'
-		tag = string.lower(string.strip(data[2:-1]))
-		try:
-			method = getattr(self, 'end_' + tag)
+		    method = getattr(self, 'end_' + tag)
 		except AttributeError:
-			self.unknown_endtag(tag)
-			return
-		# XXX Should invoke end methods when popping their
-		# XXX stack entry, not when encountering the tag!
-		if self.stack and self.stack[-1] == tag:
-			del self.stack[-1]
-		else:
-			self.report_unbalanced(tag)
-			# Now repair it
-			found = None
-			for i in range(len(self.stack)):
-				if self.stack[i] == tag: found = i
-			if found <> None:
-				del self.stack[found:]
-		method()
-
-	# Example -- report an unbalanced </...> tag.
-	def report_unbalanced(self, tag):
-		if self.verbose:
-			print '*** Unbalanced </' + tag + '>'
-			print '*** Stack:', self.stack
-
-	# Example -- handle character reference, no need to override
-	def handle_charref(self, name):
-		try:
-			n = string.atoi(name)
-		except string.atoi_error:
-			self.unknown_charref(name)
-			return
-		if not 0 <= n <= 255:
-			self.unknown_charref(name)
-			return
-		self.handle_data(chr(n))
-
-	# Definition of entities -- derived classes may override
-	entitydefs = \
-		{'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
-
-	# Example -- handle entity reference, no need to override
-	def handle_entityref(self, name):
-		table = self.entitydefs
-		if table.has_key(name):
-			self.handle_data(table[name])
-		else:
-			self.unknown_entityref(name)
-			return
-
-	# Example -- handle data, should be overridden
-	def handle_data(self, data):
-		pass
-
-	# Example -- handle comment, could be overridden
-	def handle_comment(self, data):
-		pass
-
-	# To be overridden -- handlers for unknown objects
-	def unknown_starttag(self, tag, attrs): pass
-	def unknown_endtag(self, tag): pass
-	def unknown_charref(self, ref): pass
-	def unknown_entityref(self, ref): pass
-
-
-class TestSGML(SGMLParser):
-
-	def handle_data(self, data):
-		r = repr(data)
-		if len(r) > 72:
-			r = r[:35] + '...' + r[-35:]
-		print 'data:', r
-
-	def handle_comment(self, data):
-		r = repr(data)
-		if len(r) > 68:
-			r = r[:32] + '...' + r[-32:]
-		print 'comment:', r
-
-	def unknown_starttag(self, tag, attrs):
-		print 'start tag: <' + tag,
-		for name, value in attrs:
-			print name + '=' + '"' + value + '"',
-		print '>'
-
-	def unknown_endtag(self, tag):
-		print 'end tag: </' + tag + '>'
-
-	def unknown_entityref(self, ref):
-		print '*** unknown entity ref: &' + ref + ';'
-
-	def unknown_charref(self, ref):
-		print '*** unknown char ref: &#' + ref + ';'
-
-
-def test():
+		    self.unknown_endtag(tag)
+		return
+	    found = len(self.stack)
+	    for i in range(found):
+		if self.stack[i] == tag: found = i
+	while len(self.stack) > found:
+	    tag = self.stack[-1]
+	    try:
+		method = getattr(self, 'end_' + tag)
+	    except AttributeError:
+		method = None
+	    if method:
+		self.handle_endtag(tag, method)
+	    else:
+		self.unknown_endtag(tag)
+	    del self.stack[-1]
+
+    # Overridable -- handle start tag
+    def handle_starttag(self, tag, method, attrs):
+	method(attrs)
+
+    # Overridable -- handle end tag
+    def handle_endtag(self, tag, method):
+	method()
+
+    # Example -- report an unbalanced </...> tag.
+    def report_unbalanced(self, tag):
+	if self.verbose:
+	    print '*** Unbalanced </' + tag + '>'
+	    print '*** Stack:', self.stack
+
+    # Example -- handle character reference, no need to override
+    def handle_charref(self, name):
+	try:
+	    n = string.atoi(name)
+	except string.atoi_error:
+	    self.unknown_charref(name)
+	    return
+	if not 0 <= n <= 255:
+	    self.unknown_charref(name)
+	    return
+	self.handle_data(chr(n))
+
+    # Definition of entities -- derived classes may override
+    entitydefs = \
+	    {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
+
+    # Example -- handle entity reference, no need to override
+    def handle_entityref(self, name):
+	table = self.entitydefs
+	if table.has_key(name):
+	    self.handle_data(table[name])
+	else:
+	    self.unknown_entityref(name)
+	    return
+
+    # Example -- handle data, should be overridden
+    def handle_data(self, data):
+	pass
+
+    # Example -- handle comment, could be overridden
+    def handle_comment(self, data):
+	pass
+
+    # To be overridden -- handlers for unknown objects
+    def unknown_starttag(self, tag, attrs): pass
+    def unknown_endtag(self, tag): pass
+    def unknown_charref(self, ref): pass
+    def unknown_entityref(self, ref): pass
+
+
+class TestSGMLParser(SGMLParser):
+
+    def __init__(self, verbose=0):
+	self.testdata = ""
+	SGMLParser.__init__(self, verbose)
+
+    def handle_data(self, data):
+	self.testdata = self.testdata + data
+	if len(`self.testdata`) >= 70:
+	    self.flush()
+
+    def flush(self):
+	data = self.testdata
+	if data:
+	    self.testdata = ""
+	    print 'data:', `data`
+
+    def handle_comment(self, data):
+	self.flush()
+	r = `data`
+	if len(r) > 68:
+	    r = r[:32] + '...' + r[-32:]
+	print 'comment:', r
+
+    def unknown_starttag(self, tag, attrs):
+	self.flush()
+	if not attrs:
+	    print 'start tag: <' + tag + '>'
+	else:
+	    print 'start tag: <' + tag,
+	    for name, value in attrs:
+		print name + '=' + '"' + value + '"',
+	    print '>'
+
+    def unknown_endtag(self, tag):
+	self.flush()
+	print 'end tag: </' + tag + '>'
+
+    def unknown_entityref(self, ref):
+	self.flush()
+	print '*** unknown entity ref: &' + ref + ';'
+
+    def unknown_charref(self, ref):
+	self.flush()
+	print '*** unknown char ref: &#' + ref + ';'
+
+    def close(self):
+	SGMLParser.close(self)
+	self.flush()
+
+
+def test(args = None):
+    import sys
+
+    if not args:
+	args = sys.argv[1:]
+
+    if args and args[0] == '-s':
+	args = args[1:]
+	klass = SGMLParser
+    else:
+	klass = TestSGMLParser
+
+    if args:
+	file = args[0]
+    else:
 	file = 'test.html'
-	f = open(file, 'r')
-	x = TestSGML()
-	while 1:
-		line = f.readline()
-		if not line:
-			x.close()
-			break
-		x.feed(line)
+
+    if file == '-':
+	f = sys.stdin
+    else:
+	try:
+	    f = open(file, 'r')
+	except IOError, msg:
+	    print file, ":", msg
+	    sys.exit(1)
+
+    data = f.read()
+    if f is not sys.stdin:
+	f.close()
+
+    x = klass()
+    for c in data:
+	x.feed(c)
+    x.close()
 
 
 if __name__ == '__main__':
-	test()
+    test()