summaryrefslogtreecommitdiffstats
path: root/Lib
diff options
context:
space:
mode:
authorGeorg Brandl <georg@python.org>2008-06-01 21:25:55 (GMT)
committerGeorg Brandl <georg@python.org>2008-06-01 21:25:55 (GMT)
commit877b10add4a676c3f868b86bd31e92a181b5a5b1 (patch)
tree9a7455ce3aa5f07e8f433a8aa9b59540b2cf2b40 /Lib
parent6b38daa80dc0b63a089ac4557e25abe1f76b95af (diff)
downloadcpython-877b10add4a676c3f868b86bd31e92a181b5a5b1.zip
cpython-877b10add4a676c3f868b86bd31e92a181b5a5b1.tar.gz
cpython-877b10add4a676c3f868b86bd31e92a181b5a5b1.tar.bz2
Remove the htmllib and sgmllib modules as per PEP 3108.
Diffstat (limited to 'Lib')
-rw-r--r--Lib/_markupbase.py5
-rw-r--r--Lib/htmllib.py486
-rw-r--r--Lib/sgmllib.py548
-rw-r--r--Lib/test/test___all__.py2
-rw-r--r--Lib/test/test_htmllib.py69
-rw-r--r--Lib/test/test_sgmllib.py438
6 files changed, 2 insertions, 1546 deletions
diff --git a/Lib/_markupbase.py b/Lib/_markupbase.py
index b178470..adf6ba2 100644
--- a/Lib/_markupbase.py
+++ b/Lib/_markupbase.py
@@ -1,8 +1,7 @@
"""Shared support for scanning document type declarations in HTML and XHTML.
-This module is used as a foundation for the HTMLParser and sgmllib
-modules (indirectly, for htmllib as well). It has no documented
-public API and should not be used directly.
+This module is used as a foundation for the html.parser module. It has no
+documented public API and should not be used directly.
"""
diff --git a/Lib/htmllib.py b/Lib/htmllib.py
deleted file mode 100644
index a580006..0000000
--- a/Lib/htmllib.py
+++ /dev/null
@@ -1,486 +0,0 @@
-"""HTML 2.0 parser.
-
-See the HTML 2.0 specification:
-http://www.w3.org/hypertext/WWW/MarkUp/html-spec/html-spec_toc.html
-"""
-
-import sgmllib
-
-from formatter import AS_IS
-
-__all__ = ["HTMLParser", "HTMLParseError"]
-
-
-class HTMLParseError(sgmllib.SGMLParseError):
- """Error raised when an HTML document can't be parsed."""
-
-
-class HTMLParser(sgmllib.SGMLParser):
- """This is the basic HTML parser class.
-
- It supports all entity names required by the XHTML 1.0 Recommendation.
- It also defines handlers for all HTML 2.0 and many HTML 3.0 and 3.2
- elements.
-
- """
-
- from html.entities import entitydefs
-
- def __init__(self, formatter, verbose=0):
- """Creates an instance of the HTMLParser class.
-
- The formatter parameter is the formatter instance associated with
- the parser.
-
- """
- sgmllib.SGMLParser.__init__(self, verbose)
- self.formatter = formatter
-
- def error(self, message):
- raise HTMLParseError(message)
-
- def reset(self):
- sgmllib.SGMLParser.reset(self)
- self.savedata = None
- self.isindex = 0
- self.title = None
- self.base = None
- self.anchor = None
- self.anchorlist = []
- self.nofill = 0
- self.list_stack = []
-
- # ------ Methods used internally; some may be overridden
-
- # --- Formatter interface, taking care of 'savedata' mode;
- # shouldn't need to be overridden
-
- def handle_data(self, data):
- if self.savedata is not None:
- self.savedata = self.savedata + data
- else:
- if self.nofill:
- self.formatter.add_literal_data(data)
- else:
- self.formatter.add_flowing_data(data)
-
- # --- Hooks to save data; shouldn't need to be overridden
-
- def save_bgn(self):
- """Begins saving character data in a buffer instead of sending it
- to the formatter object.
-
- Retrieve the stored data via the save_end() method. Use of the
- save_bgn() / save_end() pair may not be nested.
-
- """
- self.savedata = ''
-
- def save_end(self):
- """Ends buffering character data and returns all data saved since
- the preceding call to the save_bgn() method.
-
- If the nofill flag is false, whitespace is collapsed to single
- spaces. A call to this method without a preceding call to the
- save_bgn() method will raise a TypeError exception.
-
- """
- data = self.savedata
- self.savedata = None
- if not self.nofill:
- data = ' '.join(data.split())
- return data
-
- # --- Hooks for anchors; should probably be overridden
-
- def anchor_bgn(self, href, name, type):
- """This method is called at the start of an anchor region.
-
- The arguments correspond to the attributes of the <A> tag with
- the same names. The default implementation maintains a list of
- hyperlinks (defined by the HREF attribute for <A> tags) within
- the document. The list of hyperlinks is available as the data
- attribute anchorlist.
-
- """
- self.anchor = href
- if self.anchor:
- self.anchorlist.append(href)
-
- def anchor_end(self):
- """This method is called at the end of an anchor region.
-
- The default implementation adds a textual footnote marker using an
- index into the list of hyperlinks created by the anchor_bgn()method.
-
- """
- if self.anchor:
- self.handle_data("[%d]" % len(self.anchorlist))
- self.anchor = None
-
- # --- Hook for images; should probably be overridden
-
- def handle_image(self, src, alt, *args):
- """This method is called to handle images.
-
- The default implementation simply passes the alt value to the
- handle_data() method.
-
- """
- self.handle_data(alt)
-
- # --------- Top level elememts
-
- def start_html(self, attrs): pass
- def end_html(self): pass
-
- def start_head(self, attrs): pass
- def end_head(self): pass
-
- def start_body(self, attrs): pass
- def end_body(self): pass
-
- # ------ Head elements
-
- def start_title(self, attrs):
- self.save_bgn()
-
- def end_title(self):
- self.title = self.save_end()
-
- def do_base(self, attrs):
- for a, v in attrs:
- if a == 'href':
- self.base = v
-
- def do_isindex(self, attrs):
- self.isindex = 1
-
- def do_link(self, attrs):
- pass
-
- def do_meta(self, attrs):
- pass
-
- def do_nextid(self, attrs): # Deprecated
- pass
-
- # ------ Body elements
-
- # --- Headings
-
- def start_h1(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h1', 0, 1, 0))
-
- def end_h1(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- def start_h2(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h2', 0, 1, 0))
-
- def end_h2(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- def start_h3(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h3', 0, 1, 0))
-
- def end_h3(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- def start_h4(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h4', 0, 1, 0))
-
- def end_h4(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- def start_h5(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h5', 0, 1, 0))
-
- def end_h5(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- def start_h6(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font(('h6', 0, 1, 0))
-
- def end_h6(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
-
- # --- Block Structuring Elements
-
- def do_p(self, attrs):
- self.formatter.end_paragraph(1)
-
- def start_pre(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))
- self.nofill = self.nofill + 1
-
- def end_pre(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_font()
- self.nofill = max(0, self.nofill - 1)
-
- def start_xmp(self, attrs):
- self.start_pre(attrs)
- self.setliteral('xmp') # Tell SGML parser
-
- def end_xmp(self):
- self.end_pre()
-
- def start_listing(self, attrs):
- self.start_pre(attrs)
- self.setliteral('listing') # Tell SGML parser
-
- def end_listing(self):
- self.end_pre()
-
- def start_address(self, attrs):
- self.formatter.end_paragraph(0)
- self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))
-
- def end_address(self):
- self.formatter.end_paragraph(0)
- self.formatter.pop_font()
-
- def start_blockquote(self, attrs):
- self.formatter.end_paragraph(1)
- self.formatter.push_margin('blockquote')
-
- def end_blockquote(self):
- self.formatter.end_paragraph(1)
- self.formatter.pop_margin()
-
- # --- List Elements
-
- def start_ul(self, attrs):
- self.formatter.end_paragraph(not self.list_stack)
- self.formatter.push_margin('ul')
- self.list_stack.append(['ul', '*', 0])
-
- def end_ul(self):
- if self.list_stack: del self.list_stack[-1]
- self.formatter.end_paragraph(not self.list_stack)
- self.formatter.pop_margin()
-
- def do_li(self, attrs):
- self.formatter.end_paragraph(0)
- if self.list_stack:
- [dummy, label, counter] = top = self.list_stack[-1]
- top[2] = counter = counter+1
- else:
- label, counter = '*', 0
- self.formatter.add_label_data(label, counter)
-
- def start_ol(self, attrs):
- self.formatter.end_paragraph(not self.list_stack)
- self.formatter.push_margin('ol')
- label = '1.'
- for a, v in attrs:
- if a == 'type':
- if len(v) == 1: v = v + '.'
- label = v
- self.list_stack.append(['ol', label, 0])
-
- def end_ol(self):
- if self.list_stack: del self.list_stack[-1]
- self.formatter.end_paragraph(not self.list_stack)
- self.formatter.pop_margin()
-
- def start_menu(self, attrs):
- self.start_ul(attrs)
-
- def end_menu(self):
- self.end_ul()
-
- def start_dir(self, attrs):
- self.start_ul(attrs)
-
- def end_dir(self):
- self.end_ul()
-
- def start_dl(self, attrs):
- self.formatter.end_paragraph(1)
- self.list_stack.append(['dl', '', 0])
-
- def end_dl(self):
- self.ddpop(1)
- if self.list_stack: del self.list_stack[-1]
-
- def do_dt(self, attrs):
- self.ddpop()
-
- def do_dd(self, attrs):
- self.ddpop()
- self.formatter.push_margin('dd')
- self.list_stack.append(['dd', '', 0])
-
- def ddpop(self, bl=0):
- self.formatter.end_paragraph(bl)
- if self.list_stack:
- if self.list_stack[-1][0] == 'dd':
- del self.list_stack[-1]
- self.formatter.pop_margin()
-
- # --- Phrase Markup
-
- # Idiomatic Elements
-
- def start_cite(self, attrs): self.start_i(attrs)
- def end_cite(self): self.end_i()
-
- def start_code(self, attrs): self.start_tt(attrs)
- def end_code(self): self.end_tt()
-
- def start_em(self, attrs): self.start_i(attrs)
- def end_em(self): self.end_i()
-
- def start_kbd(self, attrs): self.start_tt(attrs)
- def end_kbd(self): self.end_tt()
-
- def start_samp(self, attrs): self.start_tt(attrs)
- def end_samp(self): self.end_tt()
-
- def start_strong(self, attrs): self.start_b(attrs)
- def end_strong(self): self.end_b()
-
- def start_var(self, attrs): self.start_i(attrs)
- def end_var(self): self.end_i()
-
- # Typographic Elements
-
- def start_i(self, attrs):
- self.formatter.push_font((AS_IS, 1, AS_IS, AS_IS))
- def end_i(self):
- self.formatter.pop_font()
-
- def start_b(self, attrs):
- self.formatter.push_font((AS_IS, AS_IS, 1, AS_IS))
- def end_b(self):
- self.formatter.pop_font()
-
- def start_tt(self, attrs):
- self.formatter.push_font((AS_IS, AS_IS, AS_IS, 1))
- def end_tt(self):
- self.formatter.pop_font()
-
- def start_a(self, attrs):
- href = ''
- name = ''
- type = ''
- for attrname, value in attrs:
- value = value.strip()
- if attrname == 'href':
- href = value
- if attrname == 'name':
- name = value
- if attrname == 'type':
- type = value.lower()
- self.anchor_bgn(href, name, type)
-
- def end_a(self):
- self.anchor_end()
-
- # --- Line Break
-
- def do_br(self, attrs):
- self.formatter.add_line_break()
-
- # --- Horizontal Rule
-
- def do_hr(self, attrs):
- self.formatter.add_hor_rule()
-
- # --- Image
-
- def do_img(self, attrs):
- align = ''
- alt = '(image)'
- ismap = ''
- src = ''
- width = 0
- height = 0
- for attrname, value in attrs:
- if attrname == 'align':
- align = value
- if attrname == 'alt':
- alt = value
- if attrname == 'ismap':
- ismap = value
- if attrname == 'src':
- src = value
- if attrname == 'width':
- try: width = int(value)
- except ValueError: pass
- if attrname == 'height':
- try: height = int(value)
- except ValueError: pass
- self.handle_image(src, alt, ismap, align, width, height)
-
- # --- Really Old Unofficial Deprecated Stuff
-
- def do_plaintext(self, attrs):
- self.start_pre(attrs)
- self.setnomoretags() # Tell SGML parser
-
- # --- Unhandled tags
-
- def unknown_starttag(self, tag, attrs):
- pass
-
- def unknown_endtag(self, tag):
- pass
-
-
-def test(args = None):
- import sys, formatter
-
- if not args:
- args = sys.argv[1:]
-
- silent = args and args[0] == '-s'
- if silent:
- del args[0]
-
- if args:
- file = args[0]
- else:
- file = 'test.html'
-
- if file == '-':
- f = sys.stdin
- else:
- try:
- f = open(file, 'r')
- except IOError as msg:
- print(file, ":", msg)
- sys.exit(1)
-
- data = f.read()
-
- if f is not sys.stdin:
- f.close()
-
- if silent:
- f = formatter.NullFormatter()
- else:
- f = formatter.AbstractFormatter(formatter.DumbWriter())
-
- p = HTMLParser(f)
- p.feed(data)
- p.close()
-
-
-if __name__ == '__main__':
- test()
diff --git a/Lib/sgmllib.py b/Lib/sgmllib.py
deleted file mode 100644
index 7eca9de..0000000
--- a/Lib/sgmllib.py
+++ /dev/null
@@ -1,548 +0,0 @@
-"""A parser for SGML, using the derived class as a static DTD."""
-
-# XXX This only supports those SGML features used by HTML.
-
-# XXX There should be a way to distinguish between PCDATA (parsed
-# character data -- the normal case), RCDATA (replaceable character
-# data -- only char and entity references and end tags are special)
-# and CDATA (character data -- only end tags are special). RCDATA is
-# not supported at all.
-
-
-import _markupbase
-import re
-
-__all__ = ["SGMLParser", "SGMLParseError"]
-
-# Regular expressions used for parsing
-
-interesting = re.compile('[&<]')
-incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
- '<([a-zA-Z][^<>]*|'
- '/([a-zA-Z][^<>]*)?|'
- '![^<>]*)?')
-
-entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
-charref = re.compile('&#([0-9]+)[^0-9]')
-
-starttagopen = re.compile('<[>a-zA-Z]')
-shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
-shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
-piclose = re.compile('>')
-endbracket = re.compile('[<>]')
-tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*')
-attrfind = re.compile(
- r'\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)(\s*=\s*'
- r'(\'[^\']*\'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?')
-
-
-class SGMLParseError(RuntimeError):
- """Exception raised for all parse errors."""
- pass
-
-
-# SGML parser base class -- find tags and call handler functions.
-# Usage: p = SGMLParser(); p.feed(data); ...; p.close().
-# The dtd is defined by deriving a class which defines methods
-# with special names to handle tags: start_foo and end_foo to handle
-# <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
-# (Tags are converted to lower case for this purpose.) The data
-# between tags is passed to the parser by calling self.handle_data()
-# with some data as argument (the data may be split up in arbitrary
-# chunks). Entity references are passed by calling
-# self.handle_entityref() with the entity reference as argument.
-
-class SGMLParser(_markupbase.ParserBase):
- # Definition of entities -- derived classes may override
- entity_or_charref = re.compile('&(?:'
- '([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+)'
- ')(;?)')
-
- def __init__(self, verbose=0):
- """Initialize and reset this instance."""
- self.verbose = verbose
- self.reset()
-
- def reset(self):
- """Reset this instance. Loses all unprocessed data."""
- self.__starttag_text = None
- self.rawdata = ''
- self.stack = []
- self.lasttag = '???'
- self.nomoretags = 0
- self.literal = 0
- _markupbase.ParserBase.reset(self)
-
- def setnomoretags(self):
- """Enter literal mode (CDATA) till EOF.
-
- Intended for derived classes only.
- """
- self.nomoretags = self.literal = 1
-
- def setliteral(self, *args):
- """Enter literal mode (CDATA).
-
- Intended for derived classes only.
- """
- self.literal = 1
-
- def feed(self, data):
- """Feed some data to the parser.
-
- Call this as often as you want, with as little or as much text
- as you want (may include '\n'). (This just saves the text,
- all the processing is done by goahead().)
- """
-
- self.rawdata = self.rawdata + data
- self.goahead(0)
-
- def close(self):
- """Handle the remaining data."""
- self.goahead(1)
-
- def error(self, message):
- raise SGMLParseError(message)
-
- # Internal -- handle data as far as reasonable. May leave state
- # and data to be processed by a subsequent call. If 'end' is
- # true, force handling all data as if followed by EOF marker.
- def goahead(self, end):
- rawdata = self.rawdata
- i = 0
- n = len(rawdata)
- while i < n:
- if self.nomoretags:
- self.handle_data(rawdata[i:n])
- i = n
- break
- match = interesting.search(rawdata, i)
- if match: j = match.start()
- else: j = n
- if i < j:
- self.handle_data(rawdata[i:j])
- i = j
- if i == n: break
- if rawdata[i] == '<':
- if starttagopen.match(rawdata, i):
- if self.literal:
- self.handle_data(rawdata[i])
- i = i+1
- continue
- k = self.parse_starttag(i)
- if k < 0: break
- i = k
- continue
- if rawdata.startswith("</", i):
- k = self.parse_endtag(i)
- if k < 0: break
- i = k
- self.literal = 0
- continue
- if self.literal:
- if n > (i + 1):
- self.handle_data("<")
- i = i+1
- else:
- # incomplete
- break
- continue
- if rawdata.startswith("<!--", i):
- # Strictly speaking, a comment is --.*--
- # within a declaration tag <!...>.
- # This should be removed,
- # and comments handled only in parse_declaration.
- k = self.parse_comment(i)
- if k < 0: break
- i = k
- continue
- if rawdata.startswith("<?", i):
- k = self.parse_pi(i)
- if k < 0: break
- i = i+k
- continue
- if rawdata.startswith("<!", i):
- # This is some sort of declaration; in "HTML as
- # deployed," this should only be the document type
- # declaration ("<!DOCTYPE html...>").
- k = self.parse_declaration(i)
- if k < 0: break
- i = k
- continue
- elif rawdata[i] == '&':
- if self.literal:
- self.handle_data(rawdata[i])
- i = i+1
- continue
- match = charref.match(rawdata, i)
- if match:
- name = match.group(1)
- self.handle_charref(name)
- i = match.end(0)
- if rawdata[i-1] != ';': i = i-1
- continue
- match = entityref.match(rawdata, i)
- if match:
- name = match.group(1)
- self.handle_entityref(name)
- i = match.end(0)
- if rawdata[i-1] != ';': i = i-1
- continue
- else:
- self.error('neither < nor & ??')
- # We get here only if incomplete matches but
- # nothing else
- match = incomplete.match(rawdata, i)
- if not match:
- self.handle_data(rawdata[i])
- i = i+1
- continue
- j = match.end(0)
- if j == n:
- break # Really incomplete
- self.handle_data(rawdata[i:j])
- i = j
- # end while
- if end and i < n:
- self.handle_data(rawdata[i:n])
- i = n
- self.rawdata = rawdata[i:]
- # XXX if end: check for empty stack
-
- # Extensions for the DOCTYPE scanner:
- _decl_otherchars = '='
-
- # Internal -- parse processing instr, return length or -1 if not terminated
- def parse_pi(self, i):
- rawdata = self.rawdata
- if rawdata[i:i+2] != '<?':
- self.error('unexpected call to parse_pi()')
- match = piclose.search(rawdata, i+2)
- if not match:
- return -1
- j = match.start(0)
- self.handle_pi(rawdata[i+2: j])
- j = match.end(0)
- return j-i
-
- def get_starttag_text(self):
- return self.__starttag_text
-
- # Internal -- handle starttag, return length or -1 if not terminated
- def parse_starttag(self, i):
- self.__starttag_text = None
- start_pos = i
- rawdata = self.rawdata
- if shorttagopen.match(rawdata, i):
- # SGML shorthand: <tag/data/ == <tag>data</tag>
- # XXX Can data contain &... (entity or char refs)?
- # XXX Can data contain < or > (tag characters)?
- # XXX Can there be whitespace before the first /?
- match = shorttag.match(rawdata, i)
- if not match:
- return -1
- tag, data = match.group(1, 2)
- self.__starttag_text = '<%s/' % tag
- tag = tag.lower()
- k = match.end(0)
- self.finish_shorttag(tag, data)
- self.__starttag_text = rawdata[start_pos:match.end(1) + 1]
- return k
- # XXX The following should skip matching quotes (' or ")
- # As a shortcut way to exit, this isn't so bad, but shouldn't
- # be used to locate the actual end of the start tag since the
- # < or > characters may be embedded in an attribute value.
- match = endbracket.search(rawdata, i+1)
- if not match:
- return -1
- j = match.start(0)
- # Now parse the data between i+1 and j into a tag and attrs
- attrs = []
- if rawdata[i:i+2] == '<>':
- # SGML shorthand: <> == <last open tag seen>
- k = j
- tag = self.lasttag
- else:
- match = tagfind.match(rawdata, i+1)
- if not match:
- self.error('unexpected call to parse_starttag')
- k = match.end(0)
- tag = rawdata[i+1:k].lower()
- self.lasttag = tag
- while k < j:
- match = attrfind.match(rawdata, k)
- if not match: break
- attrname, rest, attrvalue = match.group(1, 2, 3)
- if not rest:
- attrvalue = attrname
- else:
- if (attrvalue[:1] == "'" == attrvalue[-1:] or
- attrvalue[:1] == '"' == attrvalue[-1:]):
- # strip quotes
- attrvalue = attrvalue[1:-1]
- attrvalue = self.entity_or_charref.sub(
- self._convert_ref, attrvalue)
- attrs.append((attrname.lower(), attrvalue))
- k = match.end(0)
- if rawdata[j] == '>':
- j = j+1
- self.__starttag_text = rawdata[start_pos:j]
- self.finish_starttag(tag, attrs)
- return j
-
- # Internal -- convert entity or character reference
- def _convert_ref(self, match):
- if match.group(2):
- return self.convert_charref(match.group(2)) or \
- '&#%s%s' % match.groups()[1:]
- elif match.group(3):
- return self.convert_entityref(match.group(1)) or \
- '&%s;' % match.group(1)
- else:
- return '&%s' % match.group(1)
-
- # Internal -- parse endtag
- def parse_endtag(self, i):
- rawdata = self.rawdata
- match = endbracket.search(rawdata, i+1)
- if not match:
- return -1
- j = match.start(0)
- tag = rawdata[i+2:j].strip().lower()
- if rawdata[j] == '>':
- j = j+1
- self.finish_endtag(tag)
- return j
-
- # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
- def finish_shorttag(self, tag, data):
- self.finish_starttag(tag, [])
- self.handle_data(data)
- self.finish_endtag(tag)
-
- # Internal -- finish processing of start tag
- # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
- def finish_starttag(self, tag, attrs):
- try:
- method = getattr(self, 'start_' + tag)
- except AttributeError:
- try:
- method = getattr(self, 'do_' + tag)
- except AttributeError:
- self.unknown_starttag(tag, attrs)
- return -1
- else:
- self.handle_starttag(tag, method, attrs)
- return 0
- else:
- self.stack.append(tag)
- self.handle_starttag(tag, method, attrs)
- return 1
-
- # Internal -- finish processing of end tag
- def finish_endtag(self, tag):
- if not tag:
- found = len(self.stack) - 1
- if found < 0:
- self.unknown_endtag(tag)
- return
- else:
- if tag not in self.stack:
- try:
- method = getattr(self, 'end_' + tag)
- except AttributeError:
- self.unknown_endtag(tag)
- else:
- self.report_unbalanced(tag)
- return
- found = len(self.stack)
- for i in range(found):
- if self.stack[i] == tag: found = i
- while len(self.stack) > found:
- tag = self.stack[-1]
- try:
- method = getattr(self, 'end_' + tag)
- except AttributeError:
- method = None
- if method:
- self.handle_endtag(tag, method)
- else:
- self.unknown_endtag(tag)
- del self.stack[-1]
-
- # Overridable -- handle start tag
- def handle_starttag(self, tag, method, attrs):
- method(attrs)
-
- # Overridable -- handle end tag
- def handle_endtag(self, tag, method):
- method()
-
- # Example -- report an unbalanced </...> tag.
- def report_unbalanced(self, tag):
- if self.verbose:
- print('*** Unbalanced </' + tag + '>')
- print('*** Stack:', self.stack)
-
- def convert_charref(self, name):
- """Convert character reference, may be overridden."""
- try:
- n = int(name)
- except ValueError:
- return
- if not 0 <= n <= 255:
- return
- return self.convert_codepoint(n)
-
- def convert_codepoint(self, codepoint):
- return chr(codepoint)
-
- def handle_charref(self, name):
- """Handle character reference, no need to override."""
- replacement = self.convert_charref(name)
- if replacement is None:
- self.unknown_charref(name)
- else:
- self.handle_data(replacement)
-
- # Definition of entities -- derived classes may override
- entitydefs = \
- {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
-
- def convert_entityref(self, name):
- """Convert entity references.
-
- As an alternative to overriding this method; one can tailor the
- results by setting up the self.entitydefs mapping appropriately.
- """
- table = self.entitydefs
- if name in table:
- return table[name]
- else:
- return
-
- def handle_entityref(self, name):
- """Handle entity references, no need to override."""
- replacement = self.convert_entityref(name)
- if replacement is None:
- self.unknown_entityref(name)
- else:
- self.handle_data(replacement)
-
- # Example -- handle data, should be overridden
- def handle_data(self, data):
- pass
-
- # Example -- handle comment, could be overridden
- def handle_comment(self, data):
- pass
-
- # Example -- handle declaration, could be overridden
- def handle_decl(self, decl):
- pass
-
- # Example -- handle processing instruction, could be overridden
- def handle_pi(self, data):
- pass
-
- # To be overridden -- handlers for unknown objects
- def unknown_starttag(self, tag, attrs): pass
- def unknown_endtag(self, tag): pass
- def unknown_charref(self, ref): pass
- def unknown_entityref(self, ref): pass
-
-
-class TestSGMLParser(SGMLParser):
-
- def __init__(self, verbose=0):
- self.testdata = ""
- SGMLParser.__init__(self, verbose)
-
- def handle_data(self, data):
- self.testdata = self.testdata + data
- if len(repr(self.testdata)) >= 70:
- self.flush()
-
- def flush(self):
- data = self.testdata
- if data:
- self.testdata = ""
- print('data:', repr(data))
-
- def handle_comment(self, data):
- self.flush()
- r = repr(data)
- if len(r) > 68:
- r = r[:32] + '...' + r[-32:]
- print('comment:', r)
-
- def unknown_starttag(self, tag, attrs):
- self.flush()
- if not attrs:
- print('start tag: <' + tag + '>')
- else:
- print('start tag: <' + tag, end=' ')
- for name, value in attrs:
- print(name + '=' + '"' + value + '"', end=' ')
- print('>')
-
- def unknown_endtag(self, tag):
- self.flush()
- print('end tag: </' + tag + '>')
-
- def unknown_entityref(self, ref):
- self.flush()
- print('*** unknown entity ref: &' + ref + ';')
-
- def unknown_charref(self, ref):
- self.flush()
- print('*** unknown char ref: &#' + ref + ';')
-
- def unknown_decl(self, data):
- self.flush()
- print('*** unknown decl: [' + data + ']')
-
- def close(self):
- SGMLParser.close(self)
- self.flush()
-
-
-def test(args = None):
- import sys
-
- if args is None:
- args = sys.argv[1:]
-
- if args and args[0] == '-s':
- args = args[1:]
- klass = SGMLParser
- else:
- klass = TestSGMLParser
-
- if args:
- file = args[0]
- else:
- file = 'test.html'
-
- if file == '-':
- f = sys.stdin
- else:
- try:
- f = open(file, 'r')
- except IOError as msg:
- print(file, ":", msg)
- sys.exit(1)
-
- data = f.read()
- if f is not sys.stdin:
- f.close()
-
- x = klass()
- for c in data:
- x.feed(c)
- x.close()
-
-
-if __name__ == '__main__':
- test()
diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py
index e33c54a..491f159 100644
--- a/Lib/test/test___all__.py
+++ b/Lib/test/test___all__.py
@@ -73,7 +73,6 @@ class AllTest(unittest.TestCase):
self.check_all("glob")
self.check_all("gzip")
self.check_all("heapq")
- self.check_all("htmllib")
self.check_all("http.client")
self.check_all("ihooks")
self.check_all("imaplib")
@@ -116,7 +115,6 @@ class AllTest(unittest.TestCase):
self.check_all("rlcompleter")
self.check_all("robotparser")
self.check_all("sched")
- self.check_all("sgmllib")
self.check_all("shelve")
self.check_all("shlex")
self.check_all("shutil")
diff --git a/Lib/test/test_htmllib.py b/Lib/test/test_htmllib.py
deleted file mode 100644
index 61f3a13..0000000
--- a/Lib/test/test_htmllib.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import formatter
-import htmllib
-import unittest
-
-from test import support
-
-
-class AnchorCollector(htmllib.HTMLParser):
- def __init__(self, *args, **kw):
- self.__anchors = []
- htmllib.HTMLParser.__init__(self, *args, **kw)
-
- def get_anchor_info(self):
- return self.__anchors
-
- def anchor_bgn(self, *args):
- self.__anchors.append(args)
-
-class DeclCollector(htmllib.HTMLParser):
- def __init__(self, *args, **kw):
- self.__decls = []
- htmllib.HTMLParser.__init__(self, *args, **kw)
-
- def get_decl_info(self):
- return self.__decls
-
- def unknown_decl(self, data):
- self.__decls.append(data)
-
-
-class HTMLParserTestCase(unittest.TestCase):
- def test_anchor_collection(self):
- # See SF bug #467059.
- parser = AnchorCollector(formatter.NullFormatter(), verbose=1)
- parser.feed(
- """<a href='http://foo.org/' name='splat'> </a>
- <a href='http://www.python.org/'> </a>
- <a name='frob'> </a>
- """)
- parser.close()
- self.assertEquals(parser.get_anchor_info(),
- [('http://foo.org/', 'splat', ''),
- ('http://www.python.org/', '', ''),
- ('', 'frob', ''),
- ])
-
- def test_decl_collection(self):
- # See SF patch #545300
- parser = DeclCollector(formatter.NullFormatter(), verbose=1)
- parser.feed(
- """<html>
- <body>
- hallo
- <![if !supportEmptyParas]>&nbsp;<![endif]>
- </body>
- </html>
- """)
- parser.close()
- self.assertEquals(parser.get_decl_info(),
- ["if !supportEmptyParas",
- "endif"
- ])
-
-def test_main():
- support.run_unittest(HTMLParserTestCase)
-
-
-if __name__ == "__main__":
- test_main()
diff --git a/Lib/test/test_sgmllib.py b/Lib/test/test_sgmllib.py
deleted file mode 100644
index 6015878..0000000
--- a/Lib/test/test_sgmllib.py
+++ /dev/null
@@ -1,438 +0,0 @@
-import pprint
-import re
-import sgmllib
-import unittest
-from test import support
-
-
-class EventCollector(sgmllib.SGMLParser):
-
- def __init__(self):
- self.events = []
- self.append = self.events.append
- sgmllib.SGMLParser.__init__(self)
-
- def get_events(self):
- # Normalize the list of events so that buffer artefacts don't
- # separate runs of contiguous characters.
- L = []
- prevtype = None
- for event in self.events:
- type = event[0]
- if type == prevtype == "data":
- L[-1] = ("data", L[-1][1] + event[1])
- else:
- L.append(event)
- prevtype = type
- self.events = L
- return L
-
- # structure markup
-
- def unknown_starttag(self, tag, attrs):
- self.append(("starttag", tag, attrs))
-
- def unknown_endtag(self, tag):
- self.append(("endtag", tag))
-
- # all other markup
-
- def handle_comment(self, data):
- self.append(("comment", data))
-
- def handle_charref(self, data):
- self.append(("charref", data))
-
- def handle_data(self, data):
- self.append(("data", data))
-
- def handle_decl(self, decl):
- self.append(("decl", decl))
-
- def handle_entityref(self, data):
- self.append(("entityref", data))
-
- def handle_pi(self, data):
- self.append(("pi", data))
-
- def unknown_decl(self, decl):
- self.append(("unknown decl", decl))
-
-
-class CDATAEventCollector(EventCollector):
- def start_cdata(self, attrs):
- self.append(("starttag", "cdata", attrs))
- self.setliteral()
-
-
-class HTMLEntityCollector(EventCollector):
-
- entity_or_charref = re.compile('(?:&([a-zA-Z][-.a-zA-Z0-9]*)'
- '|&#(x[0-9a-zA-Z]+|[0-9]+))(;?)')
-
- def convert_charref(self, name):
- self.append(("charref", "convert", name))
- if name[0] != "x":
- return EventCollector.convert_charref(self, name)
-
- def convert_codepoint(self, codepoint):
- self.append(("codepoint", "convert", codepoint))
- EventCollector.convert_codepoint(self, codepoint)
-
- def convert_entityref(self, name):
- self.append(("entityref", "convert", name))
- return EventCollector.convert_entityref(self, name)
-
- # These to record that they were called, then pass the call along
- # to the default implementation so that it's actions can be
- # recorded.
-
- def handle_charref(self, data):
- self.append(("charref", data))
- sgmllib.SGMLParser.handle_charref(self, data)
-
- def handle_entityref(self, data):
- self.append(("entityref", data))
- sgmllib.SGMLParser.handle_entityref(self, data)
-
-
-class SGMLParserTestCase(unittest.TestCase):
-
- collector = EventCollector
-
- def get_events(self, source):
- parser = self.collector()
- try:
- for s in source:
- parser.feed(s)
- parser.close()
- except:
- #self.events = parser.events
- raise
- return parser.get_events()
-
- def check_events(self, source, expected_events):
- try:
- events = self.get_events(source)
- except:
- #import sys
- #print >>sys.stderr, pprint.pformat(self.events)
- raise
- if events != expected_events:
- self.fail("received events did not match expected events\n"
- "Expected:\n" + pprint.pformat(expected_events) +
- "\nReceived:\n" + pprint.pformat(events))
-
- def check_parse_error(self, source):
- parser = EventCollector()
- try:
- parser.feed(source)
- parser.close()
- except sgmllib.SGMLParseError:
- pass
- else:
- self.fail("expected SGMLParseError for %r\nReceived:\n%s"
- % (source, pprint.pformat(parser.get_events())))
-
- def test_doctype_decl_internal(self):
- inside = """\
-DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'
- SYSTEM 'http://www.w3.org/TR/html401/strict.dtd' [
- <!ELEMENT html - O EMPTY>
- <!ATTLIST html
- version CDATA #IMPLIED
- profile CDATA 'DublinCore'>
- <!NOTATION datatype SYSTEM 'http://xml.python.org/notations/python-module'>
- <!ENTITY myEntity 'internal parsed entity'>
- <!ENTITY anEntity SYSTEM 'http://xml.python.org/entities/something.xml'>
- <!ENTITY % paramEntity 'name|name|name'>
- %paramEntity;
- <!-- comment -->
-]"""
- self.check_events(["<!%s>" % inside], [
- ("decl", inside),
- ])
-
- def test_doctype_decl_external(self):
- inside = "DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'"
- self.check_events("<!%s>" % inside, [
- ("decl", inside),
- ])
-
- def test_underscore_in_attrname(self):
- # SF bug #436621
- """Make sure attribute names with underscores are accepted"""
- self.check_events("<a has_under _under>", [
- ("starttag", "a", [("has_under", "has_under"),
- ("_under", "_under")]),
- ])
-
- def test_underscore_in_tagname(self):
- # SF bug #436621
- """Make sure tag names with underscores are accepted"""
- self.check_events("<has_under></has_under>", [
- ("starttag", "has_under", []),
- ("endtag", "has_under"),
- ])
-
- def test_quotes_in_unquoted_attrs(self):
- # SF bug #436621
- """Be sure quotes in unquoted attributes are made part of the value"""
- self.check_events("<a href=foo'bar\"baz>", [
- ("starttag", "a", [("href", "foo'bar\"baz")]),
- ])
-
- def test_xhtml_empty_tag(self):
- """Handling of XHTML-style empty start tags"""
- self.check_events("<br />text<i></i>", [
- ("starttag", "br", []),
- ("data", "text"),
- ("starttag", "i", []),
- ("endtag", "i"),
- ])
-
- def test_processing_instruction_only(self):
- self.check_events("<?processing instruction>", [
- ("pi", "processing instruction"),
- ])
-
- def test_bad_nesting(self):
- self.check_events("<a><b></a></b>", [
- ("starttag", "a", []),
- ("starttag", "b", []),
- ("endtag", "a"),
- ("endtag", "b"),
- ])
-
- def test_bare_ampersands(self):
- self.check_events("this text & contains & ampersands &", [
- ("data", "this text & contains & ampersands &"),
- ])
-
- def test_bare_pointy_brackets(self):
- self.check_events("this < text > contains < bare>pointy< brackets", [
- ("data", "this < text > contains < bare>pointy< brackets"),
- ])
-
- def test_attr_syntax(self):
- output = [
- ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", "e")])
- ]
- self.check_events("""<a b='v' c="v" d=v e>""", output)
- self.check_events("""<a b = 'v' c = "v" d = v e>""", output)
- self.check_events("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output)
- self.check_events("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output)
-
- def test_attr_values(self):
- self.check_events("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""",
- [("starttag", "a", [("b", "xxx\n\txxx"),
- ("c", "yyy\t\nyyy"),
- ("d", "\txyz\n")])
- ])
- self.check_events("""<a b='' c="">""", [
- ("starttag", "a", [("b", ""), ("c", "")]),
- ])
- # URL construction stuff from RFC 1808:
- safe = "$-_.+"
- extra = "!*'(),"
- reserved = ";/?:@&="
- url = "http://example.com:8080/path/to/file?%s%s%s" % (
- safe, extra, reserved)
- self.check_events("""<e a=%s>""" % url, [
- ("starttag", "e", [("a", url)]),
- ])
- # Regression test for SF patch #669683.
- self.check_events("<e a=rgb(1,2,3)>", [
- ("starttag", "e", [("a", "rgb(1,2,3)")]),
- ])
-
- def test_attr_values_entities(self):
- """Substitution of entities and charrefs in attribute values"""
- # SF bug #1452246
- self.check_events("""<a b=&lt; c=&lt;&gt; d=&lt-&gt; e='&lt; '
- f="&xxx;" g='&#32;&#33;' h='&#500;'
- i='x?a=b&c=d;'
- j='&amp;#42;' k='&#38;#42;'>""",
- [("starttag", "a", [("b", "<"),
- ("c", "<>"),
- ("d", "&lt->"),
- ("e", "< "),
- ("f", "&xxx;"),
- ("g", " !"),
- ("h", "&#500;"),
- ("i", "x?a=b&c=d;"),
- ("j", "&#42;"),
- ("k", "&#42;"),
- ])])
-
- def test_convert_overrides(self):
- # This checks that the character and entity reference
- # conversion helpers are called at the documented times. No
- # attempt is made to really change what the parser accepts.
- #
- self.collector = HTMLEntityCollector
- self.check_events(('<a title="&ldquo;test&#x201d;">foo</a>'
- '&foobar;&#42;'), [
- ('entityref', 'convert', 'ldquo'),
- ('charref', 'convert', 'x201d'),
- ('starttag', 'a', [('title', '&ldquo;test&#x201d;')]),
- ('data', 'foo'),
- ('endtag', 'a'),
- ('entityref', 'foobar'),
- ('entityref', 'convert', 'foobar'),
- ('charref', '42'),
- ('charref', 'convert', '42'),
- ('codepoint', 'convert', 42),
- ])
-
- def test_attr_funky_names(self):
- self.check_events("""<a a.b='v' c:d=v e-f=v>""", [
- ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]),
- ])
-
- def test_attr_value_ip6_url(self):
- # http://www.python.org/sf/853506
- self.check_events(("<a href='http://[1080::8:800:200C:417A]/'>"
- "<a href=http://[1080::8:800:200C:417A]/>"), [
- ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]),
- ("starttag", "a", [("href", "http://[1080::8:800:200C:417A]/")]),
- ])
-
- def test_illegal_declarations(self):
- s = 'abc<!spacer type="block" height="25">def'
- self.check_events(s, [
- ("data", "abc"),
- ("unknown decl", 'spacer type="block" height="25"'),
- ("data", "def"),
- ])
-
- def test_weird_starttags(self):
- self.check_events("<a<a>", [
- ("starttag", "a", []),
- ("starttag", "a", []),
- ])
- self.check_events("</a<a>", [
- ("endtag", "a"),
- ("starttag", "a", []),
- ])
-
- def test_declaration_junk_chars(self):
- self.check_parse_error("<!DOCTYPE foo $ >")
-
- def test_get_starttag_text(self):
- s = """<foobar \n one="1"\ttwo=2 >"""
- self.check_events(s, [
- ("starttag", "foobar", [("one", "1"), ("two", "2")]),
- ])
-
- def test_cdata_content(self):
- s = ("<cdata> <!-- not a comment --> &not-an-entity-ref; </cdata>"
- "<notcdata> <!-- comment --> </notcdata>")
- self.collector = CDATAEventCollector
- self.check_events(s, [
- ("starttag", "cdata", []),
- ("data", " <!-- not a comment --> &not-an-entity-ref; "),
- ("endtag", "cdata"),
- ("starttag", "notcdata", []),
- ("data", " "),
- ("comment", " comment "),
- ("data", " "),
- ("endtag", "notcdata"),
- ])
- s = """<cdata> <not a='start tag'> </cdata>"""
- self.check_events(s, [
- ("starttag", "cdata", []),
- ("data", " <not a='start tag'> "),
- ("endtag", "cdata"),
- ])
-
- def test_illegal_declarations(self):
- s = 'abc<!spacer type="block" height="25">def'
- self.check_events(s, [
- ("data", "abc"),
- ("unknown decl", 'spacer type="block" height="25"'),
- ("data", "def"),
- ])
-
- def test_enumerated_attr_type(self):
- s = "<!DOCTYPE doc [<!ATTLIST doc attr (a | b) >]>"
- self.check_events(s, [
- ('decl', 'DOCTYPE doc [<!ATTLIST doc attr (a | b) >]'),
- ])
-
- def test_read_chunks(self):
- # SF bug #1541697, this caused sgml parser to hang
- # Just verify this code doesn't cause a hang.
- CHUNK = 1024 # increasing this to 8212 makes the problem go away
-
- f = open(support.findfile('sgml_input.html'), encoding="latin-1")
- fp = sgmllib.SGMLParser()
- while 1:
- data = f.read(CHUNK)
- fp.feed(data)
- if len(data) != CHUNK:
- break
-
- # XXX These tests have been disabled by prefixing their names with
- # an underscore. The first two exercise outstanding bugs in the
- # sgmllib module, and the third exhibits questionable behavior
- # that needs to be carefully considered before changing it.
-
- def _test_starttag_end_boundary(self):
- self.check_events("<a b='<'>", [("starttag", "a", [("b", "<")])])
- self.check_events("<a b='>'>", [("starttag", "a", [("b", ">")])])
-
- def _test_buffer_artefacts(self):
- output = [("starttag", "a", [("b", "<")])]
- self.check_events(["<a b='<'>"], output)
- self.check_events(["<a ", "b='<'>"], output)
- self.check_events(["<a b", "='<'>"], output)
- self.check_events(["<a b=", "'<'>"], output)
- self.check_events(["<a b='<", "'>"], output)
- self.check_events(["<a b='<'", ">"], output)
-
- output = [("starttag", "a", [("b", ">")])]
- self.check_events(["<a b='>'>"], output)
- self.check_events(["<a ", "b='>'>"], output)
- self.check_events(["<a b", "='>'>"], output)
- self.check_events(["<a b=", "'>'>"], output)
- self.check_events(["<a b='>", "'>"], output)
- self.check_events(["<a b='>'", ">"], output)
-
- output = [("comment", "abc")]
- self.check_events(["", "<!--abc-->"], output)
- self.check_events(["<", "!--abc-->"], output)
- self.check_events(["<!", "--abc-->"], output)
- self.check_events(["<!-", "-abc-->"], output)
- self.check_events(["<!--", "abc-->"], output)
- self.check_events(["<!--a", "bc-->"], output)
- self.check_events(["<!--ab", "c-->"], output)
- self.check_events(["<!--abc", "-->"], output)
- self.check_events(["<!--abc-", "->"], output)
- self.check_events(["<!--abc--", ">"], output)
- self.check_events(["<!--abc-->", ""], output)
-
- def _test_starttag_junk_chars(self):
- self.check_parse_error("<")
- self.check_parse_error("<>")
- self.check_parse_error("</$>")
- self.check_parse_error("</")
- self.check_parse_error("</a")
- self.check_parse_error("<$")
- self.check_parse_error("<$>")
- self.check_parse_error("<!")
- self.check_parse_error("<a $>")
- self.check_parse_error("<a")
- self.check_parse_error("<a foo='bar'")
- self.check_parse_error("<a foo='bar")
- self.check_parse_error("<a foo='>'")
- self.check_parse_error("<a foo='>")
- self.check_parse_error("<a foo=>")
-
-
-def test_main():
- support.run_unittest(SGMLParserTestCase)
-
-
-if __name__ == "__main__":
- test_main()