From b245ed1cdf769354b2454cc3fbd34e9b9233cb0a Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 14 Nov 2011 18:13:22 +0200 Subject: Group tests about attributes in a separate class. --- Lib/test/test_htmlparser.py | 121 +++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index b587ab8..ff3ada0 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -196,60 +196,6 @@ DOCTYPE html [ ("data", "this < text > contains < bare>pointy< brackets"), ]) - def test_attr_syntax(self): - output = [ - ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) - ] - self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) - - def test_attr_values(self): - self._run_check("""""", - [("starttag", "a", [("b", "xxx\n\txxx"), - ("c", "yyy\t\nyyy"), - ("d", "\txyz\n")]) - ]) - self._run_check("""""", [ - ("starttag", "a", [("b", ""), ("c", "")]), - ]) - # Regression test for SF patch #669683. - self._run_check("", [ - ("starttag", "e", [("a", "rgb(1,2,3)")]), - ]) - # Regression test for SF bug #921657. - self._run_check("", [ - ("starttag", "a", [("href", "mailto:xyz@example.com")]), - ]) - - def test_attr_nonascii(self): - # see issue 7311 - self._run_check("\u4e2d\u6587", [ - ("starttag", "img", [("src", "/foo/bar.png"), - ("alt", "\u4e2d\u6587")]), - ]) - self._run_check("", [ - ("starttag", "a", [("title", "\u30c6\u30b9\u30c8"), - ("href", "\u30c6\u30b9\u30c8.html")]), - ]) - self._run_check('', [ - ("starttag", "a", [("title", "\u30c6\u30b9\u30c8"), - ("href", "\u30c6\u30b9\u30c8.html")]), - ]) - - def test_attr_entity_replacement(self): - self._run_check("""""", [ - ("starttag", "a", [("b", "&><\"'")]), - ]) - - def test_attr_funky_names(self): - self._run_check("""""", [ - ("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")]), - ]) - def test_illegal_declarations(self): self._parse_error('') @@ -358,10 +304,6 @@ DOCTYPE html [ ("endtag", element_lower)]) - def test_entityrefs_in_attributes(self): - self._run_check("", - [("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])]) - class HTMLParserTolerantTestCase(HTMLParserStrictTestCase): @@ -458,8 +400,69 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase): self.assertEqual(p.unescape('{ ' * 1050), '{ ' * 1050) +class AttributesStrictTestCase(TestCaseBase): + + def get_collector(self): + return EventCollector(strict=True) + + def test_attr_syntax(self): + output = [ + ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) + ] + self._run_check("""""", output) + self._run_check("""""", output) + self._run_check("""""", output) + self._run_check("""""", output) + + def test_attr_values(self): + self._run_check("""""", + [("starttag", "a", [("b", "xxx\n\txxx"), + ("c", "yyy\t\nyyy"), + ("d", "\txyz\n")])]) + self._run_check("""""", + [("starttag", "a", [("b", ""), ("c", "")])]) + # Regression test for SF patch #669683. + self._run_check("", + [("starttag", "e", [("a", "rgb(1,2,3)")])]) + # Regression test for SF bug #921657. + self._run_check( + "", + [("starttag", "a", [("href", "mailto:xyz@example.com")])]) + + def test_attr_nonascii(self): + # see issue 7311 + self._run_check( + "\u4e2d\u6587", + [("starttag", "img", [("src", "/foo/bar.png"), + ("alt", "\u4e2d\u6587")])]) + self._run_check( + "", + [("starttag", "a", [("title", "\u30c6\u30b9\u30c8"), + ("href", "\u30c6\u30b9\u30c8.html")])]) + self._run_check( + '', + [("starttag", "a", [("title", "\u30c6\u30b9\u30c8"), + ("href", "\u30c6\u30b9\u30c8.html")])]) + + def test_attr_entity_replacement(self): + self._run_check( + "", + [("starttag", "a", [("b", "&><\"'")])]) + + def test_attr_funky_names(self): + self._run_check( + "", + [("starttag", "a", [("a.b", "v"), ("c:d", "v"), ("e-f", "v")])]) + + def test_entityrefs_in_attributes(self): + self._run_check( + "", + [("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])]) + + def test_main(): - support.run_unittest(HTMLParserStrictTestCase, HTMLParserTolerantTestCase) + support.run_unittest(HTMLParserStrictTestCase, HTMLParserTolerantTestCase, + AttributesStrictTestCase) if __name__ == "__main__": -- cgit v0.12 From c2fe57762b6cfa8849908e1a0475036cd0b058ba Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Mon, 14 Nov 2011 18:53:33 +0200 Subject: #1745761, #755670, #13357, #12629, #1200313: improve attribute handling in HTMLParser. --- Lib/html/parser.py | 19 ++++---- Lib/test/test_htmlparser.py | 107 ++++++++++++++++++++++++++++++++++++++------ Misc/NEWS | 3 ++ 3 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Lib/html/parser.py b/Lib/html/parser.py index afdb305..662e855 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -30,8 +30,8 @@ attrfind = re.compile( r'\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' r'(\'[^\']*\'|"[^"]*"|[^\s"\'=<>`]*))?') attrfind_tolerant = re.compile( - r',?\s*([a-zA-Z_][-.:a-zA-Z_0-9]*)(\s*=\s*' - r'(\'[^\']*\'|"[^"]*"|[^>\s]*))?') + r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?') locatestarttagend = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:\s+ # whitespace before attribute name @@ -49,16 +49,16 @@ locatestarttagend = re.compile(r""" locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][-.a-zA-Z0-9:_]* # tag name (?:\s* # optional whitespace before attribute name - (?:[a-zA-Z_][-.:a-zA-Z0-9_]* # attribute name - (?:\s*=\s* # value indicator + (?:(?<=['"\s])[^\s/>][^\s/=>]* # attribute name + (?:\s*=+\s* # value indicator (?:'[^']*' # LITA-enclosed value - |\"[^\"]*\" # LIT-enclosed value - |[^'\">\s]+ # bare value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\s]* # bare value ) (?:\s*,)* # possibly followed by a comma - )? - ) - )* + )?\s* + )* + )? \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') @@ -295,6 +295,7 @@ class HTMLParser(_markupbase.ParserBase): elif attrvalue[:1] == '\'' == attrvalue[-1:] or \ attrvalue[:1] == '"' == attrvalue[-1:]: attrvalue = attrvalue[1:-1] + if attrvalue: attrvalue = self.unescape(attrvalue) attrs.append((attrname.lower(), attrvalue)) k = m.end() diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index ff3ada0..1ce4594 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -241,13 +241,11 @@ DOCTYPE html [ self._parse_error("") self._parse_error("") self._parse_error("") self._parse_error("'") self._parse_error("te>>xt&a<\n' '>xt'), + ('entityref', 'a'), + ('data', '<', [ ('starttag', 'form', [('action', '/xxx.php?a=1&b=2&'), - ('method', 'post')])]) + (',', None), ('method', 'post')])]) def test_weird_chars_in_unquoted_attribute_values(self): self._run_check('
', [ @@ -383,7 +380,7 @@ class HTMLParserTolerantTestCase(HTMLParserStrictTestCase): html = '
The rain' expected = [ - ('starttag', 'div', [('style', ''), ('foo', 'bar')]), + ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]), ('starttag', 'b', []), ('data', 'The '), ('starttag', 'a', [('href', 'some_url')]), @@ -460,9 +457,91 @@ class AttributesStrictTestCase(TestCaseBase): [("starttag", "html", [("foo", "\u20AC&aa&unsupported;")])]) + +class AttributesTolerantTestCase(AttributesStrictTestCase): + + def get_collector(self): + return EventCollector(strict=False) + + def test_attr_funky_names2(self): + self._run_check( + "", + [("starttag", "a", [("$", None)]), + ("starttag", "b", [("$", "%")]), + ("starttag", "c", [("\\", "/")])]) + + def test_entities_in_attribute_value(self): + # see #1200313 + for entity in ['&', '&', '&', '&']: + self._run_check('' % entity, + [("starttag", "a", [("href", "&")])]) + self._run_check("" % entity, + [("starttag", "a", [("href", "&")])]) + self._run_check("" % entity, + [("starttag", "a", [("href", "&")])]) + + def test_malformed_attributes(self): + # see #13357 + html = ( + "test - bad1" + "test - bad2" + "test - bad3" + "test - bad4" + ) + expected = [ + ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), + ('data', 'test - bad1'), ('endtag', 'a'), + ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), + ('data', 'test - bad2'), ('endtag', 'a'), + ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), + ('data', 'test - bad3'), ('endtag', 'a'), + ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), + ('data', 'test - bad4'), ('endtag', 'a') + ] + self._run_check(html, expected) + + def test_malformed_adjacent_attributes(self): + # see #12629 + self._run_check('', + [('starttag', 'x', []), + ('startendtag', 'y', [('z', ''), ('o""', None)]), + ('endtag', 'x')]) + self._run_check('', + [('starttag', 'x', []), + ('startendtag', 'y', [('z', ''), ('""', None)]), + ('endtag', 'x')]) + + # see #755670 for the following 3 tests + def test_adjacent_attributes(self): + self._run_check('', + [("starttag", "a", + [("width", "100%"), ("cellspacing","0")])]) + + self._run_check('', + [("starttag", "a", + [("id", "foo"), ("class","bar")])]) + + def test_missing_attribute_value(self): + self._run_check('', + [("starttag", "a", [("v", "")])]) + + def test_javascript_attribute_value(self): + self._run_check("", + [("starttag", "a", + [("href", "javascript:popup('/popup/help.html')")])]) + + def test_end_tag_in_attribute_value(self): + # see #1745761 + self._run_check("spam", + [("starttag", "a", + [("href", "http://www.example.org/\">;")]), + ("data", "spam"), ("endtag", "a")]) + + + def test_main(): support.run_unittest(HTMLParserStrictTestCase, HTMLParserTolerantTestCase, - AttributesStrictTestCase) + AttributesStrictTestCase, AttributesTolerantTestCase) if __name__ == "__main__": diff --git a/Misc/NEWS b/Misc/NEWS index 6f526f6..ca8d4cb 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -73,6 +73,9 @@ Core and Builtins Library ------- +- Issues #1745761, #755670, #13357, #12629, #1200313: HTMLParser now correctly + handles non-valid attributes, including adjacent and unquoted attributes. + - Issue #13193: Fix distutils.filelist.FileList under Windows. The "recursive-include" directive now recognizes both legal path separators. -- cgit v0.12