From b579dba1195df97f87ba868a5987f18fb7509bff Mon Sep 17 00:00:00 2001 From: "R. David Murray" Date: Fri, 3 Dec 2010 04:06:39 +0000 Subject: #1486713: Add a tolerant mode to HTMLParser. The motivation for adding this option is that the the functionality it provides used to be provided by sgmllib in Python2, and was used by, for example, BeautifulSoup. Without this option, the Python3 version of BeautifulSoup and the many programs that use it are crippled. The original patch was by 'kxroberto'. I modified it heavily but kept his heuristics and test. I also added additional heuristics to fix #975556, #1046092, and part of #6191. This patch should be completely backward compatible: the behavior with the default strict=True is unchanged. --- Lib/test/test_htmlparser.py | 48 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) (limited to 'Lib/test/test_htmlparser.py') diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index e982218dba..beaf6b63a2 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -8,10 +8,10 @@ from test import support class EventCollector(html.parser.HTMLParser): - def __init__(self): + def __init__(self, *args, **kw): self.events = [] self.append = self.events.append - html.parser.HTMLParser.__init__(self) + html.parser.HTMLParser.__init__(self, *args, **kw) def get_events(self): # Normalize the list of events so that buffer artefacts don't @@ -72,8 +72,10 @@ class EventCollectorExtra(EventCollector): class TestCaseBase(unittest.TestCase): - def _run_check(self, source, expected_events, collector=EventCollector): - parser = collector() + def _run_check(self, source, expected_events, collector=None): + if collector is None: + collector = EventCollector() + parser = collector for s in source: parser.feed(s) parser.close() @@ -84,7 +86,7 @@ class TestCaseBase(unittest.TestCase): "\nReceived:\n" + pprint.pformat(events)) def _run_check_extra(self, source, events): - self._run_check(source, events, EventCollectorExtra) + self._run_check(source, events, EventCollectorExtra()) def _parse_error(self, source): def parse(source=source): @@ -321,8 +323,42 @@ DOCTYPE html [ ]) +class HTMLParserTolerantTestCase(TestCaseBase): + + def setUp(self): + self.collector = EventCollector(strict=False) + + def test_tolerant_parsing(self): + self._run_check('te>>xt&a<\n' + '', [ + ('starttag', 'form', + [('action', '/xxx.php?a=1&b=2&'), + ('method', 'post')])], + collector = self.collector) + + def test_weird_chars_in_unquoted_attribute_values(self): + self._run_check('
', [ + ('starttag', 'form', + [('action', 'bogus|&#()value')])], + collector = self.collector) + + def test_main(): - support.run_unittest(HTMLParserTestCase) + support.run_unittest(HTMLParserTestCase, HTMLParserTolerantTestCase) if __name__ == "__main__": -- cgit v1.2.1