diff options
Diffstat (limited to 'bitbake/lib/bs4/tests/test_tree.py')
-rw-r--r-- | bitbake/lib/bs4/tests/test_tree.py | 294 |
1 files changed, 238 insertions, 56 deletions
diff --git a/bitbake/lib/bs4/tests/test_tree.py b/bitbake/lib/bs4/tests/test_tree.py index f8515c0ea1..6d3e67f311 100644 --- a/bitbake/lib/bs4/tests/test_tree.py +++ b/bitbake/lib/bs4/tests/test_tree.py | |||
@@ -9,6 +9,7 @@ same markup, but all Beautiful Soup trees can be traversed with the | |||
9 | methods tested here. | 9 | methods tested here. |
10 | """ | 10 | """ |
11 | 11 | ||
12 | from pdb import set_trace | ||
12 | import copy | 13 | import copy |
13 | import pickle | 14 | import pickle |
14 | import re | 15 | import re |
@@ -19,8 +20,10 @@ from bs4.builder import ( | |||
19 | HTMLParserTreeBuilder, | 20 | HTMLParserTreeBuilder, |
20 | ) | 21 | ) |
21 | from bs4.element import ( | 22 | from bs4.element import ( |
23 | PY3K, | ||
22 | CData, | 24 | CData, |
23 | Comment, | 25 | Comment, |
26 | Declaration, | ||
24 | Doctype, | 27 | Doctype, |
25 | NavigableString, | 28 | NavigableString, |
26 | SoupStrainer, | 29 | SoupStrainer, |
@@ -67,8 +70,14 @@ class TestFind(TreeTest): | |||
67 | self.assertEqual(soup.find("b").string, "2") | 70 | self.assertEqual(soup.find("b").string, "2") |
68 | 71 | ||
69 | def test_unicode_text_find(self): | 72 | def test_unicode_text_find(self): |
70 | soup = self.soup(u'<h1>Räksmörgås</h1>') | 73 | soup = self.soup('<h1>Räksmörgås</h1>') |
71 | self.assertEqual(soup.find(text=u'Räksmörgås'), u'Räksmörgås') | 74 | self.assertEqual(soup.find(string='Räksmörgås'), 'Räksmörgås') |
75 | |||
76 | def test_unicode_attribute_find(self): | ||
77 | soup = self.soup('<h1 id="Räksmörgås">here it is</h1>') | ||
78 | str(soup) | ||
79 | self.assertEqual("here it is", soup.find(id='Räksmörgås').text) | ||
80 | |||
72 | 81 | ||
73 | def test_find_everything(self): | 82 | def test_find_everything(self): |
74 | """Test an optimization that finds all tags.""" | 83 | """Test an optimization that finds all tags.""" |
@@ -87,16 +96,17 @@ class TestFindAll(TreeTest): | |||
87 | """You can search the tree for text nodes.""" | 96 | """You can search the tree for text nodes.""" |
88 | soup = self.soup("<html>Foo<b>bar</b>\xbb</html>") | 97 | soup = self.soup("<html>Foo<b>bar</b>\xbb</html>") |
89 | # Exact match. | 98 | # Exact match. |
90 | self.assertEqual(soup.find_all(text="bar"), [u"bar"]) | 99 | self.assertEqual(soup.find_all(string="bar"), ["bar"]) |
100 | self.assertEqual(soup.find_all(text="bar"), ["bar"]) | ||
91 | # Match any of a number of strings. | 101 | # Match any of a number of strings. |
92 | self.assertEqual( | 102 | self.assertEqual( |
93 | soup.find_all(text=["Foo", "bar"]), [u"Foo", u"bar"]) | 103 | soup.find_all(text=["Foo", "bar"]), ["Foo", "bar"]) |
94 | # Match a regular expression. | 104 | # Match a regular expression. |
95 | self.assertEqual(soup.find_all(text=re.compile('.*')), | 105 | self.assertEqual(soup.find_all(text=re.compile('.*')), |
96 | [u"Foo", u"bar", u'\xbb']) | 106 | ["Foo", "bar", '\xbb']) |
97 | # Match anything. | 107 | # Match anything. |
98 | self.assertEqual(soup.find_all(text=True), | 108 | self.assertEqual(soup.find_all(text=True), |
99 | [u"Foo", u"bar", u'\xbb']) | 109 | ["Foo", "bar", '\xbb']) |
100 | 110 | ||
101 | def test_find_all_limit(self): | 111 | def test_find_all_limit(self): |
102 | """You can limit the number of items returned by find_all.""" | 112 | """You can limit the number of items returned by find_all.""" |
@@ -227,8 +237,8 @@ class TestFindAllByAttribute(TreeTest): | |||
227 | ["Matching a.", "Matching b."]) | 237 | ["Matching a.", "Matching b."]) |
228 | 238 | ||
229 | def test_find_all_by_utf8_attribute_value(self): | 239 | def test_find_all_by_utf8_attribute_value(self): |
230 | peace = u"םולש".encode("utf8") | 240 | peace = "םולש".encode("utf8") |
231 | data = u'<a title="םולש"></a>'.encode("utf8") | 241 | data = '<a title="םולש"></a>'.encode("utf8") |
232 | soup = self.soup(data) | 242 | soup = self.soup(data) |
233 | self.assertEqual([soup.a], soup.find_all(title=peace)) | 243 | self.assertEqual([soup.a], soup.find_all(title=peace)) |
234 | self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8"))) | 244 | self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8"))) |
@@ -688,7 +698,7 @@ class TestTagCreation(SoupTest): | |||
688 | 698 | ||
689 | def test_tag_inherits_self_closing_rules_from_builder(self): | 699 | def test_tag_inherits_self_closing_rules_from_builder(self): |
690 | if XML_BUILDER_PRESENT: | 700 | if XML_BUILDER_PRESENT: |
691 | xml_soup = BeautifulSoup("", "xml") | 701 | xml_soup = BeautifulSoup("", "lxml-xml") |
692 | xml_br = xml_soup.new_tag("br") | 702 | xml_br = xml_soup.new_tag("br") |
693 | xml_p = xml_soup.new_tag("p") | 703 | xml_p = xml_soup.new_tag("p") |
694 | 704 | ||
@@ -697,7 +707,7 @@ class TestTagCreation(SoupTest): | |||
697 | self.assertEqual(b"<br/>", xml_br.encode()) | 707 | self.assertEqual(b"<br/>", xml_br.encode()) |
698 | self.assertEqual(b"<p/>", xml_p.encode()) | 708 | self.assertEqual(b"<p/>", xml_p.encode()) |
699 | 709 | ||
700 | html_soup = BeautifulSoup("", "html") | 710 | html_soup = BeautifulSoup("", "html.parser") |
701 | html_br = html_soup.new_tag("br") | 711 | html_br = html_soup.new_tag("br") |
702 | html_p = html_soup.new_tag("p") | 712 | html_p = html_soup.new_tag("p") |
703 | 713 | ||
@@ -773,6 +783,14 @@ class TestTreeModification(SoupTest): | |||
773 | new_a = a.unwrap() | 783 | new_a = a.unwrap() |
774 | self.assertEqual(a, new_a) | 784 | self.assertEqual(a, new_a) |
775 | 785 | ||
786 | def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self): | ||
787 | soup = self.soup("<a><b>Foo</b></a><c>Bar</c>") | ||
788 | a = soup.a | ||
789 | a.extract() | ||
790 | self.assertEqual(None, a.parent) | ||
791 | self.assertRaises(ValueError, a.unwrap) | ||
792 | self.assertRaises(ValueError, a.replace_with, soup.c) | ||
793 | |||
776 | def test_replace_tag_with_itself(self): | 794 | def test_replace_tag_with_itself(self): |
777 | text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>" | 795 | text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>" |
778 | soup = self.soup(text) | 796 | soup = self.soup(text) |
@@ -1067,6 +1085,31 @@ class TestTreeModification(SoupTest): | |||
1067 | self.assertEqual(foo_2, soup.a.string) | 1085 | self.assertEqual(foo_2, soup.a.string) |
1068 | self.assertEqual(bar_2, soup.b.string) | 1086 | self.assertEqual(bar_2, soup.b.string) |
1069 | 1087 | ||
1088 | def test_extract_multiples_of_same_tag(self): | ||
1089 | soup = self.soup(""" | ||
1090 | <html> | ||
1091 | <head> | ||
1092 | <script>foo</script> | ||
1093 | </head> | ||
1094 | <body> | ||
1095 | <script>bar</script> | ||
1096 | <a></a> | ||
1097 | </body> | ||
1098 | <script>baz</script> | ||
1099 | </html>""") | ||
1100 | [soup.script.extract() for i in soup.find_all("script")] | ||
1101 | self.assertEqual("<body>\n\n<a></a>\n</body>", str(soup.body)) | ||
1102 | |||
1103 | |||
1104 | def test_extract_works_when_element_is_surrounded_by_identical_strings(self): | ||
1105 | soup = self.soup( | ||
1106 | '<html>\n' | ||
1107 | '<body>hi</body>\n' | ||
1108 | '</html>') | ||
1109 | soup.find('body').extract() | ||
1110 | self.assertEqual(None, soup.find('body')) | ||
1111 | |||
1112 | |||
1070 | def test_clear(self): | 1113 | def test_clear(self): |
1071 | """Tag.clear()""" | 1114 | """Tag.clear()""" |
1072 | soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>") | 1115 | soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>") |
@@ -1287,27 +1330,72 @@ class TestPersistence(SoupTest): | |||
1287 | 1330 | ||
1288 | def test_unicode_pickle(self): | 1331 | def test_unicode_pickle(self): |
1289 | # A tree containing Unicode characters can be pickled. | 1332 | # A tree containing Unicode characters can be pickled. |
1290 | html = u"<b>\N{SNOWMAN}</b>" | 1333 | html = "<b>\N{SNOWMAN}</b>" |
1291 | soup = self.soup(html) | 1334 | soup = self.soup(html) |
1292 | dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL) | 1335 | dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL) |
1293 | loaded = pickle.loads(dumped) | 1336 | loaded = pickle.loads(dumped) |
1294 | self.assertEqual(loaded.decode(), soup.decode()) | 1337 | self.assertEqual(loaded.decode(), soup.decode()) |
1295 | 1338 | ||
1339 | def test_copy_navigablestring_is_not_attached_to_tree(self): | ||
1340 | html = "<b>Foo<a></a></b><b>Bar</b>" | ||
1341 | soup = self.soup(html) | ||
1342 | s1 = soup.find(string="Foo") | ||
1343 | s2 = copy.copy(s1) | ||
1344 | self.assertEqual(s1, s2) | ||
1345 | self.assertEqual(None, s2.parent) | ||
1346 | self.assertEqual(None, s2.next_element) | ||
1347 | self.assertNotEqual(None, s1.next_sibling) | ||
1348 | self.assertEqual(None, s2.next_sibling) | ||
1349 | self.assertEqual(None, s2.previous_element) | ||
1350 | |||
1351 | def test_copy_navigablestring_subclass_has_same_type(self): | ||
1352 | html = "<b><!--Foo--></b>" | ||
1353 | soup = self.soup(html) | ||
1354 | s1 = soup.string | ||
1355 | s2 = copy.copy(s1) | ||
1356 | self.assertEqual(s1, s2) | ||
1357 | self.assertTrue(isinstance(s2, Comment)) | ||
1358 | |||
1359 | def test_copy_entire_soup(self): | ||
1360 | html = "<div><b>Foo<a></a></b><b>Bar</b></div>end" | ||
1361 | soup = self.soup(html) | ||
1362 | soup_copy = copy.copy(soup) | ||
1363 | self.assertEqual(soup, soup_copy) | ||
1364 | |||
1365 | def test_copy_tag_copies_contents(self): | ||
1366 | html = "<div><b>Foo<a></a></b><b>Bar</b></div>end" | ||
1367 | soup = self.soup(html) | ||
1368 | div = soup.div | ||
1369 | div_copy = copy.copy(div) | ||
1370 | |||
1371 | # The two tags look the same, and evaluate to equal. | ||
1372 | self.assertEqual(str(div), str(div_copy)) | ||
1373 | self.assertEqual(div, div_copy) | ||
1374 | |||
1375 | # But they're not the same object. | ||
1376 | self.assertFalse(div is div_copy) | ||
1377 | |||
1378 | # And they don't have the same relation to the parse tree. The | ||
1379 | # copy is not associated with a parse tree at all. | ||
1380 | self.assertEqual(None, div_copy.parent) | ||
1381 | self.assertEqual(None, div_copy.previous_element) | ||
1382 | self.assertEqual(None, div_copy.find(string='Bar').next_element) | ||
1383 | self.assertNotEqual(None, div.find(string='Bar').next_element) | ||
1296 | 1384 | ||
1297 | class TestSubstitutions(SoupTest): | 1385 | class TestSubstitutions(SoupTest): |
1298 | 1386 | ||
1299 | def test_default_formatter_is_minimal(self): | 1387 | def test_default_formatter_is_minimal(self): |
1300 | markup = u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" | 1388 | markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" |
1301 | soup = self.soup(markup) | 1389 | soup = self.soup(markup) |
1302 | decoded = soup.decode(formatter="minimal") | 1390 | decoded = soup.decode(formatter="minimal") |
1303 | # The < is converted back into < but the e-with-acute is left alone. | 1391 | # The < is converted back into < but the e-with-acute is left alone. |
1304 | self.assertEqual( | 1392 | self.assertEqual( |
1305 | decoded, | 1393 | decoded, |
1306 | self.document_for( | 1394 | self.document_for( |
1307 | u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) | 1395 | "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) |
1308 | 1396 | ||
1309 | def test_formatter_html(self): | 1397 | def test_formatter_html(self): |
1310 | markup = u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" | 1398 | markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" |
1311 | soup = self.soup(markup) | 1399 | soup = self.soup(markup) |
1312 | decoded = soup.decode(formatter="html") | 1400 | decoded = soup.decode(formatter="html") |
1313 | self.assertEqual( | 1401 | self.assertEqual( |
@@ -1315,49 +1403,49 @@ class TestSubstitutions(SoupTest): | |||
1315 | self.document_for("<b><<Sacré bleu!>></b>")) | 1403 | self.document_for("<b><<Sacré bleu!>></b>")) |
1316 | 1404 | ||
1317 | def test_formatter_minimal(self): | 1405 | def test_formatter_minimal(self): |
1318 | markup = u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" | 1406 | markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" |
1319 | soup = self.soup(markup) | 1407 | soup = self.soup(markup) |
1320 | decoded = soup.decode(formatter="minimal") | 1408 | decoded = soup.decode(formatter="minimal") |
1321 | # The < is converted back into < but the e-with-acute is left alone. | 1409 | # The < is converted back into < but the e-with-acute is left alone. |
1322 | self.assertEqual( | 1410 | self.assertEqual( |
1323 | decoded, | 1411 | decoded, |
1324 | self.document_for( | 1412 | self.document_for( |
1325 | u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) | 1413 | "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) |
1326 | 1414 | ||
1327 | def test_formatter_null(self): | 1415 | def test_formatter_null(self): |
1328 | markup = u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" | 1416 | markup = "<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>" |
1329 | soup = self.soup(markup) | 1417 | soup = self.soup(markup) |
1330 | decoded = soup.decode(formatter=None) | 1418 | decoded = soup.decode(formatter=None) |
1331 | # Neither the angle brackets nor the e-with-acute are converted. | 1419 | # Neither the angle brackets nor the e-with-acute are converted. |
1332 | # This is not valid HTML, but it's what the user wanted. | 1420 | # This is not valid HTML, but it's what the user wanted. |
1333 | self.assertEqual(decoded, | 1421 | self.assertEqual(decoded, |
1334 | self.document_for(u"<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) | 1422 | self.document_for("<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>")) |
1335 | 1423 | ||
1336 | def test_formatter_custom(self): | 1424 | def test_formatter_custom(self): |
1337 | markup = u"<b><foo></b><b>bar</b>" | 1425 | markup = "<b><foo></b><b>bar</b>" |
1338 | soup = self.soup(markup) | 1426 | soup = self.soup(markup) |
1339 | decoded = soup.decode(formatter = lambda x: x.upper()) | 1427 | decoded = soup.decode(formatter = lambda x: x.upper()) |
1340 | # Instead of normal entity conversion code, the custom | 1428 | # Instead of normal entity conversion code, the custom |
1341 | # callable is called on every string. | 1429 | # callable is called on every string. |
1342 | self.assertEqual( | 1430 | self.assertEqual( |
1343 | decoded, | 1431 | decoded, |
1344 | self.document_for(u"<b><FOO></b><b>BAR</b>")) | 1432 | self.document_for("<b><FOO></b><b>BAR</b>")) |
1345 | 1433 | ||
1346 | def test_formatter_is_run_on_attribute_values(self): | 1434 | def test_formatter_is_run_on_attribute_values(self): |
1347 | markup = u'<a href="http://a.com?a=b&c=é">e</a>' | 1435 | markup = '<a href="http://a.com?a=b&c=é">e</a>' |
1348 | soup = self.soup(markup) | 1436 | soup = self.soup(markup) |
1349 | a = soup.a | 1437 | a = soup.a |
1350 | 1438 | ||
1351 | expect_minimal = u'<a href="http://a.com?a=b&c=é">e</a>' | 1439 | expect_minimal = '<a href="http://a.com?a=b&c=é">e</a>' |
1352 | 1440 | ||
1353 | self.assertEqual(expect_minimal, a.decode()) | 1441 | self.assertEqual(expect_minimal, a.decode()) |
1354 | self.assertEqual(expect_minimal, a.decode(formatter="minimal")) | 1442 | self.assertEqual(expect_minimal, a.decode(formatter="minimal")) |
1355 | 1443 | ||
1356 | expect_html = u'<a href="http://a.com?a=b&c=é">e</a>' | 1444 | expect_html = '<a href="http://a.com?a=b&c=é">e</a>' |
1357 | self.assertEqual(expect_html, a.decode(formatter="html")) | 1445 | self.assertEqual(expect_html, a.decode(formatter="html")) |
1358 | 1446 | ||
1359 | self.assertEqual(markup, a.decode(formatter=None)) | 1447 | self.assertEqual(markup, a.decode(formatter=None)) |
1360 | expect_upper = u'<a href="HTTP://A.COM?A=B&C=É">E</a>' | 1448 | expect_upper = '<a href="HTTP://A.COM?A=B&C=É">E</a>' |
1361 | self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper())) | 1449 | self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper())) |
1362 | 1450 | ||
1363 | def test_formatter_skips_script_tag_for_html_documents(self): | 1451 | def test_formatter_skips_script_tag_for_html_documents(self): |
@@ -1366,7 +1454,7 @@ class TestSubstitutions(SoupTest): | |||
1366 | console.log("< < hey > > "); | 1454 | console.log("< < hey > > "); |
1367 | </script> | 1455 | </script> |
1368 | """ | 1456 | """ |
1369 | encoded = BeautifulSoup(doc).encode() | 1457 | encoded = BeautifulSoup(doc, 'html.parser').encode() |
1370 | self.assertTrue(b"< < hey > >" in encoded) | 1458 | self.assertTrue(b"< < hey > >" in encoded) |
1371 | 1459 | ||
1372 | def test_formatter_skips_style_tag_for_html_documents(self): | 1460 | def test_formatter_skips_style_tag_for_html_documents(self): |
@@ -1375,7 +1463,7 @@ class TestSubstitutions(SoupTest): | |||
1375 | console.log("< < hey > > "); | 1463 | console.log("< < hey > > "); |
1376 | </style> | 1464 | </style> |
1377 | """ | 1465 | """ |
1378 | encoded = BeautifulSoup(doc).encode() | 1466 | encoded = BeautifulSoup(doc, 'html.parser').encode() |
1379 | self.assertTrue(b"< < hey > >" in encoded) | 1467 | self.assertTrue(b"< < hey > >" in encoded) |
1380 | 1468 | ||
1381 | def test_prettify_leaves_preformatted_text_alone(self): | 1469 | def test_prettify_leaves_preformatted_text_alone(self): |
@@ -1383,24 +1471,24 @@ class TestSubstitutions(SoupTest): | |||
1383 | # Everything outside the <pre> tag is reformatted, but everything | 1471 | # Everything outside the <pre> tag is reformatted, but everything |
1384 | # inside is left alone. | 1472 | # inside is left alone. |
1385 | self.assertEqual( | 1473 | self.assertEqual( |
1386 | u'<div>\n foo\n <pre> \tbar\n \n </pre>\n baz\n</div>', | 1474 | '<div>\n foo\n <pre> \tbar\n \n </pre>\n baz\n</div>', |
1387 | soup.div.prettify()) | 1475 | soup.div.prettify()) |
1388 | 1476 | ||
1389 | def test_prettify_accepts_formatter(self): | 1477 | def test_prettify_accepts_formatter(self): |
1390 | soup = BeautifulSoup("<html><body>foo</body></html>") | 1478 | soup = BeautifulSoup("<html><body>foo</body></html>", 'html.parser') |
1391 | pretty = soup.prettify(formatter = lambda x: x.upper()) | 1479 | pretty = soup.prettify(formatter = lambda x: x.upper()) |
1392 | self.assertTrue("FOO" in pretty) | 1480 | self.assertTrue("FOO" in pretty) |
1393 | 1481 | ||
1394 | def test_prettify_outputs_unicode_by_default(self): | 1482 | def test_prettify_outputs_unicode_by_default(self): |
1395 | soup = self.soup("<a></a>") | 1483 | soup = self.soup("<a></a>") |
1396 | self.assertEqual(unicode, type(soup.prettify())) | 1484 | self.assertEqual(str, type(soup.prettify())) |
1397 | 1485 | ||
1398 | def test_prettify_can_encode_data(self): | 1486 | def test_prettify_can_encode_data(self): |
1399 | soup = self.soup("<a></a>") | 1487 | soup = self.soup("<a></a>") |
1400 | self.assertEqual(bytes, type(soup.prettify("utf-8"))) | 1488 | self.assertEqual(bytes, type(soup.prettify("utf-8"))) |
1401 | 1489 | ||
1402 | def test_html_entity_substitution_off_by_default(self): | 1490 | def test_html_entity_substitution_off_by_default(self): |
1403 | markup = u"<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>" | 1491 | markup = "<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>" |
1404 | soup = self.soup(markup) | 1492 | soup = self.soup(markup) |
1405 | encoded = soup.b.encode("utf-8") | 1493 | encoded = soup.b.encode("utf-8") |
1406 | self.assertEqual(encoded, markup.encode('utf-8')) | 1494 | self.assertEqual(encoded, markup.encode('utf-8')) |
@@ -1444,45 +1532,53 @@ class TestEncoding(SoupTest): | |||
1444 | """Test the ability to encode objects into strings.""" | 1532 | """Test the ability to encode objects into strings.""" |
1445 | 1533 | ||
1446 | def test_unicode_string_can_be_encoded(self): | 1534 | def test_unicode_string_can_be_encoded(self): |
1447 | html = u"<b>\N{SNOWMAN}</b>" | 1535 | html = "<b>\N{SNOWMAN}</b>" |
1448 | soup = self.soup(html) | 1536 | soup = self.soup(html) |
1449 | self.assertEqual(soup.b.string.encode("utf-8"), | 1537 | self.assertEqual(soup.b.string.encode("utf-8"), |
1450 | u"\N{SNOWMAN}".encode("utf-8")) | 1538 | "\N{SNOWMAN}".encode("utf-8")) |
1451 | 1539 | ||
1452 | def test_tag_containing_unicode_string_can_be_encoded(self): | 1540 | def test_tag_containing_unicode_string_can_be_encoded(self): |
1453 | html = u"<b>\N{SNOWMAN}</b>" | 1541 | html = "<b>\N{SNOWMAN}</b>" |
1454 | soup = self.soup(html) | 1542 | soup = self.soup(html) |
1455 | self.assertEqual( | 1543 | self.assertEqual( |
1456 | soup.b.encode("utf-8"), html.encode("utf-8")) | 1544 | soup.b.encode("utf-8"), html.encode("utf-8")) |
1457 | 1545 | ||
1458 | def test_encoding_substitutes_unrecognized_characters_by_default(self): | 1546 | def test_encoding_substitutes_unrecognized_characters_by_default(self): |
1459 | html = u"<b>\N{SNOWMAN}</b>" | 1547 | html = "<b>\N{SNOWMAN}</b>" |
1460 | soup = self.soup(html) | 1548 | soup = self.soup(html) |
1461 | self.assertEqual(soup.b.encode("ascii"), b"<b>☃</b>") | 1549 | self.assertEqual(soup.b.encode("ascii"), b"<b>☃</b>") |
1462 | 1550 | ||
1463 | def test_encoding_can_be_made_strict(self): | 1551 | def test_encoding_can_be_made_strict(self): |
1464 | html = u"<b>\N{SNOWMAN}</b>" | 1552 | html = "<b>\N{SNOWMAN}</b>" |
1465 | soup = self.soup(html) | 1553 | soup = self.soup(html) |
1466 | self.assertRaises( | 1554 | self.assertRaises( |
1467 | UnicodeEncodeError, soup.encode, "ascii", errors="strict") | 1555 | UnicodeEncodeError, soup.encode, "ascii", errors="strict") |
1468 | 1556 | ||
1469 | def test_decode_contents(self): | 1557 | def test_decode_contents(self): |
1470 | html = u"<b>\N{SNOWMAN}</b>" | 1558 | html = "<b>\N{SNOWMAN}</b>" |
1471 | soup = self.soup(html) | 1559 | soup = self.soup(html) |
1472 | self.assertEqual(u"\N{SNOWMAN}", soup.b.decode_contents()) | 1560 | self.assertEqual("\N{SNOWMAN}", soup.b.decode_contents()) |
1473 | 1561 | ||
1474 | def test_encode_contents(self): | 1562 | def test_encode_contents(self): |
1475 | html = u"<b>\N{SNOWMAN}</b>" | 1563 | html = "<b>\N{SNOWMAN}</b>" |
1476 | soup = self.soup(html) | 1564 | soup = self.soup(html) |
1477 | self.assertEqual( | 1565 | self.assertEqual( |
1478 | u"\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents( | 1566 | "\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents( |
1479 | encoding="utf8")) | 1567 | encoding="utf8")) |
1480 | 1568 | ||
1481 | def test_deprecated_renderContents(self): | 1569 | def test_deprecated_renderContents(self): |
1482 | html = u"<b>\N{SNOWMAN}</b>" | 1570 | html = "<b>\N{SNOWMAN}</b>" |
1483 | soup = self.soup(html) | 1571 | soup = self.soup(html) |
1484 | self.assertEqual( | 1572 | self.assertEqual( |
1485 | u"\N{SNOWMAN}".encode("utf8"), soup.b.renderContents()) | 1573 | "\N{SNOWMAN}".encode("utf8"), soup.b.renderContents()) |
1574 | |||
1575 | def test_repr(self): | ||
1576 | html = "<b>\N{SNOWMAN}</b>" | ||
1577 | soup = self.soup(html) | ||
1578 | if PY3K: | ||
1579 | self.assertEqual(html, repr(soup)) | ||
1580 | else: | ||
1581 | self.assertEqual(b'<b>\\u2603</b>', repr(soup)) | ||
1486 | 1582 | ||
1487 | class TestNavigableStringSubclasses(SoupTest): | 1583 | class TestNavigableStringSubclasses(SoupTest): |
1488 | 1584 | ||
@@ -1522,6 +1618,9 @@ class TestNavigableStringSubclasses(SoupTest): | |||
1522 | soup.insert(1, doctype) | 1618 | soup.insert(1, doctype) |
1523 | self.assertEqual(soup.encode(), b"<!DOCTYPE foo>\n") | 1619 | self.assertEqual(soup.encode(), b"<!DOCTYPE foo>\n") |
1524 | 1620 | ||
1621 | def test_declaration(self): | ||
1622 | d = Declaration("foo") | ||
1623 | self.assertEqual("<?foo?>", d.output_ready()) | ||
1525 | 1624 | ||
1526 | class TestSoupSelector(TreeTest): | 1625 | class TestSoupSelector(TreeTest): |
1527 | 1626 | ||
@@ -1534,7 +1633,7 @@ class TestSoupSelector(TreeTest): | |||
1534 | <link rel="stylesheet" href="blah.css" type="text/css" id="l1"> | 1633 | <link rel="stylesheet" href="blah.css" type="text/css" id="l1"> |
1535 | </head> | 1634 | </head> |
1536 | <body> | 1635 | <body> |
1537 | 1636 | <custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag> | |
1538 | <div id="main" class="fancy"> | 1637 | <div id="main" class="fancy"> |
1539 | <div id="inner"> | 1638 | <div id="inner"> |
1540 | <h1 id="header1">An H1</h1> | 1639 | <h1 id="header1">An H1</h1> |
@@ -1552,8 +1651,18 @@ class TestSoupSelector(TreeTest): | |||
1552 | <a href="#" id="s2a1">span2a1</a> | 1651 | <a href="#" id="s2a1">span2a1</a> |
1553 | </span> | 1652 | </span> |
1554 | <span class="span3"></span> | 1653 | <span class="span3"></span> |
1654 | <custom-dashed-tag class="dashed" id="dash2"/> | ||
1655 | <div data-tag="dashedvalue" id="data1"/> | ||
1555 | </span> | 1656 | </span> |
1556 | </div> | 1657 | </div> |
1658 | <x id="xid"> | ||
1659 | <z id="zida"/> | ||
1660 | <z id="zidab"/> | ||
1661 | <z id="zidac"/> | ||
1662 | </x> | ||
1663 | <y id="yid"> | ||
1664 | <z id="zidb"/> | ||
1665 | </y> | ||
1557 | <p lang="en" id="lang-en">English</p> | 1666 | <p lang="en" id="lang-en">English</p> |
1558 | <p lang="en-gb" id="lang-en-gb">English UK</p> | 1667 | <p lang="en-gb" id="lang-en-gb">English UK</p> |
1559 | <p lang="en-us" id="lang-en-us">English US</p> | 1668 | <p lang="en-us" id="lang-en-us">English US</p> |
@@ -1565,7 +1674,7 @@ class TestSoupSelector(TreeTest): | |||
1565 | """ | 1674 | """ |
1566 | 1675 | ||
1567 | def setUp(self): | 1676 | def setUp(self): |
1568 | self.soup = BeautifulSoup(self.HTML) | 1677 | self.soup = BeautifulSoup(self.HTML, 'html.parser') |
1569 | 1678 | ||
1570 | def assertSelects(self, selector, expected_ids): | 1679 | def assertSelects(self, selector, expected_ids): |
1571 | el_ids = [el['id'] for el in self.soup.select(selector)] | 1680 | el_ids = [el['id'] for el in self.soup.select(selector)] |
@@ -1587,21 +1696,29 @@ class TestSoupSelector(TreeTest): | |||
1587 | els = self.soup.select('title') | 1696 | els = self.soup.select('title') |
1588 | self.assertEqual(len(els), 1) | 1697 | self.assertEqual(len(els), 1) |
1589 | self.assertEqual(els[0].name, 'title') | 1698 | self.assertEqual(els[0].name, 'title') |
1590 | self.assertEqual(els[0].contents, [u'The title']) | 1699 | self.assertEqual(els[0].contents, ['The title']) |
1591 | 1700 | ||
1592 | def test_one_tag_many(self): | 1701 | def test_one_tag_many(self): |
1593 | els = self.soup.select('div') | 1702 | els = self.soup.select('div') |
1594 | self.assertEqual(len(els), 3) | 1703 | self.assertEqual(len(els), 4) |
1595 | for div in els: | 1704 | for div in els: |
1596 | self.assertEqual(div.name, 'div') | 1705 | self.assertEqual(div.name, 'div') |
1597 | 1706 | ||
1707 | el = self.soup.select_one('div') | ||
1708 | self.assertEqual('main', el['id']) | ||
1709 | |||
1710 | def test_select_one_returns_none_if_no_match(self): | ||
1711 | match = self.soup.select_one('nonexistenttag') | ||
1712 | self.assertEqual(None, match) | ||
1713 | |||
1714 | |||
1598 | def test_tag_in_tag_one(self): | 1715 | def test_tag_in_tag_one(self): |
1599 | els = self.soup.select('div div') | 1716 | els = self.soup.select('div div') |
1600 | self.assertSelects('div div', ['inner']) | 1717 | self.assertSelects('div div', ['inner', 'data1']) |
1601 | 1718 | ||
1602 | def test_tag_in_tag_many(self): | 1719 | def test_tag_in_tag_many(self): |
1603 | for selector in ('html div', 'html body div', 'body div'): | 1720 | for selector in ('html div', 'html body div', 'body div'): |
1604 | self.assertSelects(selector, ['main', 'inner', 'footer']) | 1721 | self.assertSelects(selector, ['data1', 'main', 'inner', 'footer']) |
1605 | 1722 | ||
1606 | def test_tag_no_match(self): | 1723 | def test_tag_no_match(self): |
1607 | self.assertEqual(len(self.soup.select('del')), 0) | 1724 | self.assertEqual(len(self.soup.select('del')), 0) |
@@ -1609,6 +1726,20 @@ class TestSoupSelector(TreeTest): | |||
1609 | def test_invalid_tag(self): | 1726 | def test_invalid_tag(self): |
1610 | self.assertRaises(ValueError, self.soup.select, 'tag%t') | 1727 | self.assertRaises(ValueError, self.soup.select, 'tag%t') |
1611 | 1728 | ||
1729 | def test_select_dashed_tag_ids(self): | ||
1730 | self.assertSelects('custom-dashed-tag', ['dash1', 'dash2']) | ||
1731 | |||
1732 | def test_select_dashed_by_id(self): | ||
1733 | dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]') | ||
1734 | self.assertEqual(dashed[0].name, 'custom-dashed-tag') | ||
1735 | self.assertEqual(dashed[0]['id'], 'dash2') | ||
1736 | |||
1737 | def test_dashed_tag_text(self): | ||
1738 | self.assertEqual(self.soup.select('body > custom-dashed-tag')[0].text, 'Hello there.') | ||
1739 | |||
1740 | def test_select_dashed_matches_find_all(self): | ||
1741 | self.assertEqual(self.soup.select('custom-dashed-tag'), self.soup.find_all('custom-dashed-tag')) | ||
1742 | |||
1612 | def test_header_tags(self): | 1743 | def test_header_tags(self): |
1613 | self.assertSelectMultiple( | 1744 | self.assertSelectMultiple( |
1614 | ('h1', ['header1']), | 1745 | ('h1', ['header1']), |
@@ -1709,6 +1840,7 @@ class TestSoupSelector(TreeTest): | |||
1709 | ('[id^="m"]', ['me', 'main']), | 1840 | ('[id^="m"]', ['me', 'main']), |
1710 | ('div[id^="m"]', ['main']), | 1841 | ('div[id^="m"]', ['main']), |
1711 | ('a[id^="m"]', ['me']), | 1842 | ('a[id^="m"]', ['me']), |
1843 | ('div[data-tag^="dashed"]', ['data1']) | ||
1712 | ) | 1844 | ) |
1713 | 1845 | ||
1714 | def test_attribute_endswith(self): | 1846 | def test_attribute_endswith(self): |
@@ -1716,8 +1848,8 @@ class TestSoupSelector(TreeTest): | |||
1716 | ('[href$=".css"]', ['l1']), | 1848 | ('[href$=".css"]', ['l1']), |
1717 | ('link[href$=".css"]', ['l1']), | 1849 | ('link[href$=".css"]', ['l1']), |
1718 | ('link[id$="1"]', ['l1']), | 1850 | ('link[id$="1"]', ['l1']), |
1719 | ('[id$="1"]', ['l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1']), | 1851 | ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']), |
1720 | ('div[id$="1"]', []), | 1852 | ('div[id$="1"]', ['data1']), |
1721 | ('[id$="noending"]', []), | 1853 | ('[id$="noending"]', []), |
1722 | ) | 1854 | ) |
1723 | 1855 | ||
@@ -1730,7 +1862,6 @@ class TestSoupSelector(TreeTest): | |||
1730 | ('[rel*="notstyle"]', []), | 1862 | ('[rel*="notstyle"]', []), |
1731 | ('link[rel*="notstyle"]', []), | 1863 | ('link[rel*="notstyle"]', []), |
1732 | ('link[href*="bla"]', ['l1']), | 1864 | ('link[href*="bla"]', ['l1']), |
1733 | ('a[href*="http://"]', ['bob', 'me']), | ||
1734 | ('[href*="http://"]', ['bob', 'me']), | 1865 | ('[href*="http://"]', ['bob', 'me']), |
1735 | ('[id*="p"]', ['pmulti', 'p1']), | 1866 | ('[id*="p"]', ['pmulti', 'p1']), |
1736 | ('div[id*="m"]', ['main']), | 1867 | ('div[id*="m"]', ['main']), |
@@ -1739,8 +1870,8 @@ class TestSoupSelector(TreeTest): | |||
1739 | ('[href*=".css"]', ['l1']), | 1870 | ('[href*=".css"]', ['l1']), |
1740 | ('link[href*=".css"]', ['l1']), | 1871 | ('link[href*=".css"]', ['l1']), |
1741 | ('link[id*="1"]', ['l1']), | 1872 | ('link[id*="1"]', ['l1']), |
1742 | ('[id*="1"]', ['l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1']), | 1873 | ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']), |
1743 | ('div[id*="1"]', []), | 1874 | ('div[id*="1"]', ['data1']), |
1744 | ('[id*="noending"]', []), | 1875 | ('[id*="noending"]', []), |
1745 | # New for this test | 1876 | # New for this test |
1746 | ('[href*="."]', ['bob', 'me', 'l1']), | 1877 | ('[href*="."]', ['bob', 'me', 'l1']), |
@@ -1748,6 +1879,7 @@ class TestSoupSelector(TreeTest): | |||
1748 | ('link[href*="."]', ['l1']), | 1879 | ('link[href*="."]', ['l1']), |
1749 | ('div[id*="n"]', ['main', 'inner']), | 1880 | ('div[id*="n"]', ['main', 'inner']), |
1750 | ('div[id*="nn"]', ['inner']), | 1881 | ('div[id*="nn"]', ['inner']), |
1882 | ('div[data-tag*="edval"]', ['data1']) | ||
1751 | ) | 1883 | ) |
1752 | 1884 | ||
1753 | def test_attribute_exact_or_hypen(self): | 1885 | def test_attribute_exact_or_hypen(self): |
@@ -1767,18 +1899,27 @@ class TestSoupSelector(TreeTest): | |||
1767 | ('p[class]', ['p1', 'pmulti']), | 1899 | ('p[class]', ['p1', 'pmulti']), |
1768 | ('[blah]', []), | 1900 | ('[blah]', []), |
1769 | ('p[blah]', []), | 1901 | ('p[blah]', []), |
1902 | ('div[data-tag]', ['data1']) | ||
1770 | ) | 1903 | ) |
1771 | 1904 | ||
1905 | def test_unsupported_pseudoclass(self): | ||
1906 | self.assertRaises( | ||
1907 | NotImplementedError, self.soup.select, "a:no-such-pseudoclass") | ||
1908 | |||
1909 | self.assertRaises( | ||
1910 | NotImplementedError, self.soup.select, "a:nth-of-type(a)") | ||
1911 | |||
1912 | |||
1772 | def test_nth_of_type(self): | 1913 | def test_nth_of_type(self): |
1773 | # Try to select first paragraph | 1914 | # Try to select first paragraph |
1774 | els = self.soup.select('div#inner p:nth-of-type(1)') | 1915 | els = self.soup.select('div#inner p:nth-of-type(1)') |
1775 | self.assertEqual(len(els), 1) | 1916 | self.assertEqual(len(els), 1) |
1776 | self.assertEqual(els[0].string, u'Some text') | 1917 | self.assertEqual(els[0].string, 'Some text') |
1777 | 1918 | ||
1778 | # Try to select third paragraph | 1919 | # Try to select third paragraph |
1779 | els = self.soup.select('div#inner p:nth-of-type(3)') | 1920 | els = self.soup.select('div#inner p:nth-of-type(3)') |
1780 | self.assertEqual(len(els), 1) | 1921 | self.assertEqual(len(els), 1) |
1781 | self.assertEqual(els[0].string, u'Another') | 1922 | self.assertEqual(els[0].string, 'Another') |
1782 | 1923 | ||
1783 | # Try to select (non-existent!) fourth paragraph | 1924 | # Try to select (non-existent!) fourth paragraph |
1784 | els = self.soup.select('div#inner p:nth-of-type(4)') | 1925 | els = self.soup.select('div#inner p:nth-of-type(4)') |
@@ -1791,7 +1932,7 @@ class TestSoupSelector(TreeTest): | |||
1791 | def test_nth_of_type_direct_descendant(self): | 1932 | def test_nth_of_type_direct_descendant(self): |
1792 | els = self.soup.select('div#inner > p:nth-of-type(1)') | 1933 | els = self.soup.select('div#inner > p:nth-of-type(1)') |
1793 | self.assertEqual(len(els), 1) | 1934 | self.assertEqual(len(els), 1) |
1794 | self.assertEqual(els[0].string, u'Some text') | 1935 | self.assertEqual(els[0].string, 'Some text') |
1795 | 1936 | ||
1796 | def test_id_child_selector_nth_of_type(self): | 1937 | def test_id_child_selector_nth_of_type(self): |
1797 | self.assertSelects('#inner > p:nth-of-type(2)', ['p1']) | 1938 | self.assertSelects('#inner > p:nth-of-type(2)', ['p1']) |
@@ -1803,7 +1944,7 @@ class TestSoupSelector(TreeTest): | |||
1803 | selected = inner.select("div") | 1944 | selected = inner.select("div") |
1804 | # The <div id="inner"> tag was selected. The <div id="footer"> | 1945 | # The <div id="inner"> tag was selected. The <div id="footer"> |
1805 | # tag was not. | 1946 | # tag was not. |
1806 | self.assertSelectsIDs(selected, ['inner']) | 1947 | self.assertSelectsIDs(selected, ['inner', 'data1']) |
1807 | 1948 | ||
1808 | def test_overspecified_child_id(self): | 1949 | def test_overspecified_child_id(self): |
1809 | self.assertSelects(".fancy #inner", ['inner']) | 1950 | self.assertSelects(".fancy #inner", ['inner']) |
@@ -1827,3 +1968,44 @@ class TestSoupSelector(TreeTest): | |||
1827 | 1968 | ||
1828 | def test_sibling_combinator_wont_select_same_tag_twice(self): | 1969 | def test_sibling_combinator_wont_select_same_tag_twice(self): |
1829 | self.assertSelects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr']) | 1970 | self.assertSelects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr']) |
1971 | |||
1972 | # Test the selector grouping operator (the comma) | ||
1973 | def test_multiple_select(self): | ||
1974 | self.assertSelects('x, y', ['xid', 'yid']) | ||
1975 | |||
1976 | def test_multiple_select_with_no_space(self): | ||
1977 | self.assertSelects('x,y', ['xid', 'yid']) | ||
1978 | |||
1979 | def test_multiple_select_with_more_space(self): | ||
1980 | self.assertSelects('x, y', ['xid', 'yid']) | ||
1981 | |||
1982 | def test_multiple_select_duplicated(self): | ||
1983 | self.assertSelects('x, x', ['xid']) | ||
1984 | |||
1985 | def test_multiple_select_sibling(self): | ||
1986 | self.assertSelects('x, y ~ p[lang=fr]', ['xid', 'lang-fr']) | ||
1987 | |||
1988 | def test_multiple_select_tag_and_direct_descendant(self): | ||
1989 | self.assertSelects('x, y > z', ['xid', 'zidb']) | ||
1990 | |||
1991 | def test_multiple_select_direct_descendant_and_tags(self): | ||
1992 | self.assertSelects('div > x, y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac']) | ||
1993 | |||
1994 | def test_multiple_select_indirect_descendant(self): | ||
1995 | self.assertSelects('div x,y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac']) | ||
1996 | |||
1997 | def test_invalid_multiple_select(self): | ||
1998 | self.assertRaises(ValueError, self.soup.select, ',x, y') | ||
1999 | self.assertRaises(ValueError, self.soup.select, 'x,,y') | ||
2000 | |||
2001 | def test_multiple_select_attrs(self): | ||
2002 | self.assertSelects('p[lang=en], p[lang=en-gb]', ['lang-en', 'lang-en-gb']) | ||
2003 | |||
2004 | def test_multiple_select_ids(self): | ||
2005 | self.assertSelects('x, y > z[id=zida], z[id=zidab], z[id=zidb]', ['xid', 'zidb', 'zidab']) | ||
2006 | |||
2007 | def test_multiple_select_nested(self): | ||
2008 | self.assertSelects('body > div > x, y > z', ['xid', 'zidb']) | ||
2009 | |||
2010 | |||
2011 | |||