BeautifulSoup 4 + soupsieve CSS selector scraping: working versions (tested 2026-10-16)

beautifulsoup4==4.15.0
soupsieve==3.0.3
pytest==9.1.1

import soupsieve
from bs4 import BeautifulSoup

HTML = """
<!doctype html>
<html><body>
<table id="projects">
  <thead><tr><th>Name</th><th>Owner</th><th>Cost</th><th>Docs</th></tr></thead>
  <tbody>
    <tr data-status="active">
      <td class="name">Apollo</td><td class="owner">Ada</td><td class="cost">10</td>
      <td><a href="/docs/apollo.pdf">spec</a></td>
    </tr>
    <tr data-status="archived">
      <td class="name">Borealis</td><td class="owner">Ada</td><td class="cost">20</td>
      <td><a href="/docs/borealis.html">site</a></td>
    </tr>
    <tr data-status="active">
      <td class="name">Ceres</td><td class="owner">Bo</td><td class="cost">5</td>
      <td><a href="/docs/ceres.pdf">spec</a> <a href="https://ext.example.com/ceres.pdf">mirror</a></td>
    </tr>
  </tbody>
</table>
<a href="/about" data-status="active">About</a>
</body></html>
"""


def parse(html: str = HTML) -> BeautifulSoup:
    return BeautifulSoup(html, "html.parser")


def project_names(soup: BeautifulSoup) -> list[str]:
    return [c.get_text(strip=True) for c in soup.select("table#projects tbody tr td.name")]


def pdf_links(soup: BeautifulSoup) -> list[str]:
    return [a["href"] for a in soup.select('a[href$=".pdf"]')]


def rows_with_pdf(soup: BeautifulSoup) -> list[str]:
    rows = soup.select("tbody tr:has(a[href$='.pdf'])")
    return [r.select_one("td.name").get_text(strip=True) for r in rows]


def active_names(soup: BeautifulSoup) -> list[str]:
    rows = soup.select("tbody tr[data-status=active]")
    return [r.select_one("td.name").get_text(strip=True) for r in rows if soupsieve.match("tr[data-status=active]", r)]

import pytest
import soupsieve
from bs4 import BeautifulSoup

from scrape import active_names, parse, pdf_links, project_names, rows_with_pdf


def test_extracts_table_cells():
    assert project_names(parse()) == ["Apollo", "Borealis", "Ceres"]


def test_pdf_links_use_attribute_suffix():
    assert pdf_links(parse()) == [
        "/docs/apollo.pdf",
        "/docs/ceres.pdf",
        "https://ext.example.com/ceres.pdf",
    ]


def test_comma_selector_keeps_document_order():
    classes = [c["class"][0] for c in parse().select("td.cost, td.name")]
    assert classes == ["name", "cost", "name", "cost", "name", "cost"]


def test_has_finds_rows_with_pdf():
    assert rows_with_pdf(parse()) == ["Apollo", "Ceres"]
    assert active_names(parse()) == ["Apollo", "Ceres"]


def test_data_attr_is_case_sensitive_unless_flag():
    soup = parse()
    assert len(soup.select("tbody tr[data-status=ACTIVE]")) == 0
    assert len(soup.select("tbody tr[data-status=active i]")) == 2
    row = soup.select_one("tbody tr")
    assert soupsieve.match("tr[data-status=active]", row)
    assert not soupsieve.match("tr[data-status=archived]", row)


def test_html_parser_does_not_insert_tbody():
    soup = BeautifulSoup("<table><tr><td>x</td></tr></table>", "html.parser")
    assert len(soup.select("table tr")) == 1
    assert soup.select("table > tbody > tr") == []


def test_missing_attribute_raises_but_get_is_safe():
    cell = parse().select_one("td.name")
    assert cell.get("href") is None
    with pytest.raises(KeyError):
        _ = cell["href"]

[pytest]
pythonpath = .