lxml + cssselect HTML fixture scraping into typed records: working versions (tested 2026-10-15)

lxml==6.1.3
cssselect==1.6.0
pytest==9.1.1

from dataclasses import dataclass
from decimal import Decimal

from lxml import html
from lxml.cssselect import CSSSelector

PRODUCTS = CSSSelector("ul.products > li.product")


@dataclass(frozen=True)
class Product:
    sku: str | None
    name: str
    price: Decimal
    in_stock: bool
    tags: tuple[str, ...]


def _one(node, selector: str):
    matches = node.cssselect(selector)
    return matches[0] if matches else None


def _parse_price(text: str) -> Decimal:
    return Decimal(text.strip().replace("$", "").replace(",", ""))


def parse_products(markup: bytes) -> list[Product]:
    doc = html.document_fromstring(markup)
    products: list[Product] = []
    for li in PRODUCTS(doc):
        price_el = _one(li, ".price")
        if price_el is None:
            continue
        name_el = _one(li, ".name")
        stock_el = _one(li, ".stock")
        products.append(
            Product(
                sku=li.get("data-sku"),
                name=(name_el.text or "").strip() if name_el is not None else "",
                price=_parse_price(price_el.text_content()),
                in_stock=stock_el is not None and stock_el.text_content().strip() == "in stock",
                tags=tuple(tag.text_content().strip() for tag in li.cssselect(".tag")),
            )
        )
    return products

from decimal import Decimal
from pathlib import Path

import pytest
from cssselect import ExpressionError
from lxml import etree, html

from scrape import parse_products

FIXTURE = Path(__file__).parent / "fixtures" / "products.html"


def load() -> bytes:
    return FIXTURE.read_bytes()


def test_parses_typed_fields():
    products = parse_products(load())
    assert [p.sku for p in products] == ["A1", "B2"]
    assert products[0].price == Decimal("1299.50")
    assert products[1].price == Decimal("10.00")
    assert products[1].tags == ()
    assert products[1].in_stock is False


def test_direct_text_skips_child_badge():
    first = parse_products(load())[0]
    assert first.name == "Widget"
    assert first.tags == ("tools", "sale")


def test_missing_attribute_is_none():
    markup = b'<ul class="products"><li class="product"><span class="price">$1</span></li></ul>'
    (product,) = parse_products(markup)
    assert product.sku is None
    assert product.in_stock is False


def test_pseudo_elements_are_not_supported():
    doc = html.document_fromstring(load())
    with pytest.raises(ExpressionError):
        doc.cssselect("li.product::before")


def test_encoding_declaration_needs_bytes():
    unicode_markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
    with pytest.raises(ValueError):
        etree.fromstring(unicode_markup)
    assert etree.fromstring(unicode_markup.encode()).tag == "html"

<!DOCTYPE html>
<html lang="en">
  <body>
    <ul class="products">
      <li class="product" data-sku="A1">
        <h2 class="name"> Widget <span class="badge">new</span></h2>
        <span class="price">$1,299.50</span>
        <span class="stock">in stock</span>
        <span class="tag">tools</span>
        <span class="tag">sale</span>
      </li>
      <li class="product" data-sku="B2">
        <h2 class="name">Gadget</h2>
        <span class="price">$10.00</span>
        <span class="stock">out of stock</span>
      </li>
    </ul>
  </body>
</html>

[pytest]
pythonpath = .