Cheerio HTML scraping with Zod-validated records and saved fixtures: working versions (tested 2026-10-15)

{
  "name": "cheerio-zod-scrape-fixtures",
  "private": true,
  "type": "module",
  "scripts": {
    "test": "node --import tsx --test test/*.test.ts"
  },
  "dependencies": {
    "cheerio": "1.2.0",
    "zod": "4.6.7"
  },
  "devDependencies": {
    "tsx": "4.23.15",
    "@types/node": "26.6.5"
  }
}

import * as cheerio from "cheerio";
import { z } from "zod";

export const Product = z.object({
  sku: z.string().regex(/^[A-Z]{3}-\d{3}$/),
  name: z.string().min(1),
  priceCents: z.number().int().nonnegative(),
  inStock: z.boolean(),
  url: z.url(),
  rating: z.number().min(0).max(5).nullable(),
});
export type Product = z.infer<typeof Product>;

/** "$1,299.50" -> 129950 */
export function parsePrice(text: string): number {
  const m = text.replace(/[,\s]/g, "").match(/(\d+)(?:\.(\d{1,2}))?/);
  if (!m) return NaN;
  return Number(m[1]) * 100 + Number((m[2] ?? "0").padEnd(2, "0"));
}

/**
 * Scrapes a product listing page. Each card is validated with Zod; bad cards are reported
 * instead of silently producing half-filled objects.
 */
export function scrapeListing(html: string, pageUrl: string) {
  const $ = cheerio.load(html);
  const products: Product[] = [];
  const errors: { index: number; issues: string[] }[] = [];

  $("ul.products > li.product").each((index, el) => {
    const card = $(el);
    const href = card.find("a.title").attr("href");
    const ratingAttr = card.find("[data-rating]").attr("data-rating");
    const candidate = {
      sku: card.attr("data-sku"),
      name: card.find("a.title").text().trim(),
      priceCents: parsePrice(card.find(".price").first().text()),
      inStock: !card.hasClass("sold-out"),
      url: href ? new URL(href, pageUrl).toString() : undefined,
      rating: ratingAttr === undefined ? null : Number(ratingAttr),
    };
    const r = Product.safeParse(candidate);
    if (r.success) products.push(r.data);
    else errors.push({ index, issues: r.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`) });
  });

  const next = $("nav.pagination a[rel=next]").attr("href");
  return { title: $("title").text().trim(), products, errors, nextPage: next ? new URL(next, pageUrl).toString() : null };
}

/** Reads a definition table (<dl>) into a record, e.g. a product spec sheet. */
export function scrapeSpecs(html: string): Record<string, string> {
  const $ = cheerio.load(html);
  const out: Record<string, string> = {};
  $("dl.specs dt").each((_, dt) => {
    out[$(dt).text().trim().toLowerCase()] = $(dt).next("dd").text().replace(/\s+/g, " ").trim();
  });
  return out;
}

<!doctype html>
<html>
<head><title> Hand Tools | Example Store </title></head>
<body>
  <ul class="products">
    <li class="product" data-sku="HMR-001">
      <a class="title" href="/p/claw-hammer">Claw Hammer</a>
      <span class="price">$24.99</span>
      <span data-rating="4.5"></span>
    </li>
    <li class="product sold-out" data-sku="WRN-210">
      <a class="title" href="https://shop.example.test/p/torque-wrench">Torque Wrench</a>
      <span class="price">$1,299.50</span>
      <span class="price was">$1,499.00</span>
    </li>
    <li class="product" data-sku="bad sku">
      <a class="title" href="/p/mystery"></a>
      <span class="price">call us</span>
    </li>
  </ul>
  <nav class="pagination"><a rel="next" href="?page=2">Next</a></nav>
</body>
</html>

<dl class="specs">
  <dt>Weight</dt><dd>1.2 kg</dd>
  <dt>Material</dt><dd>Forged
     steel</dd>
  <dt>Warranty</dt><dd>Lifetime</dd>
</dl>

import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { scrapeListing, scrapeSpecs, parsePrice } from "../src/scrape.ts";

const fixture = (name: string) => readFileSync(new URL(`./fixtures/${name}`, import.meta.url), "utf8");
const PAGE = "https://shop.example.test/c/hand-tools";

test("parsePrice handles currency symbols, thousands separators and cents", () => {
  assert.equal(parsePrice("$24.99"), 2499);
  assert.equal(parsePrice("$1,299.50"), 129950);
  assert.equal(parsePrice("USD 7.5"), 750);
  assert.ok(Number.isNaN(parsePrice("call us")));
});

test("scrapes valid cards with absolute URLs, stock and optional rating", () => {
  const r = scrapeListing(fixture("listing.html"), PAGE);
  assert.equal(r.title, "Hand Tools | Example Store");
  assert.deepEqual(r.products, [
    { sku: "HMR-001", name: "Claw Hammer", priceCents: 2499, inStock: true, url: "https://shop.example.test/p/claw-hammer", rating: 4.5 },
    { sku: "WRN-210", name: "Torque Wrench", priceCents: 129950, inStock: false, url: "https://shop.example.test/p/torque-wrench", rating: null },
  ]);
});

test("invalid cards are reported with zod issue paths", () => {
  const r = scrapeListing(fixture("listing.html"), PAGE);
  assert.equal(r.errors.length, 1);
  assert.equal(r.errors[0].index, 2);
  const fields = r.errors[0].issues.map((i) => i.split(":")[0]).sort();
  assert.deepEqual(fields, ["name", "priceCents", "sku"]);
});

test("resolves the next page link relative to the page URL", () => {
  assert.equal(scrapeListing(fixture("listing.html"), PAGE).nextPage, "https://shop.example.test/c/hand-tools?page=2");
  assert.equal(scrapeListing("<ul class=products></ul>", PAGE).nextPage, null);
});

test("reads a dl spec table with whitespace normalised", () => {
  assert.deepEqual(scrapeSpecs(fixture("specs.html")), { weight: "1.2 kg", material: "Forged steel", warranty: "Lifetime" });
});