Validating polars DataFrame rows with pydantic v2 TypeAdapter and collecting errors: working versions (tested 2026-10-15)

polars==2.0.0
pydantic==2.14.1
pytest==9.1.1

from typing import Literal

import polars as pl
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError

class Order(BaseModel):
    # strict: no "5" -> 5 coercion; extra="forbid": unexpected columns are errors
    model_config = ConfigDict(strict=True, extra="forbid")

    order_id: int
    customer: str = Field(min_length=1)
    amount: float = Field(ge=0, allow_inf_nan=False)
    status: Literal["paid", "refunded"]
    note: str | None = None

ORDER = TypeAdapter(Order)

def validate_frame(df: pl.DataFrame) -> tuple[list[Order], list[dict]]:
    # Validate row by row so one bad row does not hide the rest.
    # iter_rows(named=True) gives dicts; plain tuples would not map to field names.
    valid, errors = [], []
    for i, row in enumerate(df.iter_rows(named=True)):
        try:
            valid.append(ORDER.validate_python(row))
        except ValidationError as e:
            errors.append({"row": i, "errors": e.errors(include_url=False)})
    return valid, errors

import polars as pl

from rows import validate_frame

GOOD = {
    "order_id": [1, 2],
    "customer": ["ann", "bo"],
    "amount": [9.5, 0.0],
    "status": ["paid", "refunded"],
    "note": [None, "gift"],
}

def test_all_valid():
    valid, errors = validate_frame(pl.DataFrame(GOOD))
    assert [o.order_id for o in valid] == [1, 2]
    assert valid[1].note == "gift"
    assert errors == []

def test_bad_rows_collected_not_raised():
    df = pl.DataFrame({
        "order_id": [1, None, 3],        # null -> int error
        "customer": ["ann", "bo", ""],   # empty -> min_length error
        "amount": [1.0, 2.0, 3.0],
        "status": ["paid", "paid", "void"],  # "void" -> literal error
        "note": [None, None, None],
    })
    valid, errors = validate_frame(df)
    assert [o.order_id for o in valid] == [1]
    assert [e["row"] for e in errors] == [1, 2]
    assert errors[0]["errors"][0]["loc"] == ("order_id",)
    assert {e["loc"] for e in errors[1]["errors"]} == {("customer",), ("status",)}

def test_strict_mode_rejects_numeric_strings():
    # Polars keeps "1" as a string column; lax pydantic would silently coerce it.
    df = pl.DataFrame({**GOOD, "order_id": ["1", "2"]})
    valid, errors = validate_frame(df)
    assert valid == []
    assert len(errors) == 2
    assert errors[0]["errors"][0]["type"] == "int_type"

def test_nan_rejected_by_allow_inf_nan_false():
    # NaN is not null in polars, so a null check alone would let it through.
    df = pl.DataFrame({**GOOD, "amount": [float("nan"), 1.0]})
    valid, errors = validate_frame(df)
    assert [o.order_id for o in valid] == [2]
    assert errors[0]["errors"][0]["type"] == "finite_number"

def test_extra_column_is_an_error():
    df = pl.DataFrame({**GOOD, "debug": ["x", "y"]})
    valid, errors = validate_frame(df)
    assert valid == []
    assert errors[0]["errors"][0]["type"] == "extra_forbidden"
    assert errors[0]["errors"][0]["loc"] == ("debug",)

[pytest]
pythonpath = .