ftfy mojibake repair and curly-quote normalization: working versions (tested 2026-10-16)

ftfy==6.3.1
pytest==9.1.1

import ftfy
from ftfy import TextFixerConfig, fix_and_explain, fix_text

KEEP_TYPOGRAPHY = TextFixerConfig(uncurl_quotes=False, unescape_html=False)


def clean(text: str) -> str:
    """Fix mojibake and reduce typographic characters to ASCII."""
    return fix_text(text)


def clean_keep_typography(text: str) -> str:
    """Fix mojibake but keep curly quotes and HTML entities as written."""
    return fix_text(text, config=KEEP_TYPOGRAPHY)


def explain(text: str) -> tuple[str, list[str]]:
    """Return the fixed text plus the labels of the fixers that fired."""
    fixed = fix_and_explain(text)
    return fixed.text, [f"{kind}:{name}" for kind, name in fixed.explanation]

from fixer import clean, clean_keep_typography, explain


def mojibake(text: str) -> str:
    """Reproduce the classic bug: UTF-8 bytes decoded as Latin-1."""
    return text.encode("utf-8").decode("latin-1")


def test_fixes_single_encoded_mojibake():
    assert mojibake("café") == "café"
    assert clean(mojibake("café")) == "café"
    assert clean("plain ASCII") == "plain ASCII"


def test_fixes_double_encoded_mojibake():
    assert clean(mojibake(mojibake("café"))) == "café"


def test_uncurls_quotes_by_default():
    # fix_text does more than repair encoding: it also straightens quotes.
    assert clean(mojibake("The “smart” quote")) == 'The "smart" quote'


def test_keep_typography_config_preserves_quotes():
    assert clean_keep_typography("“smart”") == "“smart”"


def test_html_entities_unescaped_by_default():
    assert clean("Tom & Jerry") == "Tom & Jerry"
    assert clean_keep_typography("Tom & Jerry") == "Tom & Jerry"


def test_explain_reports_encoding_steps():
    text, steps = explain(mojibake("café"))
    assert text == "café"
    assert steps == ["encode:latin-1", "decode:utf-8"]

[pytest]
pythonpath = .