cmos/tests/test_formatter.py

"""Tests for src/cmos/formatter.py — the inner-loop iterable artifact.

Because `formatter.py` is edited every iteration of the dev-time autoresearch
loop, these tests intentionally test STABLE pieces: the prompt skeleton, the
caller-injection seam, and a smoke check that the system prompt mentions key
CMOS 18 rules. The actual LLM output shape is tested via the harness (running
all exemplars through `score`) rather than pinned here — exact string
matches on LLM output belong in the canary exact-match rate, not in unit
tests.

No real API calls in this file. Tests that hit the OpenAI API live in
`tests/test_formatter_integration.py` (not yet created) and are gated on an
OPENAI_API_KEY being set.
"""

from cmos.formatter import (
    MODEL,
    SYSTEM_PROMPT,
    build_user_message,
    format_bibliography_entry,
)


def test_default_model_is_gpt5():
    # The user specified "GPT-5 / frontier reasoning" as the default. Override
    # via the OPENAI_MODEL env var if gpt-5 is unavailable in your account.
    assert MODEL == "gpt-5"


def test_system_prompt_mentions_cmos_18():
    assert "CMOS" in SYSTEM_PROMPT or "Chicago Manual of Style" in SYSTEM_PROMPT
    assert "18" in SYSTEM_PROMPT


def test_system_prompt_mentions_no_place_of_publication():
    # CMOS 14.30 / 18th ed. change. The formatter MUST know this.
    lower = SYSTEM_PROMPT.lower()
    assert "place of publication" in lower


def test_system_prompt_mentions_doi_preference():
    assert "doi" in SYSTEM_PROMPT.lower()


def test_system_prompt_mentions_italic_markers():
    # Plan v1 uses Markdown `*Title*` for italics.
    assert "*" in SYSTEM_PROMPT and "italic" in SYSTEM_PROMPT.lower()


def test_system_prompt_mentions_month_or_season_preservation():
    """Polish fix #5: when the source provides month or season information
    in a scholarly journal date, the formatter should preserve it in CMOS
    form `(Month YEAR)` or `(Season YEAR)` rather than collapsing to
    year-only. Surfaced by Reading_Disrepair (v1) and Anti-Communist (v2)
    real-draft runs where dates like `(Spring, 1993)` and `August 1952`
    were silently collapsed to `(1993)` and `1952`. Both forms are valid
    CMOS but month/season is more informative when the source has it.
    """
    lower = SYSTEM_PROMPT.lower()
    assert "month" in lower
    assert "season" in lower
    # And some indication of preservation, not collapse.
    assert "preserve" in lower


def test_build_user_message_contains_the_messy_input():
    msg = build_user_message("yu, charles. interior chinatown. 2020")
    assert "yu, charles. interior chinatown. 2020" in msg


def test_formatter_uses_injected_caller():
    """The formatter accepts a caller shim so tests (and the harness) can
    substitute a fake OpenAI call. This is how the whole pipeline can run
    without an API key during tests."""
    recorded: dict = {}

    def fake_caller(system: str, user: str) -> str:
        recorded["system"] = system
        recorded["user"] = user
        return "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."

    output = format_bibliography_entry(
        "yu, charles. interior chinatown. New York: Pantheon Books, 2020.",
        caller=fake_caller,
    )
    assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
    assert recorded["system"] == SYSTEM_PROMPT
    assert "yu, charles" in recorded["user"]


def test_formatter_strips_whitespace_from_caller_output():
    # Language-model output often has leading/trailing whitespace or
    # surrounding code fences. v0 only strips whitespace; code-fence
    # stripping can be added when an exemplar forces it.
    def fake_caller(system: str, user: str) -> str:
        return "  Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.  \n"

    output = format_bibliography_entry("anything", caller=fake_caller)
    assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."


def test_formatter_retries_on_validator_failure():
    """If the runtime validator rejects the first attempt (e.g., italics
    missing), the formatter should retry and return a passing later attempt.
    Mirrors the iter 7 magazine_mead variance: GPT-5 sometimes drops italics
    around a magazine name, and a re-call typically succeeds."""
    attempts = []

    def flaky_caller(system: str, user: str) -> str:
        attempts.append(len(attempts) + 1)
        if len(attempts) == 1:
            # First attempt: italics dropped (validator should fail).
            return 'Mead, Rebecca. "Terms of Aggrievement." New Yorker, December 18, 2023.'
        # Retry: clean.
        return 'Mead, Rebecca. "Terms of Aggrievement." *New Yorker*, December 18, 2023.'

    output = format_bibliography_entry("messy mead", caller=flaky_caller)
    assert "*New Yorker*" in output
    assert len(attempts) == 2


def test_formatter_returns_last_attempt_after_max_retries():
    """If every attempt fails validation, return the last attempt and don't
    loop forever. We surface the failure later via scoring/canary, not by
    raising at runtime — the user should still get *something* back."""
    attempts = []

    def always_bad(system: str, user: str) -> str:
        attempts.append(1)
        return 'Mead, Rebecca. "Terms of Aggrievement." New Yorker, December 18, 2023.'

    output = format_bibliography_entry("messy", caller=always_bad, max_retries=2)
    # 1 initial + 2 retries = 3 calls total.
    assert len(attempts) == 3
    assert "*" not in output  # the bad output is what we got