cmos/tests/test_note_formatter.py

"""Tests for src/cmos/note_formatter.py — the v2 inner-loop iterable artifact.

Mirrors tests/test_formatter.py in shape and discipline. The note formatter
is iterated independently of the bibliography formatter (Path B); these
tests pin the stable seams (caller injection, retry loop, prompt smoke
checks) and leave LLM output shape testing to the v2 exemplar canary.

No real API calls in this file. Tests use injected fake callers.
"""

from cmos.note_formatter import (
    MODEL,
    SYSTEM_PROMPT_NOTES,
    build_user_message,
    format_note_entry,
)


def test_default_model_is_gpt5():
    # Same default as v1. Override with OPENAI_MODEL=... in .env.
    assert MODEL == "gpt-5"


def test_system_prompt_mentions_cmos_18_note_form():
    assert "CMOS" in SYSTEM_PROMPT_NOTES
    assert "18" in SYSTEM_PROMPT_NOTES
    assert "NOTE" in SYSTEM_PROMPT_NOTES


def test_system_prompt_mentions_no_inversion():
    # The single most distinctive difference from bibliography form: notes
    # use normal-order author names ("First Last"), not inverted.
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "normal" in lower or "not inverted" in lower


def test_system_prompt_mentions_comma_separation():
    # The second most distinctive difference: commas between elements,
    # not periods.
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "comma" in lower


def test_system_prompt_excludes_leading_number():
    # The note number prefix ("1. ", "2. ") is supplied externally; the
    # formatter must not include it. The prompt must say so.
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "leading number" in lower or "do not include" in lower


def test_system_prompt_mentions_no_ibid():
    # CMOS 18 deprecates Ibid., same as v1.
    assert "Ibid" in SYSTEM_PROMPT_NOTES or "ibid" in SYSTEM_PROMPT_NOTES.lower()


def test_system_prompt_mentions_broader_deprecated_latin_set():
    """Rule 10 was rewritten in fix #2 to cover the broad CMOS 18
    deprecated set, not just Ibid. The prompt must mention idem,
    op. cit., loc. cit. so the LLM knows to refuse all of them
    consistently — the Python guard catches pure-token cases, but
    the prompt must cover the cases where a deprecated abbreviation
    appears alongside other content (e.g. "Ibid., 47")."""
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "idem" in lower
    assert "op. cit" in lower or "op cit" in lower
    assert "loc. cit" in lower or "loc cit" in lower


def test_system_prompt_rule_10_says_return_verbatim_not_empty():
    """Rule 10 must explicitly say "return verbatim" (or equivalent)
    when the input contains a deprecated abbreviation the formatter
    cannot resolve. The previous wording said "return the cleanest
    possible full-form note from whatever information the messy input
    contains" which the model interpreted as "return empty when there's
    no information" — destroying the marker the user needs for review.
    """
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "verbatim" in lower
    # And the prompt should explicitly forbid the empty-string output.
    assert "do not return an empty" in lower or "not return empty" in lower


def test_system_prompt_mentions_publisher_full_form():
    """Surfaced by chunk 1 smoke test on chapter_first_doyle: the formatter
    preserved 'U of Chicago Press' from the messy input instead of
    expanding it to 'University of Chicago Press'. v1 formatter.py rule 14
    covers this; the v2 prompt must too. The rule is "publisher names must
    be in full canonical form, not abbreviated."
    """
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "publisher" in lower
    # Either "do not abbreviate" or "full canonical" or "full form" — any
    # phrasing that conveys the prohibition.
    assert (
        "do not abbreviate" in lower
        or "not abbreviated" in lower
        or "full canonical" in lower
        or "full form" in lower
    )


def test_system_prompt_mentions_month_or_season_preservation():
    """Polish fix #5 (v2 mirror): preserve month/season info in scholarly
    journal dates rather than collapsing to year-only. Mirror of the v1
    test in tests/test_formatter.py — both prompts get the same rule.
    Surfaced by entries [19] and [54] in the Anti-Communist Formations
    of LIS draft where (Spring, 1993) and August 1952 were collapsed.
    """
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "month" in lower
    assert "season" in lower
    assert "preserve" in lower


def test_system_prompt_publisher_exception_list_includes_nyu_press():
    """Chunk 4 task 2: rule 18's publisher exception list should include
    NYU Press alongside the existing MIT Press / ALA Editions / MLA.
    Surfaced by [6] Batterson in the chunk 3 verification re-run, where
    "NYU Press" got expanded to "New York University Press" — losing
    the publisher's canonical brand name. NYU Press's official self-
    presentation IS "NYU Press", not the long form.
    """
    assert "NYU Press" in SYSTEM_PROMPT_NOTES


def test_system_prompt_mentions_government_reports_rule():
    """Polish fix #7: v2 should explicitly handle government documents
    and institutional reports the same way v1's rule 27 does — italicize
    the title like a book, do NOT wrap in quotes. Implicit handling
    worked in the chunk 2c real-draft test (entries [11] and [83]) but
    explicit rule provides regression protection if a future prompt
    iteration loses this implicit knowledge.
    """
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "government" in lower or "institutional" in lower
    assert "report" in lower
    # And the key behavior: treat like a book (italicize title), not a
    # quoted article.
    assert "italic" in lower  # already in the prompt for general italics


def test_system_prompt_mentions_ed_invariant_for_multiple_editors():
    """Surfaced by chunk 1 smoke test on chapter_first_doyle: the formatter
    pluralized 'ed.' to 'eds.' when there were two editors. CMOS NB form
    uses 'ed.' invariantly regardless of editor count. The prompt must
    explicitly state this so the model does not grammatically pluralize
    by default.
    """
    # Search for an explicit mention of "eds." being wrong, OR an explicit
    # statement that "ed." is invariant / not pluralized.
    lower = SYSTEM_PROMPT_NOTES.lower()
    assert "eds." in lower or "invariant" in lower or "do not pluralize" in lower or "not pluralize" in lower


def test_build_user_message_contains_the_messy_input():
    msg = build_user_message("yu, charles. interior chinatown. 2020, 45")
    assert "yu, charles. interior chinatown. 2020, 45" in msg


def test_note_formatter_uses_injected_caller():
    """Same caller-injection seam as v1. Tests can substitute a fake without
    touching the network."""
    recorded: dict = {}

    def fake_caller(system: str, user: str) -> str:
        recorded["system"] = system
        recorded["user"] = user
        return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."

    output = format_note_entry(
        "yu, charles. interior chinatown. New York: Pantheon Books, 2020, p 45.",
        caller=fake_caller,
    )
    assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
    assert recorded["system"] == SYSTEM_PROMPT_NOTES
    assert "yu, charles" in recorded["user"]


def test_note_formatter_strips_whitespace_from_caller_output():
    def fake_caller(system: str, user: str) -> str:
        return "  Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45.  \n"

    output = format_note_entry("anything", caller=fake_caller)
    assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."


def test_note_formatter_retries_on_validator_failure():
    """If the runtime validator rejects the first attempt (e.g., italics
    dropped), the formatter retries. Same retry semantics as v1."""
    attempts = []

    def flaky_caller(system: str, user: str) -> str:
        attempts.append(len(attempts) + 1)
        if len(attempts) == 1:
            # First attempt: italics dropped — validator should fail.
            return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45."
        return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."

    output = format_note_entry("messy yu", caller=flaky_caller)
    assert "*Interior Chinatown*" in output
    assert len(attempts) == 2


def test_note_formatter_returns_last_attempt_after_max_retries():
    """If every attempt fails validation, return the last attempt and don't
    loop forever. Same as v1 — the failure surfaces via scoring or human
    review, not via a runtime exception."""
    attempts = []

    def always_bad(system: str, user: str) -> str:
        attempts.append(1)
        return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45."

    output = format_note_entry("messy", caller=always_bad, max_retries=2)
    assert len(attempts) == 3  # 1 initial + 2 retries
    assert "*" not in output


def test_format_note_entry_short_circuits_deprecated_latin_abbreviations():
    """When the input is ONLY a CMOS-18-deprecated Latin citation
    abbreviation (ibid, idem, id., op. cit., loc. cit.), the formatter
    must short-circuit the API and return the input verbatim. Surfaced
    by note [61] in the Anti-Communist Formations of LIS draft, where
    the formatter was given "Ibid." and (after retrying through the
    runtime validator) returned an empty string — silently destroying
    the marker that the user needs to find later for manual resolution.

    The deprecated set per CMOS 18:
        ibid, ibid., idem, id., op. cit., loc. cit.
    plus common spelling/spacing variants (Op.Cit., loc cit, etc.).

    The guard MUST be deterministic (Python-side, not LLM-side) and
    MUST NOT call the API. Cost guarantee enforced via counting caller.
    The whitespace of the original input is preserved verbatim so
    reassembly in cmos.cli.reformat_notes is byte-exact.
    """
    call_count = 0

    def counting_caller(system: str, user: str) -> str:
        nonlocal call_count
        call_count += 1
        return "should not be called"

    deprecated_inputs = [
        "Ibid.",
        "ibid",
        "IBID.",
        "Idem",
        "idem.",
        "id.",
        "Id.",
        "op. cit.",
        "Op. Cit.",
        "op.cit.",
        "Op Cit",
        "loc. cit.",
        "Loc. Cit.",
        "loc.cit.",
        # With surrounding whitespace preserved verbatim
        "  Ibid.  ",
        "\tIbid.\n",
    ]
    for inp in deprecated_inputs:
        result = format_note_entry(inp, caller=counting_caller)
        assert result == inp, (
            f"Input {inp!r} must round-trip verbatim; got {result!r}"
        )

    assert call_count == 0, (
        f"API caller was invoked {call_count} times for deprecated "
        "Latin abbreviations — the short-circuit guard is missing or "
        "not effective. Deprecated tokens must NEVER reach the API."
    )


def test_format_note_entry_does_not_short_circuit_normal_citations():
    """The deprecated-token guard must NOT match real citations that
    happen to start with one of the tokens. e.g. 'Ibid., 47' is a
    common usage where a page number is appended; for now we let the
    LLM see those (the prompt rule covers them). Pure-token-only
    inputs short-circuit; anything with extra content goes to the API.
    """
    calls: list[str] = []

    def recording_caller(system: str, user: str) -> str:
        calls.append(user)
        return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."

    # Real-looking inputs that contain "ibid" but are not pure tokens
    not_short_circuited = [
        "Ibid., 47.",  # ibid + page — common usage
        "Smith, Ibid., 47.",
        "Title with the word ibid in it.",
        "Charles Yu, Interior Chinatown.",  # totally unrelated
    ]
    for inp in not_short_circuited:
        format_note_entry(inp, caller=recording_caller)

    assert len(calls) == len(not_short_circuited), (
        "Some non-pure-token inputs were short-circuited — guard is "
        "too aggressive."
    )


def test_format_note_entry_passes_through_substantive_prose_verbatim():
    """When the input is substantive prose — long discursive commentary
    with no obvious citation skeleton — the formatter must return it
    VERBATIM without calling the API. CMOS 14.39 explicitly allows
    substantive notes (commentary, qualifications, cross-references).
    The previous behavior extracted whatever citation it could find and
    silently discarded the surrounding prose, which is data loss.

    Surfaced by notes [8], [9], and [75] in the Anti-Communist
    Formations of LIS draft. The user's explicit policy (chunk 3
    fix #4): preserve ALL free text, as long as that does not break
    other formatting.

    Detection rule (conservative): length > 150 chars AND no obvious
    citation skeleton markers (parenthesized year, URL, vol./no./pp.,
    DOI, terminal page reference, terminal year). This catches the
    egregious cases without false-positiving on legitimate citations,
    which all have at least one skeleton marker.
    """
    call_count = 0

    def counting_caller(system: str, user: str) -> str:
        nonlocal call_count
        call_count += 1
        return "should not be called"

    # [8]-style: long discursive note about DuBois/Morehouse, no markers
    note_8_style = (
        "W.E.B. DuBois' popularization of philanthropist Henry Lyman "
        "Morehouse's theory of the talented tenth is celebrated, missing "
        "the note that he himself retracted such a theory later in his "
        "life, as written about by Joy James. The violence of Jim Crow "
        "becomes diminished into many monstrosities, since traditionally "
        "black colleges and universities became monuments to human hope "
        "through philanthropic funding."
    )
    assert format_note_entry(note_8_style, caller=counting_caller) == note_8_style

    # [9]-style: discursive about Wilder/Murch, no markers
    note_9_style = (
        "Craig Steven Wilder's Ebony and Ivy discusses the political "
        "economy of anti-Blackness in higher education, and the influence "
        "of not only capital and capitalists in the creation of "
        "Historically Black Colleges and Universities and white elite "
        "universities, but also enslaved labor to support the latter."
    )
    assert format_note_entry(note_9_style, caller=counting_caller) == note_9_style

    # [75]-style: shorter cross-archive prose, no markers
    note_75_style = (
        "This letter appeared in multiple archives, including California "
        "State Archives, Berkeley and UCLA special collections, and is "
        "uploaded on UC Santa Barbara Servers."
    )
    assert format_note_entry(note_75_style, caller=counting_caller) == note_75_style

    assert call_count == 0, (
        f"API caller was invoked {call_count} times for substantive prose — "
        "the substantive-note pass-through guard is missing or not "
        "effective. Free text must NEVER be sent to the API for "
        "reformatting under this fix."
    )


def test_format_note_entry_does_not_pass_through_real_citations():
    """The substantive-prose guard must NOT match real citations that
    happen to be long. The detection requires absence of citation
    skeleton markers; real citations have at least one (year, URL,
    vol/no/pp, DOI, terminal page). This test pins the negative cases
    so the guard doesn't drift toward over-aggression.
    """
    calls: list[str] = []

    def recording_caller(system: str, user: str) -> str:
        calls.append(user)
        # Return validator-passing output so the retry loop doesn't fire
        # and inflate the call count.
        return "Fake Author, *Fake Title* (Fake Publisher, 2024), 1."

    # Real first-occurrence book note (158 chars, ends with year+period)
    note_6_style = (
        "Steve Batterson, The Prosecution of Professor Chandler Davis: "
        "McCarthyism, Communism, and the Myth of Academic Freedom. "
        "New York: NYU Press, 2023."
    )
    format_note_entry(note_6_style, caller=recording_caller)

    # Real journal article note (URL marker)
    note_44_style = (
        "Lawrence W.S. Auld, \"The King Report: New Directions in Library "
        "and Information Science Education.\" ACRL College & Research "
        "Libraries News 48, no. 4, 1987. https://crln.acrl.org/example."
    )
    format_note_entry(note_44_style, caller=recording_caller)

    # Real govt report (parenthesized year + URL)
    note_54_style = (
        "Robert D. Leigh, The California Librarian Education Survey: a "
        "Report to President Robert G. Sproul of the University of "
        "California. (New York: Columbia University, August 1952). "
        "https://hdl.handle.net/example"
    )
    format_note_entry(note_54_style, caller=recording_caller)

    # Real master's thesis (year at end)
    note_107_style = (
        "Christine Michele Curley, \"The School of the Library,\" "
        "Unpublished Master Thesis, University of California Los "
        "Angeles, 2017."
    )
    format_note_entry(note_107_style, caller=recording_caller)

    assert len(calls) == 4, (
        "Some real first-occurrence notes were short-circuited as "
        "substantive prose — the detection is too aggressive. Each of "
        "these has at least one citation skeleton marker and must be "
        "sent to the API for reformatting."
    )


def test_format_note_entry_returns_empty_input_verbatim_without_calling_api():
    """Empty or whitespace-only input must short-circuit the API call and
    return the input verbatim. Without this guard, GPT-5 occasionally breaks
    character on empty input and returns a conversational meta-reply asking
    for content (e.g., "Please paste the citation entry you'd like
    formatted..."). Surfaced by note [66] in the Anti-Communist Formations
    of LIS draft.

    The whitespace must be preserved verbatim so reassembly in
    cmos.cli.reformat_notes is byte-exact on empty notes — the source's
    spacing is part of the input contract, not noise to be normalized.

    Critically, the API must NOT be called for empty input — the counting
    caller asserts call_count == 0 to lock in the cost guarantee.
    """
    call_count = 0

    def counting_caller(system: str, user: str) -> str:
        nonlocal call_count
        call_count += 1
        return "should not be called"

    # Each whitespace flavor must round-trip verbatim and not call the API.
    assert format_note_entry("", caller=counting_caller) == ""
    assert format_note_entry("   ", caller=counting_caller) == "   "
    assert format_note_entry("\n", caller=counting_caller) == "\n"
    assert format_note_entry("\t \n", caller=counting_caller) == "\t \n"

    assert call_count == 0, (
        "API caller was invoked on empty input — the short-circuit "
        "guard is missing or not effective. Empty input must NEVER "
        "reach the API."
    )