"""Tests for src/cmos/note_formatter.py — the v2 inner-loop iterable artifact. Mirrors tests/test_formatter.py in shape and discipline. The note formatter is iterated independently of the bibliography formatter (Path B); these tests pin the stable seams (caller injection, retry loop, prompt smoke checks) and leave LLM output shape testing to the v2 exemplar canary. No real API calls in this file. Tests use injected fake callers. """ from cmos.note_formatter import ( MODEL, SYSTEM_PROMPT_NOTES, build_user_message, format_note_entry, ) def test_default_model_is_gpt5(): # Same default as v1. Override with OPENAI_MODEL=... in .env. assert MODEL == "gpt-5" def test_system_prompt_mentions_cmos_18_note_form(): assert "CMOS" in SYSTEM_PROMPT_NOTES assert "18" in SYSTEM_PROMPT_NOTES assert "NOTE" in SYSTEM_PROMPT_NOTES def test_system_prompt_mentions_no_inversion(): # The single most distinctive difference from bibliography form: notes # use normal-order author names ("First Last"), not inverted. lower = SYSTEM_PROMPT_NOTES.lower() assert "normal" in lower or "not inverted" in lower def test_system_prompt_mentions_comma_separation(): # The second most distinctive difference: commas between elements, # not periods. lower = SYSTEM_PROMPT_NOTES.lower() assert "comma" in lower def test_system_prompt_excludes_leading_number(): # The note number prefix ("1. ", "2. ") is supplied externally; the # formatter must not include it. The prompt must say so. lower = SYSTEM_PROMPT_NOTES.lower() assert "leading number" in lower or "do not include" in lower def test_system_prompt_mentions_no_ibid(): # CMOS 18 deprecates Ibid., same as v1. assert "Ibid" in SYSTEM_PROMPT_NOTES or "ibid" in SYSTEM_PROMPT_NOTES.lower() def test_system_prompt_mentions_broader_deprecated_latin_set(): """Rule 10 was rewritten in fix #2 to cover the broad CMOS 18 deprecated set, not just Ibid. The prompt must mention idem, op. cit., loc. cit. so the LLM knows to refuse all of them consistently — the Python guard catches pure-token cases, but the prompt must cover the cases where a deprecated abbreviation appears alongside other content (e.g. "Ibid., 47").""" lower = SYSTEM_PROMPT_NOTES.lower() assert "idem" in lower assert "op. cit" in lower or "op cit" in lower assert "loc. cit" in lower or "loc cit" in lower def test_system_prompt_rule_10_says_return_verbatim_not_empty(): """Rule 10 must explicitly say "return verbatim" (or equivalent) when the input contains a deprecated abbreviation the formatter cannot resolve. The previous wording said "return the cleanest possible full-form note from whatever information the messy input contains" which the model interpreted as "return empty when there's no information" — destroying the marker the user needs for review. """ lower = SYSTEM_PROMPT_NOTES.lower() assert "verbatim" in lower # And the prompt should explicitly forbid the empty-string output. assert "do not return an empty" in lower or "not return empty" in lower def test_system_prompt_mentions_publisher_full_form(): """Surfaced by chunk 1 smoke test on chapter_first_doyle: the formatter preserved 'U of Chicago Press' from the messy input instead of expanding it to 'University of Chicago Press'. v1 formatter.py rule 14 covers this; the v2 prompt must too. The rule is "publisher names must be in full canonical form, not abbreviated." """ lower = SYSTEM_PROMPT_NOTES.lower() assert "publisher" in lower # Either "do not abbreviate" or "full canonical" or "full form" — any # phrasing that conveys the prohibition. assert ( "do not abbreviate" in lower or "not abbreviated" in lower or "full canonical" in lower or "full form" in lower ) def test_system_prompt_mentions_month_or_season_preservation(): """Polish fix #5 (v2 mirror): preserve month/season info in scholarly journal dates rather than collapsing to year-only. Mirror of the v1 test in tests/test_formatter.py — both prompts get the same rule. Surfaced by entries [19] and [54] in the Anti-Communist Formations of LIS draft where (Spring, 1993) and August 1952 were collapsed. """ lower = SYSTEM_PROMPT_NOTES.lower() assert "month" in lower assert "season" in lower assert "preserve" in lower def test_system_prompt_publisher_exception_list_includes_nyu_press(): """Chunk 4 task 2: rule 18's publisher exception list should include NYU Press alongside the existing MIT Press / ALA Editions / MLA. Surfaced by [6] Batterson in the chunk 3 verification re-run, where "NYU Press" got expanded to "New York University Press" — losing the publisher's canonical brand name. NYU Press's official self- presentation IS "NYU Press", not the long form. """ assert "NYU Press" in SYSTEM_PROMPT_NOTES def test_system_prompt_mentions_government_reports_rule(): """Polish fix #7: v2 should explicitly handle government documents and institutional reports the same way v1's rule 27 does — italicize the title like a book, do NOT wrap in quotes. Implicit handling worked in the chunk 2c real-draft test (entries [11] and [83]) but explicit rule provides regression protection if a future prompt iteration loses this implicit knowledge. """ lower = SYSTEM_PROMPT_NOTES.lower() assert "government" in lower or "institutional" in lower assert "report" in lower # And the key behavior: treat like a book (italicize title), not a # quoted article. assert "italic" in lower # already in the prompt for general italics def test_system_prompt_mentions_ed_invariant_for_multiple_editors(): """Surfaced by chunk 1 smoke test on chapter_first_doyle: the formatter pluralized 'ed.' to 'eds.' when there were two editors. CMOS NB form uses 'ed.' invariantly regardless of editor count. The prompt must explicitly state this so the model does not grammatically pluralize by default. """ # Search for an explicit mention of "eds." being wrong, OR an explicit # statement that "ed." is invariant / not pluralized. lower = SYSTEM_PROMPT_NOTES.lower() assert "eds." in lower or "invariant" in lower or "do not pluralize" in lower or "not pluralize" in lower def test_build_user_message_contains_the_messy_input(): msg = build_user_message("yu, charles. interior chinatown. 2020, 45") assert "yu, charles. interior chinatown. 2020, 45" in msg def test_note_formatter_uses_injected_caller(): """Same caller-injection seam as v1. Tests can substitute a fake without touching the network.""" recorded: dict = {} def fake_caller(system: str, user: str) -> str: recorded["system"] = system recorded["user"] = user return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45." output = format_note_entry( "yu, charles. interior chinatown. New York: Pantheon Books, 2020, p 45.", caller=fake_caller, ) assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45." assert recorded["system"] == SYSTEM_PROMPT_NOTES assert "yu, charles" in recorded["user"] def test_note_formatter_strips_whitespace_from_caller_output(): def fake_caller(system: str, user: str) -> str: return " Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45. \n" output = format_note_entry("anything", caller=fake_caller) assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45." def test_note_formatter_retries_on_validator_failure(): """If the runtime validator rejects the first attempt (e.g., italics dropped), the formatter retries. Same retry semantics as v1.""" attempts = [] def flaky_caller(system: str, user: str) -> str: attempts.append(len(attempts) + 1) if len(attempts) == 1: # First attempt: italics dropped — validator should fail. return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45." return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45." output = format_note_entry("messy yu", caller=flaky_caller) assert "*Interior Chinatown*" in output assert len(attempts) == 2 def test_note_formatter_returns_last_attempt_after_max_retries(): """If every attempt fails validation, return the last attempt and don't loop forever. Same as v1 — the failure surfaces via scoring or human review, not via a runtime exception.""" attempts = [] def always_bad(system: str, user: str) -> str: attempts.append(1) return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45." output = format_note_entry("messy", caller=always_bad, max_retries=2) assert len(attempts) == 3 # 1 initial + 2 retries assert "*" not in output def test_format_note_entry_short_circuits_deprecated_latin_abbreviations(): """When the input is ONLY a CMOS-18-deprecated Latin citation abbreviation (ibid, idem, id., op. cit., loc. cit.), the formatter must short-circuit the API and return the input verbatim. Surfaced by note [61] in the Anti-Communist Formations of LIS draft, where the formatter was given "Ibid." and (after retrying through the runtime validator) returned an empty string — silently destroying the marker that the user needs to find later for manual resolution. The deprecated set per CMOS 18: ibid, ibid., idem, id., op. cit., loc. cit. plus common spelling/spacing variants (Op.Cit., loc cit, etc.). The guard MUST be deterministic (Python-side, not LLM-side) and MUST NOT call the API. Cost guarantee enforced via counting caller. The whitespace of the original input is preserved verbatim so reassembly in cmos.cli.reformat_notes is byte-exact. """ call_count = 0 def counting_caller(system: str, user: str) -> str: nonlocal call_count call_count += 1 return "should not be called" deprecated_inputs = [ "Ibid.", "ibid", "IBID.", "Idem", "idem.", "id.", "Id.", "op. cit.", "Op. Cit.", "op.cit.", "Op Cit", "loc. cit.", "Loc. Cit.", "loc.cit.", # With surrounding whitespace preserved verbatim " Ibid. ", "\tIbid.\n", ] for inp in deprecated_inputs: result = format_note_entry(inp, caller=counting_caller) assert result == inp, ( f"Input {inp!r} must round-trip verbatim; got {result!r}" ) assert call_count == 0, ( f"API caller was invoked {call_count} times for deprecated " "Latin abbreviations — the short-circuit guard is missing or " "not effective. Deprecated tokens must NEVER reach the API." ) def test_format_note_entry_does_not_short_circuit_normal_citations(): """The deprecated-token guard must NOT match real citations that happen to start with one of the tokens. e.g. 'Ibid., 47' is a common usage where a page number is appended; for now we let the LLM see those (the prompt rule covers them). Pure-token-only inputs short-circuit; anything with extra content goes to the API. """ calls: list[str] = [] def recording_caller(system: str, user: str) -> str: calls.append(user) return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45." # Real-looking inputs that contain "ibid" but are not pure tokens not_short_circuited = [ "Ibid., 47.", # ibid + page — common usage "Smith, Ibid., 47.", "Title with the word ibid in it.", "Charles Yu, Interior Chinatown.", # totally unrelated ] for inp in not_short_circuited: format_note_entry(inp, caller=recording_caller) assert len(calls) == len(not_short_circuited), ( "Some non-pure-token inputs were short-circuited — guard is " "too aggressive." ) def test_format_note_entry_passes_through_substantive_prose_verbatim(): """When the input is substantive prose — long discursive commentary with no obvious citation skeleton — the formatter must return it VERBATIM without calling the API. CMOS 14.39 explicitly allows substantive notes (commentary, qualifications, cross-references). The previous behavior extracted whatever citation it could find and silently discarded the surrounding prose, which is data loss. Surfaced by notes [8], [9], and [75] in the Anti-Communist Formations of LIS draft. The user's explicit policy (chunk 3 fix #4): preserve ALL free text, as long as that does not break other formatting. Detection rule (conservative): length > 150 chars AND no obvious citation skeleton markers (parenthesized year, URL, vol./no./pp., DOI, terminal page reference, terminal year). This catches the egregious cases without false-positiving on legitimate citations, which all have at least one skeleton marker. """ call_count = 0 def counting_caller(system: str, user: str) -> str: nonlocal call_count call_count += 1 return "should not be called" # [8]-style: long discursive note about DuBois/Morehouse, no markers note_8_style = ( "W.E.B. DuBois' popularization of philanthropist Henry Lyman " "Morehouse's theory of the talented tenth is celebrated, missing " "the note that he himself retracted such a theory later in his " "life, as written about by Joy James. The violence of Jim Crow " "becomes diminished into many monstrosities, since traditionally " "black colleges and universities became monuments to human hope " "through philanthropic funding." ) assert format_note_entry(note_8_style, caller=counting_caller) == note_8_style # [9]-style: discursive about Wilder/Murch, no markers note_9_style = ( "Craig Steven Wilder's Ebony and Ivy discusses the political " "economy of anti-Blackness in higher education, and the influence " "of not only capital and capitalists in the creation of " "Historically Black Colleges and Universities and white elite " "universities, but also enslaved labor to support the latter." ) assert format_note_entry(note_9_style, caller=counting_caller) == note_9_style # [75]-style: shorter cross-archive prose, no markers note_75_style = ( "This letter appeared in multiple archives, including California " "State Archives, Berkeley and UCLA special collections, and is " "uploaded on UC Santa Barbara Servers." ) assert format_note_entry(note_75_style, caller=counting_caller) == note_75_style assert call_count == 0, ( f"API caller was invoked {call_count} times for substantive prose — " "the substantive-note pass-through guard is missing or not " "effective. Free text must NEVER be sent to the API for " "reformatting under this fix." ) def test_format_note_entry_does_not_pass_through_real_citations(): """The substantive-prose guard must NOT match real citations that happen to be long. The detection requires absence of citation skeleton markers; real citations have at least one (year, URL, vol/no/pp, DOI, terminal page). This test pins the negative cases so the guard doesn't drift toward over-aggression. """ calls: list[str] = [] def recording_caller(system: str, user: str) -> str: calls.append(user) # Return validator-passing output so the retry loop doesn't fire # and inflate the call count. return "Fake Author, *Fake Title* (Fake Publisher, 2024), 1." # Real first-occurrence book note (158 chars, ends with year+period) note_6_style = ( "Steve Batterson, The Prosecution of Professor Chandler Davis: " "McCarthyism, Communism, and the Myth of Academic Freedom. " "New York: NYU Press, 2023." ) format_note_entry(note_6_style, caller=recording_caller) # Real journal article note (URL marker) note_44_style = ( "Lawrence W.S. Auld, \"The King Report: New Directions in Library " "and Information Science Education.\" ACRL College & Research " "Libraries News 48, no. 4, 1987. https://crln.acrl.org/example." ) format_note_entry(note_44_style, caller=recording_caller) # Real govt report (parenthesized year + URL) note_54_style = ( "Robert D. Leigh, The California Librarian Education Survey: a " "Report to President Robert G. Sproul of the University of " "California. (New York: Columbia University, August 1952). " "https://hdl.handle.net/example" ) format_note_entry(note_54_style, caller=recording_caller) # Real master's thesis (year at end) note_107_style = ( "Christine Michele Curley, \"The School of the Library,\" " "Unpublished Master Thesis, University of California Los " "Angeles, 2017." ) format_note_entry(note_107_style, caller=recording_caller) assert len(calls) == 4, ( "Some real first-occurrence notes were short-circuited as " "substantive prose — the detection is too aggressive. Each of " "these has at least one citation skeleton marker and must be " "sent to the API for reformatting." ) def test_format_note_entry_returns_empty_input_verbatim_without_calling_api(): """Empty or whitespace-only input must short-circuit the API call and return the input verbatim. Without this guard, GPT-5 occasionally breaks character on empty input and returns a conversational meta-reply asking for content (e.g., "Please paste the citation entry you'd like formatted..."). Surfaced by note [66] in the Anti-Communist Formations of LIS draft. The whitespace must be preserved verbatim so reassembly in cmos.cli.reformat_notes is byte-exact on empty notes — the source's spacing is part of the input contract, not noise to be normalized. Critically, the API must NOT be called for empty input — the counting caller asserts call_count == 0 to lock in the cost guarantee. """ call_count = 0 def counting_caller(system: str, user: str) -> str: nonlocal call_count call_count += 1 return "should not be called" # Each whitespace flavor must round-trip verbatim and not call the API. assert format_note_entry("", caller=counting_caller) == "" assert format_note_entry(" ", caller=counting_caller) == " " assert format_note_entry("\n", caller=counting_caller) == "\n" assert format_note_entry("\t \n", caller=counting_caller) == "\t \n" assert call_count == 0, ( "API caller was invoked on empty input — the short-circuit " "guard is missing or not effective. Empty input must NEVER " "reach the API." )