diff --git a/src/cmos/note_formatter.py b/src/cmos/note_formatter.py index 3a8068b..5835831 100644 --- a/src/cmos/note_formatter.py +++ b/src/cmos/note_formatter.py @@ -59,33 +59,48 @@ _DEPRECATED_LATIN_RE = re.compile( re.IGNORECASE, ) -# Substantive note pass-through (chunk 3 fix #4, Option A). When the -# input is long discursive prose with no obvious citation skeleton, the -# formatter returns it VERBATIM without calling the API. CMOS 14.39 -# explicitly allows substantive notes (commentary, qualifications, -# cross-references), and the previous behavior extracted whatever -# citation it could find and silently discarded the surrounding prose. -# Surfaced by notes [8], [9], and [75] in the Anti-Communist Formations -# of LIS draft. +# Substantive note pass-through (chunk 3 fix #4, Option A; tightened in +# chunk 5 task 1). When the input is long discursive prose with no +# obvious citation skeleton, the formatter returns it VERBATIM without +# calling the API. CMOS 14.39 explicitly allows substantive notes +# (commentary, qualifications, cross-references), and the previous +# behavior extracted whatever citation it could find and silently +# discarded the surrounding prose. Surfaced by notes [8], [9], and [75] +# in the Anti-Communist Formations of LIS draft. # # Detection is conservative: input must be longer than the threshold -# AND must NOT contain ANY of the citation skeleton markers below. +# AND must NOT contain ANY of the citation skeleton markers below +# AND must NOT contain a semicolon (the chunk 5 tightening). # Real first-occurrence citations virtually always have at least one # skeleton marker (terminal year, terminal page reference, URL, vol., # no., pp., DOI, parenthesized year). Substantive prose typically has -# none of these. The combination of length-floor + marker-absence -# catches the egregious cases without false-positiving on legitimate -# citations that happen to be long. +# none of these. # -# The user's explicit policy (chunk 3 fix #4): preserve ALL free text -# as long as that does not break other formatting. Erring on the side -# of preservation is intentional. +# The semicolon check (chunk 5 task 1) was added after the HML draft +# verification revealed ~17 of 72 notes were COMPOUND SHORTENED +# REFERENCES — multiple shortened citations separated by semicolons — +# that had no skeleton markers and were >150 chars, so they matched the +# original (chunk 3) detector and got passed through verbatim. Compound +# references look like: +# +# Author1, "Short Title 1;" Author2, "Short Title 2;" Author3, "..." +# +# Substantive prose almost never uses semicolons as item separators +# (they use commas inside English sentences). Adding "no semicolons" as +# a third condition cleanly separates the two classes without affecting +# the genuine substantive notes (which contain none). +# +# The user's explicit policy: preserve ALL free text as long as that +# does not break other formatting. The chunk 5 tightening reflects the +# observation that the original detector WAS breaking other formatting +# (preventing legit compound references from being processed). _SUBSTANTIVE_PROSE_LENGTH_THRESHOLD = 150 _CITATION_SKELETON_MARKERS_RE = re.compile( r"(?:" r"\([^)]*\d{4}[^)]*\)" # parenthesized year — (2024), (NYU Press, 2023), (March 2024) - r"|https?://" # URL + r"|https?://" # URL with scheme + r"|\b\w{2,}(?:\.\w{2,})+/\S*" # bare-domain URL — files.gao.gov/path, example.com/abc r"|\bvol\.\s*\d+" # volume number — vol. 7 r"|\bno\.\s*\d+" # issue number — no. 4 r"|\bpp?\.\s*\d+" # page reference — p. 45, pp. 45-67 @@ -100,23 +115,30 @@ _CITATION_SKELETON_MARKERS_RE = re.compile( def _looks_like_substantive_prose(text: str) -> bool: """True if ``text`` looks like substantive note prose, not a citation. - Conservative detection: only matches input that is BOTH long AND - lacks any obvious citation skeleton markers. Designed to catch - long discursive notes (689+ chars in the Anti-Communist draft) - while preserving legitimate first-occurrence notes — which all - have at least one skeleton marker even when long. The shorter - [75]-style cases (~163 chars, no markers) are also caught because - the threshold is set well below the typical citation length floor - when markers are present. + Conservative detection: only matches input that meets ALL of: + - longer than ``_SUBSTANTIVE_PROSE_LENGTH_THRESHOLD`` chars + - contains no citation skeleton markers (parenthesized year, URL, + vol./no./pp., DOI, terminal page reference, terminal year) + - contains no semicolons - See ``_SUBSTANTIVE_PROSE_LENGTH_THRESHOLD`` and - ``_CITATION_SKELETON_MARKERS_RE`` above for the detection details. + Designed to catch long discursive notes (689+ chars in the + Anti-Communist draft) while preserving legitimate first-occurrence + citations (which always have at least one skeleton marker) AND + legitimate compound shortened references (which use semicolons as + item separators — see the chunk 5 tightening note in the constants + block above). + + See ``_SUBSTANTIVE_PROSE_LENGTH_THRESHOLD``, + ``_CITATION_SKELETON_MARKERS_RE``, and the module constants for + the detection details. """ stripped = text.strip() if len(stripped) < _SUBSTANTIVE_PROSE_LENGTH_THRESHOLD: return False if _CITATION_SKELETON_MARKERS_RE.search(stripped): return False + if ";" in stripped: + return False return True # Default model. Override with OPENAI_MODEL=... in your .env. @@ -147,9 +169,23 @@ Rules you must follow (CMOS 18th edition specifically, NOTE form): - One author: "Charles Yu" - Two authors: "Charles Yu and Hyeyoung Kwon" - Three authors: "Charles Yu, Hyeyoung Kwon, and Kathleen Doyle" - - Four or more authors: first author followed by ", et al." (CMOS - 14.76: notes use first author + et al. for 4+ authors, even + - Four or more authors: the first author's name followed DIRECTLY + by "et al." with NO comma between the name and "et al." (CMOS + 14.76; notes use first author + et al. for 4+ authors, even though bibliography lists more before truncating). + - Correct: "Charles Yu et al." + - Correct: "Bignoli et al." + - WRONG: "Charles Yu, et al." (extra comma) + - WRONG: "Bignoli, et al." (extra comma) + The comma before "et al." is a feature of INVERTED bibliography + form ("Bignoli, A., et al."), not non-inverted note form. In note + form there is no comma between the author's last name (or full + name) and "et al." + + This applies equally to shortened-form notes — when the input is + just "Author et al., Short Title" the output should also be + "Author et al., 'Short Title'" with no comma before "et al." + Corporate authors (e.g., "Google", "Modern Language Association") appear as-is, not reordered. Non-Western names already in family-first order (e.g., "Liu Xinwu", @@ -164,9 +200,31 @@ Rules you must follow (CMOS 18th edition specifically, NOTE form): 3. ITALIC AND QUOTE CONVENTIONS are the same as bibliography form: book titles and journal titles are italicized using Markdown asterisks (*Title*); article titles and chapter titles are - wrapped in straight double quotes. In NOTE form the closing - comma goes INSIDE the closing quote of the article/chapter - title: "Article Title," (not "Article Title",). + wrapped in straight double quotes. + + The CLOSING PUNCTUATION INSIDE the article/chapter title's + closing quote depends on what comes next in the note: + + (a) If MORE CONTENT FOLLOWS the title (a journal name, a page + number, a publisher, etc.), use a COMMA inside the closing + quote, NOT outside: + - Correct: "Article Title," *Journal* 12... + - WRONG: "Article Title", *Journal* 12... + + (b) If the title is the LAST ELEMENT of the note (nothing + follows except the entry-terminal period), use a PERIOD + inside the closing quote, NOT a comma: + - Correct: Mattern, "Maintenance and Care." + - WRONG: Mattern, "Maintenance and Care,". + - WRONG: Mattern, "Maintenance and Care". + + The (b) form is especially common in shortened-form notes that + give just an author and a short title (no page number, no + journal). The model commonly errs on this case by producing + `"Title,".` (comma inside quote followed by stray period + outside) — that is incorrect punctuation. The correct form is + `"Title."` with the period inside the closing quote and nothing + after. 4. BOOK NOTE FORM: Author, *Book Title* (Publisher, Year), specific-page. @@ -356,6 +414,36 @@ Rules you must follow (CMOS 18th edition specifically, NOTE form): Investigating Committee on Education, the 1951 Hearings*, 66, https://hdl.handle.net/2027/mdp.39015070361673. +20. SIGNAL PHRASES — preserve them verbatim. CMOS notes commonly + open with a signal phrase that tells the reader how the cited + source relates to the surrounding argument. Common signal phrases + that appear at the start of notes: + + - "See" / "See also" / "See, e.g.," + - "For example," + - "Cf." (compare) + - "Compare" + - "But see" / "But cf." + - "Contra" + - "Accord" + - "Quoted in" + + These phrases are MEANINGFUL — they tell the reader whether the + citation supports, contradicts, exemplifies, or qualifies the + author's claim. PRESERVE them VERBATIM at the start of the note. + Do NOT strip them, do NOT normalize one to another (don't convert + "For example," to "See" or vice versa), and do NOT move them to + a different position in the note. + + Examples: + - Source: `For example, Bignoli et al., "Status in Academic Libraries."` + - Correct: `For example, Bignoli et al., "Status in Academic Libraries."` + - WRONG: `Bignoli et al., "Status in Academic Libraries."` (signal phrase stripped) + + - Source: `See Smith, "Article Title;" Jones, "Other Title."` + - Correct: `See Smith, "Article Title"; Jones, "Other Title."` + - WRONG: `Smith, "Article Title"; Jones, "Other Title."` (signal phrase stripped) + Output ONLY the single reformatted note. No preamble, no explanation, no leading number, no code fences. """ diff --git a/tests/test_note_formatter.py b/tests/test_note_formatter.py index 465fc7f..d3563fd 100644 --- a/tests/test_note_formatter.py +++ b/tests/test_note_formatter.py @@ -112,6 +112,108 @@ def test_system_prompt_mentions_month_or_season_preservation(): assert "preserve" in lower +def test_system_prompt_rule_3_distinguishes_comma_and_period_at_title_end(): + """Chunk 5 task 2: rule 3's punctuation guidance for note form must + distinguish between (a) title followed by more content (comma inside + quote) and (b) title as the last element of the entry (period inside + quote). Surfaced by 8+ entries across drafts producing the wrong + `"Title,".` pattern instead of `"Title."`. + """ + lower = SYSTEM_PROMPT_NOTES.lower() + # Either explicit "last element" wording or both forms shown + assert "last element" in lower or "nothing follows" in lower + # Both punctuation patterns should be mentioned + assert "period inside" in lower or 'title."' in SYSTEM_PROMPT_NOTES.lower() + + +def test_system_prompt_mentions_signal_phrases_to_preserve(): + """Chunk 5 task 3: explicit rule that signal phrases at the start + of notes must be preserved verbatim. Surfaced by HML [69] and [70] + where "For example," was stripped while "See" was preserved + elsewhere — inconsistent behavior that loses meaningful authorial + intent. + """ + lower = SYSTEM_PROMPT_NOTES.lower() + assert "signal phrase" in lower + # The list should mention at least the common ones that were + # actually observed in the drafts + assert "see also" in lower + assert "for example" in lower + assert "cf." in lower or "compare" in lower + + +def test_system_prompt_rule_1_says_no_comma_before_et_al(): + """Chunk 5 task 4: rule 1 must explicitly say "et al." follows the + author name with NO comma in note form (non-inverted). Surfaced by + HML [69] where "Bignoli et al." became "Bignoli, et al." with a + spurious comma. CMOS 14.76: in non-inverted note form, the comma + that would precede "et al." in inverted bibliography form is NOT + used. + + Test logic: the "no comma" prohibition must appear NEAR the + "et al." mention in rule 1 (within 300 chars), not just somewhere + else in the prompt (the polish #5 month/season rule also says "no + comma" but that's a different context). + """ + text = SYSTEM_PROMPT_NOTES.lower() + # Find all occurrences of "et al" and check if "no comma" is within + # 300 chars before or after each one + found_proximity = False + idx = text.find("et al") + while idx != -1: + window = text[max(0, idx - 300) : min(len(text), idx + 300)] + if "no comma" in window: + found_proximity = True + break + idx = text.find("et al", idx + 1) + + # Also accept an explicit wrong-form example + has_wrong_example = ( + '"Bignoli, et al."' in SYSTEM_PROMPT_NOTES + or '"Smith, et al."' in SYSTEM_PROMPT_NOTES + or 'wrong: "' in text and "et al" in text + ) + + assert found_proximity or has_wrong_example, ( + "rule 1 must explicitly forbid the comma between author name " + "and 'et al.' in non-inverted note form. The 'no comma' " + "prohibition must appear near the 'et al.' rule, or an " + "explicit wrong-form example must be present." + ) + + +def test_substantive_prose_detector_recognizes_domain_only_urls(): + """Chunk 5 task 5: the citation skeleton marker for URLs must also + recognize bare-domain URLs (without the https:// scheme), so that + real first-occurrence citations with bare-domain URLs aren't + false-positively passed through as substantive prose. Surfaced by + Reading_Disrepair [1] GAO entry which had files.gao.gov/... instead + of https://files.gao.gov/... + """ + # 190+ char input with no parens-year, no vol/no/pp, no DOI marker, + # no terminal page reference — but WITH a bare-domain URL. + input_with_bare_domain = ( + "Government Accountability Office. Public Libraries: Many Buildings " + "Are Reported to Be in Poor Condition. GAO-26-107262. 18 December " + "2025. files.gao.gov/reports/GAO-26-107262/index.html" + ) + assert len(input_with_bare_domain) > 150 # confirms it would otherwise hit the threshold + + call_count = 0 + + def counting_caller(system: str, user: str) -> str: + nonlocal call_count + call_count += 1 + return "Fake Author, *Fake Title* (Fake Publisher, 2024), 1." + + format_note_entry(input_with_bare_domain, caller=counting_caller) + assert call_count == 1, ( + "Citation with bare-domain URL was incorrectly passed through " + "as substantive prose; the URL marker should also recognize " + "bare-domain URLs without the https:// scheme." + ) + + def test_system_prompt_publisher_exception_list_includes_nyu_press(): """Chunk 4 task 2: rule 18's publisher exception list should include NYU Press alongside the existing MIT Press / ALA Editions / MLA. @@ -366,6 +468,92 @@ def test_format_note_entry_passes_through_substantive_prose_verbatim(): ) +def test_format_note_entry_does_not_pass_through_compound_shortened_references(): + """Chunk 5 task 1: long compound shortened references (multiple + citations separated by semicolons) must NOT be treated as substantive + prose, even when they exceed the length threshold and have no + citation skeleton markers. Surfaced by the HML draft, where ~17 of + 72 notes were compound references like: + + See Meadowbrooke et al., "Information Behavior and HIV Testing;" + Kim and Syn, "Health Information on Facebook;" ... + + These have no parenthesized year, no URL, no vol/no/pp markers, and + they're 200-400 chars long — so they matched the chunk 3 fix #4 + substantive prose detector and were passed through verbatim, leaving + curly quotes and incorrect semicolon-quote ordering unfixed. + + The discriminator: substantive prose doesn't typically use + semicolons as item separators. Compound references do. So adding + "no semicolons" as a third condition (alongside "long" and "no + skeleton markers") cleanly separates the two classes. + """ + calls: list[str] = [] + + def recording_caller(system: str, user: str) -> str: + calls.append(user) + return "Fake Author, *Fake Title* (Fake Publisher, 2024), 1." + + # All of these are real-shaped HML compound references (synthesized to + # match observed patterns). Each is over the 150-char threshold and has + # no citation skeleton markers (no parenthesized year, no URL, no + # vol/no/pp). They MUST be sent to the LLM for formatting. + compound_refs = [ + # 2-reference compound (smallest) + 'Knapp, "Creating Safe and Inclusive Spacing;" Padrón, "Defending Queer Youth;" Pinsky and Brenner, "Threats to Accessing Sexuality Information."', + # 4-reference compound with "See" prefix + 'See Dilevko and Gottlieb, "Pornography and Bibliographic Access;" Drabinski, "Queering the Catalog;" Billey et al., "Critique of RDA 9.7;" Baucom, "Archival Descriptions of LGBTQ Materials."', + # 3-reference compound, no prefix + 'Ajayi and Omotayo, "Challenges of HIV/AIDS;" Kachota and Kassim, "Sexual and Reproductive Health Information-Seeking Behaviour;" Bankole and Busayo, "Reproductive Health Information Seeking Behaviour."', + ] + for ref in compound_refs: + format_note_entry(ref, caller=recording_caller) + + assert len(calls) == len(compound_refs), ( + f"some compound references were short-circuited as substantive prose; " + f"expected {len(compound_refs)} calls, got {len(calls)}. The " + f"detector is over-matching." + ) + + +def test_format_note_entry_still_passes_through_genuine_substantive_prose(): + """Regression test for chunk 5 task 1: tightening the detector with a + semicolon check must NOT break the genuine substantive-prose pass- + through that chunk 3 fix #4 was designed for. The Anti-Communist + notes [8], [9], and [75] (long discursive commentary, NO semicolons) + must still be preserved verbatim. + """ + call_count = 0 + + def counting_caller(system: str, user: str) -> str: + nonlocal call_count + call_count += 1 + return "should not be called" + + # Synthesized to match Anti-Communist note shapes — long discursive + # prose, no semicolons, no skeleton markers. + note_8_style = ( + "W.E.B. DuBois' popularization of philanthropist Henry Lyman " + "Morehouse's theory of the talented tenth is celebrated, missing " + "the note that he himself retracted such a theory later in his " + "life, as written about by Joy James. The violence of Jim Crow " + "becomes diminished into many monstrosities, since traditionally " + "black colleges and universities became monuments to human hope " + "through philanthropic funding." + ) + note_75_style = ( + "This letter appeared in multiple archives, including California " + "State Archives, Berkeley and UCLA special collections, and is " + "uploaded on UC Santa Barbara Servers." + ) + assert format_note_entry(note_8_style, caller=counting_caller) == note_8_style + assert format_note_entry(note_75_style, caller=counting_caller) == note_75_style + assert call_count == 0, ( + "genuine substantive prose was incorrectly sent to the API; the " + "semicolon-tightening should not have affected these (no semicolons)." + ) + + def test_format_note_entry_does_not_pass_through_real_citations(): """The substantive-prose guard must NOT match real citations that happen to be long. The detection requires absence of citation