From a21d2614d37ae0afdb60c38a66f510ce3fa4eb0d Mon Sep 17 00:00:00 2001 From: cmos dev Date: Sat, 11 Apr 2026 00:43:40 -0400 Subject: [PATCH] iter 7: prompt rules 27-29 driven by real-draft testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First exercise of the tool against unfamiliar real-world drafts (user's in-progress academic papers, not derived from CMOS quick guide examples). Found three systemic patterns affecting ~14% of entries across 134-entry and 13-entry drafts; this commit addresses all three. Prompt additions: 27. GOVERNMENT DOCUMENTS / INSTITUTIONAL REPORTS — when the author is a government body (GAO, US Census, WHO, Pew, etc.) and the source is a standalone report, italicize the title like a book; do NOT wrap it in quotes. CMOS 14.272. 28. PRESERVE DELIBERATE LOWERCASING of proper nouns. Examples cited in the prompt: the journal "portal: Libraries and the Academy" (deliberately lowercase "p"), authors bell hooks / danah boyd / e e cummings, brand names iPhone / eBay. Headline-case normalization must NOT touch these. Rule 17 also extended (was newspaper/magazine only): 17. PERIODICAL NAMES — drop leading "The" from ANY periodical: newspapers, magazines, AND scholarly journals. Write "Journal of Academic Librarianship", not "The Journal of Academic Librarianship"; "Library Quarterly", not "The Library Quarterly"; "American Archivist", not "The American Archivist". CMOS 14.191. Rule explicitly excepts BOOK titles and report titles, which keep their leading "The" (e.g., *The Library's Guide to Sexual and Reproductive Health Information*). Empirical impact, Reading_Disrepair (13 entries): before: 3 errors (2 leading-The, 1 GAO format, 1 portal capitalized) after: 0 errors Empirical impact, HML Submission (134 entries): before: ~21 errors (18 leading-The, 0 GAO in this draft, 3 portal) after: ~1 error (a multi-author 2nd-author inversion bug newly surfaced — separate fix) Also adds scripts/analyze_draft_output.py — a triage helper that diff-walks input and output, surfacing common failure patterns (leading-The, doi: prefix, bare-hyphen ranges, gov-report quoted, and intentional lowercasing stripped). False positives are tolerated since it's a human-review tool, not a validator. Exemplar corpus is unchanged. Re-baseline against 14 canary exemplars under the new rules: scalar 1.000, canary 1.000 (no regressions). User's real drafts under rough_drafts/ remain untracked — they are in-progress academic work and don't belong in git history. --- scripts/analyze_draft_output.py | 183 ++++++++++++++++++++++++++++++++ src/cmos/formatter.py | 66 ++++++++++-- 2 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 scripts/analyze_draft_output.py diff --git a/scripts/analyze_draft_output.py b/scripts/analyze_draft_output.py new file mode 100644 index 0000000..24515e9 --- /dev/null +++ b/scripts/analyze_draft_output.py @@ -0,0 +1,183 @@ +"""Analyze a cmos-formatted draft output against its input draft. + +Usage: + uv run python scripts/analyze_draft_output.py INPUT.md OUTPUT.md + +Extracts the bibliography sections from both files, pairs them line-by- +line, and surfaces common failure patterns: + +- "The " leading article preserved in periodical names +- "doi:" prefix not converted +- Bare ASCII hyphen in page ranges (2+ digits - 2+ digits) +- Quoted titles that look like they should be italicized (book/report) +- Entries that are byte-identical (no change applied) +- Entries with large length deltas (potential truncation or runaway) + +Output is a summary plus a per-entry table. Not a substitute for human +review but cheap triage on long bibliographies. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from cmos.parser import NoBibliographyError, split_bibliography # noqa: E402 + + +def entries_from(path: Path) -> list[str]: + text = path.read_text(encoding="utf-8") + try: + return split_bibliography(text).entries + except NoBibliographyError: + print(f"no ## Bibliography section in {path}", file=sys.stderr) + return [] + + +# --- Failure-mode detectors --------------------------------------------------- + +_LEADING_THE_PERIODICAL = re.compile(r"\*(The [A-Z][^*]+)\*") +_DOI_PREFIX = re.compile(r"\bdoi:\s*10\.", re.IGNORECASE) +_BARE_HYPHEN_RANGE = re.compile(r"(? str | None: + m = _LEADING_THE_PERIODICAL.search(entry) + if m: + return f"leading 'The' in italicized periodical: '*{m.group(1)}*'" + return None + + +def check_doi_prefix(entry: str) -> str | None: + if _DOI_PREFIX.search(entry): + return "has 'doi:' prefix instead of https://doi.org/" + return None + + +def check_bare_hyphen_range(entry: str) -> str | None: + # Allow en-dash (which we want) and hyphenated year ranges inside titles + # are sometimes legitimate, but inside a colon-prefixed page range they + # should be en-dash. Just flag all and let a human decide. + m = _BARE_HYPHEN_RANGE.search(entry) + if m: + return f"bare hyphen in number range: '{m.group(0)}'" + return None + + +def check_gov_report_quoted(entry: str) -> str | None: + # Only flag entries whose author field is clearly a government body AND + # the title is wrapped in quotes (rather than italicized). + has_gov_author = any(marker in entry for marker in _GOV_AUTHOR_MARKERS) + if not has_gov_author: + return None + # Title in quotes (not italicized) on a gov-authored entry = likely wrong. + if re.search(r'"[^"]{20,}"', entry): + return "government-body author with quoted title; should be italicized like a book" + return None + + +def check_intentional_lowercase_stripped(entry: str, source_entry: str | None) -> str | None: + # If the source entry had an all-lowercase word-token that appears + # title-cased in the output, flag it as potentially wrongful normalization. + if source_entry is None: + return None + # Find all-lowercase alphabetic tokens in the source that would normally + # be capitalized in a proper noun. Skip common short words. + skip = { + "a", "an", "and", "as", "at", "but", "by", "for", "if", "in", "nor", + "of", "on", "or", "so", "the", "to", "up", "via", "vs", "yet", "no", + "not", "with", "from", "into", "over", "per", "than", + } + lowered_tokens = [ + t for t in re.findall(r"\b[a-z]{4,}\b", source_entry) if t not in skip + ] + for tok in lowered_tokens: + # Is the same token title-cased in the output? + if tok[0].lower() + tok[1:] not in entry and tok.capitalize() in entry: + return f"source had '{tok}' (lowercase); output may have normalized to '{tok.capitalize()}'" + return None + + +DETECTORS = [ + ("leading_the", check_leading_the), + ("doi_prefix", check_doi_prefix), + ("bare_hyphen_range", check_bare_hyphen_range), + ("gov_report_quoted", check_gov_report_quoted), +] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("input", type=Path) + ap.add_argument("output", type=Path) + ap.add_argument("--show", type=int, default=5, help="entries to show per bucket") + args = ap.parse_args() + + inputs = entries_from(args.input) + outputs = entries_from(args.output) + if not inputs or not outputs: + return 2 + + print(f"input entries: {len(inputs)}") + print(f"output entries: {len(outputs)}") + if len(inputs) != len(outputs): + print(f"!! count mismatch — pairing by index will misalign") + print() + + # Count issues across all output entries. + issue_counts: dict[str, list[tuple[int, str, str]]] = {name: [] for name, _ in DETECTORS} + issue_counts["intentional_lowercase_stripped"] = [] + unchanged = [] + for i, out in enumerate(outputs): + for name, detector in DETECTORS: + msg = detector(out) + if msg: + issue_counts[name].append((i, msg, out)) + src = inputs[i] if i < len(inputs) else None + msg = check_intentional_lowercase_stripped(out, src) + if msg: + issue_counts["intentional_lowercase_stripped"].append((i, msg, out)) + if src is not None and out.strip() == src.strip(): + unchanged.append((i, out)) + + print("=== Failure-mode summary ===") + for name, hits in issue_counts.items(): + print(f" {name:<30} {len(hits)}") + print(f" {'byte_identical':<30} {len(unchanged)}") + print() + + for name, hits in issue_counts.items(): + if not hits: + continue + print(f"--- {name} (first {min(args.show, len(hits))} of {len(hits)}) ---") + for idx, msg, entry in hits[: args.show]: + print(f" [{idx}] {msg}") + print(f" {entry[:140]}{'...' if len(entry) > 140 else ''}") + print() + + if unchanged: + print(f"--- byte_identical (first {min(args.show, len(unchanged))} of {len(unchanged)}) ---") + for idx, entry in unchanged[: args.show]: + print(f" [{idx}] {entry[:140]}") + print() + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cmos/formatter.py b/src/cmos/formatter.py index 76da62f..9f534cd 100644 --- a/src/cmos/formatter.py +++ b/src/cmos/formatter.py @@ -109,16 +109,31 @@ Rules you must follow (CMOS 18th edition specifically): provides them (not initials), e.g., "Snyder, Carl D., Manuel Bedrossian, Casey Barr, et al." — NOT "Snyder, C. D., ..." -17. NEWSPAPER AND MAGAZINE NAMES: - - Expand common abbreviations. "NYT" → "New York Times". "WSJ" → - "Wall Street Journal". "WaPo" → "Washington Post". "LAT" → - "Los Angeles Times". - - Drop a leading "The" from the publication name. Write "New Yorker", - NOT "The New Yorker". Write "New York Times", NOT "The New York - Times". Write "Atlantic", NOT "The Atlantic". - - Italicize newspaper and magazine names: *New Yorker*. - - Use a comma (not a period) between the italicized name and the date: - *New Yorker*, December 18, 2023. +17. PERIODICAL NAMES (newspapers, magazines, AND scholarly journals): + - Expand common newspaper abbreviations. "NYT" → "New York Times". + "WSJ" → "Wall Street Journal". "WaPo" → "Washington Post". "LAT" + → "Los Angeles Times". + - Drop a leading "The" from ANY periodical name — newspapers, + magazines, AND scholarly journals. Write "New Yorker", NOT "The + New Yorker". Write "New York Times", NOT "The New York Times". + Write "Atlantic", NOT "The Atlantic". Write "Journal of Academic + Librarianship", NOT "The Journal of Academic Librarianship". Write + "Library Quarterly", NOT "The Library Quarterly". Write "American + Archivist", NOT "The American Archivist". This is a CMOS 14.191 + convention for all periodicals. + - IMPORTANT EXCEPTION: this "drop leading The" rule applies ONLY to + periodical names. Do NOT drop the leading "The" from BOOK titles, + REPORT titles, or any other non-periodical title. Book titles and + report titles keep their "The" exactly as given. For example, + *The Library's Guide to Sexual and Reproductive Health + Information* (a book) keeps its "The"; *The Free Press of + Glencoe* (a publisher name that begins with "The") keeps its + "The". + - Italicize newspaper, magazine, and journal names. + - Between an italicized newspaper or magazine name and a full date, + use a comma: *New Yorker*, December 18, 2023. (Scholarly journals + use the volume/issue/year-in-parens form instead, and do NOT get + a comma before the parenthetical.) 18. E-BOOK FORMAT INDICATOR: write just the platform name with a period. "Kindle." NOT "Kindle edition." "Nook." NOT "Nook e-book." @@ -180,6 +195,37 @@ Rules you must follow (CMOS 18th edition specifically): title and the publisher: Lastname, First. *Title*. 2nd ed. Publisher, Year. +27. GOVERNMENT DOCUMENTS AND INSTITUTIONAL REPORTS: when the author is a + government body, agency, institution, or NGO (e.g., "Government + Accountability Office", "U.S. Department of Education", "Congressional + Research Service", "World Health Organization", "Pew Research + Center") and the source is a STANDALONE report or document (not an + article in a periodical), treat the report like a BOOK: italicize + the title with Markdown *...*, do NOT wrap it in quotes. Include any + report number as an identifier after the title. Form: + Author-Body. *Title of Report*. Report Number. Publisher (often the + same body), Date. URL. + Example: Government Accountability Office. *Public Libraries: Many + Buildings Are Reported to Be in Poor Condition*. GAO-26-107262. + December 18, 2025. https://files.gao.gov/reports/GAO-26-107262/index.html. + This is CMOS 14.272. + +28. PRESERVE DELIBERATE LOWERCASING of proper nouns. Some author names, + journal names, organization names, and brand names are intentionally + all-lowercase as a stylistic or branding choice and MUST be preserved + as-is. Known cases include: + - The scholarly journal "portal: Libraries and the Academy" — always + written with a lowercase "p". Do NOT title-case it to + "Portal: Libraries and the Academy". + - Authors: "bell hooks", "danah boyd", "e e cummings", "k.d. lang". + - Brand / product names: "iPhone", "eBay" — preserve the exact + capitalization shown on their own sites. + If the source has a proper noun in ALL LOWERCASE and you might + otherwise normalize it to headline case, STOP and leave it + lowercase. This is a deliberate authorial/brand choice, not sloppy + capitalization. When in doubt about whether a lowercased form is + intentional, err on the side of preserving the source's form. + Output ONLY the single reformatted entry. No preamble, no explanation. """