iter 7: prompt rules 27-29 driven by real-draft testing

First exercise of the tool against unfamiliar real-world drafts (user's in-progress academic papers, not derived from CMOS quick guide examples). Found three systemic patterns affecting ~14% of entries across 134-entry and 13-entry drafts; this commit addresses all three. Prompt additions: 27. GOVERNMENT DOCUMENTS / INSTITUTIONAL REPORTS — when the author is a government body (GAO, US Census, WHO, Pew, etc.) and the source is a standalone report, italicize the title like a book; do NOT wrap it in quotes. CMOS 14.272. 28. PRESERVE DELIBERATE LOWERCASING of proper nouns. Examples cited in the prompt: the journal "portal: Libraries and the Academy" (deliberately lowercase "p"), authors bell hooks / danah boyd / e e cummings, brand names iPhone / eBay. Headline-case normalization must NOT touch these. Rule 17 also extended (was newspaper/magazine only): 17. PERIODICAL NAMES — drop leading "The" from ANY periodical: newspapers, magazines, AND scholarly journals. Write "Journal of Academic Librarianship", not "The Journal of Academic Librarianship"; "Library Quarterly", not "The Library Quarterly"; "American Archivist", not "The American Archivist". CMOS 14.191. Rule explicitly excepts BOOK titles and report titles, which keep their leading "The" (e.g., *The Library's Guide to Sexual and Reproductive Health Information*). Empirical impact, Reading_Disrepair (13 entries): before: 3 errors (2 leading-The, 1 GAO format, 1 portal capitalized) after: 0 errors Empirical impact, HML Submission (134 entries): before: ~21 errors (18 leading-The, 0 GAO in this draft, 3 portal) after: ~1 error (a multi-author 2nd-author inversion bug newly surfaced — separate fix) Also adds scripts/analyze_draft_output.py — a triage helper that diff-walks input and output, surfacing common failure patterns (leading-The, doi: prefix, bare-hyphen ranges, gov-report quoted, and intentional lowercasing stripped). False positives are tolerated since it's a human-review tool, not a validator. Exemplar corpus is unchanged. Re-baseline against 14 canary exemplars under the new rules: scalar 1.000, canary 1.000 (no regressions). User's real drafts under rough_drafts/ remain untracked — they are in-progress academic work and don't belong in git history.
2026-04-11 00:43:40 -04:00
parent 2bd1bdf0d4
commit a21d2614d3
2 changed files with 239 additions and 10 deletions
@@ -0,0 +1,183 @@
+"""Analyze a cmos-formatted draft output against its input draft.
+
+Usage:
+    uv run python scripts/analyze_draft_output.py INPUT.md OUTPUT.md
+
+Extracts the bibliography sections from both files, pairs them line-by-
+line, and surfaces common failure patterns:
+
+- "The " leading article preserved in periodical names
+- "doi:" prefix not converted
+- Bare ASCII hyphen in page ranges (2+ digits - 2+ digits)
+- Quoted titles that look like they should be italicized (book/report)
+- Entries that are byte-identical (no change applied)
+- Entries with large length deltas (potential truncation or runaway)
+
+Output is a summary plus a per-entry table. Not a substitute for human
+review but cheap triage on long bibliographies.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from cmos.parser import NoBibliographyError, split_bibliography  # noqa: E402
+
+
+def entries_from(path: Path) -> list[str]:
+    text = path.read_text(encoding="utf-8")
+    try:
+        return split_bibliography(text).entries
+    except NoBibliographyError:
+        print(f"no ## Bibliography section in {path}", file=sys.stderr)
+        return []
+
+
+# --- Failure-mode detectors ---------------------------------------------------
+
+_LEADING_THE_PERIODICAL = re.compile(r"\*(The [A-Z][^*]+)\*")
+_DOI_PREFIX = re.compile(r"\bdoi:\s*10\.", re.IGNORECASE)
+_BARE_HYPHEN_RANGE = re.compile(r"(?<!\w)(\d{2,4})-(\d{2,4})(?!\w)")
+_GOV_AUTHOR_MARKERS = (
+    "Accountability Office",
+    "Department of ",
+    "Bureau of ",
+    "Government of ",
+    "U.S. ",
+    "United States ",
+    "Census Bureau",
+    "National Institute",
+    "Congressional Research Service",
+)
+
+
+def check_leading_the(entry: str) -> str | None:
+    m = _LEADING_THE_PERIODICAL.search(entry)
+    if m:
+        return f"leading 'The' in italicized periodical: '*{m.group(1)}*'"
+    return None
+
+
+def check_doi_prefix(entry: str) -> str | None:
+    if _DOI_PREFIX.search(entry):
+        return "has 'doi:' prefix instead of https://doi.org/"
+    return None
+
+
+def check_bare_hyphen_range(entry: str) -> str | None:
+    # Allow en-dash (which we want) and hyphenated year ranges inside titles
+    # are sometimes legitimate, but inside a colon-prefixed page range they
+    # should be en-dash. Just flag all and let a human decide.
+    m = _BARE_HYPHEN_RANGE.search(entry)
+    if m:
+        return f"bare hyphen in number range: '{m.group(0)}'"
+    return None
+
+
+def check_gov_report_quoted(entry: str) -> str | None:
+    # Only flag entries whose author field is clearly a government body AND
+    # the title is wrapped in quotes (rather than italicized).
+    has_gov_author = any(marker in entry for marker in _GOV_AUTHOR_MARKERS)
+    if not has_gov_author:
+        return None
+    # Title in quotes (not italicized) on a gov-authored entry = likely wrong.
+    if re.search(r'"[^"]{20,}"', entry):
+        return "government-body author with quoted title; should be italicized like a book"
+    return None
+
+
+def check_intentional_lowercase_stripped(entry: str, source_entry: str | None) -> str | None:
+    # If the source entry had an all-lowercase word-token that appears
+    # title-cased in the output, flag it as potentially wrongful normalization.
+    if source_entry is None:
+        return None
+    # Find all-lowercase alphabetic tokens in the source that would normally
+    # be capitalized in a proper noun. Skip common short words.
+    skip = {
+        "a", "an", "and", "as", "at", "but", "by", "for", "if", "in", "nor",
+        "of", "on", "or", "so", "the", "to", "up", "via", "vs", "yet", "no",
+        "not", "with", "from", "into", "over", "per", "than",
+    }
+    lowered_tokens = [
+        t for t in re.findall(r"\b[a-z]{4,}\b", source_entry) if t not in skip
+    ]
+    for tok in lowered_tokens:
+        # Is the same token title-cased in the output?
+        if tok[0].lower() + tok[1:] not in entry and tok.capitalize() in entry:
+            return f"source had '{tok}' (lowercase); output may have normalized to '{tok.capitalize()}'"
+    return None
+
+
+DETECTORS = [
+    ("leading_the", check_leading_the),
+    ("doi_prefix", check_doi_prefix),
+    ("bare_hyphen_range", check_bare_hyphen_range),
+    ("gov_report_quoted", check_gov_report_quoted),
+]
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("input", type=Path)
+    ap.add_argument("output", type=Path)
+    ap.add_argument("--show", type=int, default=5, help="entries to show per bucket")
+    args = ap.parse_args()
+
+    inputs = entries_from(args.input)
+    outputs = entries_from(args.output)
+    if not inputs or not outputs:
+        return 2
+
+    print(f"input entries:  {len(inputs)}")
+    print(f"output entries: {len(outputs)}")
+    if len(inputs) != len(outputs):
+        print(f"!! count mismatch — pairing by index will misalign")
+    print()
+
+    # Count issues across all output entries.
+    issue_counts: dict[str, list[tuple[int, str, str]]] = {name: [] for name, _ in DETECTORS}
+    issue_counts["intentional_lowercase_stripped"] = []
+    unchanged = []
+    for i, out in enumerate(outputs):
+        for name, detector in DETECTORS:
+            msg = detector(out)
+            if msg:
+                issue_counts[name].append((i, msg, out))
+        src = inputs[i] if i < len(inputs) else None
+        msg = check_intentional_lowercase_stripped(out, src)
+        if msg:
+            issue_counts["intentional_lowercase_stripped"].append((i, msg, out))
+        if src is not None and out.strip() == src.strip():
+            unchanged.append((i, out))
+
+    print("=== Failure-mode summary ===")
+    for name, hits in issue_counts.items():
+        print(f"  {name:<30} {len(hits)}")
+    print(f"  {'byte_identical':<30} {len(unchanged)}")
+    print()
+
+    for name, hits in issue_counts.items():
+        if not hits:
+            continue
+        print(f"--- {name} (first {min(args.show, len(hits))} of {len(hits)}) ---")
+        for idx, msg, entry in hits[: args.show]:
+            print(f"  [{idx}] {msg}")
+            print(f"       {entry[:140]}{'...' if len(entry) > 140 else ''}")
+        print()
+
+    if unchanged:
+        print(f"--- byte_identical (first {min(args.show, len(unchanged))} of {len(unchanged)}) ---")
+        for idx, entry in unchanged[: args.show]:
+            print(f"  [{idx}] {entry[:140]}")
+        print()
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())