"""Analyze a cmos-formatted draft output against its input draft. Usage: uv run python scripts/analyze_draft_output.py INPUT.md OUTPUT.md Extracts the bibliography sections from both files, pairs them line-by- line, and surfaces common failure patterns: - "The " leading article preserved in periodical names - "doi:" prefix not converted - Bare ASCII hyphen in page ranges (2+ digits - 2+ digits) - Quoted titles that look like they should be italicized (book/report) - Entries that are byte-identical (no change applied) - Entries with large length deltas (potential truncation or runaway) Output is a summary plus a per-entry table. Not a substitute for human review but cheap triage on long bibliographies. """ from __future__ import annotations import argparse import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from cmos.parser import NoBibliographyError, split_bibliography # noqa: E402 def entries_from(path: Path) -> list[str]: text = path.read_text(encoding="utf-8") try: return split_bibliography(text).entries except NoBibliographyError: print(f"no ## Bibliography section in {path}", file=sys.stderr) return [] # --- Failure-mode detectors --------------------------------------------------- _LEADING_THE_PERIODICAL = re.compile(r"\*(The [A-Z][^*]+)\*") _DOI_PREFIX = re.compile(r"\bdoi:\s*10\.", re.IGNORECASE) _BARE_HYPHEN_RANGE = re.compile(r"(? str | None: m = _LEADING_THE_PERIODICAL.search(entry) if m: return f"leading 'The' in italicized periodical: '*{m.group(1)}*'" return None def check_doi_prefix(entry: str) -> str | None: if _DOI_PREFIX.search(entry): return "has 'doi:' prefix instead of https://doi.org/" return None def check_bare_hyphen_range(entry: str) -> str | None: # Allow en-dash (which we want) and hyphenated year ranges inside titles # are sometimes legitimate, but inside a colon-prefixed page range they # should be en-dash. Just flag all and let a human decide. m = _BARE_HYPHEN_RANGE.search(entry) if m: return f"bare hyphen in number range: '{m.group(0)}'" return None def check_gov_report_quoted(entry: str) -> str | None: # Only flag entries whose author field is clearly a government body AND # the title is wrapped in quotes (rather than italicized). has_gov_author = any(marker in entry for marker in _GOV_AUTHOR_MARKERS) if not has_gov_author: return None # Title in quotes (not italicized) on a gov-authored entry = likely wrong. if re.search(r'"[^"]{20,}"', entry): return "government-body author with quoted title; should be italicized like a book" return None def check_intentional_lowercase_stripped(entry: str, source_entry: str | None) -> str | None: # If the source entry had an all-lowercase word-token that appears # title-cased in the output, flag it as potentially wrongful normalization. if source_entry is None: return None # Find all-lowercase alphabetic tokens in the source that would normally # be capitalized in a proper noun. Skip common short words. skip = { "a", "an", "and", "as", "at", "but", "by", "for", "if", "in", "nor", "of", "on", "or", "so", "the", "to", "up", "via", "vs", "yet", "no", "not", "with", "from", "into", "over", "per", "than", } lowered_tokens = [ t for t in re.findall(r"\b[a-z]{4,}\b", source_entry) if t not in skip ] for tok in lowered_tokens: # Is the same token title-cased in the output? if tok[0].lower() + tok[1:] not in entry and tok.capitalize() in entry: return f"source had '{tok}' (lowercase); output may have normalized to '{tok.capitalize()}'" return None DETECTORS = [ ("leading_the", check_leading_the), ("doi_prefix", check_doi_prefix), ("bare_hyphen_range", check_bare_hyphen_range), ("gov_report_quoted", check_gov_report_quoted), ] def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("input", type=Path) ap.add_argument("output", type=Path) ap.add_argument("--show", type=int, default=5, help="entries to show per bucket") args = ap.parse_args() inputs = entries_from(args.input) outputs = entries_from(args.output) if not inputs or not outputs: return 2 print(f"input entries: {len(inputs)}") print(f"output entries: {len(outputs)}") if len(inputs) != len(outputs): print(f"!! count mismatch — pairing by index will misalign") print() # Count issues across all output entries. issue_counts: dict[str, list[tuple[int, str, str]]] = {name: [] for name, _ in DETECTORS} issue_counts["intentional_lowercase_stripped"] = [] unchanged = [] for i, out in enumerate(outputs): for name, detector in DETECTORS: msg = detector(out) if msg: issue_counts[name].append((i, msg, out)) src = inputs[i] if i < len(inputs) else None msg = check_intentional_lowercase_stripped(out, src) if msg: issue_counts["intentional_lowercase_stripped"].append((i, msg, out)) if src is not None and out.strip() == src.strip(): unchanged.append((i, out)) print("=== Failure-mode summary ===") for name, hits in issue_counts.items(): print(f" {name:<30} {len(hits)}") print(f" {'byte_identical':<30} {len(unchanged)}") print() for name, hits in issue_counts.items(): if not hits: continue print(f"--- {name} (first {min(args.show, len(hits))} of {len(hits)}) ---") for idx, msg, entry in hits[: args.show]: print(f" [{idx}] {msg}") print(f" {entry[:140]}{'...' if len(entry) > 140 else ''}") print() if unchanged: print(f"--- byte_identical (first {min(args.show, len(unchanged))} of {len(unchanged)}) ---") for idx, entry in unchanged[: args.show]: print(f" [{idx}] {entry[:140]}") print() return 0 if __name__ == "__main__": raise SystemExit(main())