Scaffold CMOS 18 reformatter: harness, linter, parser, formatter, CLI

Initial phase-1 baseline of the karpathy/autoresearch-style loop.
The formatter module is the inner-loop artifact; parser and linter
are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that
will force a re-baseline on any rule change.

Components:
- harness/diff.py: case-sensitive field-level substring diff
- harness/score.py: three-axis scoring (field, linter, canary exact)
- src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited
- src/cmos/parser.py: locate ## Bibliography section, split entries
- src/cmos/formatter.py: prompt + OpenAI call with caller injection
- src/cmos/cli.py: cmos format path/to/draft.md
- scripts/run_loop.py: loop runner with --fake mode for no-API runs
- exemplars/: 3 canary seed exemplars (book, journal w/DOI, web),
  sourced from chicagomanualofstyle.org quick guide

Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed
exemplars (identity caller fails the linter on every rule). This is
the floor the real GPT-5 formatter needs to improve from.

tests/test_cli.py

@@ -0,0 +1,68 @@
+"""Tests for src/cmos/cli.py — end-to-end draft reformatting.
+
+The CLI orchestrates parser → formatter → reassemble. Tests inject a fake
+formatter so no API calls happen; the goal is to pin the glue logic, not the
+LLM behavior.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from cmos.cli import reformat_draft
+from cmos.parser import NoBibliographyError
+
+
+def _fake_formatter(messy: str) -> str:
+    # Deterministic fake: return a cleaned-up marker string so the output is
+    # recognizable.
+    return f"FORMATTED({messy.strip()})"
+
+
+def test_reformat_draft_rewrites_only_bibliography_section():
+    draft = """\
+# My Paper
+
+Some prose with citations.
+
+## Bibliography
+
+yu, charles. interior chinatown. 2020.
+kwon, hyeyoung. inclusion work. 2022.
+"""
+    output = reformat_draft(draft, formatter=_fake_formatter)
+    assert "# My Paper" in output
+    assert "Some prose with citations." in output
+    assert "FORMATTED(yu, charles. interior chinatown. 2020.)" in output
+    assert "FORMATTED(kwon, hyeyoung. inclusion work. 2022.)" in output
+    # Original messy lines must not also survive.
+    assert "yu, charles. interior chinatown. 2020." not in output.replace(
+        "FORMATTED(yu, charles. interior chinatown. 2020.)", ""
+    )
+
+
+def test_reformat_draft_preserves_sections_after_bibliography():
+    draft = """\
+## Bibliography
+
+one entry.
+
+## Appendix
+
+appendix text.
+"""
+    output = reformat_draft(draft, formatter=_fake_formatter)
+    assert "FORMATTED(one entry.)" in output
+    assert "## Appendix" in output
+    assert "appendix text." in output
+
+
+def test_reformat_draft_raises_when_no_bibliography():
+    with pytest.raises(NoBibliographyError):
+        reformat_draft("## Intro\n\nno bib here.\n", formatter=_fake_formatter)
+
+
+def test_reformat_draft_bibliography_heading_preserved():
+    draft = "## Bibliography\n\nentry.\n"
+    output = reformat_draft(draft, formatter=_fake_formatter)
+    assert "## Bibliography" in output

tests/test_diff.py

@@ -0,0 +1,53 @@
+"""Tests for harness/diff.py — field-level structured diff.
+
+The diff checks, for each (field, expected_value) in a canonical record,
+whether the candidate formatted string contains that value as a case-sensitive
+substring. This is intentionally simpler than parsing — the linter catches
+structural issues; the diff catches "did the right facts survive?"
+"""
+
+from harness.diff import field_diff
+
+
+def test_all_fields_present_in_candidate_match():
+    canonical = {
+        "author": "Smith, Jane",
+        "title": "The History of Nothing",
+        "publisher": "University of Chicago Press",
+        "year": 2023,
+    }
+    candidate = "Smith, Jane. *The History of Nothing*. University of Chicago Press, 2023."
+    result = field_diff(candidate, canonical)
+    assert result.matched == ["author", "title", "publisher", "year"]
+    assert result.missing == []
+    assert result.rate == 1.0
+
+
+def test_missing_publisher_is_reported():
+    canonical = {
+        "author": "Smith, Jane",
+        "title": "The History of Nothing",
+        "publisher": "University of Chicago Press",
+        "year": 2023,
+    }
+    # Publisher dropped.
+    candidate = "Smith, Jane. *The History of Nothing*. 2023."
+    result = field_diff(candidate, canonical)
+    assert "publisher" in result.missing
+    assert "author" in result.matched
+    assert result.rate == 0.75
+
+
+def test_title_case_mismatch_counts_as_missing():
+    # CMOS 18 requires headline-style caps. If the formatter emits sentence
+    # case, the title should not match its canonical headline-case form.
+    canonical = {"title": "The History of Nothing"}
+    candidate = "Smith, Jane. *The history of nothing*. 2023."
+    result = field_diff(candidate, canonical)
+    assert result.missing == ["title"]
+    assert result.rate == 0.0
+
+
+def test_empty_canonical_is_rate_one():
+    # Degenerate case — no fields to check means nothing is wrong.
+    assert field_diff("whatever", {}).rate == 1.0

tests/test_formatter.py

@@ -0,0 +1,83 @@
+"""Tests for src/cmos/formatter.py — the inner-loop iterable artifact.
+
+Because `formatter.py` is edited every iteration of the dev-time autoresearch
+loop, these tests intentionally test STABLE pieces: the prompt skeleton, the
+caller-injection seam, and a smoke check that the system prompt mentions key
+CMOS 18 rules. The actual LLM output shape is tested via the harness (running
+all exemplars through `score`) rather than pinned here — exact string
+matches on LLM output belong in the canary exact-match rate, not in unit
+tests.
+
+No real API calls in this file. Tests that hit the OpenAI API live in
+`tests/test_formatter_integration.py` (not yet created) and are gated on an
+OPENAI_API_KEY being set.
+"""
+
+from cmos.formatter import (
+    MODEL,
+    SYSTEM_PROMPT,
+    build_user_message,
+    format_bibliography_entry,
+)
+
+
+def test_default_model_is_gpt5():
+    # The user specified "GPT-5 / frontier reasoning" as the default. Override
+    # via the OPENAI_MODEL env var if gpt-5 is unavailable in your account.
+    assert MODEL == "gpt-5"
+
+
+def test_system_prompt_mentions_cmos_18():
+    assert "CMOS" in SYSTEM_PROMPT or "Chicago Manual of Style" in SYSTEM_PROMPT
+    assert "18" in SYSTEM_PROMPT
+
+
+def test_system_prompt_mentions_no_place_of_publication():
+    # CMOS 14.30 / 18th ed. change. The formatter MUST know this.
+    lower = SYSTEM_PROMPT.lower()
+    assert "place of publication" in lower
+
+
+def test_system_prompt_mentions_doi_preference():
+    assert "doi" in SYSTEM_PROMPT.lower()
+
+
+def test_system_prompt_mentions_italic_markers():
+    # Plan v1 uses Markdown `*Title*` for italics.
+    assert "*" in SYSTEM_PROMPT and "italic" in SYSTEM_PROMPT.lower()
+
+
+def test_build_user_message_contains_the_messy_input():
+    msg = build_user_message("yu, charles. interior chinatown. 2020")
+    assert "yu, charles. interior chinatown. 2020" in msg
+
+
+def test_formatter_uses_injected_caller():
+    """The formatter accepts a caller shim so tests (and the harness) can
+    substitute a fake OpenAI call. This is how the whole pipeline can run
+    without an API key during tests."""
+    recorded: dict = {}
+
+    def fake_caller(system: str, user: str) -> str:
+        recorded["system"] = system
+        recorded["user"] = user
+        return "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
+
+    output = format_bibliography_entry(
+        "yu, charles. interior chinatown. New York: Pantheon Books, 2020.",
+        caller=fake_caller,
+    )
+    assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
+    assert recorded["system"] == SYSTEM_PROMPT
+    assert "yu, charles" in recorded["user"]
+
+
+def test_formatter_strips_whitespace_from_caller_output():
+    # Language-model output often has leading/trailing whitespace or
+    # surrounding code fences. v0 only strips whitespace; code-fence
+    # stripping can be added when an exemplar forces it.
+    def fake_caller(system: str, user: str) -> str:
+        return "  Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.  \n"
+
+    output = format_bibliography_entry("anything", caller=fake_caller)
+    assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."

tests/test_linter.py

@@ -0,0 +1,266 @@
+"""Tests for src/cmos/linter.py — deterministic CMOS 18 structural rules.
+
+Each rule has a positive and a negative unit test. Doc comments cite the
+source that justifies the rule (Purdue OWL page or CMOS quick guide URL).
+
+The linter only checks structural properties of the RAW output string
+(italic markers as emitted, punctuation as emitted). Fact survival
+(author/title/year/etc. matching canonical) is the diff's job, not the
+linter's, so we deliberately don't duplicate it here.
+"""
+
+from cmos.linter import (
+    LINTER_VERSION,
+    check_passes,
+    lint,
+    rule_article_title_quoted,
+    rule_book_title_italicized,
+    rule_doi_is_https_url,
+    rule_ends_with_period,
+    rule_journal_title_italicized,
+    rule_no_ibid,
+    rule_no_place_of_publication_for_books,
+    rule_page_range_uses_en_dash,
+    rule_web_page_title_quoted,
+)
+from harness.score import Exemplar
+
+BOOK = Exemplar(
+    name="book",
+    source="test",
+    type="book",
+    tags=["book"],
+    canary=False,
+    messy_input="",
+    canonical={
+        "author": "Yu, Charles",
+        "title": "Interior Chinatown",
+        "publisher": "Pantheon Books",
+        "year": 2020,
+    },
+    expected_bibliography="Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.",
+)
+
+JOURNAL = Exemplar(
+    name="journal",
+    source="test",
+    type="journal",
+    tags=["journal", "doi"],
+    canary=False,
+    messy_input="",
+    canonical={
+        "author": "Kwon, Hyeyoung",
+        "article_title": "Inclusion Work: Children of Immigrants Claiming Membership in Everyday Life",
+        "journal": "American Journal of Sociology",
+        "volume": 127,
+        "issue": 6,
+        "year": 2022,
+        "pages": "1818–59",
+        "doi": "https://doi.org/10.1086/720277",
+    },
+    expected_bibliography=(
+        'Kwon, Hyeyoung. "Inclusion Work: Children of Immigrants Claiming Membership in Everyday Life." '
+        "*American Journal of Sociology* 127, no. 6 (2022): 1818–59. https://doi.org/10.1086/720277."
+    ),
+)
+
+WEB = Exemplar(
+    name="web",
+    source="test",
+    type="web",
+    tags=["web"],
+    canary=False,
+    messy_input="",
+    canonical={
+        "author": "Google",
+        "title": "Privacy Policy",
+        "site": "Privacy & Terms",
+        "date": "November 15, 2023",
+        "url": "https://policies.google.com/privacy",
+    },
+    expected_bibliography='Google. "Privacy Policy." Privacy & Terms. Effective November 15, 2023. https://policies.google.com/privacy.',
+)
+
+
+def test_linter_version_is_set():
+    assert LINTER_VERSION
+    assert isinstance(LINTER_VERSION, str)
+
+
+def test_ends_with_period_passes_on_well_formed_entry():
+    # CMOS 18: bibliography entries separate major elements by periods and
+    # end with a period. Source: chicagomanualofstyle.org quick guide.
+    result = rule_ends_with_period("Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", BOOK)
+    assert result.applicable is True
+    assert result.passed is True
+
+
+def test_ends_with_period_fails_on_missing_period():
+    result = rule_ends_with_period("Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020", BOOK)
+    assert result.applicable is True
+    assert result.passed is False
+
+
+# --- No place of publication for books (CMOS 18 change from 17, CMOS 14.30) ----
+
+
+def test_no_place_of_publication_passes_when_absent():
+    # CMOS 18: place of publication no longer required for books.
+    # Source: CMOS 14.30 (cited on chicagomanualofstyle.org quick guide).
+    result = rule_no_place_of_publication_for_books(
+        "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", BOOK
+    )
+    assert result.applicable is True
+    assert result.passed is True
+
+
+def test_no_place_of_publication_fails_when_city_prefix_present():
+    result = rule_no_place_of_publication_for_books(
+        "Yu, Charles. *Interior Chinatown*. New York: Pantheon Books, 2020.", BOOK
+    )
+    assert result.passed is False
+
+
+def test_no_place_of_publication_not_applicable_to_journals():
+    # The rule should not fire on non-book types even if their output happens
+    # to contain a colon (which journals routinely do, e.g., article title).
+    result = rule_no_place_of_publication_for_books("anything: anything.", JOURNAL)
+    assert result.applicable is False
+
+
+# --- Book title italicized ----------------------------------------------------
+
+
+def test_book_title_italicized_passes_when_wrapped_in_asterisks():
+    # CMOS 18 bibliography basics: titles of books and journals are italicized.
+    # Source: chicagomanualofstyle.org quick guide; Purdue OWL CMOS 18 NB poster
+    # page 3 "Bibliography: Basics".
+    result = rule_book_title_italicized(
+        "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", BOOK
+    )
+    assert result.passed is True
+
+
+def test_book_title_italicized_fails_when_title_plain():
+    result = rule_book_title_italicized(
+        "Yu, Charles. Interior Chinatown. Pantheon Books, 2020.", BOOK
+    )
+    assert result.passed is False
+
+
+def test_book_title_italicized_not_applicable_to_web():
+    result = rule_book_title_italicized("whatever", WEB)
+    assert result.applicable is False
+
+
+# --- No "Ibid." (CMOS 18 deprecation) -----------------------------------------
+
+
+def test_no_ibid_passes_on_clean_entry():
+    # CMOS 18 deprecates "Ibid." in favor of shortened notes.
+    # Source: Widener "New in 18th" LibGuide.
+    result = rule_no_ibid("Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", BOOK)
+    assert result.passed is True
+
+
+def test_no_ibid_fails_when_ibid_present():
+    result = rule_no_ibid("Ibid., 42.", BOOK)
+    assert result.passed is False
+
+
+# --- Journal title italicized --------------------------------------------------
+
+
+def test_journal_title_italicized_passes_when_wrapped():
+    result = rule_journal_title_italicized(JOURNAL.expected_bibliography, JOURNAL)
+    assert result.passed is True
+
+
+def test_journal_title_italicized_fails_when_plain():
+    bad = JOURNAL.expected_bibliography.replace(
+        "*American Journal of Sociology*", "American Journal of Sociology"
+    )
+    result = rule_journal_title_italicized(bad, JOURNAL)
+    assert result.passed is False
+
+
+# --- Article title in quotes ---------------------------------------------------
+
+
+def test_article_title_quoted_passes_when_wrapped_in_double_quotes():
+    # CMOS 18: titles of articles, chapters, poems are in quotation marks.
+    # Source: Purdue OWL CMOS 18 NB poster page 3 Bibliography Basics.
+    result = rule_article_title_quoted(JOURNAL.expected_bibliography, JOURNAL)
+    assert result.passed is True
+
+
+def test_article_title_quoted_fails_when_unquoted():
+    bad = JOURNAL.expected_bibliography.replace(
+        '"Inclusion Work: Children of Immigrants Claiming Membership in Everyday Life."',
+        "Inclusion Work: Children of Immigrants Claiming Membership in Everyday Life.",
+    )
+    result = rule_article_title_quoted(bad, JOURNAL)
+    assert result.passed is False
+
+
+# --- Page range uses en-dash ---------------------------------------------------
+
+
+def test_page_range_uses_en_dash_passes():
+    # CMOS convention: page ranges use en-dash (–), not ASCII hyphen (-).
+    result = rule_page_range_uses_en_dash(JOURNAL.expected_bibliography, JOURNAL)
+    assert result.passed is True
+
+
+def test_page_range_uses_en_dash_fails_on_ascii_hyphen():
+    bad = JOURNAL.expected_bibliography.replace("1818–59", "1818-59")
+    result = rule_page_range_uses_en_dash(bad, JOURNAL)
+    assert result.passed is False
+
+
+# --- DOI as https://doi.org/ URL -----------------------------------------------
+
+
+def test_doi_is_https_url_passes_when_canonical_prefix_present():
+    # CMOS 18 prefers DOIs formatted as https://doi.org/... rather than bare
+    # DOIs or "DOI:" prefixes. Source: CMOS 18 quick guide.
+    result = rule_doi_is_https_url(JOURNAL.expected_bibliography, JOURNAL)
+    assert result.passed is True
+
+
+def test_doi_is_https_url_fails_on_bare_doi():
+    bad = JOURNAL.expected_bibliography.replace("https://doi.org/10.1086/720277", "10.1086/720277")
+    result = rule_doi_is_https_url(bad, JOURNAL)
+    assert result.passed is False
+
+
+def test_doi_rule_not_applicable_without_doi():
+    # Web exemplar has no DOI field.
+    result = rule_doi_is_https_url(WEB.expected_bibliography, WEB)
+    assert result.applicable is False
+
+
+# --- Web page title in quotes --------------------------------------------------
+
+
+def test_web_page_title_quoted_passes():
+    result = rule_web_page_title_quoted(WEB.expected_bibliography, WEB)
+    assert result.passed is True
+
+
+def test_web_page_title_quoted_fails_when_unquoted():
+    bad = WEB.expected_bibliography.replace('"Privacy Policy."', "Privacy Policy.")
+    result = rule_web_page_title_quoted(bad, WEB)
+    assert result.passed is False
+
+
+# --- Aggregate check_passes ---------------------------------------------------
+
+
+def test_check_passes_on_canonical_outputs():
+    # All three seed exemplars' expected_bibliography strings must lint clean.
+    # If any of them fail, either the linter is too strict or the exemplar is
+    # wrong — either way, we want to know immediately.
+    assert check_passes(BOOK.expected_bibliography, BOOK)
+    assert check_passes(JOURNAL.expected_bibliography, JOURNAL)
+    assert check_passes(WEB.expected_bibliography, WEB)

tests/test_parser.py

@@ -0,0 +1,76 @@
+"""Tests for src/cmos/parser.py — extract the bibliography section from a draft.
+
+v1 scope: locate a ``## Bibliography`` heading (case-insensitive) and return
+the prose before it, the list of entries inside it, and the prose after it.
+The section ends at the next ``##`` (level-2) heading or end of file.
+Blank lines and bullet markers (``- `` or ``* ``) are stripped from entries.
+"""
+
+import pytest
+
+from cmos.parser import NoBibliographyError, split_bibliography
+
+
+def test_three_entries_under_bibliography_heading():
+    draft = """\
+Some prose here.
+
+More prose.
+
+## Bibliography
+
+yu, charles. interior chinatown. New York: Pantheon Books, 2020.
+Kwon, Hyeyoung. "inclusion work." American Journal of Sociology 127, no. 6 (2022): 1818-1859.
+google, "privacy policy," privacy & terms, nov 15 2023, policies.google.com/privacy
+"""
+    result = split_bibliography(draft)
+    assert len(result.entries) == 3
+    assert result.entries[0].startswith("yu, charles")
+    assert result.entries[1].startswith("Kwon, Hyeyoung")
+    assert result.entries[2].startswith("google")
+    assert "Some prose here." in result.before
+    assert result.after == ""
+
+
+def test_section_ends_at_next_level_two_heading():
+    draft = """\
+## Bibliography
+
+First entry.
+Second entry.
+
+## Appendix
+
+Appendix content here.
+"""
+    result = split_bibliography(draft)
+    assert result.entries == ["First entry.", "Second entry."]
+    assert "## Appendix" in result.after
+    assert "Appendix content here." in result.after
+
+
+def test_bullet_prefixes_stripped():
+    draft = """\
+## Bibliography
+
+- First entry.
+- Second entry.
+* Third entry.
+"""
+    result = split_bibliography(draft)
+    assert result.entries == ["First entry.", "Second entry.", "Third entry."]
+
+
+def test_heading_match_is_case_insensitive():
+    draft = """\
+## bibliography
+
+Only entry.
+"""
+    result = split_bibliography(draft)
+    assert result.entries == ["Only entry."]
+
+
+def test_no_bibliography_raises():
+    with pytest.raises(NoBibliographyError):
+        split_bibliography("## Introduction\n\nJust prose.\n")

tests/test_score.py

@@ -0,0 +1,116 @@
+"""Tests for harness/score.py — the frozen scoring function.
+
+The score function takes a list of exemplars and a formatter callable,
+produces per-exemplar results, and aggregates a scalar for the loop.
+"""
+
+from pathlib import Path
+
+from harness.score import Exemplar, load_exemplars, score
+
+
+def identity_formatter(messy: str) -> str:
+    return messy
+
+
+def perfect_formatter_for(fixtures: dict) -> callable:
+    """Return a formatter that looks up the expected output by messy input."""
+
+    def _fmt(messy: str) -> str:
+        return fixtures[messy]
+
+    return _fmt
+
+
+BOOK = Exemplar(
+    name="book_single_author",
+    source="toy fixture",
+    type="book",
+    tags=["book"],
+    canary=False,
+    messy_input="smith, jane. the history of nothing. u of chicago press, 2023.",
+    canonical={
+        "author": "Smith, Jane",
+        "title": "The History of Nothing",
+        "publisher": "University of Chicago Press",
+        "year": 2023,
+    },
+    expected_bibliography="Smith, Jane. *The History of Nothing*. University of Chicago Press, 2023.",
+)
+
+WEB = Exemplar(
+    name="web_page",
+    source="toy fixture",
+    type="web",
+    tags=["web"],
+    canary=False,
+    messy_input="modern language association. mla style center. 2024.",
+    canonical={
+        "author": "Modern Language Association",
+        "title": "MLA Style Center",
+        "year": 2024,
+    },
+    expected_bibliography='Modern Language Association. "MLA Style Center." 2024. https://style.mla.org/.',
+)
+
+
+def test_identity_formatter_scores_below_one():
+    # Identity returns the messy input unchanged, so canonical fields should
+    # mostly NOT appear in the output.
+    result = score([BOOK, WEB], identity_formatter)
+    assert result.scalar < 1.0
+    assert len(result.per_exemplar) == 2
+
+
+def test_perfect_formatter_scores_one():
+    fixtures = {
+        BOOK.messy_input: BOOK.expected_bibliography,
+        WEB.messy_input: WEB.expected_bibliography,
+    }
+    result = score([BOOK, WEB], perfect_formatter_for(fixtures))
+    assert result.scalar == 1.0
+    assert result.field_match_rate == 1.0
+
+
+def test_load_exemplars_reads_toml_files(tmp_path: Path):
+    (tmp_path / "example.toml").write_text("""
+source = "toy"
+type = "book"
+tags = ["book"]
+canary = true
+messy_input = "smith jane, nothing, 2023"
+expected_bibliography = "Smith, Jane. *Nothing*. Publisher, 2023."
+
+[canonical]
+author = "Smith, Jane"
+year = 2023
+""".strip())
+    loaded = load_exemplars(tmp_path)
+    assert len(loaded) == 1
+    ex = loaded[0]
+    assert ex.name == "example"
+    assert ex.canary is True
+    assert ex.canonical["author"] == "Smith, Jane"
+    assert ex.canonical["year"] == 2023
+
+
+def test_canary_exact_match_tracked_separately():
+    canary_book = Exemplar(
+        name="book_canary",
+        source="toy",
+        type="book",
+        tags=["book"],
+        canary=True,
+        messy_input=BOOK.messy_input,
+        canonical=BOOK.canonical,
+        expected_bibliography=BOOK.expected_bibliography,
+    )
+
+    # Formatter produces correct fields but with a trailing space — fields
+    # match, exact-string doesn't.
+    def nearly(_messy: str) -> str:
+        return BOOK.expected_bibliography + " "
+
+    result = score([canary_book], nearly)
+    assert result.field_match_rate == 1.0
+    assert result.canary_exact_match_rate == 0.0