Scaffold CMOS 18 reformatter: harness, linter, parser, formatter, CLI
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
"""Tests for harness/score.py — the frozen scoring function.
|
||||
|
||||
The score function takes a list of exemplars and a formatter callable,
|
||||
produces per-exemplar results, and aggregates a scalar for the loop.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from harness.score import Exemplar, load_exemplars, score
|
||||
|
||||
|
||||
def identity_formatter(messy: str) -> str:
|
||||
return messy
|
||||
|
||||
|
||||
def perfect_formatter_for(fixtures: dict) -> callable:
|
||||
"""Return a formatter that looks up the expected output by messy input."""
|
||||
|
||||
def _fmt(messy: str) -> str:
|
||||
return fixtures[messy]
|
||||
|
||||
return _fmt
|
||||
|
||||
|
||||
BOOK = Exemplar(
|
||||
name="book_single_author",
|
||||
source="toy fixture",
|
||||
type="book",
|
||||
tags=["book"],
|
||||
canary=False,
|
||||
messy_input="smith, jane. the history of nothing. u of chicago press, 2023.",
|
||||
canonical={
|
||||
"author": "Smith, Jane",
|
||||
"title": "The History of Nothing",
|
||||
"publisher": "University of Chicago Press",
|
||||
"year": 2023,
|
||||
},
|
||||
expected_bibliography="Smith, Jane. *The History of Nothing*. University of Chicago Press, 2023.",
|
||||
)
|
||||
|
||||
WEB = Exemplar(
|
||||
name="web_page",
|
||||
source="toy fixture",
|
||||
type="web",
|
||||
tags=["web"],
|
||||
canary=False,
|
||||
messy_input="modern language association. mla style center. 2024.",
|
||||
canonical={
|
||||
"author": "Modern Language Association",
|
||||
"title": "MLA Style Center",
|
||||
"year": 2024,
|
||||
},
|
||||
expected_bibliography='Modern Language Association. "MLA Style Center." 2024. https://style.mla.org/.',
|
||||
)
|
||||
|
||||
|
||||
def test_identity_formatter_scores_below_one():
|
||||
# Identity returns the messy input unchanged, so canonical fields should
|
||||
# mostly NOT appear in the output.
|
||||
result = score([BOOK, WEB], identity_formatter)
|
||||
assert result.scalar < 1.0
|
||||
assert len(result.per_exemplar) == 2
|
||||
|
||||
|
||||
def test_perfect_formatter_scores_one():
|
||||
fixtures = {
|
||||
BOOK.messy_input: BOOK.expected_bibliography,
|
||||
WEB.messy_input: WEB.expected_bibliography,
|
||||
}
|
||||
result = score([BOOK, WEB], perfect_formatter_for(fixtures))
|
||||
assert result.scalar == 1.0
|
||||
assert result.field_match_rate == 1.0
|
||||
|
||||
|
||||
def test_load_exemplars_reads_toml_files(tmp_path: Path):
|
||||
(tmp_path / "example.toml").write_text("""
|
||||
source = "toy"
|
||||
type = "book"
|
||||
tags = ["book"]
|
||||
canary = true
|
||||
messy_input = "smith jane, nothing, 2023"
|
||||
expected_bibliography = "Smith, Jane. *Nothing*. Publisher, 2023."
|
||||
|
||||
[canonical]
|
||||
author = "Smith, Jane"
|
||||
year = 2023
|
||||
""".strip())
|
||||
loaded = load_exemplars(tmp_path)
|
||||
assert len(loaded) == 1
|
||||
ex = loaded[0]
|
||||
assert ex.name == "example"
|
||||
assert ex.canary is True
|
||||
assert ex.canonical["author"] == "Smith, Jane"
|
||||
assert ex.canonical["year"] == 2023
|
||||
|
||||
|
||||
def test_canary_exact_match_tracked_separately():
|
||||
canary_book = Exemplar(
|
||||
name="book_canary",
|
||||
source="toy",
|
||||
type="book",
|
||||
tags=["book"],
|
||||
canary=True,
|
||||
messy_input=BOOK.messy_input,
|
||||
canonical=BOOK.canonical,
|
||||
expected_bibliography=BOOK.expected_bibliography,
|
||||
)
|
||||
|
||||
# Formatter produces correct fields but with a trailing space — fields
|
||||
# match, exact-string doesn't.
|
||||
def nearly(_messy: str) -> str:
|
||||
return BOOK.expected_bibliography + " "
|
||||
|
||||
result = score([canary_book], nearly)
|
||||
assert result.field_match_rate == 1.0
|
||||
assert result.canary_exact_match_rate == 0.0
|
||||
Reference in New Issue
Block a user