cmos dev
4cad38ef30
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
117 lines
3.3 KiB
Python
117 lines
3.3 KiB
Python
"""Tests for harness/score.py — the frozen scoring function.
|
|
|
|
The score function takes a list of exemplars and a formatter callable,
|
|
produces per-exemplar results, and aggregates a scalar for the loop.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
|
|
from harness.score import Exemplar, load_exemplars, score
|
|
|
|
|
|
def identity_formatter(messy: str) -> str:
|
|
return messy
|
|
|
|
|
|
def perfect_formatter_for(fixtures: dict) -> callable:
|
|
"""Return a formatter that looks up the expected output by messy input."""
|
|
|
|
def _fmt(messy: str) -> str:
|
|
return fixtures[messy]
|
|
|
|
return _fmt
|
|
|
|
|
|
BOOK = Exemplar(
|
|
name="book_single_author",
|
|
source="toy fixture",
|
|
type="book",
|
|
tags=["book"],
|
|
canary=False,
|
|
messy_input="smith, jane. the history of nothing. u of chicago press, 2023.",
|
|
canonical={
|
|
"author": "Smith, Jane",
|
|
"title": "The History of Nothing",
|
|
"publisher": "University of Chicago Press",
|
|
"year": 2023,
|
|
},
|
|
expected_bibliography="Smith, Jane. *The History of Nothing*. University of Chicago Press, 2023.",
|
|
)
|
|
|
|
WEB = Exemplar(
|
|
name="web_page",
|
|
source="toy fixture",
|
|
type="web",
|
|
tags=["web"],
|
|
canary=False,
|
|
messy_input="modern language association. mla style center. 2024.",
|
|
canonical={
|
|
"author": "Modern Language Association",
|
|
"title": "MLA Style Center",
|
|
"year": 2024,
|
|
},
|
|
expected_bibliography='Modern Language Association. "MLA Style Center." 2024. https://style.mla.org/.',
|
|
)
|
|
|
|
|
|
def test_identity_formatter_scores_below_one():
|
|
# Identity returns the messy input unchanged, so canonical fields should
|
|
# mostly NOT appear in the output.
|
|
result = score([BOOK, WEB], identity_formatter)
|
|
assert result.scalar < 1.0
|
|
assert len(result.per_exemplar) == 2
|
|
|
|
|
|
def test_perfect_formatter_scores_one():
|
|
fixtures = {
|
|
BOOK.messy_input: BOOK.expected_bibliography,
|
|
WEB.messy_input: WEB.expected_bibliography,
|
|
}
|
|
result = score([BOOK, WEB], perfect_formatter_for(fixtures))
|
|
assert result.scalar == 1.0
|
|
assert result.field_match_rate == 1.0
|
|
|
|
|
|
def test_load_exemplars_reads_toml_files(tmp_path: Path):
|
|
(tmp_path / "example.toml").write_text("""
|
|
source = "toy"
|
|
type = "book"
|
|
tags = ["book"]
|
|
canary = true
|
|
messy_input = "smith jane, nothing, 2023"
|
|
expected_bibliography = "Smith, Jane. *Nothing*. Publisher, 2023."
|
|
|
|
[canonical]
|
|
author = "Smith, Jane"
|
|
year = 2023
|
|
""".strip())
|
|
loaded = load_exemplars(tmp_path)
|
|
assert len(loaded) == 1
|
|
ex = loaded[0]
|
|
assert ex.name == "example"
|
|
assert ex.canary is True
|
|
assert ex.canonical["author"] == "Smith, Jane"
|
|
assert ex.canonical["year"] == 2023
|
|
|
|
|
|
def test_canary_exact_match_tracked_separately():
|
|
canary_book = Exemplar(
|
|
name="book_canary",
|
|
source="toy",
|
|
type="book",
|
|
tags=["book"],
|
|
canary=True,
|
|
messy_input=BOOK.messy_input,
|
|
canonical=BOOK.canonical,
|
|
expected_bibliography=BOOK.expected_bibliography,
|
|
)
|
|
|
|
# Formatter produces correct fields but with a trailing space — fields
|
|
# match, exact-string doesn't.
|
|
def nearly(_messy: str) -> str:
|
|
return BOOK.expected_bibliography + " "
|
|
|
|
result = score([canary_book], nearly)
|
|
assert result.field_match_rate == 1.0
|
|
assert result.canary_exact_match_rate == 0.0
|