cmos dev
4cad38ef30
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
123 lines
3.5 KiB
Python
123 lines
3.5 KiB
Python
"""Frozen scoring harness.
|
|
|
|
Runs a formatter callable against a list of exemplars and aggregates:
|
|
- field-level match rate (via harness.diff.field_diff)
|
|
- linter pass rate (if a linter function is provided)
|
|
- canary exact-match rate (on exemplars marked canary=True)
|
|
|
|
The loop scalar is `field_match_rate * linter_pass_rate` (linter defaults to
|
|
1.0 when no linter is wired in yet). Canary exact-match is tracked separately
|
|
and treated as a regression gate at the loop level, not folded into the
|
|
scalar.
|
|
|
|
This module is FROZEN with respect to the autoresearch loop — edits here are
|
|
infra commits that invalidate prior scores.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tomllib
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from harness.diff import field_diff
|
|
|
|
|
|
@dataclass
|
|
class Exemplar:
|
|
name: str
|
|
source: str
|
|
type: str
|
|
tags: list[str]
|
|
canary: bool
|
|
messy_input: str
|
|
canonical: dict
|
|
expected_bibliography: str
|
|
|
|
|
|
@dataclass
|
|
class ExemplarResult:
|
|
name: str
|
|
candidate: str
|
|
field_match_rate: float
|
|
linter_passed: bool
|
|
canary: bool
|
|
exact_match: bool
|
|
|
|
|
|
@dataclass
|
|
class ScoreResult:
|
|
per_exemplar: list[ExemplarResult] = field(default_factory=list)
|
|
field_match_rate: float = 0.0
|
|
linter_pass_rate: float = 1.0
|
|
canary_exact_match_rate: float = 1.0
|
|
scalar: float = 0.0
|
|
|
|
|
|
LinterFn = Callable[[str, Exemplar], bool]
|
|
FormatterFn = Callable[[str], str]
|
|
|
|
|
|
def _mean(values: list[float]) -> float:
|
|
return sum(values) / len(values) if values else 1.0
|
|
|
|
|
|
def score(
|
|
exemplars: list[Exemplar],
|
|
formatter: FormatterFn,
|
|
linter: LinterFn | None = None,
|
|
) -> ScoreResult:
|
|
"""Run `formatter` against each exemplar and aggregate a score."""
|
|
results: list[ExemplarResult] = []
|
|
for ex in exemplars:
|
|
candidate = formatter(ex.messy_input)
|
|
diff = field_diff(candidate, ex.canonical)
|
|
linter_ok = True if linter is None else bool(linter(candidate, ex))
|
|
results.append(
|
|
ExemplarResult(
|
|
name=ex.name,
|
|
candidate=candidate,
|
|
field_match_rate=diff.rate,
|
|
linter_passed=linter_ok,
|
|
canary=ex.canary,
|
|
exact_match=(candidate == ex.expected_bibliography),
|
|
)
|
|
)
|
|
|
|
field_rate = _mean([r.field_match_rate for r in results])
|
|
linter_rate = _mean([1.0 if r.linter_passed else 0.0 for r in results])
|
|
canary_results = [r for r in results if r.canary]
|
|
canary_rate = (
|
|
_mean([1.0 if r.exact_match else 0.0 for r in canary_results]) if canary_results else 1.0
|
|
)
|
|
|
|
return ScoreResult(
|
|
per_exemplar=results,
|
|
field_match_rate=field_rate,
|
|
linter_pass_rate=linter_rate,
|
|
canary_exact_match_rate=canary_rate,
|
|
scalar=field_rate * linter_rate,
|
|
)
|
|
|
|
|
|
def load_exemplars(directory: Path) -> list[Exemplar]:
|
|
"""Load every `.toml` file in `directory` as an Exemplar."""
|
|
exemplars: list[Exemplar] = []
|
|
for path in sorted(directory.glob("*.toml")):
|
|
with path.open("rb") as fh:
|
|
data = tomllib.load(fh)
|
|
exemplars.append(
|
|
Exemplar(
|
|
name=path.stem,
|
|
source=data["source"],
|
|
type=data["type"],
|
|
tags=list(data.get("tags", [])),
|
|
canary=bool(data.get("canary", False)),
|
|
messy_input=data["messy_input"],
|
|
canonical=dict(data["canonical"]),
|
|
expected_bibliography=data["expected_bibliography"],
|
|
)
|
|
)
|
|
return exemplars
|