Scaffold CMOS 18 reformatter: harness, linter, parser, formatter, CLI
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
"""Frozen scoring harness.
|
||||
|
||||
Runs a formatter callable against a list of exemplars and aggregates:
|
||||
- field-level match rate (via harness.diff.field_diff)
|
||||
- linter pass rate (if a linter function is provided)
|
||||
- canary exact-match rate (on exemplars marked canary=True)
|
||||
|
||||
The loop scalar is `field_match_rate * linter_pass_rate` (linter defaults to
|
||||
1.0 when no linter is wired in yet). Canary exact-match is tracked separately
|
||||
and treated as a regression gate at the loop level, not folded into the
|
||||
scalar.
|
||||
|
||||
This module is FROZEN with respect to the autoresearch loop — edits here are
|
||||
infra commits that invalidate prior scores.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from harness.diff import field_diff
|
||||
|
||||
|
||||
@dataclass
|
||||
class Exemplar:
|
||||
name: str
|
||||
source: str
|
||||
type: str
|
||||
tags: list[str]
|
||||
canary: bool
|
||||
messy_input: str
|
||||
canonical: dict
|
||||
expected_bibliography: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExemplarResult:
|
||||
name: str
|
||||
candidate: str
|
||||
field_match_rate: float
|
||||
linter_passed: bool
|
||||
canary: bool
|
||||
exact_match: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScoreResult:
|
||||
per_exemplar: list[ExemplarResult] = field(default_factory=list)
|
||||
field_match_rate: float = 0.0
|
||||
linter_pass_rate: float = 1.0
|
||||
canary_exact_match_rate: float = 1.0
|
||||
scalar: float = 0.0
|
||||
|
||||
|
||||
LinterFn = Callable[[str, Exemplar], bool]
|
||||
FormatterFn = Callable[[str], str]
|
||||
|
||||
|
||||
def _mean(values: list[float]) -> float:
|
||||
return sum(values) / len(values) if values else 1.0
|
||||
|
||||
|
||||
def score(
|
||||
exemplars: list[Exemplar],
|
||||
formatter: FormatterFn,
|
||||
linter: LinterFn | None = None,
|
||||
) -> ScoreResult:
|
||||
"""Run `formatter` against each exemplar and aggregate a score."""
|
||||
results: list[ExemplarResult] = []
|
||||
for ex in exemplars:
|
||||
candidate = formatter(ex.messy_input)
|
||||
diff = field_diff(candidate, ex.canonical)
|
||||
linter_ok = True if linter is None else bool(linter(candidate, ex))
|
||||
results.append(
|
||||
ExemplarResult(
|
||||
name=ex.name,
|
||||
candidate=candidate,
|
||||
field_match_rate=diff.rate,
|
||||
linter_passed=linter_ok,
|
||||
canary=ex.canary,
|
||||
exact_match=(candidate == ex.expected_bibliography),
|
||||
)
|
||||
)
|
||||
|
||||
field_rate = _mean([r.field_match_rate for r in results])
|
||||
linter_rate = _mean([1.0 if r.linter_passed else 0.0 for r in results])
|
||||
canary_results = [r for r in results if r.canary]
|
||||
canary_rate = (
|
||||
_mean([1.0 if r.exact_match else 0.0 for r in canary_results]) if canary_results else 1.0
|
||||
)
|
||||
|
||||
return ScoreResult(
|
||||
per_exemplar=results,
|
||||
field_match_rate=field_rate,
|
||||
linter_pass_rate=linter_rate,
|
||||
canary_exact_match_rate=canary_rate,
|
||||
scalar=field_rate * linter_rate,
|
||||
)
|
||||
|
||||
|
||||
def load_exemplars(directory: Path) -> list[Exemplar]:
|
||||
"""Load every `.toml` file in `directory` as an Exemplar."""
|
||||
exemplars: list[Exemplar] = []
|
||||
for path in sorted(directory.glob("*.toml")):
|
||||
with path.open("rb") as fh:
|
||||
data = tomllib.load(fh)
|
||||
exemplars.append(
|
||||
Exemplar(
|
||||
name=path.stem,
|
||||
source=data["source"],
|
||||
type=data["type"],
|
||||
tags=list(data.get("tags", [])),
|
||||
canary=bool(data.get("canary", False)),
|
||||
messy_input=data["messy_input"],
|
||||
canonical=dict(data["canonical"]),
|
||||
expected_bibliography=data["expected_bibliography"],
|
||||
)
|
||||
)
|
||||
return exemplars
|
||||
Reference in New Issue
Block a user