cmos dev
4cad38ef30
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
126 lines
4.1 KiB
Python
126 lines
4.1 KiB
Python
"""Run one iteration of the autoresearch loop.
|
|
|
|
Loads all exemplars from ``exemplars/``, runs the formatter against each,
|
|
scores with the field diff + linter, and appends a JSON-lines log entry to
|
|
``logs/runs.jsonl``.
|
|
|
|
Usage:
|
|
uv run python scripts/run_loop.py # real OpenAI call
|
|
uv run python scripts/run_loop.py --fake # identity caller (no API key)
|
|
|
|
The fake mode is what we use to verify the pipeline end-to-end before
|
|
adding an OpenAI key, and for smoke-testing harness changes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from dataclasses import asdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# Ensure project root on sys.path so imports work when run as a script.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from cmos.formatter import MODEL, format_bibliography_entry # noqa: E402
|
|
from cmos.linter import LINTER_VERSION, check_passes # noqa: E402
|
|
from harness.score import load_exemplars, score # noqa: E402
|
|
|
|
|
|
def _fake_caller(system: str, user: str) -> str:
|
|
"""Identity-ish caller: extracts the messy entry from the user message."""
|
|
marker = "\n\n"
|
|
idx = user.rfind(marker)
|
|
return user[idx + len(marker) :] if idx != -1 else user
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__)
|
|
ap.add_argument(
|
|
"--fake",
|
|
action="store_true",
|
|
help="Use an identity caller instead of the real OpenAI API.",
|
|
)
|
|
ap.add_argument(
|
|
"--exemplars",
|
|
default="exemplars",
|
|
help="Directory of .toml exemplar files.",
|
|
)
|
|
ap.add_argument(
|
|
"--logs",
|
|
default="logs/runs.jsonl",
|
|
help="JSON-lines log file to append the run summary to.",
|
|
)
|
|
ap.add_argument(
|
|
"--note",
|
|
default="",
|
|
help="Free-form note to attach to this run (why you ran it).",
|
|
)
|
|
args = ap.parse_args()
|
|
|
|
exemplars = load_exemplars(Path(args.exemplars))
|
|
if not exemplars:
|
|
print(f"no exemplars found in {args.exemplars}", file=sys.stderr)
|
|
return 2
|
|
|
|
def formatter(messy: str) -> str:
|
|
if args.fake:
|
|
return format_bibliography_entry(messy, caller=_fake_caller)
|
|
return format_bibliography_entry(messy)
|
|
|
|
result = score(exemplars, formatter, linter=check_passes)
|
|
|
|
log_entry = {
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"fake": args.fake,
|
|
"model": None if args.fake else MODEL,
|
|
"linter_version": LINTER_VERSION,
|
|
"note": args.note,
|
|
"scalar": result.scalar,
|
|
"field_match_rate": result.field_match_rate,
|
|
"linter_pass_rate": result.linter_pass_rate,
|
|
"canary_exact_match_rate": result.canary_exact_match_rate,
|
|
"per_exemplar": [
|
|
{
|
|
"name": r.name,
|
|
"field_match_rate": r.field_match_rate,
|
|
"linter_passed": r.linter_passed,
|
|
"exact_match": r.exact_match,
|
|
"canary": r.canary,
|
|
"candidate": r.candidate,
|
|
}
|
|
for r in result.per_exemplar
|
|
],
|
|
}
|
|
|
|
log_path = Path(args.logs)
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with log_path.open("a") as fh:
|
|
fh.write(json.dumps(log_entry) + "\n")
|
|
|
|
# Human-readable summary.
|
|
mode = "FAKE" if args.fake else f"OpenAI {MODEL}"
|
|
print(f"=== run ({mode}) — linter {LINTER_VERSION} ===")
|
|
print(f"scalar: {result.scalar:.3f}")
|
|
print(f"field_match_rate: {result.field_match_rate:.3f}")
|
|
print(f"linter_pass_rate: {result.linter_pass_rate:.3f}")
|
|
print(f"canary_exact_match_rate: {result.canary_exact_match_rate:.3f}")
|
|
print()
|
|
print(f"{'exemplar':<35} {'field':>6} {'lint':>6} {'exact':>6}")
|
|
for r in result.per_exemplar:
|
|
print(
|
|
f"{r.name:<35} "
|
|
f"{r.field_match_rate:>6.2f} "
|
|
f"{'PASS' if r.linter_passed else 'FAIL':>6} "
|
|
f"{'YES' if r.exact_match else 'no':>6}"
|
|
)
|
|
print()
|
|
print(f"logged to {log_path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|