"""Run one iteration of the autoresearch loop. Loads all exemplars from ``exemplars/``, runs the formatter against each, scores with the field diff + linter, and appends a JSON-lines log entry to ``logs/runs.jsonl``. Usage: uv run python scripts/run_loop.py # real OpenAI call uv run python scripts/run_loop.py --fake # identity caller (no API key) The fake mode is what we use to verify the pipeline end-to-end before adding an OpenAI key, and for smoke-testing harness changes. """ from __future__ import annotations import argparse import json import sys from dataclasses import asdict from datetime import datetime, timezone from pathlib import Path # Ensure project root on sys.path so imports work when run as a script. sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from cmos.formatter import MODEL, format_bibliography_entry # noqa: E402 from cmos.linter import LINTER_VERSION, check_passes # noqa: E402 from harness.score import load_exemplars, score # noqa: E402 def _fake_caller(system: str, user: str) -> str: """Identity-ish caller: extracts the messy entry from the user message.""" marker = "\n\n" idx = user.rfind(marker) return user[idx + len(marker) :] if idx != -1 else user def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument( "--fake", action="store_true", help="Use an identity caller instead of the real OpenAI API.", ) ap.add_argument( "--exemplars", default="exemplars", help="Directory of .toml exemplar files.", ) ap.add_argument( "--logs", default="logs/runs.jsonl", help="JSON-lines log file to append the run summary to.", ) ap.add_argument( "--note", default="", help="Free-form note to attach to this run (why you ran it).", ) args = ap.parse_args() exemplars = load_exemplars(Path(args.exemplars)) if not exemplars: print(f"no exemplars found in {args.exemplars}", file=sys.stderr) return 2 def formatter(messy: str) -> str: if args.fake: return format_bibliography_entry(messy, caller=_fake_caller) return format_bibliography_entry(messy) result = score(exemplars, formatter, linter=check_passes) log_entry = { "timestamp": datetime.now(timezone.utc).isoformat(), "fake": args.fake, "model": None if args.fake else MODEL, "linter_version": LINTER_VERSION, "note": args.note, "scalar": result.scalar, "field_match_rate": result.field_match_rate, "linter_pass_rate": result.linter_pass_rate, "canary_exact_match_rate": result.canary_exact_match_rate, "per_exemplar": [ { "name": r.name, "field_match_rate": r.field_match_rate, "linter_passed": r.linter_passed, "exact_match": r.exact_match, "canary": r.canary, "candidate": r.candidate, } for r in result.per_exemplar ], } log_path = Path(args.logs) log_path.parent.mkdir(parents=True, exist_ok=True) with log_path.open("a") as fh: fh.write(json.dumps(log_entry) + "\n") # Human-readable summary. mode = "FAKE" if args.fake else f"OpenAI {MODEL}" print(f"=== run ({mode}) — linter {LINTER_VERSION} ===") print(f"scalar: {result.scalar:.3f}") print(f"field_match_rate: {result.field_match_rate:.3f}") print(f"linter_pass_rate: {result.linter_pass_rate:.3f}") print(f"canary_exact_match_rate: {result.canary_exact_match_rate:.3f}") print() print(f"{'exemplar':<35} {'field':>6} {'lint':>6} {'exact':>6}") for r in result.per_exemplar: print( f"{r.name:<35} " f"{r.field_match_rate:>6.2f} " f"{'PASS' if r.linter_passed else 'FAIL':>6} " f"{'YES' if r.exact_match else 'no':>6}" ) print() print(f"logged to {log_path}") return 0 if __name__ == "__main__": raise SystemExit(main())