diff --git a/harness/judge.py b/harness/judge.py new file mode 100644 index 0000000..dc14b30 --- /dev/null +++ b/harness/judge.py @@ -0,0 +1,132 @@ +"""LLM-as-judge for canary failures — advisory triage only. + +This module is NOT part of the hot autoresearch loop. It exists to help a +human (or script) classify canary-exact-match failures: was the formatter +genuinely wrong (a "regression"), or did it produce a CMOS-18-legal +variant that happens to differ from the frozen canonical string (a +"variant")? + +Judge verdicts are advisory and stored as data in ``logs/triage.jsonl``. +They never feed back into the scoring harness — if they did, we'd be +optimizing the formatter against a judge built on the same model family, +which invites shared-bias drift. + +Caller-injection pattern matches ``cmos.formatter`` so tests can run +without touching the network. +""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass +from typing import Callable + +from dotenv import load_dotenv + +JUDGE_MODEL = os.environ.get("CMOS_JUDGE_MODEL", os.environ.get("OPENAI_MODEL", "gpt-5")) + +Caller = Callable[[str, str], str] + + +JUDGE_SYSTEM_PROMPT = """\ +You are a strict Chicago Manual of Style, 18th edition, notes-and-bibliography \ +adjudicator. You will be given TWO strings: an EXPECTED bibliography entry \ +(the frozen canonical) and a CANDIDATE bibliography entry (what a formatter \ +produced). Your job is to decide whether the candidate is: + +- "regression": the candidate contains a real CMOS-18 error OR has dropped, \ +added, or corrupted a piece of source metadata (author, title, publisher, \ +year, page range, DOI, URL, edition, etc.) relative to the expected. Examples: \ +a missing "et al.", a wrong year, "Second edition" instead of "2nd ed.", \ +dropping "Effective" from a web policy date, adding a place of publication \ +(CMOS 18 does not require one), using a hyphen where CMOS 9.61 prescribes an \ +en-dash, omitting the "Podcast," format label. + +- "variant": the candidate differs from the expected only in ways that are \ +explicitly CMOS-18-legal. Examples include harmless spelling out vs. \ +abbreviation that CMOS treats as equivalent, genuinely optional fields, \ +equivalent punctuation choices where CMOS 18 allows either, or differences in \ +whitespace. A variant is NOT a regression; the formatter is still correct. + +- "unclear": you cannot determine from the strings alone whether the \ +difference is a regression or a legal variant. Use this sparingly. + +Respond with ONLY a JSON object of the form: +{"label": "regression" | "variant" | "unclear", "reasoning": "one or two sentences citing the specific CMOS 18 rule that applies"} + +No preamble, no markdown fences, no commentary outside the JSON. +""" + + +@dataclass +class Verdict: + label: str + reasoning: str + + +def build_judge_prompt(expected: str, candidate: str, cmos_type: str) -> str: + return ( + f"Source type: {cmos_type}\n\n" + f"EXPECTED:\n{expected}\n\n" + f"CANDIDATE:\n{candidate}\n\n" + f"Classify the candidate per CMOS 18 rules. Respond with the JSON object described in the system prompt." + ) + + +_CODE_FENCE_RE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL) + + +def parse_verdict(raw: str) -> Verdict: + """Parse a judge response into a Verdict. Tolerates code fences and + normalizes unknown labels to 'unclear'.""" + stripped = raw.strip() + m = _CODE_FENCE_RE.match(stripped) + if m: + stripped = m.group(1).strip() + try: + data = json.loads(stripped) + except json.JSONDecodeError: + return Verdict(label="unclear", reasoning=f"could not parse judge output: {raw[:200]}") + label = str(data.get("label", "unclear")).lower() + if label not in ("regression", "variant", "unclear"): + label = "unclear" + reasoning = str(data.get("reasoning", "")) + return Verdict(label=label, reasoning=reasoning) + + +def _is_reasoning_model(model: str) -> bool: + prefixes = ("gpt-5", "o1", "o3", "o4") + return any(model.startswith(p) for p in prefixes) + + +def _openai_judge_caller(system: str, user: str) -> str: + load_dotenv() + from openai import OpenAI + + client = OpenAI() + kwargs: dict = { + "model": JUDGE_MODEL, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + } + if not _is_reasoning_model(JUDGE_MODEL): + kwargs["temperature"] = 0 + response = client.chat.completions.create(**kwargs) + return response.choices[0].message.content or "" + + +def judge( + expected: str, + candidate: str, + cmos_type: str, + caller: Caller | None = None, +) -> Verdict: + """Ask the judge to classify a (expected, candidate) pair.""" + call = caller or _openai_judge_caller + user = build_judge_prompt(expected, candidate, cmos_type) + raw = call(JUDGE_SYSTEM_PROMPT, user) + return parse_verdict(raw) diff --git a/scripts/triage.py b/scripts/triage.py new file mode 100644 index 0000000..30ca5b3 --- /dev/null +++ b/scripts/triage.py @@ -0,0 +1,119 @@ +"""Triage canary failures from the most recent loop run using an LLM judge. + +Reads ``logs/runs.jsonl``, finds the latest run, identifies exemplars whose +``exact_match`` is False (canary failures), calls the judge on each, and +appends verdicts to ``logs/triage.jsonl``. + +The judge output is ADVISORY — it never feeds into the loop scalar. +Verdicts are for a human to review ("is this a variant or a real +regression?") after the fact. + +Usage: + uv run python scripts/triage.py + uv run python scripts/triage.py --run-index -1 # last run (default) + uv run python scripts/triage.py --run-index -2 # second-last + uv run python scripts/triage.py --dry-run # no API calls + +If there are no canary failures in the target run, the script prints a +note and exits 0 without calling the judge. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from harness.judge import JUDGE_MODEL, Verdict, judge # noqa: E402 +from harness.score import load_exemplars # noqa: E402 + + +def _dry_run_judge(expected: str, candidate: str, cmos_type: str) -> Verdict: + return Verdict(label="unclear", reasoning="(dry run — judge not called)") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--runs-log", default="logs/runs.jsonl") + ap.add_argument("--triage-log", default="logs/triage.jsonl") + ap.add_argument("--exemplars", default="exemplars") + ap.add_argument("--run-index", type=int, default=-1, help="which run to triage") + ap.add_argument("--dry-run", action="store_true", help="do not call the judge") + args = ap.parse_args() + + runs_path = Path(args.runs_log) + if not runs_path.exists(): + print(f"no runs log at {runs_path}", file=sys.stderr) + return 2 + + with runs_path.open() as fh: + runs = [json.loads(line) for line in fh if line.strip()] + if not runs: + print("runs log is empty", file=sys.stderr) + return 2 + try: + run = runs[args.run_index] + except IndexError: + print(f"run index {args.run_index} out of range (have {len(runs)} runs)", file=sys.stderr) + return 2 + + # Map exemplar name → canonical expected_bibliography and type. + ex_map = {ex.name: ex for ex in load_exemplars(Path(args.exemplars))} + + failures = [r for r in run["per_exemplar"] if r["canary"] and not r["exact_match"]] + if not failures: + print(f"no canary failures in run at {run['timestamp']} — nothing to triage") + return 0 + + judge_fn = _dry_run_judge if args.dry_run else judge + + print(f"triaging {len(failures)} canary failure(s) from run {run['timestamp']}") + print(f"judge model: {'(dry run)' if args.dry_run else JUDGE_MODEL}") + print() + + triage_path = Path(args.triage_log) + triage_path.parent.mkdir(parents=True, exist_ok=True) + with triage_path.open("a") as out: + for r in failures: + name = r["name"] + ex = ex_map.get(name) + if ex is None: + print(f" [{name}] exemplar not found on disk, skipping") + continue + expected = ex.expected_bibliography + candidate = r["candidate"] + verdict = judge_fn(expected, candidate, ex.type) + + entry = { + "triage_timestamp": datetime.now(timezone.utc).isoformat(), + "run_timestamp": run["timestamp"], + "exemplar": name, + "cmos_type": ex.type, + "expected": expected, + "candidate": candidate, + "judge_label": verdict.label, + "judge_reasoning": verdict.reasoning, + "judge_model": None if args.dry_run else JUDGE_MODEL, + } + out.write(json.dumps(entry) + "\n") + + icon = { + "regression": "!!", + "variant": "ok", + "unclear": "??", + }.get(verdict.label, "??") + print(f" [{icon}] {name}: {verdict.label}") + if verdict.reasoning: + print(f" {verdict.reasoning}") + + print() + print(f"verdicts appended to {triage_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 0000000..c18d969 --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,81 @@ +"""Tests for harness/judge.py — LLM-as-judge for canary failures. + +The judge is an advisory triage tool, not part of the hot loop. It +classifies canary-mismatch candidates as either "regression" (the +formatter made a real CMOS error) or "variant" (the formatter produced a +CMOS-legal rendering that happens to differ from the frozen canonical +string). + +These tests use a fake caller — no real API calls. +""" + +import json + +from harness.judge import Verdict, build_judge_prompt, judge, parse_verdict + + +def test_build_judge_prompt_includes_both_strings_and_type(): + prompt = build_judge_prompt( + expected="Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", + candidate="Yu, Charles. *Interior Chinatown*. New York: Pantheon Books, 2020.", + cmos_type="book", + ) + assert "Yu, Charles" in prompt + assert "New York" in prompt + assert "book" in prompt.lower() + assert "cmos" in prompt.lower() or "chicago" in prompt.lower() + + +def test_parse_verdict_extracts_label_and_reasoning(): + raw = json.dumps( + {"label": "regression", "reasoning": "place of publication added; CMOS 18 drops it."} + ) + v = parse_verdict(raw) + assert v.label == "regression" + assert "place of publication" in v.reasoning + + +def test_parse_verdict_accepts_label_only(): + raw = json.dumps({"label": "variant", "reasoning": ""}) + v = parse_verdict(raw) + assert v.label == "variant" + assert v.reasoning == "" + + +def test_parse_verdict_normalizes_unexpected_labels_to_unclear(): + # A judge that returns something other than regression/variant should + # end up labeled 'unclear' so the human reviewer knows to look at it. + raw = json.dumps({"label": "maybe", "reasoning": "..."}) + v = parse_verdict(raw) + assert v.label == "unclear" + + +def test_parse_verdict_handles_code_fences(): + # LLM sometimes wraps JSON in ``` fences; parse_verdict should survive. + raw = '```json\n{"label": "variant", "reasoning": "abbreviated publisher"}\n```' + v = parse_verdict(raw) + assert v.label == "variant" + assert "abbreviated" in v.reasoning + + +def test_judge_uses_injected_caller(): + captured = {} + + def fake_caller(system: str, user: str) -> str: + captured["system"] = system + captured["user"] = user + return json.dumps( + {"label": "regression", "reasoning": "inserted place of publication"} + ) + + v = judge( + expected="Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020.", + candidate="Yu, Charles. *Interior Chinatown*. New York: Pantheon Books, 2020.", + cmos_type="book", + caller=fake_caller, + ) + assert isinstance(v, Verdict) + assert v.label == "regression" + assert "place of publication" in v.reasoning + assert "18" in captured["system"] # CMOS 18 grounding in the system prompt + assert "Yu, Charles" in captured["user"]