cmos/harness/judge.py

"""LLM-as-judge for canary failures — advisory triage only.

This module is NOT part of the hot autoresearch loop. It exists to help a
human (or script) classify canary-exact-match failures: was the formatter
genuinely wrong (a "regression"), or did it produce a CMOS-18-legal
variant that happens to differ from the frozen canonical string (a
"variant")?

Judge verdicts are advisory and stored as data in ``logs/triage.jsonl``.
They never feed back into the scoring harness — if they did, we'd be
optimizing the formatter against a judge built on the same model family,
which invites shared-bias drift.

Caller-injection pattern matches ``cmos.formatter`` so tests can run
without touching the network.
"""

from __future__ import annotations

import json
import os
import re
from dataclasses import dataclass
from typing import Callable

from dotenv import load_dotenv

JUDGE_MODEL = os.environ.get("CMOS_JUDGE_MODEL", os.environ.get("OPENAI_MODEL", "gpt-5"))

Caller = Callable[[str, str], str]


JUDGE_SYSTEM_PROMPT = """\
You are a strict Chicago Manual of Style, 18th edition, notes-and-bibliography \
adjudicator. You will be given TWO strings: an EXPECTED bibliography entry \
(the frozen canonical) and a CANDIDATE bibliography entry (what a formatter \
produced). Your job is to decide whether the candidate is:

- "regression": the candidate contains a real CMOS-18 error OR has dropped, \
added, or corrupted a piece of source metadata (author, title, publisher, \
year, page range, DOI, URL, edition, etc.) relative to the expected. Examples: \
a missing "et al.", a wrong year, "Second edition" instead of "2nd ed.", \
dropping "Effective" from a web policy date, adding a place of publication \
(CMOS 18 does not require one), using a hyphen where CMOS 9.61 prescribes an \
en-dash, omitting the "Podcast," format label.

- "variant": the candidate differs from the expected only in ways that are \
explicitly CMOS-18-legal. Examples include harmless spelling out vs. \
abbreviation that CMOS treats as equivalent, genuinely optional fields, \
equivalent punctuation choices where CMOS 18 allows either, or differences in \
whitespace. A variant is NOT a regression; the formatter is still correct.

- "unclear": you cannot determine from the strings alone whether the \
difference is a regression or a legal variant. Use this sparingly.

Respond with ONLY a JSON object of the form:
{"label": "regression" | "variant" | "unclear", "reasoning": "one or two sentences citing the specific CMOS 18 rule that applies"}

No preamble, no markdown fences, no commentary outside the JSON.
"""


@dataclass
class Verdict:
    label: str
    reasoning: str


def build_judge_prompt(expected: str, candidate: str, cmos_type: str) -> str:
    return (
        f"Source type: {cmos_type}\n\n"
        f"EXPECTED:\n{expected}\n\n"
        f"CANDIDATE:\n{candidate}\n\n"
        f"Classify the candidate per CMOS 18 rules. Respond with the JSON object described in the system prompt."
    )


_CODE_FENCE_RE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL)


def parse_verdict(raw: str) -> Verdict:
    """Parse a judge response into a Verdict. Tolerates code fences and
    normalizes unknown labels to 'unclear'."""
    stripped = raw.strip()
    m = _CODE_FENCE_RE.match(stripped)
    if m:
        stripped = m.group(1).strip()
    try:
        data = json.loads(stripped)
    except json.JSONDecodeError:
        return Verdict(label="unclear", reasoning=f"could not parse judge output: {raw[:200]}")
    label = str(data.get("label", "unclear")).lower()
    if label not in ("regression", "variant", "unclear"):
        label = "unclear"
    reasoning = str(data.get("reasoning", ""))
    return Verdict(label=label, reasoning=reasoning)


def _is_reasoning_model(model: str) -> bool:
    prefixes = ("gpt-5", "o1", "o3", "o4")
    return any(model.startswith(p) for p in prefixes)


def _openai_judge_caller(system: str, user: str) -> str:
    load_dotenv()
    from openai import OpenAI

    client = OpenAI()
    kwargs: dict = {
        "model": JUDGE_MODEL,
        "messages": [
            {"role": "system", "content": system},
            {"role": "user", "content": user},
        ],
    }
    if not _is_reasoning_model(JUDGE_MODEL):
        kwargs["temperature"] = 0
    response = client.chat.completions.create(**kwargs)
    return response.choices[0].message.content or ""


def judge(
    expected: str,
    candidate: str,
    cmos_type: str,
    caller: Caller | None = None,
) -> Verdict:
    """Ask the judge to classify a (expected, candidate) pair."""
    call = caller or _openai_judge_caller
    user = build_judge_prompt(expected, candidate, cmos_type)
    raw = call(JUDGE_SYSTEM_PROMPT, user)
    return parse_verdict(raw)