"""Tests for harness/score.py — the frozen scoring function. The score function takes a list of exemplars and a formatter callable, produces per-exemplar results, and aggregates a scalar for the loop. """ from pathlib import Path from harness.score import Exemplar, load_exemplars, score def identity_formatter(messy: str) -> str: return messy def perfect_formatter_for(fixtures: dict) -> callable: """Return a formatter that looks up the expected output by messy input.""" def _fmt(messy: str) -> str: return fixtures[messy] return _fmt BOOK = Exemplar( name="book_single_author", source="toy fixture", type="book", tags=["book"], canary=False, messy_input="smith, jane. the history of nothing. u of chicago press, 2023.", canonical={ "author": "Smith, Jane", "title": "The History of Nothing", "publisher": "University of Chicago Press", "year": 2023, }, expected_bibliography="Smith, Jane. *The History of Nothing*. University of Chicago Press, 2023.", ) WEB = Exemplar( name="web_page", source="toy fixture", type="web", tags=["web"], canary=False, messy_input="modern language association. mla style center. 2024.", canonical={ "author": "Modern Language Association", "title": "MLA Style Center", "year": 2024, }, expected_bibliography='Modern Language Association. "MLA Style Center." 2024. https://style.mla.org/.', ) def test_identity_formatter_scores_below_one(): # Identity returns the messy input unchanged, so canonical fields should # mostly NOT appear in the output. result = score([BOOK, WEB], identity_formatter) assert result.scalar < 1.0 assert len(result.per_exemplar) == 2 def test_perfect_formatter_scores_one(): fixtures = { BOOK.messy_input: BOOK.expected_bibliography, WEB.messy_input: WEB.expected_bibliography, } result = score([BOOK, WEB], perfect_formatter_for(fixtures)) assert result.scalar == 1.0 assert result.field_match_rate == 1.0 def test_load_exemplars_reads_toml_files(tmp_path: Path): (tmp_path / "example.toml").write_text(""" source = "toy" type = "book" tags = ["book"] canary = true messy_input = "smith jane, nothing, 2023" expected_bibliography = "Smith, Jane. *Nothing*. Publisher, 2023." [canonical] author = "Smith, Jane" year = 2023 """.strip()) loaded = load_exemplars(tmp_path) assert len(loaded) == 1 ex = loaded[0] assert ex.name == "example" assert ex.canary is True assert ex.canonical["author"] == "Smith, Jane" assert ex.canonical["year"] == 2023 def test_canary_exact_match_tracked_separately(): canary_book = Exemplar( name="book_canary", source="toy", type="book", tags=["book"], canary=True, messy_input=BOOK.messy_input, canonical=BOOK.canonical, expected_bibliography=BOOK.expected_bibliography, ) # Formatter produces correct fields but with a trailing space — fields # match, exact-string doesn't. def nearly(_messy: str) -> str: return BOOK.expected_bibliography + " " result = score([canary_book], nearly) assert result.field_match_rate == 1.0 assert result.canary_exact_match_rate == 0.0