cmos dev
4cad38ef30
Initial phase-1 baseline of the karpathy/autoresearch-style loop. The formatter module is the inner-loop artifact; parser and linter are infra. The linter carries a LINTER_VERSION hash (v0.2.0) that will force a re-baseline on any rule change. Components: - harness/diff.py: case-sensitive field-level substring diff - harness/score.py: three-axis scoring (field, linter, canary exact) - src/cmos/linter.py: 9 CMOS 18 structural rules, each Purdue/CMOS cited - src/cmos/parser.py: locate ## Bibliography section, split entries - src/cmos/formatter.py: prompt + OpenAI call with caller injection - src/cmos/cli.py: cmos format path/to/draft.md - scripts/run_loop.py: loop runner with --fake mode for no-API runs - exemplars/: 3 canary seed exemplars (book, journal w/DOI, web), sourced from chicagomanualofstyle.org quick guide Tests: 48 passing. Fake-mode baseline scalar = 0.000 on the 3 seed exemplars (identity caller fails the linter on every rule). This is the floor the real GPT-5 formatter needs to improve from.
84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
"""Tests for src/cmos/formatter.py — the inner-loop iterable artifact.
|
|
|
|
Because `formatter.py` is edited every iteration of the dev-time autoresearch
|
|
loop, these tests intentionally test STABLE pieces: the prompt skeleton, the
|
|
caller-injection seam, and a smoke check that the system prompt mentions key
|
|
CMOS 18 rules. The actual LLM output shape is tested via the harness (running
|
|
all exemplars through `score`) rather than pinned here — exact string
|
|
matches on LLM output belong in the canary exact-match rate, not in unit
|
|
tests.
|
|
|
|
No real API calls in this file. Tests that hit the OpenAI API live in
|
|
`tests/test_formatter_integration.py` (not yet created) and are gated on an
|
|
OPENAI_API_KEY being set.
|
|
"""
|
|
|
|
from cmos.formatter import (
|
|
MODEL,
|
|
SYSTEM_PROMPT,
|
|
build_user_message,
|
|
format_bibliography_entry,
|
|
)
|
|
|
|
|
|
def test_default_model_is_gpt5():
|
|
# The user specified "GPT-5 / frontier reasoning" as the default. Override
|
|
# via the OPENAI_MODEL env var if gpt-5 is unavailable in your account.
|
|
assert MODEL == "gpt-5"
|
|
|
|
|
|
def test_system_prompt_mentions_cmos_18():
|
|
assert "CMOS" in SYSTEM_PROMPT or "Chicago Manual of Style" in SYSTEM_PROMPT
|
|
assert "18" in SYSTEM_PROMPT
|
|
|
|
|
|
def test_system_prompt_mentions_no_place_of_publication():
|
|
# CMOS 14.30 / 18th ed. change. The formatter MUST know this.
|
|
lower = SYSTEM_PROMPT.lower()
|
|
assert "place of publication" in lower
|
|
|
|
|
|
def test_system_prompt_mentions_doi_preference():
|
|
assert "doi" in SYSTEM_PROMPT.lower()
|
|
|
|
|
|
def test_system_prompt_mentions_italic_markers():
|
|
# Plan v1 uses Markdown `*Title*` for italics.
|
|
assert "*" in SYSTEM_PROMPT and "italic" in SYSTEM_PROMPT.lower()
|
|
|
|
|
|
def test_build_user_message_contains_the_messy_input():
|
|
msg = build_user_message("yu, charles. interior chinatown. 2020")
|
|
assert "yu, charles. interior chinatown. 2020" in msg
|
|
|
|
|
|
def test_formatter_uses_injected_caller():
|
|
"""The formatter accepts a caller shim so tests (and the harness) can
|
|
substitute a fake OpenAI call. This is how the whole pipeline can run
|
|
without an API key during tests."""
|
|
recorded: dict = {}
|
|
|
|
def fake_caller(system: str, user: str) -> str:
|
|
recorded["system"] = system
|
|
recorded["user"] = user
|
|
return "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
|
|
|
|
output = format_bibliography_entry(
|
|
"yu, charles. interior chinatown. New York: Pantheon Books, 2020.",
|
|
caller=fake_caller,
|
|
)
|
|
assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
|
|
assert recorded["system"] == SYSTEM_PROMPT
|
|
assert "yu, charles" in recorded["user"]
|
|
|
|
|
|
def test_formatter_strips_whitespace_from_caller_output():
|
|
# Language-model output often has leading/trailing whitespace or
|
|
# surrounding code fences. v0 only strips whitespace; code-fence
|
|
# stripping can be added when an exemplar forces it.
|
|
def fake_caller(system: str, user: str) -> str:
|
|
return " Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020. \n"
|
|
|
|
output = format_bibliography_entry("anything", caller=fake_caller)
|
|
assert output == "Yu, Charles. *Interior Chinatown*. Pantheon Books, 2020."
|