v2 chunk 1: scaffold note formatter (Path B parallel artifact)
Begin v2 (in-text citation note form) as a parallel artifact to the
v1 bibliography formatter, per the Path B architectural decision:
no shared mutable state, no shared prompt content, no v1 changes.
New artifacts:
- src/cmos/parser.py: add find_notes() / NoteDefinition /
NotesParseResult as siblings to split_bibliography(). Targets
pandoc-style markdown footnote definitions [^marker]: text.
Single-line only; multi-line continuation deferred.
- src/cmos/note_formatter.py: new module mirroring formatter.py.
17-rule SYSTEM_PROMPT_NOTES for CMOS 18 first-occurrence note
form. Reuses cmos.runtime_validator.validate() unchanged — its
structural checks all apply to note form too. Caller injection,
retry loop, and model selection mirror v1.
- tests/test_parser.py: 7 new tests for find_notes().
- tests/test_note_formatter.py: 11 new tests mirroring v1 test
discipline (no API calls, fake-caller injection, retry semantics,
prompt smoke checks).
- exemplars/notes/: new subdirectory with 3 hand-synthesized
first-occurrence note exemplars (book, journal article, chapter
in edited book). Uses expected_note as the field name (not v1's
expected_bibliography). harness/score.py:load_exemplars uses a
non-recursive glob, so the v1 canary loader does not see these —
Path B isolation is automatic.
v1 untouched. v1 canary still loads exactly 17 exemplars. Full
test suite: 96 v1 + 18 new v2 = 114 passing.
Smoke-tested all 3 v2 exemplars against real GPT-5 (not fakes),
2 runs each. book_first_yu and journal_first_kwon: 4/4 byte-perfect
on the first try. chapter_first_doyle: 0/2, surfacing two known
prompt gaps for the next iteration:
1. Publisher abbreviation not expanded ("U of Chicago Press"
preserved instead of "University of Chicago Press"). v1's
formatter.py rule 14 is missing from the v2 prompt.
2. "ed." pluralized to "eds." for multiple editors. CMOS NB
uses "ed." invariantly regardless of editor count.
Both gaps are addressable prompt edits, not architectural problems —
exactly the kind of finding the dev loop is designed to surface.
Out of scope (deferred to chunk 2 and later): CLI extension,
shortened-form generation, document reassembly, harness scoring
loop integration, v2 linter rules, real-draft testing.
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
"""Tests for src/cmos/note_formatter.py — the v2 inner-loop iterable artifact.
|
||||
|
||||
Mirrors tests/test_formatter.py in shape and discipline. The note formatter
|
||||
is iterated independently of the bibliography formatter (Path B); these
|
||||
tests pin the stable seams (caller injection, retry loop, prompt smoke
|
||||
checks) and leave LLM output shape testing to the v2 exemplar canary.
|
||||
|
||||
No real API calls in this file. Tests use injected fake callers.
|
||||
"""
|
||||
|
||||
from cmos.note_formatter import (
|
||||
MODEL,
|
||||
SYSTEM_PROMPT_NOTES,
|
||||
build_user_message,
|
||||
format_note_entry,
|
||||
)
|
||||
|
||||
|
||||
def test_default_model_is_gpt5():
|
||||
# Same default as v1. Override with OPENAI_MODEL=... in .env.
|
||||
assert MODEL == "gpt-5"
|
||||
|
||||
|
||||
def test_system_prompt_mentions_cmos_18_note_form():
|
||||
assert "CMOS" in SYSTEM_PROMPT_NOTES
|
||||
assert "18" in SYSTEM_PROMPT_NOTES
|
||||
assert "NOTE" in SYSTEM_PROMPT_NOTES
|
||||
|
||||
|
||||
def test_system_prompt_mentions_no_inversion():
|
||||
# The single most distinctive difference from bibliography form: notes
|
||||
# use normal-order author names ("First Last"), not inverted.
|
||||
lower = SYSTEM_PROMPT_NOTES.lower()
|
||||
assert "normal" in lower or "not inverted" in lower
|
||||
|
||||
|
||||
def test_system_prompt_mentions_comma_separation():
|
||||
# The second most distinctive difference: commas between elements,
|
||||
# not periods.
|
||||
lower = SYSTEM_PROMPT_NOTES.lower()
|
||||
assert "comma" in lower
|
||||
|
||||
|
||||
def test_system_prompt_excludes_leading_number():
|
||||
# The note number prefix ("1. ", "2. ") is supplied externally; the
|
||||
# formatter must not include it. The prompt must say so.
|
||||
lower = SYSTEM_PROMPT_NOTES.lower()
|
||||
assert "leading number" in lower or "do not include" in lower
|
||||
|
||||
|
||||
def test_system_prompt_mentions_no_ibid():
|
||||
# CMOS 18 deprecates Ibid., same as v1.
|
||||
assert "Ibid" in SYSTEM_PROMPT_NOTES or "ibid" in SYSTEM_PROMPT_NOTES.lower()
|
||||
|
||||
|
||||
def test_build_user_message_contains_the_messy_input():
|
||||
msg = build_user_message("yu, charles. interior chinatown. 2020, 45")
|
||||
assert "yu, charles. interior chinatown. 2020, 45" in msg
|
||||
|
||||
|
||||
def test_note_formatter_uses_injected_caller():
|
||||
"""Same caller-injection seam as v1. Tests can substitute a fake without
|
||||
touching the network."""
|
||||
recorded: dict = {}
|
||||
|
||||
def fake_caller(system: str, user: str) -> str:
|
||||
recorded["system"] = system
|
||||
recorded["user"] = user
|
||||
return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
||||
|
||||
output = format_note_entry(
|
||||
"yu, charles. interior chinatown. New York: Pantheon Books, 2020, p 45.",
|
||||
caller=fake_caller,
|
||||
)
|
||||
assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
||||
assert recorded["system"] == SYSTEM_PROMPT_NOTES
|
||||
assert "yu, charles" in recorded["user"]
|
||||
|
||||
|
||||
def test_note_formatter_strips_whitespace_from_caller_output():
|
||||
def fake_caller(system: str, user: str) -> str:
|
||||
return " Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45. \n"
|
||||
|
||||
output = format_note_entry("anything", caller=fake_caller)
|
||||
assert output == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
||||
|
||||
|
||||
def test_note_formatter_retries_on_validator_failure():
|
||||
"""If the runtime validator rejects the first attempt (e.g., italics
|
||||
dropped), the formatter retries. Same retry semantics as v1."""
|
||||
attempts = []
|
||||
|
||||
def flaky_caller(system: str, user: str) -> str:
|
||||
attempts.append(len(attempts) + 1)
|
||||
if len(attempts) == 1:
|
||||
# First attempt: italics dropped — validator should fail.
|
||||
return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45."
|
||||
return "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
||||
|
||||
output = format_note_entry("messy yu", caller=flaky_caller)
|
||||
assert "*Interior Chinatown*" in output
|
||||
assert len(attempts) == 2
|
||||
|
||||
|
||||
def test_note_formatter_returns_last_attempt_after_max_retries():
|
||||
"""If every attempt fails validation, return the last attempt and don't
|
||||
loop forever. Same as v1 — the failure surfaces via scoring or human
|
||||
review, not via a runtime exception."""
|
||||
attempts = []
|
||||
|
||||
def always_bad(system: str, user: str) -> str:
|
||||
attempts.append(1)
|
||||
return "Charles Yu, Interior Chinatown (Pantheon Books, 2020), 45."
|
||||
|
||||
output = format_note_entry("messy", caller=always_bad, max_retries=2)
|
||||
assert len(attempts) == 3 # 1 initial + 2 retries
|
||||
assert "*" not in output
|
||||
+86
-1
@@ -4,11 +4,15 @@ v1 scope: locate a ``## Bibliography`` heading (case-insensitive) and return
|
||||
the prose before it, the list of entries inside it, and the prose after it.
|
||||
The section ends at the next ``##`` (level-2) heading or end of file.
|
||||
Blank lines and bullet markers (``- `` or ``* ``) are stripped from entries.
|
||||
|
||||
v2 (notes) parser scope: find pandoc-style markdown footnote definitions
|
||||
``[^marker]: text`` anywhere in the document. Single-line definitions only;
|
||||
multi-line continuation is deferred to a later chunk.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from cmos.parser import NoBibliographyError, split_bibliography
|
||||
from cmos.parser import NoBibliographyError, find_notes, split_bibliography
|
||||
|
||||
|
||||
def test_three_entries_under_bibliography_heading():
|
||||
@@ -74,3 +78,84 @@ Only entry.
|
||||
def test_no_bibliography_raises():
|
||||
with pytest.raises(NoBibliographyError):
|
||||
split_bibliography("## Introduction\n\nJust prose.\n")
|
||||
|
||||
|
||||
# --- find_notes (v2 parser, pandoc-style markdown footnote definitions) ----
|
||||
|
||||
|
||||
def test_find_notes_single_definition():
|
||||
draft = """\
|
||||
Some prose with a citation.[^1]
|
||||
|
||||
[^1]: Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45.
|
||||
"""
|
||||
result = find_notes(draft)
|
||||
assert len(result.definitions) == 1
|
||||
assert result.definitions[0].marker == "1"
|
||||
assert result.definitions[0].text == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
||||
|
||||
|
||||
def test_find_notes_multiple_definitions_in_order():
|
||||
draft = """\
|
||||
Prose.[^1] More prose.[^2] And again.[^3]
|
||||
|
||||
[^1]: First note.
|
||||
[^2]: Second note.
|
||||
[^3]: Third note.
|
||||
"""
|
||||
result = find_notes(draft)
|
||||
assert len(result.definitions) == 3
|
||||
assert [d.marker for d in result.definitions] == ["1", "2", "3"]
|
||||
assert [d.text for d in result.definitions] == [
|
||||
"First note.",
|
||||
"Second note.",
|
||||
"Third note.",
|
||||
]
|
||||
|
||||
|
||||
def test_find_notes_named_marker():
|
||||
draft = """\
|
||||
Citation.[^smith2020]
|
||||
|
||||
[^smith2020]: Smith, J. (2020). Some work.
|
||||
"""
|
||||
result = find_notes(draft)
|
||||
assert len(result.definitions) == 1
|
||||
assert result.definitions[0].marker == "smith2020"
|
||||
assert result.definitions[0].text == "Smith, J. (2020). Some work."
|
||||
|
||||
|
||||
def test_find_notes_returns_empty_when_no_definitions():
|
||||
draft = "Just prose with no footnote definitions.\n"
|
||||
result = find_notes(draft)
|
||||
assert result.definitions == []
|
||||
|
||||
|
||||
def test_find_notes_records_line_number():
|
||||
draft = """\
|
||||
Line zero.
|
||||
Line one.
|
||||
|
||||
[^1]: Note on line three.
|
||||
"""
|
||||
result = find_notes(draft)
|
||||
assert len(result.definitions) == 1
|
||||
assert result.definitions[0].line_number == 3
|
||||
|
||||
|
||||
def test_find_notes_strips_trailing_whitespace_from_text():
|
||||
draft = "[^1]: Note text with trailing spaces. \n"
|
||||
result = find_notes(draft)
|
||||
assert result.definitions[0].text == "Note text with trailing spaces."
|
||||
|
||||
|
||||
def test_find_notes_definition_can_appear_before_reference():
|
||||
# Pandoc allows definitions anywhere in the doc, not just at the bottom.
|
||||
draft = """\
|
||||
[^1]: A definition before the reference.
|
||||
|
||||
Some prose that cites it.[^1]
|
||||
"""
|
||||
result = find_notes(draft)
|
||||
assert len(result.definitions) == 1
|
||||
assert result.definitions[0].text == "A definition before the reference."
|
||||
|
||||
Reference in New Issue
Block a user