Mark Eaton
45f51c6341
Extend the v2 parser and CLI to handle [N] text footnote definitions in addition to pandoc-style [^marker]: text. The numbered format is what docx-to-text conversion of footnoted Word documents produces. The user's "Anti-Communist Formations of LIS" draft uses this format for all 107 of its notes; without this support, v2 was structurally incapable of running on real-world docx-derived inputs. src/cmos/parser.py: - Add NoteDefinition.original_prefix field so reassembly can round-trip the source's marker syntax (pandoc input → pandoc output, numbered input → numbered output) without consumers needing to know which format was matched. - Update find_notes() to populate original_prefix as "[^N]: ". - Add find_numbered_notes() targeting "[N] text" definitions. Marker must be all digits (rejects [Smith 2020], [foo], etc.); caret prefix is rejected (rejects pandoc-style cleanly). src/cmos/cli.py: - reformat_notes now auto-detects format: tries find_notes first, falls back to find_numbered_notes if no pandoc definitions found. Uses definition.original_prefix for reassembly so both formats round-trip correctly. tests/test_parser.py: - 11 new tests for find_numbered_notes covering: single/multiple definitions, multi-digit markers, line number recording, trailing whitespace stripping, ignoring pandoc/non-numeric markers, original prefix recording, and the actual Anti-Communist draft format. - 1 new test for find_notes original_prefix population. tests/test_cli.py: - 2 new tests for reformat_notes auto-detect: numbered input round- trips as numbered output, pandoc input still round-trips as pandoc. Total suite: 136/136 (was 123, +13 net new). v1 untouched, 96/96 v1 tests still passing. Real-draft validation: ran cmos format-notes against the 107-note Anti-Communist Formations of LIS draft (108 calls in parallel via the existing concurrency=8 thread pool, completed cleanly). Output saved to /tmp (not committed). All 107 notes preserved through the pipeline; ~30-40 first-occurrence full notes produced clean CMOS 18 note form; ~30 shortened-form refs correctly left unchanged; 2 real bugs surfaced for the next iteration (empty input → conver- sational reply, Ibid → empty string), plus several lower-priority issues (substantive note truncation, retry waste on shortened forms, lossy month dropping). Not addressed in this chunk per the "collect signal, don't fix" plan.
295 lines
9.1 KiB
Python
295 lines
9.1 KiB
Python
"""Tests for src/cmos/parser.py — extract the bibliography section from a draft.
|
|
|
|
v1 scope: locate a ``## Bibliography`` heading (case-insensitive) and return
|
|
the prose before it, the list of entries inside it, and the prose after it.
|
|
The section ends at the next ``##`` (level-2) heading or end of file.
|
|
Blank lines and bullet markers (``- `` or ``* ``) are stripped from entries.
|
|
|
|
v2 (notes) parser scope: two complementary functions for finding footnote
|
|
definitions in two distinct source formats:
|
|
|
|
- ``find_notes`` finds pandoc-style ``[^marker]: text`` definitions
|
|
(caret-prefixed marker, colon after the closing bracket).
|
|
- ``find_numbered_notes`` finds ``[N] text`` definitions (square bracket
|
|
numeric marker, no caret, no colon, just a space before the body) —
|
|
the format produced by docx-to-text conversion of footnoted Word docs.
|
|
|
|
Both functions return ``NotesParseResult`` with a list of ``NoteDefinition``
|
|
records carrying the marker, body text, line number, and the original
|
|
prefix string (so reassembly can round-trip the source's marker syntax).
|
|
|
|
Single-line definitions only in both formats; multi-line continuation is
|
|
deferred.
|
|
"""
|
|
|
|
import pytest
|
|
|
|
from cmos.parser import (
|
|
NoBibliographyError,
|
|
find_notes,
|
|
find_numbered_notes,
|
|
split_bibliography,
|
|
)
|
|
|
|
|
|
def test_three_entries_under_bibliography_heading():
|
|
draft = """\
|
|
Some prose here.
|
|
|
|
More prose.
|
|
|
|
## Bibliography
|
|
|
|
yu, charles. interior chinatown. New York: Pantheon Books, 2020.
|
|
Kwon, Hyeyoung. "inclusion work." American Journal of Sociology 127, no. 6 (2022): 1818-1859.
|
|
google, "privacy policy," privacy & terms, nov 15 2023, policies.google.com/privacy
|
|
"""
|
|
result = split_bibliography(draft)
|
|
assert len(result.entries) == 3
|
|
assert result.entries[0].startswith("yu, charles")
|
|
assert result.entries[1].startswith("Kwon, Hyeyoung")
|
|
assert result.entries[2].startswith("google")
|
|
assert "Some prose here." in result.before
|
|
assert result.after == ""
|
|
|
|
|
|
def test_section_ends_at_next_level_two_heading():
|
|
draft = """\
|
|
## Bibliography
|
|
|
|
First entry.
|
|
Second entry.
|
|
|
|
## Appendix
|
|
|
|
Appendix content here.
|
|
"""
|
|
result = split_bibliography(draft)
|
|
assert result.entries == ["First entry.", "Second entry."]
|
|
assert "## Appendix" in result.after
|
|
assert "Appendix content here." in result.after
|
|
|
|
|
|
def test_bullet_prefixes_stripped():
|
|
draft = """\
|
|
## Bibliography
|
|
|
|
- First entry.
|
|
- Second entry.
|
|
* Third entry.
|
|
"""
|
|
result = split_bibliography(draft)
|
|
assert result.entries == ["First entry.", "Second entry.", "Third entry."]
|
|
|
|
|
|
def test_heading_match_is_case_insensitive():
|
|
draft = """\
|
|
## bibliography
|
|
|
|
Only entry.
|
|
"""
|
|
result = split_bibliography(draft)
|
|
assert result.entries == ["Only entry."]
|
|
|
|
|
|
def test_no_bibliography_raises():
|
|
with pytest.raises(NoBibliographyError):
|
|
split_bibliography("## Introduction\n\nJust prose.\n")
|
|
|
|
|
|
# --- find_notes (v2 parser, pandoc-style markdown footnote definitions) ----
|
|
|
|
|
|
def test_find_notes_single_definition():
|
|
draft = """\
|
|
Some prose with a citation.[^1]
|
|
|
|
[^1]: Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45.
|
|
"""
|
|
result = find_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].marker == "1"
|
|
assert result.definitions[0].text == "Charles Yu, *Interior Chinatown* (Pantheon Books, 2020), 45."
|
|
|
|
|
|
def test_find_notes_multiple_definitions_in_order():
|
|
draft = """\
|
|
Prose.[^1] More prose.[^2] And again.[^3]
|
|
|
|
[^1]: First note.
|
|
[^2]: Second note.
|
|
[^3]: Third note.
|
|
"""
|
|
result = find_notes(draft)
|
|
assert len(result.definitions) == 3
|
|
assert [d.marker for d in result.definitions] == ["1", "2", "3"]
|
|
assert [d.text for d in result.definitions] == [
|
|
"First note.",
|
|
"Second note.",
|
|
"Third note.",
|
|
]
|
|
|
|
|
|
def test_find_notes_named_marker():
|
|
draft = """\
|
|
Citation.[^smith2020]
|
|
|
|
[^smith2020]: Smith, J. (2020). Some work.
|
|
"""
|
|
result = find_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].marker == "smith2020"
|
|
assert result.definitions[0].text == "Smith, J. (2020). Some work."
|
|
|
|
|
|
def test_find_notes_returns_empty_when_no_definitions():
|
|
draft = "Just prose with no footnote definitions.\n"
|
|
result = find_notes(draft)
|
|
assert result.definitions == []
|
|
|
|
|
|
def test_find_notes_records_line_number():
|
|
draft = """\
|
|
Line zero.
|
|
Line one.
|
|
|
|
[^1]: Note on line three.
|
|
"""
|
|
result = find_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].line_number == 3
|
|
|
|
|
|
def test_find_notes_strips_trailing_whitespace_from_text():
|
|
draft = "[^1]: Note text with trailing spaces. \n"
|
|
result = find_notes(draft)
|
|
assert result.definitions[0].text == "Note text with trailing spaces."
|
|
|
|
|
|
def test_find_notes_definition_can_appear_before_reference():
|
|
# Pandoc allows definitions anywhere in the doc, not just at the bottom.
|
|
draft = """\
|
|
[^1]: A definition before the reference.
|
|
|
|
Some prose that cites it.[^1]
|
|
"""
|
|
result = find_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].text == "A definition before the reference."
|
|
|
|
|
|
def test_find_notes_records_original_prefix_for_pandoc_format():
|
|
"""Reassembly needs the literal prefix string so the round-trip
|
|
preserves the source's marker syntax. For pandoc this is `[^N]: `."""
|
|
draft = "[^1]: text.\n[^smith2020]: text.\n"
|
|
result = find_notes(draft)
|
|
assert result.definitions[0].original_prefix == "[^1]: "
|
|
assert result.definitions[1].original_prefix == "[^smith2020]: "
|
|
|
|
|
|
# --- find_numbered_notes ([N] text format, no caret, no colon) -------------
|
|
|
|
|
|
def test_find_numbered_notes_single_definition():
|
|
draft = """\
|
|
Some prose.
|
|
|
|
[1] Charles Yu, Interior Chinatown. Pantheon, 2020. p. 45.
|
|
"""
|
|
result = find_numbered_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].marker == "1"
|
|
assert result.definitions[0].text == "Charles Yu, Interior Chinatown. Pantheon, 2020. p. 45."
|
|
|
|
|
|
def test_find_numbered_notes_multiple_sequential():
|
|
draft = """\
|
|
[1] First.
|
|
[2] Second.
|
|
[3] Third.
|
|
"""
|
|
result = find_numbered_notes(draft)
|
|
assert len(result.definitions) == 3
|
|
assert [d.marker for d in result.definitions] == ["1", "2", "3"]
|
|
assert [d.text for d in result.definitions] == ["First.", "Second.", "Third."]
|
|
|
|
|
|
def test_find_numbered_notes_multi_digit_markers():
|
|
"""The Anti-Communist draft has 107 notes, including [107]. Multi-digit
|
|
markers must work."""
|
|
draft = "[1] one.\n[42] forty-two.\n[107] one-oh-seven.\n"
|
|
result = find_numbered_notes(draft)
|
|
assert [d.marker for d in result.definitions] == ["1", "42", "107"]
|
|
|
|
|
|
def test_find_numbered_notes_returns_empty_when_no_definitions():
|
|
draft = "Just prose with no numbered notes.\n"
|
|
result = find_numbered_notes(draft)
|
|
assert result.definitions == []
|
|
|
|
|
|
def test_find_numbered_notes_records_line_number():
|
|
draft = """\
|
|
Line zero.
|
|
Line one.
|
|
|
|
[1] Note on line three.
|
|
"""
|
|
result = find_numbered_notes(draft)
|
|
assert result.definitions[0].line_number == 3
|
|
|
|
|
|
def test_find_numbered_notes_strips_trailing_whitespace_from_text():
|
|
draft = "[1] Note text with trailing spaces. \n"
|
|
result = find_numbered_notes(draft)
|
|
assert result.definitions[0].text == "Note text with trailing spaces."
|
|
|
|
|
|
def test_find_numbered_notes_ignores_pandoc_format():
|
|
"""`[^1]: text` is pandoc format, not numbered. find_numbered_notes
|
|
must NOT match it (or the two parsers would step on each other when
|
|
used in fallback fashion). The `^` after `[` disqualifies it."""
|
|
draft = "[^1]: pandoc-style note.\n"
|
|
result = find_numbered_notes(draft)
|
|
assert result.definitions == []
|
|
|
|
|
|
def test_find_numbered_notes_ignores_non_numeric_markers():
|
|
"""A marker like [Smith 2020] or [foo] is NOT a numbered footnote
|
|
definition — it could be many other things (citation reference, link
|
|
label, etc.). Only digit markers count."""
|
|
draft = """\
|
|
[Smith 2020] Some text — looks like an author-date reference.
|
|
[foo] some kind of label.
|
|
[1] Actual numbered note.
|
|
"""
|
|
result = find_numbered_notes(draft)
|
|
assert len(result.definitions) == 1
|
|
assert result.definitions[0].marker == "1"
|
|
|
|
|
|
def test_find_numbered_notes_records_original_prefix():
|
|
"""For numbered format the prefix is `[N] ` — square bracket, number,
|
|
closing bracket, single space. Reassembly will use this verbatim."""
|
|
draft = "[1] text.\n[107] text.\n"
|
|
result = find_numbered_notes(draft)
|
|
assert result.definitions[0].original_prefix == "[1] "
|
|
assert result.definitions[1].original_prefix == "[107] "
|
|
|
|
|
|
def test_find_numbered_notes_handles_anti_communist_draft_format():
|
|
"""Smoke test mirroring the actual Anti-Communist Formations draft:
|
|
notes follow a `## Notes` heading, separated by `________________` rule,
|
|
one definition per line, sequentially numbered."""
|
|
draft = """\
|
|
## Notes
|
|
________________
|
|
[1] First reference. https://example.org/path
|
|
[2] Second reference, abbreviated.
|
|
[3] Rosen, 7.
|
|
"""
|
|
result = find_numbered_notes(draft)
|
|
assert len(result.definitions) == 3
|
|
assert result.definitions[0].text == "First reference. https://example.org/path"
|
|
assert result.definitions[2].text == "Rosen, 7."
|