parser: support diverse md inputs (bold headings, Works Cited, blockquote notes, junk filtering)

- Bibliography heading now accepts bold markers (## **BIBLIOGRAPHY**),
  alternative names (Works Cited, References), and numbered sub-headings
  within the section.
- Original heading text preserved in output instead of hardcoded
  "## Bibliography".
- New find_blockquote_notes() parser for PDF-to-markdown footnote format
  (> N text), wired into CLI as third fallback after pandoc and numbered.
- PDF junk filtered from bibliography entries: bare page numbers,
  blockquote footnotes, download banners, CC license URLs, and short
  running headers.
This commit is contained in:
Mark Eaton
2026-04-12 00:11:52 -04:00
parent c21ca7b58e
commit cad936f576
4 changed files with 441 additions and 52 deletions
+51 -18
View File
@@ -51,14 +51,14 @@ def test_reformat_draft_preserves_sections_after_bibliography():
draft = """\
## Bibliography
one entry.
Yu, Charles. *Interior Chinatown* (New York: Pantheon, 2020).
## Appendix
appendix text.
"""
output = reformat_draft(draft, formatter=_fake_formatter)
assert "FORMATTED(one entry.)" in output
assert "FORMATTED(Yu, Charles. *Interior Chinatown* (New York: Pantheon, 2020).)" in output
assert "## Appendix" in output
assert "appendix text." in output
@@ -69,11 +69,27 @@ def test_reformat_draft_raises_when_no_bibliography():
def test_reformat_draft_bibliography_heading_preserved():
draft = "## Bibliography\n\nentry.\n"
draft = '## Bibliography\n\nYu, Charles. *Interior Chinatown* (Pantheon, 2020).\n'
output = reformat_draft(draft, formatter=_fake_formatter)
assert "## Bibliography" in output
def test_reformat_draft_preserves_works_cited_heading():
"""When the input heading is ``## Works Cited``, the output must
keep that heading — not replace it with ``## Bibliography``."""
draft = '## Works Cited\n\nDavidson, Donald. "On the Very Idea." (1974).\n'
output = reformat_draft(draft, formatter=_fake_formatter)
assert "## Works Cited" in output
assert "## Bibliography" not in output
def test_reformat_draft_preserves_bold_wrapped_heading():
"""Bold markers in the heading should be preserved in the output."""
draft = '## **Works Cited**\n\nDavidson, Donald. "On the Very Idea." (1974).\n'
output = reformat_draft(draft, formatter=_fake_formatter)
assert "## **Works Cited**" in output
def test_python_dash_m_invocation_actually_runs_main():
"""Regression test: `python -m cmos.cli` must actually invoke main().
@@ -216,6 +232,23 @@ def test_reformat_notes_auto_detects_numbered_format():
assert "[^2]" not in output
def test_reformat_notes_auto_detects_blockquote_format():
"""PDF-to-markdown drafts use ``> N text`` blockquote footnotes.
reformat_notes must auto-detect after pandoc and numbered fail."""
draft = """\
Some prose.
> 1 Davidson, "On the Very Idea of a Conceptual Scheme."
> 2 Frankenberry 1999, 526.
"""
output = reformat_notes(draft, formatter=_fake_note_formatter)
assert '> 1 NOTE_FORMATTED(Davidson, "On the Very Idea of a Conceptual Scheme.")' in output
assert "> 2 NOTE_FORMATTED(Frankenberry 1999, 526.)" in output
# No pandoc or numbered markers should appear.
assert "[^1]" not in output
assert "[1] " not in output
def test_reformat_notes_pandoc_input_still_round_trips_as_pandoc():
"""Regression: after the auto-detect change, pandoc-format input
must still produce pandoc-format output. The original_prefix path
@@ -265,28 +298,28 @@ def test_reformat_draft_preserves_order_under_concurrency():
# later ones if order were naively tied to completion.
import time
# Map each entry to a sleep duration so earlier entries finish last.
order = {"2020": 4, "2021": 3, "2022": 2, "2023": 1, "2024": 0}
def slow_fake(messy: str) -> str:
# Entries with lower index sleep longer so they finish last.
n = int(messy.split()[-1])
time.sleep(0.05 * (5 - n))
year = messy.strip()[-5:-1] # extract "2020" etc.
time.sleep(0.05 * order.get(year, 0))
return f"FORMATTED({messy})"
draft = """\
## Bibliography
entry 0
entry 1
entry 2
entry 3
entry 4
Author, A. *Title Zero* (Publisher, 2020).
Author, B. *Title One* (Publisher, 2021).
Author, C. *Title Two* (Publisher, 2022).
Author, D. *Title Three* (Publisher, 2023).
Author, E. *Title Four* (Publisher, 2024).
"""
output = reformat_draft(draft, formatter=slow_fake, concurrency=4)
# Entries must appear in input order.
lines = [l for l in output.splitlines() if l.startswith("FORMATTED(")]
assert lines == [
"FORMATTED(entry 0)",
"FORMATTED(entry 1)",
"FORMATTED(entry 2)",
"FORMATTED(entry 3)",
"FORMATTED(entry 4)",
]
assert lines[0].startswith("FORMATTED(Author, A.")
assert lines[1].startswith("FORMATTED(Author, B.")
assert lines[2].startswith("FORMATTED(Author, C.")
assert lines[3].startswith("FORMATTED(Author, D.")
assert lines[4].startswith("FORMATTED(Author, E.")