parser: support diverse md inputs (bold headings, Works Cited, blockquote notes, junk filtering)
- Bibliography heading now accepts bold markers (## **BIBLIOGRAPHY**), alternative names (Works Cited, References), and numbered sub-headings within the section. - Original heading text preserved in output instead of hardcoded "## Bibliography". - New find_blockquote_notes() parser for PDF-to-markdown footnote format (> N text), wired into CLI as third fallback after pandoc and numbered. - PDF junk filtered from bibliography entries: bare page numbers, blockquote footnotes, download banners, CC license URLs, and short running headers.
This commit is contained in:
+51
-18
@@ -51,14 +51,14 @@ def test_reformat_draft_preserves_sections_after_bibliography():
|
||||
draft = """\
|
||||
## Bibliography
|
||||
|
||||
one entry.
|
||||
Yu, Charles. *Interior Chinatown* (New York: Pantheon, 2020).
|
||||
|
||||
## Appendix
|
||||
|
||||
appendix text.
|
||||
"""
|
||||
output = reformat_draft(draft, formatter=_fake_formatter)
|
||||
assert "FORMATTED(one entry.)" in output
|
||||
assert "FORMATTED(Yu, Charles. *Interior Chinatown* (New York: Pantheon, 2020).)" in output
|
||||
assert "## Appendix" in output
|
||||
assert "appendix text." in output
|
||||
|
||||
@@ -69,11 +69,27 @@ def test_reformat_draft_raises_when_no_bibliography():
|
||||
|
||||
|
||||
def test_reformat_draft_bibliography_heading_preserved():
|
||||
draft = "## Bibliography\n\nentry.\n"
|
||||
draft = '## Bibliography\n\nYu, Charles. *Interior Chinatown* (Pantheon, 2020).\n'
|
||||
output = reformat_draft(draft, formatter=_fake_formatter)
|
||||
assert "## Bibliography" in output
|
||||
|
||||
|
||||
def test_reformat_draft_preserves_works_cited_heading():
|
||||
"""When the input heading is ``## Works Cited``, the output must
|
||||
keep that heading — not replace it with ``## Bibliography``."""
|
||||
draft = '## Works Cited\n\nDavidson, Donald. "On the Very Idea." (1974).\n'
|
||||
output = reformat_draft(draft, formatter=_fake_formatter)
|
||||
assert "## Works Cited" in output
|
||||
assert "## Bibliography" not in output
|
||||
|
||||
|
||||
def test_reformat_draft_preserves_bold_wrapped_heading():
|
||||
"""Bold markers in the heading should be preserved in the output."""
|
||||
draft = '## **Works Cited**\n\nDavidson, Donald. "On the Very Idea." (1974).\n'
|
||||
output = reformat_draft(draft, formatter=_fake_formatter)
|
||||
assert "## **Works Cited**" in output
|
||||
|
||||
|
||||
def test_python_dash_m_invocation_actually_runs_main():
|
||||
"""Regression test: `python -m cmos.cli` must actually invoke main().
|
||||
|
||||
@@ -216,6 +232,23 @@ def test_reformat_notes_auto_detects_numbered_format():
|
||||
assert "[^2]" not in output
|
||||
|
||||
|
||||
def test_reformat_notes_auto_detects_blockquote_format():
|
||||
"""PDF-to-markdown drafts use ``> N text`` blockquote footnotes.
|
||||
reformat_notes must auto-detect after pandoc and numbered fail."""
|
||||
draft = """\
|
||||
Some prose.
|
||||
|
||||
> 1 Davidson, "On the Very Idea of a Conceptual Scheme."
|
||||
> 2 Frankenberry 1999, 526.
|
||||
"""
|
||||
output = reformat_notes(draft, formatter=_fake_note_formatter)
|
||||
assert '> 1 NOTE_FORMATTED(Davidson, "On the Very Idea of a Conceptual Scheme.")' in output
|
||||
assert "> 2 NOTE_FORMATTED(Frankenberry 1999, 526.)" in output
|
||||
# No pandoc or numbered markers should appear.
|
||||
assert "[^1]" not in output
|
||||
assert "[1] " not in output
|
||||
|
||||
|
||||
def test_reformat_notes_pandoc_input_still_round_trips_as_pandoc():
|
||||
"""Regression: after the auto-detect change, pandoc-format input
|
||||
must still produce pandoc-format output. The original_prefix path
|
||||
@@ -265,28 +298,28 @@ def test_reformat_draft_preserves_order_under_concurrency():
|
||||
# later ones if order were naively tied to completion.
|
||||
import time
|
||||
|
||||
# Map each entry to a sleep duration so earlier entries finish last.
|
||||
order = {"2020": 4, "2021": 3, "2022": 2, "2023": 1, "2024": 0}
|
||||
|
||||
def slow_fake(messy: str) -> str:
|
||||
# Entries with lower index sleep longer so they finish last.
|
||||
n = int(messy.split()[-1])
|
||||
time.sleep(0.05 * (5 - n))
|
||||
year = messy.strip()[-5:-1] # extract "2020" etc.
|
||||
time.sleep(0.05 * order.get(year, 0))
|
||||
return f"FORMATTED({messy})"
|
||||
|
||||
draft = """\
|
||||
## Bibliography
|
||||
|
||||
entry 0
|
||||
entry 1
|
||||
entry 2
|
||||
entry 3
|
||||
entry 4
|
||||
Author, A. *Title Zero* (Publisher, 2020).
|
||||
Author, B. *Title One* (Publisher, 2021).
|
||||
Author, C. *Title Two* (Publisher, 2022).
|
||||
Author, D. *Title Three* (Publisher, 2023).
|
||||
Author, E. *Title Four* (Publisher, 2024).
|
||||
"""
|
||||
output = reformat_draft(draft, formatter=slow_fake, concurrency=4)
|
||||
# Entries must appear in input order.
|
||||
lines = [l for l in output.splitlines() if l.startswith("FORMATTED(")]
|
||||
assert lines == [
|
||||
"FORMATTED(entry 0)",
|
||||
"FORMATTED(entry 1)",
|
||||
"FORMATTED(entry 2)",
|
||||
"FORMATTED(entry 3)",
|
||||
"FORMATTED(entry 4)",
|
||||
]
|
||||
assert lines[0].startswith("FORMATTED(Author, A.")
|
||||
assert lines[1].startswith("FORMATTED(Author, B.")
|
||||
assert lines[2].startswith("FORMATTED(Author, C.")
|
||||
assert lines[3].startswith("FORMATTED(Author, D.")
|
||||
assert lines[4].startswith("FORMATTED(Author, E.")
|
||||
|
||||
Reference in New Issue
Block a user