337 lines
12 KiB
Python
337 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Script to download pictures of saints from Wikipedia.
|
|
Run this once to populate the pics/ directory.
|
|
"""
|
|
|
|
import csv
|
|
import requests
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
import time
|
|
import random
|
|
|
|
# Entries that are holidays, feasts, or events rather than actual saints/people
|
|
NON_PERSON_ENTRIES = {
|
|
"Epiphany",
|
|
"Holy Name of Jesus",
|
|
"Conversion of St. Paul",
|
|
"Presentation of the Lord",
|
|
"Our Lady of Lourdes",
|
|
"Chair of St. Peter",
|
|
"Annunciation",
|
|
"Our Lady of Fatima",
|
|
"Visitation of the Virgin Mary",
|
|
"Nativity of John the Baptist",
|
|
"First Martyrs of Rome",
|
|
"Our Lady of Mount Carmel",
|
|
"Dedication of St. Mary Major",
|
|
"Transfiguration",
|
|
"Assumption of Mary",
|
|
"Queenship of Mary",
|
|
"Beheading of John the Baptist",
|
|
"Nativity of Mary",
|
|
"Holy Name of Mary",
|
|
"Exaltation of the Holy Cross",
|
|
"Our Lady of Sorrows",
|
|
"Guardian Angels",
|
|
"Our Lady of the Rosary",
|
|
"All Saints",
|
|
"All Souls",
|
|
"Dedication of the Lateran Basilica",
|
|
"Dedication of the Basilicas of Peter and Paul",
|
|
"Presentation of Mary",
|
|
"Immaculate Conception",
|
|
"Our Lady of Guadalupe",
|
|
"Holy Innocents",
|
|
}
|
|
|
|
WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"
|
|
|
|
# Wikimedia requires a descriptive User-Agent per their policy
|
|
# https://meta.wikimedia.org/wiki/User-Agent_policy
|
|
USER_AGENT = "SaintsPictureDownloader/1.0 (Educational project for French saints calendar)"
|
|
|
|
# Mapping of CSV saint names to alternate Wikipedia article titles to try
|
|
# Many saints have disambiguation issues or different article naming conventions
|
|
ALTERNATE_TITLES = {
|
|
"Agatha": ["Agatha of Sicily"],
|
|
"Agnes": ["Agnes of Rome"],
|
|
"Albert the Great": ["Albertus Magnus"],
|
|
"Andrew Dung-Lac and Companions": ["Andrew Dung-Lac"],
|
|
"Andrew Kim Taegon and Companions": ["Andrew Kim Taegon"],
|
|
"Anselm": ["Anselm of Canterbury"],
|
|
"Anthony of Egypt": ["Anthony the Great"],
|
|
"Apollinaris": ["Apollinaris of Ravenna"],
|
|
"Athanasius": ["Athanasius of Alexandria"],
|
|
"Augustine": ["Augustine of Hippo"],
|
|
"Augustine Zhao Rong and Companions": ["Augustine Zhao Rong"],
|
|
"Bartholomew": ["Bartholomew the Apostle"],
|
|
"Basil the Great and Gregory Nazianzen": ["Basil of Caesarea", "Gregory of Nazianzus"],
|
|
"Bede the Venerable": ["Bede"],
|
|
"Benedict": ["Benedict of Nursia"],
|
|
"Bernardine of Siena": ["Bernardino of Siena"],
|
|
"Blaise": ["Saint Blaise"],
|
|
"Boniface": ["Saint Boniface"],
|
|
"Bruno": ["Bruno of Cologne"],
|
|
"Cajetan": ["Saint Cajetan", "Gaetano dei Conti di Thiene"],
|
|
"Callistus I": ["Pope Callixtus I"],
|
|
"Casimir": ["Casimir Jagiellon", "Saint Casimir"],
|
|
"Charles Lwanga and Companions": ["Charles Lwanga"],
|
|
"Christopher Magallanes and Companions": ["Cristóbal Magallanes Jara"],
|
|
"Clare": ["Clare of Assisi"],
|
|
"Clement I": ["Pope Clement I"],
|
|
"Columban": ["Columbanus"],
|
|
"Cornelius and Cyprian": ["Pope Cornelius", "Cyprian"],
|
|
"Damasus I": ["Pope Damasus I"],
|
|
"Denis and Companions": ["Denis of Paris", "Saint Denis"],
|
|
"Ephrem": ["Ephrem the Syrian"],
|
|
"Fabian": ["Pope Fabian"],
|
|
"George": ["Saint George"],
|
|
"Gertrude": ["Gertrude the Great"],
|
|
"Gregory VII": ["Pope Gregory VII"],
|
|
"Gregory the Great": ["Pope Gregory I"],
|
|
"Hedwig": ["Hedwig of Silesia"],
|
|
"Henry": ["Henry II, Holy Roman Emperor"],
|
|
"Isaac Jogues and John de Brebeuf": ["Isaac Jogues", "Jean de Brébeuf"],
|
|
"Isidore the Farmer": ["Isidore the Laborer"],
|
|
"James": ["James the Great", "James, son of Zebedee"],
|
|
"Jerome Emiliani": ["Jerome Emiliani"],
|
|
"Joachim and Anne": ["Saint Joachim", "Saint Anne"],
|
|
"John": ["John the Apostle"],
|
|
"John Baptist de la Salle": ["Jean-Baptiste de La Salle"],
|
|
"John Damascene": ["John of Damascus"],
|
|
"John Fisher and Thomas More": ["John Fisher", "Thomas More"],
|
|
"John I": ["Pope John I"],
|
|
"John Paul II": ["Pope John Paul II"],
|
|
"John XXIII": ["Pope John XXIII"],
|
|
"John of Kanty": ["John Cantius"],
|
|
"Josaphat": ["Josaphat Kuntsevych"],
|
|
"Joseph the Worker": ["Saint Joseph"],
|
|
"Justin": ["Justin Martyr"],
|
|
"Lawrence": ["Saint Lawrence", "Lawrence of Rome"],
|
|
"Lawrence Ruiz and Companions": ["Lorenzo Ruiz", "Lawrence Ruiz"],
|
|
"Leo the Great": ["Pope Leo I"],
|
|
"Louis IX": ["Louis IX of France"],
|
|
"Luke": ["Luke the Evangelist"],
|
|
"Margaret of Scotland": ["Saint Margaret of Scotland"],
|
|
"Mark": ["Mark the Evangelist"],
|
|
"Martin I": ["Pope Martin I"],
|
|
"Mary Magdalene de Pazzi": ["Mary Magdalene de' Pazzi"],
|
|
"Mary Mother of God": ["Mary, mother of Jesus"],
|
|
"Mary and Martha": ["Mary of Bethany", "Martha of Bethany"],
|
|
"Matthew": ["Matthew the Apostle"],
|
|
"Matthias": ["Matthias the Apostle"],
|
|
"Michael": ["Michael (archangel)"],
|
|
"Monica": ["Saint Monica"],
|
|
"Pancras": ["Pancras of Rome"],
|
|
"Patrick": ["Saint Patrick"],
|
|
"Paul Miki and Companions": ["Paul Miki"],
|
|
"Peter and Paul": ["Saint Peter", "Paul the Apostle"],
|
|
"Philip and James": ["Philip the Apostle", "James, son of Alphaeus"],
|
|
"Pius V": ["Pope Pius V"],
|
|
"Pius X": ["Pope Pius X"],
|
|
"Pius of Pietrelcina": ["Padre Pio"],
|
|
"Pontian and Hippolytus": ["Pope Pontian", "Hippolytus of Rome"],
|
|
"Raymond of Penyafort": ["Raymond of Peñafort"],
|
|
"Sebastian": ["Saint Sebastian"],
|
|
"Seven Founders of the Servite Order": ["Seven Holy Founders of the Servite Order"],
|
|
"Sharbel Makhluf": ["Charbel Makhluf"],
|
|
"Simon and Jude": ["Simon the Zealot", "Jude the Apostle"],
|
|
"Sixtus II and Companions": ["Pope Sixtus II"],
|
|
"Stanislaus": ["Stanislaus of Szczepanów"],
|
|
"Stephen of Hungary": ["Stephen I of Hungary"],
|
|
"Sylvester I": ["Pope Sylvester I"],
|
|
"Teresa Benedicta of the Cross": ["Edith Stein"],
|
|
"Teresa of Avila": ["Teresa of Ávila"],
|
|
"Therese of the Child Jesus": ["Thérèse of Lisieux"],
|
|
"Thomas": ["Thomas the Apostle"],
|
|
"Timothy and Titus": ["Saint Timothy", "Saint Titus"],
|
|
"Valentine": ["Saint Valentine"],
|
|
"Wenceslaus": ["Wenceslaus I, Duke of Bohemia"],
|
|
}
|
|
|
|
|
|
def create_session() -> requests.Session:
|
|
"""Create a requests session with proper headers for Wikimedia."""
|
|
session = requests.Session()
|
|
session.headers.update({
|
|
"User-Agent": USER_AGENT,
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
"Accept-Encoding": "gzip, deflate",
|
|
"Connection": "keep-alive",
|
|
})
|
|
return session
|
|
|
|
|
|
def get_standardized_name(saint_name: str) -> str:
|
|
"""Get the standardized name used for Wikipedia URLs and filenames."""
|
|
return saint_name.replace(" ", "_")
|
|
|
|
|
|
def get_all_saints() -> list[str]:
|
|
"""Read the CSV and return all unique saint names (excluding non-person entries)."""
|
|
csv_path = Path(__file__).parent / "saints_days_france.csv"
|
|
saints = set()
|
|
|
|
with open(csv_path, "r", encoding="utf-8") as f:
|
|
reader = csv.DictReader(f)
|
|
for row in reader:
|
|
saint_name = row["Saint Name"]
|
|
if saint_name not in NON_PERSON_ENTRIES:
|
|
saints.add(saint_name)
|
|
|
|
return sorted(saints)
|
|
|
|
|
|
def get_wikipedia_image_url(session: requests.Session, page_title: str) -> str | None:
|
|
"""
|
|
Use Wikipedia API to get the main image URL for a page.
|
|
Uses imageinfo prop to get the direct file URL.
|
|
Returns the image URL or None if no image found.
|
|
"""
|
|
# First, get the page image filename
|
|
params = {
|
|
"action": "query",
|
|
"titles": page_title,
|
|
"prop": "pageimages",
|
|
"piprop": "name", # Get the filename instead of thumbnail URL
|
|
"format": "json",
|
|
}
|
|
|
|
try:
|
|
response = session.get(WIKIPEDIA_API, params=params, timeout=10)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
pages = data.get("query", {}).get("pages", {})
|
|
image_filename = None
|
|
for page_id, page_data in pages.items():
|
|
if page_id == "-1":
|
|
return None
|
|
image_filename = page_data.get("pageimage")
|
|
break
|
|
|
|
if not image_filename:
|
|
return None
|
|
|
|
# Now get the actual image URL using imageinfo
|
|
params = {
|
|
"action": "query",
|
|
"titles": f"File:{image_filename}",
|
|
"prop": "imageinfo",
|
|
"iiprop": "url",
|
|
"iiurlwidth": 500, # Request a 500px wide thumbnail
|
|
"format": "json",
|
|
}
|
|
|
|
response = session.get(WIKIPEDIA_API, params=params, timeout=10)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
|
|
pages = data.get("query", {}).get("pages", {})
|
|
for page_id, page_data in pages.items():
|
|
# Files on Commons may show as "missing" (-1) from enwiki but still have imageinfo
|
|
imageinfo_list = page_data.get("imageinfo", [])
|
|
if imageinfo_list:
|
|
imageinfo = imageinfo_list[0]
|
|
# Prefer thumburl (resized), fall back to original url
|
|
return imageinfo.get("thumburl") or imageinfo.get("url")
|
|
|
|
except requests.RequestException as e:
|
|
print(f" Error fetching Wikipedia API: {e}")
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
def download_image(session: requests.Session, url: str, filepath: Path) -> bool:
|
|
"""Download an image from URL and save to filepath."""
|
|
try:
|
|
# Add image-specific accept header for this request
|
|
headers = {"Accept": "image/webp,image/apng,image/*,*/*;q=0.8"}
|
|
response = session.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
|
|
with open(filepath, "wb") as f:
|
|
f.write(response.content)
|
|
return True
|
|
|
|
except requests.RequestException as e:
|
|
print(f" Error downloading image: {e}")
|
|
return False
|
|
|
|
|
|
def main():
|
|
pics_dir = Path(__file__).parent / "pics"
|
|
pics_dir.mkdir(exist_ok=True)
|
|
|
|
saints = get_all_saints()
|
|
print(f"Found {len(saints)} saints to process.\n")
|
|
|
|
# Create a persistent session
|
|
session = create_session()
|
|
|
|
downloaded = 0
|
|
skipped = 0
|
|
failed = 0
|
|
|
|
for saint_name in saints:
|
|
standardized_name = get_standardized_name(saint_name)
|
|
|
|
# Check if we already have an image for this saint
|
|
existing_files = list(pics_dir.glob(f"{standardized_name}.*"))
|
|
if existing_files:
|
|
print(f"[SKIP] {saint_name} - already have {existing_files[0].name}")
|
|
skipped += 1
|
|
continue
|
|
|
|
print(f"[FETCH] {saint_name}...")
|
|
|
|
# Get image URL from Wikipedia - try primary name first, then alternates
|
|
image_url = get_wikipedia_image_url(session, saint_name)
|
|
|
|
if not image_url and saint_name in ALTERNATE_TITLES:
|
|
for alt_title in ALTERNATE_TITLES[saint_name]:
|
|
print(f" Trying alternate: {alt_title}")
|
|
image_url = get_wikipedia_image_url(session, alt_title)
|
|
if image_url:
|
|
break
|
|
time.sleep(0.3) # Small delay between alternate attempts
|
|
|
|
if not image_url:
|
|
print(f" No image found on Wikipedia")
|
|
failed += 1
|
|
continue
|
|
|
|
# Determine file extension from URL
|
|
url_path = urllib.parse.urlparse(image_url).path
|
|
ext = Path(url_path).suffix.lower()
|
|
# Handle URLs with size suffix like .jpg/500px-foo.jpg
|
|
if ext not in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
|
|
# Try to extract from the path
|
|
for known_ext in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
|
|
if known_ext in url_path.lower():
|
|
ext = known_ext
|
|
break
|
|
else:
|
|
ext = ".jpg" # Default fallback
|
|
|
|
filepath = pics_dir / f"{standardized_name}{ext}"
|
|
|
|
if download_image(session, image_url, filepath):
|
|
print(f" Saved to {filepath.name}")
|
|
downloaded += 1
|
|
else:
|
|
failed += 1
|
|
|
|
# Be nice to Wikipedia's servers - random delay between 0.5 and 1.5 seconds
|
|
time.sleep(0.5 + random.random())
|
|
|
|
print(f"\nDone! Downloaded: {downloaded}, Skipped: {skipped}, Failed: {failed}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|