saints/download_saint_pictures.py

#!/usr/bin/env python3
"""
Script to download pictures of saints from Wikipedia.
Run this once to populate the pics/ directory.
"""

import csv
import requests
import urllib.parse
from pathlib import Path
import time
import random

# Entries that are holidays, feasts, or events rather than actual saints/people
NON_PERSON_ENTRIES = {
    "Epiphany",
    "Holy Name of Jesus",
    "Conversion of St. Paul",
    "Presentation of the Lord",
    "Our Lady of Lourdes",
    "Chair of St. Peter",
    "Annunciation",
    "Our Lady of Fatima",
    "Visitation of the Virgin Mary",
    "Nativity of John the Baptist",
    "First Martyrs of Rome",
    "Our Lady of Mount Carmel",
    "Dedication of St. Mary Major",
    "Transfiguration",
    "Assumption of Mary",
    "Queenship of Mary",
    "Beheading of John the Baptist",
    "Nativity of Mary",
    "Holy Name of Mary",
    "Exaltation of the Holy Cross",
    "Our Lady of Sorrows",
    "Guardian Angels",
    "Our Lady of the Rosary",
    "All Saints",
    "All Souls",
    "Dedication of the Lateran Basilica",
    "Dedication of the Basilicas of Peter and Paul",
    "Presentation of Mary",
    "Immaculate Conception",
    "Our Lady of Guadalupe",
    "Holy Innocents",
}

WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"

# Wikimedia requires a descriptive User-Agent per their policy
# https://meta.wikimedia.org/wiki/User-Agent_policy
USER_AGENT = (
    "SaintsPictureDownloader/1.0 (Educational project for French saints calendar)"
)

# Mapping of CSV saint names to alternate Wikipedia article titles to try
# Many saints have disambiguation issues or different article naming conventions
ALTERNATE_TITLES = {
    "Agatha": ["Agatha of Sicily"],
    "Agnes": ["Agnes of Rome"],
    "Albert the Great": ["Albertus Magnus"],
    "Andrew Dung-Lac and Companions": ["Andrew Dung-Lac"],
    "Andrew Kim Taegon and Companions": ["Andrew Kim Taegon"],
    "Anselm": ["Anselm of Canterbury"],
    "Anthony of Egypt": ["Anthony the Great"],
    "Apollinaris": ["Apollinaris of Ravenna"],
    "Athanasius": ["Athanasius of Alexandria"],
    "Augustine": ["Augustine of Hippo"],
    "Augustine Zhao Rong and Companions": ["Augustine Zhao Rong"],
    "Bartholomew": ["Bartholomew the Apostle"],
    "Basil the Great and Gregory Nazianzen": [
        "Basil of Caesarea",
        "Gregory of Nazianzus",
    ],
    "Bede the Venerable": ["Bede"],
    "Benedict": ["Benedict of Nursia"],
    "Bernardine of Siena": ["Bernardino of Siena"],
    "Blaise": ["Saint Blaise"],
    "Boniface": ["Saint Boniface"],
    "Bruno": ["Bruno of Cologne"],
    "Cajetan": ["Saint Cajetan", "Gaetano dei Conti di Thiene"],
    "Callistus I": ["Pope Callixtus I"],
    "Casimir": ["Casimir Jagiellon", "Saint Casimir"],
    "Charles Lwanga and Companions": ["Charles Lwanga"],
    "Christopher Magallanes and Companions": ["Cristóbal Magallanes Jara"],
    "Clare": ["Clare of Assisi"],
    "Clement I": ["Pope Clement I"],
    "Columban": ["Columbanus"],
    "Cornelius and Cyprian": ["Pope Cornelius", "Cyprian"],
    "Damasus I": ["Pope Damasus I"],
    "Denis and Companions": ["Denis of Paris", "Saint Denis"],
    "Ephrem": ["Ephrem the Syrian"],
    "Fabian": ["Pope Fabian"],
    "George": ["Saint George"],
    "Gertrude": ["Gertrude the Great"],
    "Gregory VII": ["Pope Gregory VII"],
    "Gregory the Great": ["Pope Gregory I"],
    "Hedwig": ["Hedwig of Silesia"],
    "Henry": ["Henry II, Holy Roman Emperor"],
    "Isaac Jogues and John de Brebeuf": ["Isaac Jogues", "Jean de Brébeuf"],
    "Isidore the Farmer": ["Isidore the Laborer"],
    "James": ["James the Great", "James, son of Zebedee"],
    "Jerome Emiliani": ["Jerome Emiliani"],
    "Joachim and Anne": ["Saint Joachim", "Saint Anne"],
    "John": ["John the Apostle"],
    "John Baptist de la Salle": ["Jean-Baptiste de La Salle"],
    "John Damascene": ["John of Damascus"],
    "John Fisher and Thomas More": ["John Fisher", "Thomas More"],
    "John I": ["Pope John I"],
    "John Paul II": ["Pope John Paul II"],
    "John XXIII": ["Pope John XXIII"],
    "John of Kanty": ["John Cantius"],
    "Josaphat": ["Josaphat Kuntsevych"],
    "Joseph the Worker": ["Saint Joseph"],
    "Justin": ["Justin Martyr"],
    "Lawrence": ["Saint Lawrence", "Lawrence of Rome"],
    "Lawrence Ruiz and Companions": ["Lorenzo Ruiz", "Lawrence Ruiz"],
    "Leo the Great": ["Pope Leo I"],
    "Louis IX": ["Louis IX of France"],
    "Luke": ["Luke the Evangelist"],
    "Margaret of Scotland": ["Saint Margaret of Scotland"],
    "Mark": ["Mark the Evangelist"],
    "Martin I": ["Pope Martin I"],
    "Mary Magdalene de Pazzi": ["Mary Magdalene de' Pazzi"],
    "Mary Mother of God": ["Mary, mother of Jesus"],
    "Mary and Martha": ["Mary of Bethany", "Martha of Bethany"],
    "Matthew": ["Matthew the Apostle"],
    "Matthias": ["Matthias the Apostle"],
    "Michael": ["Michael (archangel)"],
    "Monica": ["Saint Monica"],
    "Pancras": ["Pancras of Rome"],
    "Patrick": ["Saint Patrick"],
    "Paul Miki and Companions": ["Paul Miki"],
    "Peter and Paul": ["Saint Peter", "Paul the Apostle"],
    "Philip and James": ["Philip the Apostle", "James, son of Alphaeus"],
    "Pius V": ["Pope Pius V"],
    "Pius X": ["Pope Pius X"],
    "Pius of Pietrelcina": ["Padre Pio"],
    "Pontian and Hippolytus": ["Pope Pontian", "Hippolytus of Rome"],
    "Raymond of Penyafort": ["Raymond of Peñafort"],
    "Sebastian": ["Saint Sebastian"],
    "Seven Founders of the Servite Order": ["Seven Holy Founders of the Servite Order"],
    "Sharbel Makhluf": ["Charbel Makhluf"],
    "Simon and Jude": ["Simon the Zealot", "Jude the Apostle"],
    "Sixtus II and Companions": ["Pope Sixtus II"],
    "Stanislaus": ["Stanislaus of Szczepanów"],
    "Stephen of Hungary": ["Stephen I of Hungary"],
    "Sylvester I": ["Pope Sylvester I"],
    "Teresa Benedicta of the Cross": ["Edith Stein"],
    "Teresa of Avila": ["Teresa of Ávila"],
    "Therese of the Child Jesus": ["Thérèse of Lisieux"],
    "Thomas": ["Thomas the Apostle"],
    "Timothy and Titus": ["Saint Timothy", "Saint Titus"],
    "Valentine": ["Saint Valentine"],
    "Wenceslaus": ["Wenceslaus I, Duke of Bohemia"],
}


def create_session() -> requests.Session:
    """Create a requests session with proper headers for Wikimedia."""
    session = requests.Session()
    session.headers.update(
        {
            "User-Agent": USER_AGENT,
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
            "Accept-Encoding": "gzip, deflate",
            "Connection": "keep-alive",
        }
    )
    return session


def get_standardized_name(saint_name: str) -> str:
    """Get the standardized name used for Wikipedia URLs and filenames."""
    return saint_name.replace(" ", "_")


def get_all_saints() -> list[str]:
    """Read the CSV and return all unique saint names (excluding non-person entries)."""
    csv_path = Path(__file__).parent / "saints_days_france.csv"
    saints = set()

    with open(csv_path, "r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            saint_name = row["Saint Name"]
            if saint_name not in NON_PERSON_ENTRIES:
                saints.add(saint_name)

    return sorted(saints)


def get_wikipedia_image_url(session: requests.Session, page_title: str) -> str | None:
    """
    Use Wikipedia API to get the main image URL for a page.
    Uses imageinfo prop to get the direct file URL.
    Returns the image URL or None if no image found.
    """
    # First, get the page image filename
    params = {
        "action": "query",
        "titles": page_title,
        "prop": "pageimages",
        "piprop": "name",  # Get the filename instead of thumbnail URL
        "format": "json",
    }

    try:
        response = session.get(WIKIPEDIA_API, params=params, timeout=10)
        response.raise_for_status()
        data = response.json()

        pages = data.get("query", {}).get("pages", {})
        image_filename = None
        for page_id, page_data in pages.items():
            if page_id == "-1":
                return None
            image_filename = page_data.get("pageimage")
            break

        if not image_filename:
            return None

        # Now get the actual image URL using imageinfo
        params = {
            "action": "query",
            "titles": f"File:{image_filename}",
            "prop": "imageinfo",
            "iiprop": "url",
            "iiurlwidth": 500,  # Request a 500px wide thumbnail
            "format": "json",
        }

        response = session.get(WIKIPEDIA_API, params=params, timeout=10)
        response.raise_for_status()
        data = response.json()

        pages = data.get("query", {}).get("pages", {})
        for page_id, page_data in pages.items():
            # Files on Commons may show as "missing" (-1) from enwiki but still have imageinfo
            imageinfo_list = page_data.get("imageinfo", [])
            if imageinfo_list:
                imageinfo = imageinfo_list[0]
                # Prefer thumburl (resized), fall back to original url
                return imageinfo.get("thumburl") or imageinfo.get("url")

    except requests.RequestException as e:
        print(f"  Error fetching Wikipedia API: {e}")
        return None

    return None


def download_image(session: requests.Session, url: str, filepath: Path) -> bool:
    """Download an image from URL and save to filepath."""
    try:
        # Add image-specific accept header for this request
        headers = {"Accept": "image/webp,image/apng,image/*,*/*;q=0.8"}
        response = session.get(url, headers=headers, timeout=30)
        response.raise_for_status()

        with open(filepath, "wb") as f:
            f.write(response.content)
        return True

    except requests.RequestException as e:
        print(f"  Error downloading image: {e}")
        return False


def main():
    pics_dir = Path(__file__).parent / "pics"
    pics_dir.mkdir(exist_ok=True)

    saints = get_all_saints()
    print(f"Found {len(saints)} saints to process.\n")

    # Create a persistent session
    session = create_session()

    downloaded = 0
    skipped = 0
    failed = 0

    for saint_name in saints:
        standardized_name = get_standardized_name(saint_name)

        # Check if we already have an image for this saint
        existing_files = list(pics_dir.glob(f"{standardized_name}.*"))
        if existing_files:
            print(f"[SKIP] {saint_name} - already have {existing_files[0].name}")
            skipped += 1
            continue

        print(f"[FETCH] {saint_name}...")

        # Get image URL from Wikipedia - try primary name first, then alternates
        image_url = get_wikipedia_image_url(session, saint_name)

        if not image_url and saint_name in ALTERNATE_TITLES:
            for alt_title in ALTERNATE_TITLES[saint_name]:
                print(f"  Trying alternate: {alt_title}")
                image_url = get_wikipedia_image_url(session, alt_title)
                if image_url:
                    break
                time.sleep(0.3)  # Small delay between alternate attempts

        if not image_url:
            print(f"  No image found on Wikipedia")
            failed += 1
            continue

        # Determine file extension from URL
        url_path = urllib.parse.urlparse(image_url).path
        ext = Path(url_path).suffix.lower()
        # Handle URLs with size suffix like .jpg/500px-foo.jpg
        if ext not in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
            # Try to extract from the path
            for known_ext in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
                if known_ext in url_path.lower():
                    ext = known_ext
                    break
            else:
                ext = ".jpg"  # Default fallback

        filepath = pics_dir / f"{standardized_name}{ext}"

        if download_image(session, image_url, filepath):
            print(f"  Saved to {filepath.name}")
            downloaded += 1
        else:
            failed += 1

        # Be nice to Wikipedia's servers - random delay between 0.5 and 1.5 seconds
        time.sleep(0.5 + random.random())

    print(f"\nDone! Downloaded: {downloaded}, Skipped: {skipped}, Failed: {failed}")


if __name__ == "__main__":
    main()