initial commit
This commit is contained in:
Mark Eaton
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to download pictures of saints from Wikipedia.
|
||||
Run this once to populate the pics/ directory.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import requests
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
import time
|
||||
import random
|
||||
|
||||
# Entries that are holidays, feasts, or events rather than actual saints/people
|
||||
NON_PERSON_ENTRIES = {
|
||||
"Epiphany",
|
||||
"Holy Name of Jesus",
|
||||
"Conversion of St. Paul",
|
||||
"Presentation of the Lord",
|
||||
"Our Lady of Lourdes",
|
||||
"Chair of St. Peter",
|
||||
"Annunciation",
|
||||
"Our Lady of Fatima",
|
||||
"Visitation of the Virgin Mary",
|
||||
"Nativity of John the Baptist",
|
||||
"First Martyrs of Rome",
|
||||
"Our Lady of Mount Carmel",
|
||||
"Dedication of St. Mary Major",
|
||||
"Transfiguration",
|
||||
"Assumption of Mary",
|
||||
"Queenship of Mary",
|
||||
"Beheading of John the Baptist",
|
||||
"Nativity of Mary",
|
||||
"Holy Name of Mary",
|
||||
"Exaltation of the Holy Cross",
|
||||
"Our Lady of Sorrows",
|
||||
"Guardian Angels",
|
||||
"Our Lady of the Rosary",
|
||||
"All Saints",
|
||||
"All Souls",
|
||||
"Dedication of the Lateran Basilica",
|
||||
"Dedication of the Basilicas of Peter and Paul",
|
||||
"Presentation of Mary",
|
||||
"Immaculate Conception",
|
||||
"Our Lady of Guadalupe",
|
||||
"Holy Innocents",
|
||||
}
|
||||
|
||||
WIKIPEDIA_API = "https://en.wikipedia.org/w/api.php"
|
||||
|
||||
# Wikimedia requires a descriptive User-Agent per their policy
|
||||
# https://meta.wikimedia.org/wiki/User-Agent_policy
|
||||
USER_AGENT = "SaintsPictureDownloader/1.0 (Educational project for French saints calendar)"
|
||||
|
||||
# Mapping of CSV saint names to alternate Wikipedia article titles to try
|
||||
# Many saints have disambiguation issues or different article naming conventions
|
||||
ALTERNATE_TITLES = {
|
||||
"Agatha": ["Agatha of Sicily"],
|
||||
"Agnes": ["Agnes of Rome"],
|
||||
"Albert the Great": ["Albertus Magnus"],
|
||||
"Andrew Dung-Lac and Companions": ["Andrew Dung-Lac"],
|
||||
"Andrew Kim Taegon and Companions": ["Andrew Kim Taegon"],
|
||||
"Anselm": ["Anselm of Canterbury"],
|
||||
"Anthony of Egypt": ["Anthony the Great"],
|
||||
"Apollinaris": ["Apollinaris of Ravenna"],
|
||||
"Athanasius": ["Athanasius of Alexandria"],
|
||||
"Augustine": ["Augustine of Hippo"],
|
||||
"Augustine Zhao Rong and Companions": ["Augustine Zhao Rong"],
|
||||
"Bartholomew": ["Bartholomew the Apostle"],
|
||||
"Basil the Great and Gregory Nazianzen": ["Basil of Caesarea", "Gregory of Nazianzus"],
|
||||
"Bede the Venerable": ["Bede"],
|
||||
"Benedict": ["Benedict of Nursia"],
|
||||
"Bernardine of Siena": ["Bernardino of Siena"],
|
||||
"Blaise": ["Saint Blaise"],
|
||||
"Boniface": ["Saint Boniface"],
|
||||
"Bruno": ["Bruno of Cologne"],
|
||||
"Cajetan": ["Saint Cajetan", "Gaetano dei Conti di Thiene"],
|
||||
"Callistus I": ["Pope Callixtus I"],
|
||||
"Casimir": ["Casimir Jagiellon", "Saint Casimir"],
|
||||
"Charles Lwanga and Companions": ["Charles Lwanga"],
|
||||
"Christopher Magallanes and Companions": ["Cristóbal Magallanes Jara"],
|
||||
"Clare": ["Clare of Assisi"],
|
||||
"Clement I": ["Pope Clement I"],
|
||||
"Columban": ["Columbanus"],
|
||||
"Cornelius and Cyprian": ["Pope Cornelius", "Cyprian"],
|
||||
"Damasus I": ["Pope Damasus I"],
|
||||
"Denis and Companions": ["Denis of Paris", "Saint Denis"],
|
||||
"Ephrem": ["Ephrem the Syrian"],
|
||||
"Fabian": ["Pope Fabian"],
|
||||
"George": ["Saint George"],
|
||||
"Gertrude": ["Gertrude the Great"],
|
||||
"Gregory VII": ["Pope Gregory VII"],
|
||||
"Gregory the Great": ["Pope Gregory I"],
|
||||
"Hedwig": ["Hedwig of Silesia"],
|
||||
"Henry": ["Henry II, Holy Roman Emperor"],
|
||||
"Isaac Jogues and John de Brebeuf": ["Isaac Jogues", "Jean de Brébeuf"],
|
||||
"Isidore the Farmer": ["Isidore the Laborer"],
|
||||
"James": ["James the Great", "James, son of Zebedee"],
|
||||
"Jerome Emiliani": ["Jerome Emiliani"],
|
||||
"Joachim and Anne": ["Saint Joachim", "Saint Anne"],
|
||||
"John": ["John the Apostle"],
|
||||
"John Baptist de la Salle": ["Jean-Baptiste de La Salle"],
|
||||
"John Damascene": ["John of Damascus"],
|
||||
"John Fisher and Thomas More": ["John Fisher", "Thomas More"],
|
||||
"John I": ["Pope John I"],
|
||||
"John Paul II": ["Pope John Paul II"],
|
||||
"John XXIII": ["Pope John XXIII"],
|
||||
"John of Kanty": ["John Cantius"],
|
||||
"Josaphat": ["Josaphat Kuntsevych"],
|
||||
"Joseph the Worker": ["Saint Joseph"],
|
||||
"Justin": ["Justin Martyr"],
|
||||
"Lawrence": ["Saint Lawrence", "Lawrence of Rome"],
|
||||
"Lawrence Ruiz and Companions": ["Lorenzo Ruiz", "Lawrence Ruiz"],
|
||||
"Leo the Great": ["Pope Leo I"],
|
||||
"Louis IX": ["Louis IX of France"],
|
||||
"Luke": ["Luke the Evangelist"],
|
||||
"Margaret of Scotland": ["Saint Margaret of Scotland"],
|
||||
"Mark": ["Mark the Evangelist"],
|
||||
"Martin I": ["Pope Martin I"],
|
||||
"Mary Magdalene de Pazzi": ["Mary Magdalene de' Pazzi"],
|
||||
"Mary Mother of God": ["Mary, mother of Jesus"],
|
||||
"Mary and Martha": ["Mary of Bethany", "Martha of Bethany"],
|
||||
"Matthew": ["Matthew the Apostle"],
|
||||
"Matthias": ["Matthias the Apostle"],
|
||||
"Michael": ["Michael (archangel)"],
|
||||
"Monica": ["Saint Monica"],
|
||||
"Pancras": ["Pancras of Rome"],
|
||||
"Patrick": ["Saint Patrick"],
|
||||
"Paul Miki and Companions": ["Paul Miki"],
|
||||
"Peter and Paul": ["Saint Peter", "Paul the Apostle"],
|
||||
"Philip and James": ["Philip the Apostle", "James, son of Alphaeus"],
|
||||
"Pius V": ["Pope Pius V"],
|
||||
"Pius X": ["Pope Pius X"],
|
||||
"Pius of Pietrelcina": ["Padre Pio"],
|
||||
"Pontian and Hippolytus": ["Pope Pontian", "Hippolytus of Rome"],
|
||||
"Raymond of Penyafort": ["Raymond of Peñafort"],
|
||||
"Sebastian": ["Saint Sebastian"],
|
||||
"Seven Founders of the Servite Order": ["Seven Holy Founders of the Servite Order"],
|
||||
"Sharbel Makhluf": ["Charbel Makhluf"],
|
||||
"Simon and Jude": ["Simon the Zealot", "Jude the Apostle"],
|
||||
"Sixtus II and Companions": ["Pope Sixtus II"],
|
||||
"Stanislaus": ["Stanislaus of Szczepanów"],
|
||||
"Stephen of Hungary": ["Stephen I of Hungary"],
|
||||
"Sylvester I": ["Pope Sylvester I"],
|
||||
"Teresa Benedicta of the Cross": ["Edith Stein"],
|
||||
"Teresa of Avila": ["Teresa of Ávila"],
|
||||
"Therese of the Child Jesus": ["Thérèse of Lisieux"],
|
||||
"Thomas": ["Thomas the Apostle"],
|
||||
"Timothy and Titus": ["Saint Timothy", "Saint Titus"],
|
||||
"Valentine": ["Saint Valentine"],
|
||||
"Wenceslaus": ["Wenceslaus I, Duke of Bohemia"],
|
||||
}
|
||||
|
||||
|
||||
def create_session() -> requests.Session:
|
||||
"""Create a requests session with proper headers for Wikimedia."""
|
||||
session = requests.Session()
|
||||
session.headers.update({
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Connection": "keep-alive",
|
||||
})
|
||||
return session
|
||||
|
||||
|
||||
def get_standardized_name(saint_name: str) -> str:
|
||||
"""Get the standardized name used for Wikipedia URLs and filenames."""
|
||||
return saint_name.replace(" ", "_")
|
||||
|
||||
|
||||
def get_all_saints() -> list[str]:
|
||||
"""Read the CSV and return all unique saint names (excluding non-person entries)."""
|
||||
csv_path = Path(__file__).parent / "saints_days_france.csv"
|
||||
saints = set()
|
||||
|
||||
with open(csv_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
saint_name = row["Saint Name"]
|
||||
if saint_name not in NON_PERSON_ENTRIES:
|
||||
saints.add(saint_name)
|
||||
|
||||
return sorted(saints)
|
||||
|
||||
|
||||
def get_wikipedia_image_url(session: requests.Session, page_title: str) -> str | None:
|
||||
"""
|
||||
Use Wikipedia API to get the main image URL for a page.
|
||||
Uses imageinfo prop to get the direct file URL.
|
||||
Returns the image URL or None if no image found.
|
||||
"""
|
||||
# First, get the page image filename
|
||||
params = {
|
||||
"action": "query",
|
||||
"titles": page_title,
|
||||
"prop": "pageimages",
|
||||
"piprop": "name", # Get the filename instead of thumbnail URL
|
||||
"format": "json",
|
||||
}
|
||||
|
||||
try:
|
||||
response = session.get(WIKIPEDIA_API, params=params, timeout=10)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
pages = data.get("query", {}).get("pages", {})
|
||||
image_filename = None
|
||||
for page_id, page_data in pages.items():
|
||||
if page_id == "-1":
|
||||
return None
|
||||
image_filename = page_data.get("pageimage")
|
||||
break
|
||||
|
||||
if not image_filename:
|
||||
return None
|
||||
|
||||
# Now get the actual image URL using imageinfo
|
||||
params = {
|
||||
"action": "query",
|
||||
"titles": f"File:{image_filename}",
|
||||
"prop": "imageinfo",
|
||||
"iiprop": "url",
|
||||
"iiurlwidth": 500, # Request a 500px wide thumbnail
|
||||
"format": "json",
|
||||
}
|
||||
|
||||
response = session.get(WIKIPEDIA_API, params=params, timeout=10)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
pages = data.get("query", {}).get("pages", {})
|
||||
for page_id, page_data in pages.items():
|
||||
# Files on Commons may show as "missing" (-1) from enwiki but still have imageinfo
|
||||
imageinfo_list = page_data.get("imageinfo", [])
|
||||
if imageinfo_list:
|
||||
imageinfo = imageinfo_list[0]
|
||||
# Prefer thumburl (resized), fall back to original url
|
||||
return imageinfo.get("thumburl") or imageinfo.get("url")
|
||||
|
||||
except requests.RequestException as e:
|
||||
print(f" Error fetching Wikipedia API: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def download_image(session: requests.Session, url: str, filepath: Path) -> bool:
|
||||
"""Download an image from URL and save to filepath."""
|
||||
try:
|
||||
# Add image-specific accept header for this request
|
||||
headers = {"Accept": "image/webp,image/apng,image/*,*/*;q=0.8"}
|
||||
response = session.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(filepath, "wb") as f:
|
||||
f.write(response.content)
|
||||
return True
|
||||
|
||||
except requests.RequestException as e:
|
||||
print(f" Error downloading image: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
pics_dir = Path(__file__).parent / "pics"
|
||||
pics_dir.mkdir(exist_ok=True)
|
||||
|
||||
saints = get_all_saints()
|
||||
print(f"Found {len(saints)} saints to process.\n")
|
||||
|
||||
# Create a persistent session
|
||||
session = create_session()
|
||||
|
||||
downloaded = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
for saint_name in saints:
|
||||
standardized_name = get_standardized_name(saint_name)
|
||||
|
||||
# Check if we already have an image for this saint
|
||||
existing_files = list(pics_dir.glob(f"{standardized_name}.*"))
|
||||
if existing_files:
|
||||
print(f"[SKIP] {saint_name} - already have {existing_files[0].name}")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
print(f"[FETCH] {saint_name}...")
|
||||
|
||||
# Get image URL from Wikipedia - try primary name first, then alternates
|
||||
image_url = get_wikipedia_image_url(session, saint_name)
|
||||
|
||||
if not image_url and saint_name in ALTERNATE_TITLES:
|
||||
for alt_title in ALTERNATE_TITLES[saint_name]:
|
||||
print(f" Trying alternate: {alt_title}")
|
||||
image_url = get_wikipedia_image_url(session, alt_title)
|
||||
if image_url:
|
||||
break
|
||||
time.sleep(0.3) # Small delay between alternate attempts
|
||||
|
||||
if not image_url:
|
||||
print(f" No image found on Wikipedia")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# Determine file extension from URL
|
||||
url_path = urllib.parse.urlparse(image_url).path
|
||||
ext = Path(url_path).suffix.lower()
|
||||
# Handle URLs with size suffix like .jpg/500px-foo.jpg
|
||||
if ext not in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
|
||||
# Try to extract from the path
|
||||
for known_ext in [".jpg", ".jpeg", ".png", ".gif", ".svg", ".webp"]:
|
||||
if known_ext in url_path.lower():
|
||||
ext = known_ext
|
||||
break
|
||||
else:
|
||||
ext = ".jpg" # Default fallback
|
||||
|
||||
filepath = pics_dir / f"{standardized_name}{ext}"
|
||||
|
||||
if download_image(session, image_url, filepath):
|
||||
print(f" Saved to {filepath.name}")
|
||||
downloaded += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
# Be nice to Wikipedia's servers - random delay between 0.5 and 1.5 seconds
|
||||
time.sleep(0.5 + random.random())
|
||||
|
||||
print(f"\nDone! Downloaded: {downloaded}, Skipped: {skipped}, Failed: {failed}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user