Mark Eatonandaider
272b4c0909
Co-authored-by: aider (ollama_chat/qwen3-coder:30b) <aider@aider.chat>
142 lines
3.0 KiB
Python
142 lines
3.0 KiB
Python
import csv
|
|
import json
|
|
import re
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
from collections import Counter
|
|
from pprint import pprint
|
|
|
|
stop_words = [
|
|
"(review)",
|
|
"of",
|
|
"and",
|
|
"the",
|
|
"we",
|
|
"in",
|
|
"to",
|
|
"what",
|
|
"a",
|
|
"on",
|
|
"an",
|
|
"do",
|
|
"for",
|
|
"with",
|
|
"is",
|
|
"it",
|
|
"that",
|
|
"this",
|
|
"as",
|
|
"by",
|
|
"are",
|
|
"using",
|
|
"from",
|
|
"how",
|
|
"has",
|
|
"be",
|
|
"or",
|
|
"can",
|
|
"our",
|
|
"at",
|
|
"why",
|
|
"when",
|
|
"introduction",
|
|
"your",
|
|
"through",
|
|
"et",
|
|
"al.",
|
|
"&",
|
|
"",
|
|
]
|
|
file_path = "data.csv"
|
|
data = []
|
|
|
|
|
|
def parse_csv(file_path):
|
|
try:
|
|
with open(file_path, "r") as file:
|
|
reader = csv.reader(file)
|
|
for row in reader:
|
|
data.append(row)
|
|
except FileNotFoundError:
|
|
print(f"Error: File {file_path} not found")
|
|
return
|
|
except Exception as e:
|
|
print(f"Error reading file: {e}")
|
|
return
|
|
|
|
|
|
def extract_journal_titles(data):
|
|
journal_titles = []
|
|
for row in data:
|
|
if "(edited volume)" not in row[3]:
|
|
journal_titles.append(row[3])
|
|
count_journal_titles = Counter(journal_titles).most_common(30)
|
|
pprint(count_journal_titles)
|
|
print("Total journals: " + str(len(set(journal_titles))))
|
|
|
|
|
|
def extract_title_keywords(data):
|
|
title_keywords = []
|
|
for row in data:
|
|
raw_words = row[2].split(" ")
|
|
for word in raw_words:
|
|
re_word = re.sub(r"[^a-zA-Z\-]", "", word.lower())
|
|
if re_word not in stop_words and re_word != "":
|
|
title_keywords.append(re_word)
|
|
return Counter(title_keywords).most_common(30)
|
|
|
|
|
|
def print_title_keywords(data):
|
|
title_keywords = extract_title_keywords(data)
|
|
pprint(title_keywords)
|
|
|
|
|
|
def histogram(data):
|
|
dates = []
|
|
for row in data:
|
|
try:
|
|
dates.append(int(row[4]))
|
|
except (ValueError, IndexError):
|
|
# Skip invalid date entries
|
|
continue
|
|
|
|
if not dates:
|
|
print("No valid dates found")
|
|
return
|
|
|
|
min_year = min(dates)
|
|
max_year = max(dates)
|
|
|
|
# Create bins with reasonable spacing
|
|
years = range(min_year, max_year + 1)
|
|
plt.hist(dates, bins=len(years), edgecolor='black')
|
|
plt.xlabel('Year')
|
|
plt.ylabel('Frequency')
|
|
plt.title('Publication Year Distribution')
|
|
plt.show()
|
|
|
|
|
|
def names(data):
|
|
names = []
|
|
for row in data:
|
|
name_parts = row[1].split(" ")
|
|
for name in name_parts:
|
|
name = name.rstrip(",")
|
|
if name not in stop_words:
|
|
if "." not in name and name != "":
|
|
names.append(name)
|
|
count_names = Counter(names).most_common(30)
|
|
pprint(count_names)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parse_csv(file_path)
|
|
if data: # Only process if data was loaded
|
|
data = data[1:] # Skip header row
|
|
extract_journal_titles(data)
|
|
print_title_keywords(data)
|
|
# names(data)
|
|
histogram(data)
|
|
else:
|
|
print("No data to process")
|