95 lines
2.8 KiB
Python
95 lines
2.8 KiB
Python
import csv
|
|
import json
|
|
import re
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
from collections import Counter
|
|
from pprint import pprint
|
|
|
|
stop_words = ['of', 'and', 'the', 'we', 'in', 'to', 'what', 'a', 'on', 'an',
|
|
'do', 'for', 'with', 'is', 'it', 'that', 'this', 'as', 'by',
|
|
'are', 'using', 'from', 'how', 'has', 'be', 'or', 'can', 'our',
|
|
'at', 'why', 'when', 'introduction', 'your', 'through']
|
|
file_path = 'data.csv'
|
|
data = []
|
|
|
|
|
|
def parse_csv(file_path):
|
|
with open(file_path, 'r') as file:
|
|
reader = csv.reader(file)
|
|
for row in reader:
|
|
data.append(row)
|
|
|
|
|
|
def extract_journal_titles(data):
|
|
journal_titles = []
|
|
for row in data:
|
|
if "(edited volume)" not in row[3]:
|
|
journal_titles.append(row[3])
|
|
count_journal_titles = Counter(journal_titles).most_common(20)
|
|
pprint(count_journal_titles)
|
|
print("Total journals: " + str(len(set(journal_titles))))
|
|
|
|
|
|
def extract_title_keywords(data):
|
|
title_keywords = []
|
|
re_words = []
|
|
for row in data:
|
|
raw_words = row[2].split(' ')
|
|
row_words = []
|
|
for word in raw_words:
|
|
re_word = re.sub(r'[^a-zA-Z\-]', '', word.lower())
|
|
if re_word not in stop_words:
|
|
re_words.append(re_word)
|
|
row_words.append(re_word)
|
|
title_keywords = [(word, row_words) for word in re_words]
|
|
return title_keywords
|
|
|
|
|
|
def print_title_keywords(data):
|
|
title_keywords = extract_title_keywords(data)
|
|
title_keywords = [word[0] for word in title_keywords]
|
|
count_title_words = Counter(title_keywords).most_common(20)
|
|
pprint(count_title_words)
|
|
|
|
|
|
def histogram(data):
|
|
dates = []
|
|
for row in data:
|
|
dates.append(int(row[4]))
|
|
years = range(min(dates), max(dates)+2)
|
|
plt.hist(dates, bins=years)
|
|
plt.show()
|
|
|
|
|
|
def dash_viz(data):
|
|
title_keywords = extract_title_keywords(data)
|
|
word_output = {}
|
|
for line in title_keywords:
|
|
if line[0] not in word_output:
|
|
word_output[line[0]] = [line[1]]
|
|
else:
|
|
word_output[line[0]].append(line[1])
|
|
output_data = []
|
|
for word in word_output:
|
|
if word:
|
|
output_data.append({'data': {'id': word, 'label': word}})
|
|
for all_words in word_output.values():
|
|
for title_words in all_words:
|
|
edges = [(word1, word2) for i, word1 in enumerate(title_words) for word2
|
|
in title_words[i+1:]]
|
|
for edge1, edge2 in edges:
|
|
if edge1 and edge2:
|
|
output_data.append({'data': {'source': edge1, 'target': edge2}})
|
|
with open('viz_data.json', 'w') as file:
|
|
json.dump(output_data, file)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parse_csv(file_path)
|
|
data = data[1:]
|
|
extract_journal_titles(data)
|
|
print_title_keywords(data)
|
|
dash_viz(data)
|
|
histogram(data)
|