import csv
import json
import re
import matplotlib.pyplot as plt
import numpy as np
from collections import Counter
from pprint import pprint

stop_words = ['of', 'and', 'the', 'we', 'in', 'to', 'what', 'a', 'on', 'an',
              'do', 'for', 'with', 'is', 'it', 'that', 'this', 'as', 'by',
              'are', 'using', 'from', 'how', 'has', 'be', 'or', 'can', 'our',
              'at', 'why', 'when', 'introduction', 'your', 'through']
file_path = 'data.csv'
data = []


def parse_csv(file_path):
    with open(file_path, 'r') as file:
        reader = csv.reader(file)
        for row in reader:
            data.append(row)


def extract_journal_titles(data):
    journal_titles = []
    for row in data:
        if "(edited volume)" not in row[3]:
            journal_titles.append(row[3])
    count_journal_titles = Counter(journal_titles).most_common(20)
    pprint(count_journal_titles)
    print("Total journals: " + str(len(set(journal_titles))))


def extract_title_keywords(data):
    title_keywords = []
    re_words = []
    for row in data:
        raw_words = row[2].split(' ')
        row_words = []
        for word in raw_words:
            re_word = re.sub(r'[^a-zA-Z\-]', '', word.lower()) 
            if re_word not in stop_words:
                re_words.append(re_word)
                row_words.append(re_word)
    title_keywords = [(word, row_words) for word in re_words]
    return title_keywords


def print_title_keywords(data):
    title_keywords = extract_title_keywords(data)
    title_keywords = [word[0] for word in title_keywords]
    count_title_words = Counter(title_keywords).most_common(20)
    pprint(count_title_words) 


def histogram(data):
    dates = [] 
    for row in data:
        dates.append(int(row[4]))
    years = range(min(dates), max(dates)+2)
    plt.hist(dates, bins=years)
    plt.show()


def dash_viz(data):
    title_keywords = extract_title_keywords(data)
    word_output = {}
    for line in title_keywords:
        if line[0] not in word_output:
            word_output[line[0]] = [line[1]]
        else:
            word_output[line[0]].append(line[1])
    output_data = [] 
    for word in word_output:
        if word: 
            output_data.append({'data': {'id': word, 'label': word}})
    for all_words in word_output.values():
        for title_words in all_words: 
            edges = [(word1, word2) for i, word1 in enumerate(title_words) for word2
                     in title_words[i+1:]]
            for edge1, edge2 in edges:
                if edge1 and edge2:
                    output_data.append({'data': {'source': edge1, 'target': edge2}})
    with open('viz_data.json', 'w') as file:
        json.dump(output_data, file)


if __name__ == '__main__':
    parse_csv(file_path)
    data = data[1:]
    extract_journal_titles(data)
    print_title_keywords(data)
    dash_viz(data)
    histogram(data)