generate analysis

Author: kodymoodley

docona/analyseresults.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# # Document Content and Citation Analysis (DoConA)
+# # Module: analyses "results.csv" raw output from pipeline and computes the similarity overlap between the citation network of the documents and the document texts 
+# # Output: generates "analysis.csv" file with summary of the results
+# GNU AGPLv3 - https://choosealicense.com/licenses/agpl-3.0/
+
+import pandas as pd
+import os
+
+def analyse():
+	# Import results and remove duplicate rows
+	results_df = pd.read_csv(os.path.join(os.path.join(os.path.realpath('..'), "outputdata"), "results.csv"))
+	results_df.drop_duplicates(subset=None, keep='first', inplace=True)
+
+	# Filter for the overlaps and sort the dataframe (in descending order) according to the number of citation overlaps (group by method as well)
+	overlaps_df = results_df[results_df["citation_link"] == True]
+	overlaps_df = overlaps_df.groupby(['method']).citation_link.agg('count').to_frame('num_cite_and_similar_links').reset_index()
+	overlaps_df = overlaps_df.sort_values('num_cite_and_similar_links',ascending=False)
+
+	# Import full citation network of documents
+	citations = pd.read_csv('../inputdata/citations.csv')
+
+	# Function to location the cited documents of a given document
+	def find_cited_documents(documentID):
+	    relevantsource = citations[citations['source'] == documentID]
+	    return relevantsource['target'].tolist()
+
+	# Import sample documents
+	unique_samples = pd.read_csv('../inputdata/sample.csv', header=None)
+
+	# Compute the total number of unique documents cited by the sample documents
+	cited_documents = []
+	for item in unique_samples.values.tolist():
+	    cited_documents.extend(find_cited_documents(item[0]))
+	num_cite_links = len(cited_documents)
+
+	# Aggregate all the counts into a single data frame and compute the percentage overlaps of each method
+	percentage_overlap_arr = []
+	num_cite_links_arr = []
+	num_rows = len(overlaps_df.index)
+	num_cite_and_similar_links_arr = overlaps_df['num_cite_and_similar_links'].values.tolist()
+
+	for i in range(0,num_rows):
+	    num_cite_links_arr.append(num_cite_links)
+	    current_perc_overlap = (num_cite_and_similar_links_arr[i] / num_cite_links) * 100
+	    percentage_overlap_arr.append(current_perc_overlap)
+
+	overlaps_df['num_cite_links'] = num_cite_links_arr
+	overlaps_df['percentage_overlap'] = percentage_overlap_arr
+
+	# Write the output to file
+	overlaps_df.to_csv('../outputdata/analysis.csv', index=False)
+

docona/docona.py

@@ -9,28 +9,30 @@
 print()
 print("1. Preprocessing documents...", end = '', flush=True)
 # Preprocessing full texts
-from preprocessdocs import preprocess
-preprocess()
+from preprocessdocs import preprocess, preprocessingdone
+if not preprocessingdone():
+	preprocess()
 print("Done!")
 print()
 print("2. Executing semantic measures")
 print("a) Training corpus models...", end = '', flush=True)
 # Train Doc2Vec and Word2Vec models on full texts of documents
 from trainedmodels import getdoc2vecmodel,getword2vecmodel
 doc2vecmodel = getdoc2vecmodel()
-word2vecmodel = getword2vecmodel()
+# word2vecmodel = getword2vecmodel()
 print("Done!")
 print("b) Similarity checks: corpus models...", end = '', flush=True)
+print()
 # # Trained models: check document similarity
 from trainedmodelssimilarity import dosimilaritychecks
 dosimilaritychecks("doc2vec",doc2vecmodel,"cosine")
-dosimilaritychecks("word2vec",word2vecmodel,"wmd")
+# dosimilaritychecks("word2vec",word2vecmodel,"wmd")
+print()
 print("Done!")
 
-# --------------------------------------------- #
-# --- ADD CUSTOM PRETRAINED MODEL CODE HERE --- #
-# --------------------------------------------- #
-
+# # --------------------------------------------- #
+# # --- ADD CUSTOM PRETRAINED MODEL CODE HERE --- #
+# # --------------------------------------------- #
 # print("c) Adapting GoogleNews pretrained model...", end = '', flush=True)
 # # # GoogleNews pretrained load / train
 # from pretrainedmodels import getdoc2vecmodel,getword2vecmodel
@@ -53,7 +55,8 @@
 # dosimilaritychecks("doc2vec", "law2vec", law2vecdoc2vecmodel, "cosine")
 # dosimilaritychecks("word2vec", "law2vec", law2vecword2vecmodel, "wmd")
 # print("Done!")
-print()
+# print()
+
 print("3. Executing syntactic measures")
 print("a) Training TFIDF and Ngram models...", end = '', flush=True)
 # # TFIDF, Ngram models load / train
@@ -68,4 +71,9 @@
 dosyntacticsimilaritychecks("jaccard",model=None)
 print("Done!")
 print()
+print("4. Analysing results")
+from analyseresults import analyse
+analyse()
+print("Done!")
+print()
 print("-- FINISHED --")

docona/preprocessdocs.py

@@ -21,6 +21,20 @@
 import pandas as pd
 from utility import cleantoken, stemSentence
 
+def preprocessingdone():
+    fulltextpath = "../inputdata/fulltexts/"
+    processedtextpath = "../inputdata/processedtexts/"
+    resourcespath = "../inputdata/resources/"
+
+    if path.exists(processedtextpath) and path.exists(fulltextpath):
+        numfulltexts = len([name for name in os.listdir(fulltextpath) if os.path.isfile(os.path.join(fulltextpath, name))])
+        numprocessedtexts = len([name for name in os.listdir(processedtextpath) if os.path.isfile(os.path.join(processedtextpath, name))])
+        if numfulltexts == numprocessedtexts:
+            if os.path.isfile(resourcespath + 'documentID_to_tokenized.pickle') and os.path.isfile(resourcespath + 'data_to_tokenized.pickle') and os.path.isfile(resourcespath + 'datafortraining.pickle') and os.path.isfile(resourcespath + 'documents.pickle') and os.path.isfile(resourcespath + 'documentID_to_data.pickle') and os.path.isfile(resourcespath + 'index_to_documentID.pickle') and os.path.isfile(resourcespath + 'documentID_to_index.pickle') and os.path.isfile(resourcespath + 'word2vec.model') and os.path.isfile(resourcespath + 'doc2vec.model') and os.path.isfile(resourcespath + 'word2vecsimilaritymatrix.pickle'):
+                return True
+
+    return False
+                
 def preprocess():
     textpath = "../inputdata/fulltexts/"
     index_to_documentID = {}

docona/pretrainedmodels.py

@@ -31,7 +31,7 @@ def getdoc2vecmodel(modelinputfilename,modeloutputfilename):
         min_count = 2
         sampling_threshold = 1e-5
         negative_size = 5
-        training_epochs = 5
+        training_epochs = 20
         dm = 0
         hs = 0
         worker_count = 4
@@ -41,6 +41,8 @@ def getdoc2vecmodel(modelinputfilename,modeloutputfilename):
         fname = get_tmpfile(os.path.join(os.path.join(os.path.join(os.path.realpath('..'), "inputdata"), "resources"), modeloutputfilename))
         # Train doc2vec model
         model = g.Doc2Vec(documents, vector_size=vector_size, window=window_size, min_count=min_count, sample=sampling_threshold, workers=worker_count, hs=hs, dm=dm, negative=negative_size, dbow_words=1, dm_concat=1, pretrained_emb=pretrained_emb, epochs=training_epochs)
+        model.train(documents, total_examples=model.corpus_count, epochs=model.epochs)
+        model.init_sims(replace=True)
         # Save model
         model.save(fname)
 

docona/pretrainedmodelssimilarity.py

@@ -10,6 +10,7 @@
 import os
 import os.path
 from os import path
+import time
 import csv
 import pandas as pd
 from helper import convert_to_document_references, exists_citation_link_between
@@ -19,27 +20,41 @@
 def lookup_similar_documents_docvec_cosine(sample_documents, n, model, pretrained_embedding_name):
     results = []
 
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         similar_documents = model.docvecs.most_similar(documentID_to_index[item], topn=n)
         similar_documents_references = convert_to_document_references(similar_documents)
         for reference in similar_documents_references:
             method = pretrained_embedding_name + "_doc2vec_wmd"
             results.append([item,reference[0].replace(".txt",""),reference[1],method,exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 # ### Look up top n similar documents per sample document (googlenews word2vec embeddings + word mover's distance)
 def lookup_similar_documents_word2vec_wmd(sample_documents, pretrained_embedding_name):
     results = []
 
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         similar_documents = sim[documentID_to_data[item]]
         similar_documents_references = convert_to_document_references(similar_documents)
         for reference in similar_documents_references:
             method = pretrained_embedding_name + "_word2vec_wmd"
             if (str(item) != str(reference[0])):
                 results.append([item,reference[0],reference[1],method,exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 # ### Main function

docona/syntacticmodels.py

@@ -13,6 +13,7 @@
 import os.path
 from os import path
 import csv
+import time
 import math
 import operator
 
@@ -57,20 +58,31 @@ def find_similar(tfidf_matrix, index, top_n):
 
 def lookup_similar_documents_tfidf_based(sample_documents, n, model,methodname):
     results = []
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         index = documentID_to_index[item]                                                   # Look up this documents index in the TFIDF matrix
         similar_documents = find_similar(model, index, n)                                   # Look up top n similar documents for this document
         similar_documents_references = convert_to_document_references(similar_documents)
         for reference in similar_documents_references:
             results.append([item,reference[0],reference[1],methodname,exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 def lookup_similar_documents_jaccard(sample_documents, n, methodname):
     results = []
 
-    num = len(sample_documents)
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         current_dict = {}
         for k,v in documentID_to_data.items():
             if k != item:
@@ -80,7 +92,9 @@ def lookup_similar_documents_jaccard(sample_documents, n, methodname):
         topn = sorted_dict[-n:]
         for reference in topn:
             results.append([item,reference[0],reference[1], methodname, exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 # ### Main function

docona/trainedmodels.py

@@ -40,9 +40,10 @@ def getdoc2vecmodel():
     if not model_exists:
         # No existing model
         documents = pickle.load( open( "../inputdata/resources/documents.pickle", "rb" ) )
-        model = Doc2Vec(vector_size=32, min_count=2, epochs=5)
+        model = Doc2Vec(vector_size=256, min_count=2, epochs=30)
         model.build_vocab(documents)
         model.train(documents, total_examples=model.corpus_count, epochs=model.epochs)
+        model.init_sims(replace=True)
         model_file = get_tmpfile(os.path.join(os.path.join(os.path.join(os.path.realpath('..'), "inputdata"), "resources"), "doc2vec.model"))
         model.save(model_file)
 
@@ -66,8 +67,8 @@ def getword2vecmodel():
         texts = []
         for doc in documents:
             texts.append(doc.words)
-        model = Word2Vec(texts, size=32, window=7, min_count=2, workers=2)
-        model.train(texts, total_examples=model.corpus_count,epochs=5)        
+        model = Word2Vec(texts, size=256, window=5, min_count=2, workers=4)
+        model.train(texts, total_examples=model.corpus_count,epochs=30)        
         model.init_sims(replace=True)
         model_file = get_tmpfile(os.path.join(os.path.join(os.path.join(os.path.realpath('..'), "inputdata"), "resources"), "word2vec.model"))
         model.save(model_file)

docona/trainedmodelssimilarity.py

@@ -24,36 +24,56 @@
 def lookup_similar_documents_docvec_cosine(sample_documents, n, model):
     results = []
 
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         similar_documents = model.docvecs.most_similar(documentID_to_index[item], topn=n)
         similar_documents_references = convert_to_document_references(similar_documents)
         for reference in similar_documents_references:
             method = "doc2vec_cosine"
             results.append([item,reference[0].replace(".txt",""),reference[1],method,exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 # ### Look up top n similar documents per sample document (word2vec embeddings + word mover's distance)
 def lookup_similar_documents_word2vec_wmd(sample_documents):
     results = []
 
+    num_samples = len(sample_documents)
+    count = 1
     for item in sample_documents:
+        start = time.time()
+        print(str(count) + "/" + str(num_samples) + "...", end='', flush=True)
+        count = count + 1
         similar_documents = sim[documentID_to_data[item]]
         similar_documents_references = convert_to_document_references(similar_documents)
         for reference in similar_documents_references:
             method = "word2vec_wmd"
             if (str(item) != str(reference[0])):
                 results.append([item,reference[0],reference[1],method,exists_citation_link_between(item,reference[0])])
-
+        end = time.time()
+        timetaken = end-start
+        print(str(timetaken) + "s")
     return results
 
 # ### Main function
 def dosimilaritychecks(modeltype,model,distancemeasure):
     results = []
     if (modeltype == "doc2vec" and distancemeasure == "cosine"):
+        print("doc2vec + cosine distance")
+        print()
         results = lookup_similar_documents_docvec_cosine(sampledocuments,20, model)
+        print()
     elif (modeltype == "word2vec" and distancemeasure == "wmd"):
+        print("word2vec + word movers distance")
+        print()
         results = lookup_similar_documents_word2vec_wmd(sampledocuments)
+        print()
     if os.path.exists('../outputdata/results.csv') == False:
         results.insert(0,['source_document','similar_document','similarity_score','method','citation_link'])