Testing and citation match improved

Some tests fixed, but mostly citation matching has been greatly improved.

Author: ptth222

src/academic_tracker/__main__.py

@@ -7,7 +7,6 @@
                                                       [--no-ORCID --no_ORCID] 
                                                       [--no-Crossref --no_Crossref] 
                                                       [--no-PubMed --no_PubMed]
-                                                      [--citation-match-ratio=<0-100>]
                                                       [--verbose --silent]
     academic_tracker reference_search <config_json_file> <references_file_or_URL> [--test] 
                                                                                   [--prev-pub=<file-path> --prev_pub=<file-path>]
@@ -16,7 +15,6 @@
                                                                                   [--MEDLINE-reference --MEDLINE_reference]
                                                                                   [--no-Crossref --no_Crossref]
                                                                                   [--no-PubMed --no_PubMed]
-                                                                                  [--citation-match-ratio=<0-100>]
                                                                                   [--verbose --silent]
     academic_tracker find_ORCID <config_json_file> [--verbose --silent]
     academic_tracker find_Google_Scholar <config_json_file> [--verbose --silent]
@@ -39,7 +37,6 @@
                                       Enter "ignore" for the <file_path> to not look for previous publications.json files in tracker directories.
     --prev_pub=<file-path>            Deprecated. Use --prev-pub instead.
     --save-all-queries                Save all queried results from each source in "all_results.json".
-    --citation-match-ratio=<num>      An integer from 0-100. The threshold to consider 2 citations the same using fuzzy matching. Lower is more forgiving. [default: 65]
     
 Reference Type Options:    
     --PMID-reference                  Indicates that the reference_file is a PMID file and only PubMed info will be returned.
@@ -112,8 +109,7 @@ def main():
                       args["--no_PubMed"] or args["--no-PubMed"],
                       args["--test"], 
                       args["--prev-pub"] if args["--prev-pub"] else args["--prev_pub"],
-                      args["--save-all-queries"],
-                      args["--citation-match-ratio"])
+                      args["--save-all-queries"])
     elif len(sys.argv) > 1 and sys.argv[1] == "reference_search":
         if args["--PMID_reference"] or args["--PMID-reference"]:
             PMID_reference(args["<config_json_file>"], args["<references_file_or_URL>"], args["--test"])
@@ -125,8 +121,7 @@ def main():
                              args["--no_PubMed"] or args["--no-PubMed"],
                              args["--test"], 
                              args["--prev-pub"] if args["--prev-pub"] else args["--prev_pub"],
-                             args["--save-all-queries"],
-                             args["--citation-match-ratio"])
+                             args["--save-all-queries"])
     elif len(sys.argv) > 1 and sys.argv[1] == "find_ORCID":
         find_ORCID(args["<config_json_file>"])
     elif len(sys.argv) > 1 and sys.argv[1] == "find_Google_Scholar":
@@ -151,7 +146,7 @@ def main():
 
 
 def author_search(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed, 
-                  test, prev_pub_filepath, save_all_results, citation_match_ratio):
+                  test, prev_pub_filepath, save_all_results):
     """Query sources for publications by author.
     
     Reads in the JSON config file, previous publications JSON file, and checks for errors.
@@ -169,11 +164,10 @@ def author_search(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref,
         test (bool): If True save_dir_name is tracker-test instead of tracker- and emails are not sent.
         prev_pub_filepath (str or None): filepath to the publication JSON to read in.
         save_all_results (bool): if True, save all of the queried publications from each source as "all_results.json"
-        citation_match_ratio (int): if the fuzzy ratio between 2 citations is greater than or equal to this, then consider them to match.
     """
 
     config_dict = athr_srch_modularized.input_reading_and_checking(config_json_filepath, no_ORCID, no_GoogleScholar, 
-                                                                   no_Crossref, no_PubMed, citation_match_ratio)
+                                                                   no_Crossref, no_PubMed)
 
     ## Create an authors_json for each project in the config_dict and update those authors attributes with the project attributes.
     authors_by_project_dict, config_dict = athr_srch_modularized.generate_internal_data_and_check_authors(config_dict)
@@ -184,7 +178,7 @@ def author_search(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref,
         user_input_checking.prev_pubs_file_check(prev_pubs)
 
     ## Query sources and build publication_dict.
-    publication_dict, all_queries = athr_srch_modularized.build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed, citation_match_ratio)            
+    publication_dict, all_queries = athr_srch_modularized.build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed)            
 
     save_dir_name = athr_srch_modularized.save_and_send_reports_and_emails(authors_by_project_dict, publication_dict, config_dict, test)
 
@@ -199,7 +193,7 @@ def author_search(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref,
 
 
 def reference_search(config_json_filepath, ref_path_or_URL, MEDLINE_reference, no_Crossref, no_PubMed, 
-                     test, prev_pub_filepath, save_all_results, citation_match_ratio):
+                     test, prev_pub_filepath, save_all_results):
     """Query PubMed and Crossref for publications matching a reference.
     
     Read in user inputs and check for error, query sources based on inputs, build 
@@ -214,12 +208,14 @@ def reference_search(config_json_filepath, ref_path_or_URL, MEDLINE_reference, n
         test (bool): If True save_dir_name is tracker-test instead of tracker- and emails are not sent.
         prev_pub_filepath (str or None): filepath to the publication JSON to read in.
         save_all_results (bool): if True, save all of the queried publications from each source as "all_results.json"
-        citation_match_ratio (int): if the fuzzy ratio between 2 citations is greater than or equal to this, then consider them to match.
     """
 
-    config_dict, tokenized_citations, has_previous_pubs, prev_pubs = ref_srch_modularized.input_reading_and_checking(config_json_filepath, ref_path_or_URL, MEDLINE_reference, no_Crossref, no_PubMed, prev_pub_filepath, citation_match_ratio)       
+    config_dict, tokenized_citations, has_previous_pubs, prev_pubs = \
+        ref_srch_modularized.input_reading_and_checking(config_json_filepath, ref_path_or_URL, 
+                                                        MEDLINE_reference, no_Crossref, no_PubMed, 
+                                                        prev_pub_filepath)       
 
-    publication_dict, tokenized_citations, all_queries = ref_srch_modularized.build_publication_dict(config_dict, tokenized_citations, no_Crossref, no_PubMed, citation_match_ratio)
+    publication_dict, tokenized_citations, all_queries = ref_srch_modularized.build_publication_dict(config_dict, tokenized_citations, no_Crossref, no_PubMed)
 
     save_dir_name = ref_srch_modularized.save_and_send_reports_and_emails(config_dict, tokenized_citations, publication_dict, prev_pubs, has_previous_pubs, test)
 

src/academic_tracker/athr_srch_modularized.py

@@ -20,7 +20,7 @@
 from . import webio
 
 
-def input_reading_and_checking(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed, citation_match_ratio):
+def input_reading_and_checking(config_json_filepath, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed):
     """Read in inputs from user and do error checking.
     
     Args:
@@ -29,18 +29,10 @@ def input_reading_and_checking(config_json_filepath, no_ORCID, no_GoogleScholar,
         no_GoogleScholar (bool): if True search Google Scholar else don't. Reduces checking on config JSON if True.
         no_Crossref (bool): If True search Crossref else don't. Reduces checking on config JSON if True.
         no_PubMed (bool): If True search PubMed else don't. Reduces checking on config JSON if True.
-        citation_match_ratio (int): should be an integer between 0-100.
         
     Returns:
         config_dict (dict): Matches the Configuration file JSON schema.
-    """
-    if not isinstance(citation_match_ratio, int):
-        helper_functions.vprint("Error: The given citation-match-ratio is not an integer value.")
-        sys.exit()
-    elif citation_match_ratio > 100 or citation_match_ratio < 0:
-        helper_functions.vprint("Error: The given citation-match-ratio is not within the range 0-100.")
-        sys.exit()
-    
+    """    
     ## read in config file
     config_dict = fileio.load_json(config_json_filepath)
 
@@ -93,7 +85,7 @@ def generate_internal_data_and_check_authors(config_dict):
 
 
 
-def build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed, citation_match_ratio):
+def build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, no_Crossref, no_PubMed):
     """Query PubMed, ORCID, Google Scholar, and Crossref for publications for the authors.
     
     Args:
@@ -103,7 +95,6 @@ def build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, n
         no_GoogleScholar (bool): if True search Google Scholar else don't.
         no_Crossref (bool): If True search Crossref else don't.
         no_PubMed (bool): If True search PubMed else don't.
-        citation_match_ratio (int): if the fuzzy ratio between 2 citations is greater than or equal to this, then consider them to match.
         
     Returns:
         running_pubs (dict): The dictionary matching the publication JSON schema.
@@ -116,30 +107,30 @@ def build_publication_dict(config_dict, prev_pubs, no_ORCID, no_GoogleScholar, n
     all_queries = {}
     if not no_PubMed:
         helper_functions.vprint("Searching PubMed.")
-        running_pubs, PubMed_publication_dict = athr_srch_webio.search_PubMed_for_pubs(running_pubs, config_dict["Authors"], config_dict["PubMed_search"]["PubMed_email"], citation_match_ratio)
+        running_pubs, PubMed_publication_dict = athr_srch_webio.search_PubMed_for_pubs(running_pubs, config_dict["Authors"], config_dict["PubMed_search"]["PubMed_email"])
         all_queries["PubMed"] = PubMed_publication_dict
     if not no_ORCID:
         helper_functions.vprint("Searching ORCID.")
-        running_pubs, ORCID_publication_dict = athr_srch_webio.search_ORCID_for_pubs(running_pubs, config_dict["ORCID_search"]["ORCID_key"], config_dict["ORCID_search"]["ORCID_secret"], config_dict["Authors"], citation_match_ratio)
+        running_pubs, ORCID_publication_dict = athr_srch_webio.search_ORCID_for_pubs(running_pubs, config_dict["ORCID_search"]["ORCID_key"], config_dict["ORCID_search"]["ORCID_secret"], config_dict["Authors"])
         all_queries["ORCID"] = ORCID_publication_dict
     if not no_GoogleScholar:
         helper_functions.vprint("Searching Google Scholar.")
-        running_pubs, Google_Scholar_publication_dict = athr_srch_webio.search_Google_Scholar_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], citation_match_ratio)
+        running_pubs, Google_Scholar_publication_dict = athr_srch_webio.search_Google_Scholar_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"])
         all_queries["Google Scholar"] = Google_Scholar_publication_dict
     if not no_Crossref:
         helper_functions.vprint("Searching Crossref.")
-        running_pubs, Crossref_publication_dict = athr_srch_webio.search_Crossref_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], citation_match_ratio)
+        running_pubs, Crossref_publication_dict = athr_srch_webio.search_Crossref_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"])
         all_queries["Crossref"] = Crossref_publication_dict
 
     ## Do a second pass using the saved queries.
     if not no_PubMed:
-        running_pubs, PubMed_publication_dict = athr_srch_webio.search_PubMed_for_pubs(running_pubs, config_dict["Authors"], config_dict["PubMed_search"]["PubMed_email"], citation_match_ratio, all_queries["PubMed"])
+        running_pubs, PubMed_publication_dict = athr_srch_webio.search_PubMed_for_pubs(running_pubs, config_dict["Authors"], config_dict["PubMed_search"]["PubMed_email"], all_queries["PubMed"])
     if not no_ORCID:
-        running_pubs, ORCID_publication_dict = athr_srch_webio.search_ORCID_for_pubs(running_pubs, config_dict["ORCID_search"]["ORCID_key"], config_dict["ORCID_search"]["ORCID_secret"], config_dict["Authors"], citation_match_ratio, all_queries["ORCID"])
+        running_pubs, ORCID_publication_dict = athr_srch_webio.search_ORCID_for_pubs(running_pubs, config_dict["ORCID_search"]["ORCID_key"], config_dict["ORCID_search"]["ORCID_secret"], config_dict["Authors"], all_queries["ORCID"])
     if not no_GoogleScholar:
-        running_pubs, Google_Scholar_publication_dict = athr_srch_webio.search_Google_Scholar_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], citation_match_ratio, all_queries["Google Scholar"])
+        running_pubs, Google_Scholar_publication_dict = athr_srch_webio.search_Google_Scholar_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], all_queries["Google Scholar"])
     if not no_Crossref:
-        running_pubs, Crossref_publication_dict = athr_srch_webio.search_Crossref_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], citation_match_ratio, all_queries["Crossref"])
+        running_pubs, Crossref_publication_dict = athr_srch_webio.search_Crossref_for_pubs(running_pubs, config_dict["Authors"], config_dict["Crossref_search"]["mailto_email"], all_queries["Crossref"])
 
     ## Compare current pubs with previous and only keep those that are new or updated.
     for pub_id, pub_values in prev_pubs.items():