Merge pull request #58 from jdkent/enh/add_processed_db

[FIX] re-run errors

Author: jdkent

ns_extract/pipelines/base.py

@@ -390,18 +390,20 @@ def __identify_matching_results(
 
         for dbid, study in dataset.data.items():
             # Get existing input file hashes for this study
-            existing = existing_results.get(dbid, {}).get("inputs", {})
+            existing_output = existing_results.get(dbid, {})
+            existing = existing_output.get("inputs", {})
 
-            # Skip if no existing results and there are current inputs
-            if not existing and dbid in study_inputs and study_inputs[dbid]:
+            # There is no match if there are no outputs
+            if not existing_output:
                 result_matches[dbid] = False
                 continue
-            elif not study_inputs[dbid]:
-                # no current inputs, so results are matching
+            elif not study_inputs[dbid] and existing_output:
+                # no current inputs and output file already exists
+                # so results are matching
                 result_matches[dbid] = True
                 continue
 
-            # Use __are_file_hashes_identical to compare hashes
+            # Compare file hashes to determine if the inputs have changed
             result_matches[dbid] = self.__do_file_hashes_match(
                 study_inputs[dbid], existing
             )

ns_extract/pipelines/participant_demographics/model.py

@@ -22,6 +22,9 @@ def post_process(self, results, study_inputs, **kwargs):
         # Clean known issues with GPT demographics result
         cleaned_results = super().post_process(results, study_inputs, **kwargs)
         for study_id, study_result in cleaned_results.items():
+            if study_result == dict():
+                study_result = {"groups": []}
+                continue
             meta_keys = ["pmid", "rank", "start_char", "end_char", "id"]
             meta_keys = [k for k in meta_keys if k in study_result]
 

ns_extract/pipelines/utils.py

@@ -9,6 +9,7 @@
 pipeline implementations.
 """
 
+import math
 import hashlib
 import inspect
 from pathlib import Path
@@ -66,7 +67,12 @@ def _load_json(self, file_path: Path) -> Dict[str, Any]:
             json.JSONDecodeError: If file contains invalid JSON
         """
         with file_path.open("r") as f:
-            return json.load(f)
+            data = json.load(f)
+        if isinstance(data.get("title", ""), float) and math.isnan(data["title"]):
+            data["title"] = ""
+        if isinstance(data.get("abstract", ""), float) and math.isnan(data["abstract"]):
+            data["abstract"] = ""
+        return data
 
     def _write_json(self, file_path: Path, data: Dict[str, Any]) -> None:
         """Write data to a JSON file.

tests/cassettes/test_nv_task/test_TaskExtractor.yaml

@@ -0,0 +1 @@
+{"pmid": "22410476", "doi": "10.1016/j.brs.2012.01.006", "pmcid": null}

tests/data/sample_inputs/4tm6XzAD8xSa/identifiers.json

@@ -0,0 +1,11 @@
+{
+  "title": "Modulation of verbal fluency networks by transcranial direct current stimulation (tDCS) in Parkinson\u2019s disease",
+  "authors": "Pereira, Joana B;Junque, Carme;Bartres-Faz, David;Marti, Maria J;Sala-Llonch, Roser;Compta, Yarko;Falcon, Carles;Vendrell, Pere;Pascual-Leone, Alvaro;Valls-Sole, Josep;Tolosa, Eduardo",
+  "journal": "Brain stimulation",
+  "keywords": null,
+  "abstract": null,
+  "publication_year": 2013,
+  "coordinate_space": "UNKNOWN",
+  "license": null,
+  "text": false
+}

tests/data/sample_inputs/4tm6XzAD8xSa/processed/db/metadata.json

@@ -0,0 +1 @@
+{"pmid": "28301868", "doi": "10.1097/PHM.0000000000000582", "pmcid": null}

tests/data/sample_inputs/G6qSwgM5koZp/identifiers.json

@@ -0,0 +1 @@
+

tests/data/sample_inputs/G6qSwgM5koZp/processed/ace/coordinates.csv

@@ -0,0 +1,11 @@
+{
+    "title": "Quadriplegia Due to Injury of Corticofugal Tracts from Secondary Motor Area in a Patient With Severe Traumatic Brain Injury.",
+    "authors": "Jang, Sung Ho;Kim, Seong Ho;Lee, Han Do",
+    "journal": "American journal of physical medicine & rehabilitation",
+    "keywords": null,
+    "abstract": NaN,
+    "publication_year": 2017,
+    "coordinate_space": "UNKNOWN",
+    "license": null,
+    "text": true
+}