Export service json validator fixes (#382)

* Assorted fixes for export service to get closer to passing validator

* Accept date or date-time

* Hook in the validator check to the jsonl output

* Get things passing the validator for the exported json lines

* mypy and flake8 fixes

Author: dannymandel

isamples_metadata/GEOMETransformer.py

@@ -351,7 +351,7 @@ def produced_by_responsibilities(self) -> typing.List[dict[str, str]]:
             return responsibilities
         return []
 
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         parent_record = self._source_record_parent_record()
         if parent_record is not None:
             return self._formatted_date(
@@ -633,7 +633,7 @@ def produced_by_responsibilities(self) -> list:
         # TODO: who did the tissue extract, if available -- where does this live, if anywhere?
         return []
 
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         # TODO: time the tissue extract was done, if available -- where does this live?
         return ""
 

isamples_metadata/OpenContextTransformer.py

@@ -420,7 +420,7 @@ def produced_by_responsibilities(self) -> typing.List[dict[str, str]]:
                 responsibilities.append(Transformer._responsibility_dict("collector", OpenContextTransformer._get_oc_str_or_dict_item_label(contributor)))
         return responsibilities
 
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         return self.source_record.get("published", Transformer.NOT_PROVIDED)
 
     def sampling_site_description(self) -> str:

isamples_metadata/SESARTransformer.py

@@ -467,7 +467,7 @@ def produced_by_responsibilities(self) -> list:
 
         return responsibilities
 
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         result_time = Transformer.NOT_PROVIDED
         description = self._source_record_description()
         if "collectionStartDate" in description:

isamples_metadata/SmithsonianTransformer.py

@@ -208,7 +208,7 @@ def produced_by_responsibilities(self) -> list[dict[str, str]]:
                 responsibilities.append({"role": "identified by", "name": current.strip()})
         return responsibilities
 
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         return self._formatted_date(
             self.source_record.get("year", ""),
             self.source_record.get("month", ""),

isamples_metadata/Transformer.py

@@ -260,17 +260,8 @@ def produced_by_responsibilities(self) -> typing.List[dict[str, str]]:
         """The responsibility list for the producedBy dictionary"""
         pass
 
-    def produced_by_result_time(self) -> str:
-        """The result time for the producedBy dictionary"""
-        result_time = self._produced_by_result_time_impl()
-        if result_time is not None:
-            # JSON schema expects this to be YYYY-MM-dd, so chop off any timestamps
-            if len(result_time) > 10:
-                result_time = result_time[:10]
-        return result_time
-
     @abstractmethod
-    def _produced_by_result_time_impl(self) -> str:
+    def produced_by_result_time(self) -> str:
         pass
 
     @abstractmethod

isb_lib/utilities/solr_result_transformer.py

@@ -12,8 +12,8 @@
     METADATA_REGISTRANT, METADATA_SAMPLE_LOCATION, METADATA_ELEVATION, METADATA_SAMPLING_SITE, \
     METADATA_RESULT_TIME, METADATA_HAS_FEATURE_OF_INTEREST, METADATA_DESCRIPTION, METADATA_INFORMAL_CLASSIFICATION, \
     METADATA_KEYWORDS, METADATA_HAS_SPECIMEN_CATEGORY, METADATA_HAS_MATERIAL_CATEGORY, METADATA_HAS_CONTEXT_CATEGORY, \
-    METADATA_LABEL, METADATA_SAMPLE_IDENTIFIER, METADATA_AT_ID, METADATA_RESPONSIBILITY, METADATA_PRODUCED_BY, \
-    METADATA_NAME
+    METADATA_LABEL, METADATA_SAMPLE_IDENTIFIER, METADATA_RESPONSIBILITY, METADATA_PRODUCED_BY, \
+    METADATA_NAME, METADATA_KEYWORD, METADATA_IDENTIFIER, METADATA_ROLE
 from isamples_metadata.solr_field_constants import SOLR_PRODUCED_BY_SAMPLING_SITE_PLACE_NAME, SOLR_AUTHORIZED_BY, \
     SOLR_COMPLIES_WITH, SOLR_PRODUCED_BY_SAMPLING_SITE_LOCATION_LONGITUDE, \
     SOLR_PRODUCED_BY_SAMPLING_SITE_LOCATION_LATITUDE, SOLR_RELATED_RESOURCE_ISB_CORE_ID, SOLR_CURATION_RESPONSIBILITY, \
@@ -65,6 +65,19 @@ def transform(table: Table, dest_path_no_extension: str, append: bool) -> str:
 
 
 class JSONExportTransformer(AbstractExportTransformer):
+
+    @staticmethod
+    def filter_null_values(obj):
+        """
+        Recursively filter out null values from a dictionary.
+        """
+        if isinstance(obj, dict):
+            return {k: JSONExportTransformer.filter_null_values(v) for k, v in obj.items() if v is not None}
+        elif isinstance(obj, list):
+            return [JSONExportTransformer.filter_null_values(elem) for elem in obj if elem is not None]
+        else:
+            return obj
+
     @staticmethod
     def transform(table: Table, dest_path_no_extension: str, append: bool) -> str:
         if append:
@@ -73,7 +86,7 @@ def transform(table: Table, dest_path_no_extension: str, append: bool) -> str:
         dest_path = f"{dest_path_no_extension}.{extension}"
         with open(dest_path, "w") as file:
             for row in petl.util.base.dicts(table):
-                json.dump(row, file)
+                json.dump(JSONExportTransformer.filter_null_values(row), file)
                 file.write("\n")
         return dest_path
 
@@ -90,22 +103,39 @@ def _add_to_dict(self, target_dict: dict, target_key: str, source_dict: dict, so
         if source_value is not None:
             target_dict[target_key] = source_value
 
+    def _add_responsibilities_to_container(self,
+                                           rec: dict,
+                                           responsibility_key_solr: str,
+                                           responsibility_key: str,
+                                           container: dict):
+        responsibilities = rec.get(responsibility_key_solr, [])
+        responsibility_dicts = []
+        for responsibility in responsibilities:
+            pieces = responsibility.split(":")
+            responsibility_dicts.append({METADATA_ROLE: pieces[0], METADATA_NAME: pieces[1]})
+        if len(responsibility_dicts) > 0:
+            container[responsibility_key] = responsibility_dicts
+
     def _curation_dict(self, rec: dict) -> dict:
         curation_dict: dict = {}
         self._add_to_dict(curation_dict, METADATA_LABEL, rec, SOLR_CURATION_LABEL)
         self._add_to_dict(curation_dict, METADATA_DESCRIPTION, rec, SOLR_CURATION_DESCRIPTION)
-        self._add_to_dict(curation_dict, METADATA_ACCESS_CONSTRAINTS, rec, SOLR_CURATION_ACCESS_CONSTRAINTS)
         self._add_to_dict(curation_dict, METADATA_CURATION_LOCATION, rec, SOLR_CURATION_LOCATION)
-        self._add_to_dict(curation_dict, METADATA_RESPONSIBILITY, rec, SOLR_CURATION_RESPONSIBILITY)
+        self._add_responsibilities_to_container(rec, SOLR_CURATION_RESPONSIBILITY, METADATA_RESPONSIBILITY, curation_dict)
+        access_constraints = rec.get(SOLR_CURATION_ACCESS_CONSTRAINTS, "").split("|")
+        if len(access_constraints) > 0:
+            curation_dict[METADATA_ACCESS_CONSTRAINTS] = access_constraints
         return curation_dict
 
     def _produced_by_dict(self, rec: dict) -> dict:
         produced_by_dict: dict = {}
-        self._add_to_dict(produced_by_dict, METADATA_AT_ID, rec, SOLR_PRODUCED_BY_ISB_CORE_ID)
+        self._add_to_dict(produced_by_dict, METADATA_IDENTIFIER, rec, SOLR_PRODUCED_BY_ISB_CORE_ID)
         self._add_to_dict(produced_by_dict, METADATA_LABEL, rec, SOLR_PRODUCED_BY_LABEL)
-        self._add_to_dict(produced_by_dict, METADATA_RESPONSIBILITY, rec, SOLR_PRODUCED_BY_RESPONSIBILITY)
         self._add_to_dict(produced_by_dict, METADATA_DESCRIPTION, rec, SOLR_PRODUCED_BY_DESCRIPTION)
-        self._add_to_dict(produced_by_dict, METADATA_RESULT_TIME, rec, SOLR_PRODUCED_BY_RESULT_TIME)
+        result_time = rec.get(SOLR_PRODUCED_BY_RESULT_TIME)
+        if result_time is not None:
+            result_time = result_time[:10]
+            produced_by_dict[METADATA_RESULT_TIME] = result_time
         self._add_to_dict(produced_by_dict, METADATA_HAS_FEATURE_OF_INTEREST, rec, SOLR_PRODUCED_BY_FEATURE_OF_INTEREST)
         sampling_site_dict: dict = {}
         produced_by_dict[METADATA_SAMPLING_SITE] = sampling_site_dict
@@ -117,10 +147,27 @@ def _produced_by_dict(self, rec: dict) -> dict:
         self._add_to_dict(sample_location_dict, METADATA_ELEVATION, rec, SOLR_PRODUCED_BY_SAMPLING_SITE_ELEVATION_IN_METERS)
         self._add_to_dict(sample_location_dict, METADATA_LATITUDE, rec, SOLR_PRODUCED_BY_SAMPLING_SITE_LOCATION_LATITUDE)
         self._add_to_dict(sample_location_dict, METADATA_LONGITUDE, rec, SOLR_PRODUCED_BY_SAMPLING_SITE_LOCATION_LONGITUDE)
+        self._add_responsibilities_to_container(rec, SOLR_PRODUCED_BY_RESPONSIBILITY, METADATA_RESPONSIBILITY, produced_by_dict)
         return produced_by_dict
 
+    def _formatted_controlled_vocabulary(self, rec: dict, key: str) -> list[dict]:
+        values = rec.get(key, [])
+        return [{METADATA_LABEL: value} for value in values]
+
+    def _has_specimen_categories(self, rec: dict) -> list:
+        return self._formatted_controlled_vocabulary(rec, SOLR_HAS_SPECIMEN_CATEGORY)
+
+    def _has_material_categories(self, rec: dict) -> list:
+        return self._formatted_controlled_vocabulary(rec, SOLR_HAS_MATERIAL_CATEGORY)
+
+    def _has_context_categories(self, rec: dict) -> list:
+        return self._formatted_controlled_vocabulary(rec, SOLR_HAS_CONTEXT_CATEGORY)
+
+    def _keywords(self, rec: dict) -> list:
+        return [{METADATA_KEYWORD: keyword} for keyword in rec.get(SOLR_KEYWORDS, [])]
+
     def _registrant_dict(self, rec: dict) -> dict:
-        return {METADATA_NAME: rec[SOLR_REGISTRANT]}
+        return {METADATA_NAME: rec[SOLR_REGISTRANT][0]}
 
     def _rename_table_columns_csv(self):
         """Renames the solr columns to the public names in the public metadata schema, while maintaining CSV tabular format"""
@@ -168,11 +215,11 @@ def _rename_table_columns_jsonl(self):
         mappings[METADATA_LABEL] = SOLR_LABEL
         mappings[METADATA_DESCRIPTION] = SOLR_DESCRIPTION
         mappings["source_collection"] = SOLR_SOURCE  # this isn't present in the exported metadata
-        mappings[METADATA_HAS_SPECIMEN_CATEGORY] = SOLR_HAS_SPECIMEN_CATEGORY
-        mappings[METADATA_HAS_MATERIAL_CATEGORY] = SOLR_HAS_MATERIAL_CATEGORY
-        mappings[METADATA_HAS_CONTEXT_CATEGORY] = SOLR_HAS_CONTEXT_CATEGORY
+        mappings[METADATA_HAS_SPECIMEN_CATEGORY] = self._has_specimen_categories
+        mappings[METADATA_HAS_MATERIAL_CATEGORY] = self._has_material_categories
+        mappings[METADATA_HAS_CONTEXT_CATEGORY] = self._has_context_categories
         mappings[METADATA_INFORMAL_CLASSIFICATION] = SOLR_INFORMAL_CLASSIFICATION
-        mappings[METADATA_KEYWORDS] = SOLR_KEYWORDS
+        mappings[METADATA_KEYWORDS] = self._keywords
         mappings[METADATA_PRODUCED_BY] = self._produced_by_dict
         mappings[METADATA_REGISTRANT] = self._registrant_dict
         mappings[METADATA_SAMPLING_PURPOSE] = SOLR_SAMPLING_PURPOSE

tests/test_data/OpenContext/test/ark-28722-k26d5xr5z-test.json

@@ -52,7 +52,7 @@
                 "name": "Marshall C. Agnew"
             }
         ],
-        "result_time": "2007-11-11",
+        "result_time": "2007-11-11T00:00:00Z",
         "sampling_site": {
             "description": "",
             "label": "",

tests/test_data/OpenContext/test/ark-28722-k26h4xk1f-test.json

@@ -50,7 +50,7 @@
                 "name": "Joanna Smith"
             }
         ],
-        "result_time": "2017-01-30",
+        "result_time": "2017-01-30T22:57:28Z",
         "sampling_site": {
             "description": "",
             "label": "",

tests/test_data/OpenContext/test/ark-28722-k2b85cg1p-test.json

@@ -48,7 +48,7 @@
                 "name": "Katrina M. Haile"
             }
         ],
-        "result_time": "2007-11-11",
+        "result_time": "2007-11-11T00:00:00Z",
         "sampling_site": {
             "description": "",
             "label": "",

tests/test_data/OpenContext/test/ark-28722-k2vq31x46-test.json

@@ -48,7 +48,7 @@
                 "name": "Elizabeth E. Payne"
             }
         ],
-        "result_time": "2007-11-11",
+        "result_time": "2007-11-11T00:00:00Z",
         "sampling_site": {
             "description": "",
             "label": "",