Get dict of schema (#219)

* corrected sssom.get_dict_from_mapping

* tests fixed

* reformatted

* test values changes

* avoided .astype(str) for DataFrames

* clean-up

* formatted

* duplicate drop

* reverting concat back to merge

* made NaN assignment dynamic

* fixed dataframe cell value deduction if None

* formatted

Author: hrshdhgd

setup.cfg

@@ -144,6 +144,7 @@ ignore =
     E501 # Line length
     W503 # Line break before binary operator (flake8 is wrong)
     S408 # don't worry about unsafe xml
+    S324 # Use of weak MD4, MD5, or SHA1 hash for security. Consider usedforsecurity=False
     S318 # don't worry about unsafe xml
     S310 # TODO remove this later and switch to using requests
     B018 # This is 'useless' statements which are new atm.

sssom/parsers.py

@@ -18,6 +18,8 @@
 from linkml_runtime.loaders.json_loader import JSONLoader
 from rdflib import Graph, URIRef
 
+from sssom.constants import MAPPING_SET_SLOTS, MAPPING_SLOTS
+
 from .context import (
     DEFAULT_LICENSE,
     DEFAULT_MAPPING_SET_ID,
@@ -75,7 +77,6 @@ def read_sssom_table(
         meta = sssom_metadata
 
     prefix_map, meta = _get_prefix_map_and_metadata(prefix_map=prefix_map, meta=meta)
-
     msdf = from_sssom_dataframe(df, prefix_map=prefix_map, meta=meta)
     return msdf
 
@@ -188,21 +189,23 @@ def _get_mdict_ms_and_bad_attrs(
 ) -> Tuple[dict, MappingSet, Counter]:
 
     mdict = {}
+
     for k, v in row.items():
         if v and v == v:
             ok = False
             if k:
                 k = str(k)
             v = _address_multivalued_slot(k, v)
-            if hasattr(Mapping, k):
+            # if hasattr(Mapping, k):
+            if k in MAPPING_SLOTS:
                 mdict[k] = v
                 ok = True
-            if hasattr(MappingSet, k):
+            # if hasattr(MappingSet, k):
+            if k in MAPPING_SET_SLOTS:
                 ms[k] = v
                 ok = True
             if not ok:
                 bad_attrs[k] += 1
-
     return (mdict, ms, bad_attrs)
 
 
@@ -248,7 +251,6 @@ def from_sssom_dataframe(
     for _, row in df.iterrows():
         mdict, ms, bad_attrs = _get_mdict_ms_and_bad_attrs(row, ms, bad_attrs)
         mlist.append(_prepare_mapping(Mapping(**mdict)))
-
     for k, v in bad_attrs.most_common():
         logging.warning(f"No attr for {k} [{v} instances]")
     # the autogenerated code's type annotations are _really_ messy. This is in fact okay,

sssom/util.py

@@ -35,7 +35,7 @@
 from .context import SSSOM_URI_PREFIX, get_default_metadata, get_jsonld_context
 from .internal_context import multivalued_slots
 from .sssom_datamodel import Mapping as SSSOM_Mapping
-from .sssom_datamodel import MatchTypeEnum, PredicateModifierEnum, slots
+from .sssom_datamodel import slots
 from .sssom_document import MappingSetDocument
 from .typehints import Metadata, MetadataType, PrefixMap
 
@@ -831,17 +831,49 @@ def get_dict_from_mapping(map_obj: Union[Any, Dict[Any, Any], SSSOM_Mapping]) ->
     :return: Dictionary
     """
     map_dict = {}
+    slots_with_double_as_range = [
+        s
+        for s in SCHEMA_DICT["slots"].keys()
+        if SCHEMA_DICT["slots"][s]["range"] == "double"
+    ]
     for property in map_obj:
-        if isinstance(map_obj[property], list):
-            map_dict[property] = "|".join(
-                enum_value.code.text
-                for enum_value in map_obj[property]
-                if type(enum_value).__name__ == MatchTypeEnum._defn.name
-            )
-        elif type(map_obj[property]).__name__ == PredicateModifierEnum._defn.name:
-            map_dict[property] = map_obj[property].code.text
+        if map_obj[property] is not None:
+            if isinstance(map_obj[property], list):
+                # IF object is an enum
+                if (
+                    SCHEMA_DICT["slots"][property]["range"]
+                    in SCHEMA_DICT["enums"].keys()
+                ):
+                    # IF object is a multivalued enum
+                    if SCHEMA_DICT["slots"][property]["multivalued"]:
+                        map_dict[property] = "|".join(
+                            enum_value.code.text for enum_value in map_obj[property]
+                        )
+                    # If object is NOT multivalued BUT an enum.
+                    else:
+                        map_dict[property] = map_obj[property].code.text
+                # IF object is NOT an enum but a list
+                else:
+                    map_dict[property] = "|".join(
+                        enum_value for enum_value in map_obj[property]
+                    )
+            # IF object NOT a list
+            else:
+                # IF object is an enum
+                if (
+                    SCHEMA_DICT["slots"][property]["range"]
+                    in SCHEMA_DICT["enums"].keys()
+                ):
+                    map_dict[property] = map_obj[property].code.text
+                else:
+                    map_dict[property] = map_obj[property]
         else:
-            map_dict[property] = map_obj[property]
+            # IF map_obj[property] is None:
+            if property in slots_with_double_as_range:
+                map_dict[property] = np.nan
+            else:
+                map_dict[property] = ""
+
     return map_dict
 
 

tests/test_config.yaml

@@ -2,7 +2,7 @@ tests:
   - filename: "basic.tsv"
     inputformat: "tsv"
     multiple_input: True
-    ct_json_elements: 11
+    ct_json_elements: 13
     ct_data_frame_rows: 141
     ct_graph_queries_owl:
       query_count_equivalent_classes: 90
@@ -11,7 +11,7 @@ tests:
   - filename: "basic2.tsv"
     inputformat: "tsv"
     multiple_input: True
-    ct_json_elements: 11
+    ct_json_elements: 13
     ct_data_frame_rows: 126
     ct_graph_queries_owl:
       query_count_equivalent_classes: 79

tests/test_parsers.py

@@ -1,10 +1,12 @@
 """Tests for parsers."""
 
 import json
+import math
 import os
 import unittest
 from xml.dom import minidom
 
+import numpy as np
 import pandas as pd
 import yaml
 from rdflib import Graph
@@ -176,9 +178,36 @@ def test_piped_element_to_list(self):
         input_path = os.path.join(test_data_dir, "basic.tsv")
         msdf = read_sssom_table(input_path)
         df = msdf.df
-        msdf.df = df[df["match_type"].str.contains("\\|")].reset_index()
+        msdf.df = df[df["match_type"].str.contains("\\|", na=False)].reset_index()
         old_match_type = msdf.df["match_type"]
         msdoc = to_mapping_set_document(msdf)
         new_msdf = to_mapping_set_dataframe(msdoc)
         new_match_type = new_msdf.df["match_type"]
         self.assertTrue(old_match_type.equals(new_match_type))
+
+    def test_read_sssom_table(self):
+        """Test read SSSOM method to validate import of all columns."""
+        input_path = os.path.join(test_data_dir, "basic3.tsv")
+        msdf = read_sssom_table(input_path)
+        imported_df = pd.read_csv(input_path, comment="#", sep="\t")
+        self.assertEqual(set(imported_df.columns), set(msdf.df.columns))
+        list_cols = [
+            "subject_match_field",
+            "object_match_field",
+            "match_string",
+            "match_type",
+        ]
+        for idx, row in msdf.df.iterrows():
+            for k, v in row.iteritems():
+                if v == "":
+                    self.assertTrue(math.isnan(imported_df.iloc[idx][k]))
+                else:
+                    if k not in list_cols:
+                        if v is np.nan:
+                            self.assertTrue(imported_df.iloc[idx][k] is v)
+                        else:
+                            self.assertEqual(imported_df.iloc[idx][k], v)
+                    elif k == "match_type":
+                        self.assertEqual(imported_df.iloc[idx][k], v)
+                    else:
+                        self.assertEqual(imported_df.iloc[idx][k], v)

tests/test_reconcile.py

@@ -29,7 +29,6 @@ def test_merge(self):
         """Test merging two tables."""
         msdf1 = read_sssom_table(data_dir / "basic.tsv")
         msdf2 = read_sssom_table(data_dir / "basic2.tsv")
-
         merged_msdf = merge_msdf(msdf1, msdf2)
 
         self.assertEqual(71, len(merged_msdf.df))