New Feature: SNOMED::ICD10CM Mapping Support

joeflack4 · joeflack4 · commit 7d90e91f5e19 · 2022-02-24T19:36:20.000-05:00
- Added feature to allow for conversion of these premade mappings provided by SNOMED into SSSOM format. (WIP) General updates - cli.py: Reorganized SSSOM_READ_FORMATS: Top half are plain data formats, and bottom half are special-case formats. Both halves of the list are alphabetically sorted. Tentative updates - Changed some relative imports to absolute imports, in order to speed up development and make debugging easier. It is possible that this could be a good permanent change too, though. Dev updates - requirements-unlocked.txt: Unversioned variation of requirements.txt - requirements.txt: Included this because I don't know if setup.py is sufficient to install packages in a standalone dev environment. Also, it is shown to be expected in the documentation: https://mapping-commons.github.io/sssom-py/installation.html#installation-for-developers
diff --git a/requirements-unlocked.txt b/requirements-unlocked.txt
@@ -0,0 +1,38 @@
+pyparsing
+click
+linkml-runtime
+networkx
+numpy
+pandas
+pandasql
+pyyaml
+rdflib
+recommonmark
+scikit_learn
+scipy
+setuptools
+sparqlwrapper
+validators
+# joeflack4 2022/02/24: I commented these specific versions out for now because of these errors:
+# ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
+# sparqlslurper 0.4.1 requires rdflib~=5.0, but you have rdflib 6.1.1 which is incompatible.
+# pyshexc 0.8.3 requires rdflib~=5.0, but you have rdflib 6.1.1 which is incompatible.
+# pyshex 0.7.20 requires rdflib~=5.0, but you have rdflib 6.1.1 which is incompatible.
+# linkml 1.0.2 requires rdflib~=5.0, but you have rdflib 6.1.1 which is incompatible.
+# linkml-model 1.0.0 requires rdflib~=5.0, but you have rdflib 6.1.1 which is incompatible.
+# pyparsing==2.4.7
+# click
+# linkml-runtime>=1.1.12
+# networkx
+# numpy
+# pandas
+# pandasql
+# pyyaml
+# rdflib>=6
+# recommonmark>=0.7
+# scikit_learn
+# scipy
+# setuptools
+# sparqlwrapper
+# validators
+# validators>=0.0
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,96 @@
+alabaster==0.7.12
+antlr4-python3-runtime==4.9.2
+appdirs==1.4.4
+attrs==21.2.0
+Babel==2.9.1
+cachetools==4.2.2
+certifi==2021.5.30
+CFGraph==0.2.1
+chardet==4.0.0
+charset-normalizer==2.0.3
+click==7.1.2
+commonmark==0.9.1
+decorator==4.4.2
+Deprecated==1.2.12
+distlib==0.3.2
+docutils==0.17.1
+filelock==3.0.12
+frozendict==2.0.3
+graphviz==0.17
+greenlet==1.1.0
+hbreader==0.9.1
+idna==3.2
+imagesize==1.2.0
+iniconfig==1.1.1
+isodate==0.6.0
+Jinja2==3.0.1
+joblib==1.0.1
+json-flattener==0.1.7
+jsonasobj==1.2.1
+jsonasobj2==1.0.4
+jsonschema==3.2.0
+linkml==1.0.2
+linkml-model==1.0.0
+linkml-runtime==1.1.24
+lxml==4.6.3
+MarkupSafe==2.0.1
+networkx==2.5.1
+numpy==1.21.1
+packaging==21.0
+pandas==1.1.5
+pandasql==0.7.3
+parse==1.19.0
+pbr==5.6.0
+pluggy==0.13.1
+prefixcommons==0.1.9
+prologterms==0.0.6
+py==1.10.0
+Pygments==2.9.0
+PyJSG==0.11.6
+PyLD==2.0.3
+PyLDmod==2.0.5
+pyparsing==2.4.7
+pyrsistent==0.18.0
+PyShEx==0.7.20
+PyShExC==0.8.3
+pytest==6.2.4
+python-dateutil==2.8.2
+pytz==2021.1
+PyYAML==5.4.1
+rdflib==6.1.1
+rdflib-jsonld==0.6.1
+rdflib-pyld-compat==0.1.0
+rdflib-pyldmod-compat==0.1.2
+rdflib-shim==1.0.3
+recommonmark==0.7.1
+requests==2.26.0
+scikit-learn==0.24.2
+scipy==1.7.0
+ShExJSG==0.7.1
+six==1.16.0
+snowballstemmer==2.1.0
+sparqlslurper==0.4.1
+SPARQLWrapper==1.8.5
+Sphinx==2.4.4
+sphinx-click==2.3.1
+sphinx-rtd-theme==0.4.3
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+SQLAlchemy==1.4.21
+stevedore==3.3.0
+testfixtures==6.18.0
+threadpoolctl==2.2.0
+toml==0.10.2
+tox==3.23.1
+urllib3==1.26.6
+validators==0.18.2
+virtualenv==20.4.7
+virtualenv-clone==0.5.4
+virtualenvwrapper==4.8.4
+watchdog==2.1.3
+wrapt==1.12.1
+zipp==3.5.0
diff --git a/sssom/cli.py b/sssom/cli.py
@@ -24,12 +24,12 @@
 from rdflib import Graph
 from scipy.stats import chi2_contingency
 
-from .cliques import split_into_cliques, summarize_cliques
-from .io import convert_file, parse_file, split_file, validate_file
-from .parsers import read_sssom_table
-from .rdf_util import rewire_graph
-from .sparql_util import EndpointConfig, query_mappings
-from .util import (
+from sssom.cliques import split_into_cliques, summarize_cliques
+from sssom.io import convert_file, parse_file, split_file, validate_file
+from sssom.parsers import read_sssom_table
+from sssom.rdf_util import rewire_graph
+from sssom.sparql_util import EndpointConfig, query_mappings
+from sssom.util import (
     SSSOM_EXPORT_FORMATS,
     SSSOM_READ_FORMATS,
     MappingSetDataFrame,
@@ -41,7 +41,7 @@
     remove_unmatched,
     to_mapping_set_dataframe,
 )
-from .writers import write_table
+from sssom.writers import write_table
 
 # Click input options common across commands
 input_argument = click.argument("input", required=True, type=click.Path())
diff --git a/sssom/parsers.py b/sssom/parsers.py
@@ -140,6 +140,46 @@ def read_obographs_json(
     )
 
 
+def read_snomed_icd10cm_map_tsv(
+    file_path: str,
+    prefix_map: Dict[str, str] = None,
+    meta: Dict[str, str] = None,
+) -> MappingSetDataFrame:
+    """Parse special SNOMED ICD10CM mapping file and translates it into a MappingSetDataFrame.
+
+    :param file_path: The path to the obographs file
+    :param prefix_map: an optional prefix map
+    :param meta: an optional dictionary of metadata elements
+    :return: A SSSOM MappingSetDataFrame
+    """
+    raise_for_bad_path(file_path)
+    df = read_pandas(file_path)
+
+    # TODO: DRYify: Put this in a func as is used multiple times now. - joeflack4 2022/02/24
+    # If SSSOM external metadata is provided, merge it with the internal metadata
+    sssom_metadata = _read_metadata_from_table(file_path)
+    if sssom_metadata:
+        if meta:
+            for k, v in meta.items():
+                if k in sssom_metadata:
+                    if sssom_metadata[k] != v:
+                        logging.warning(
+                            f"SSSOM internal metadata {k} ({sssom_metadata[k]}) "
+                            f"conflicts with provided ({meta[k]})."
+                        )
+                else:
+                    logging.info(
+                        f"Externally provided metadata {k}:{v} is added to metadata set."
+                    )
+                    sssom_metadata[k] = v
+        meta = sssom_metadata
+
+    prefix_map, meta = _get_prefix_map_and_metadata(prefix_map=prefix_map, meta=meta)
+
+    df2 = from_snomed_icd10cm_map_tsv(df, prefix_map=prefix_map, meta=meta)
+    return df2
+
+
 def _get_prefix_map_and_metadata(
     prefix_map: Optional[PrefixMap] = None, meta: Optional[MetadataType] = None
 ) -> Metadata:
@@ -499,6 +539,42 @@ def from_obographs(
     return to_mapping_set_dataframe(mdoc)
 
 
+def from_snomed_icd10cm_map_tsv(
+    df: pd.DataFrame,
+    *,
+    prefix_map: Optional[PrefixMap] = None,
+    meta: Optional[MetadataType] = None,
+) -> MappingSetDataFrame:
+    """Convert a snomed_icd10cm_map dataframe to a MappingSetDataFrame.
+
+    :param df: A mappings dataframe
+    :param prefix_map: A prefix map
+    :param meta: A metadata dictionary
+    :return: MappingSetDataFrame
+    """
+    prefix_map = _ensure_prefix_map(prefix_map)
+
+    # TODO: repurpose boilerplate below
+    if "confidence" in df.columns:
+        df["confidence"].replace(r"^\s*$", np.NaN, regex=True, inplace=True)
+
+    mlist: List[Mapping] = []
+    ms = _init_mapping_set(meta)
+    bad_attrs: typing.Counter[str] = Counter()
+    for _, row in df.iterrows():
+        mdict, ms, bad_attrs = _get_mdict_ms_and_bad_attrs(row, ms, bad_attrs)
+        mlist.append(_prepare_mapping(Mapping(**mdict)))
+
+    for k, v in bad_attrs.most_common():
+        logging.warning(f"No attr for {k} [{v} instances]")
+    # the autogenerated code's type annotations are _really_ messy. This is in fact okay,
+    # so with a heavy heart we employ type:ignore
+    ms.mappings = mlist  # type:ignore
+    _set_metadata_in_mapping_set(mapping_set=ms, metadata=meta)
+    doc = MappingSetDocument(mapping_set=ms, prefix_map=prefix_map)
+    return to_mapping_set_dataframe(doc)
+
+
 # All from_* take as an input a python object (data frame, json, etc) and return a MappingSetDataFrame
 # All read_* take as an input a a file handle and return a MappingSetDataFrame (usually wrapping a from_* method)
 
@@ -523,6 +599,9 @@ def get_parsing_function(input_format: Optional[str], filename: str) -> Callable
         return read_alignment_xml
     elif input_format == "obographs-json":
         return read_obographs_json
+    elif input_format == "snomed-icd10cm-map-tsv":
+        return read_snomed_icd10cm_map_tsv
+
     else:
         raise Exception(f"Unknown input format: {input_format}")
 
diff --git a/sssom/util.py b/sssom/util.py
@@ -43,12 +43,13 @@
 PREFIX_MAP_KEY = "curie_map"
 
 SSSOM_READ_FORMATS = [
-    "tsv",
-    "rdf",
+    "json",
     "owl",
+    "rdf",
+    "tsv",
     "alignment-api-xml",
     "obographs-json",
-    "json",
+    "snomed-icd10cm-map-tsv"
 ]
 SSSOM_EXPORT_FORMATS = ["tsv", "rdf", "owl", "json"]
 
