Merge branch 'master' into create-pull-request/patch

Author: matentzn

poetry.lock

@@ -13,7 +13,7 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0.0"
 click = ">=8.1.6"
-curies = ">=0.7.3"
+curies = ">=0.10.18"
 linkml-runtime = "^1.7.5"
 linkml = ">1.7.10"
 pandas = ">1.0.3"

pyproject.toml

@@ -13,6 +13,7 @@
 import yaml
 from curies import Converter
 from deprecation import deprecated
+from linkml.validator import ValidationReport
 
 from sssom.validators import validate
 
@@ -98,17 +99,23 @@ def parse_file(
     write_table(doc, output, embedded_mode)
 
 
-def validate_file(input_path: str, validation_types: List[SchemaValidationType]) -> None:
+def validate_file(
+    input_path: str,
+    validation_types: Optional[List[SchemaValidationType]] = None,
+    fail_on_error: bool = True,
+) -> dict[SchemaValidationType, ValidationReport]:
     """Validate the incoming SSSOM TSV according to the SSSOM specification.
 
     :param input_path: The path to the input file in one of the legal formats, eg obographs, aligmentapi-xml
     :param validation_types: A list of validation types to run.
+    :param fail_on_error: Should an exception be raised on error of _any_ validator?
+    :returns: A dictionary from validation types to validation reports
     """
     # Two things to check:
     # 1. All prefixes in the DataFrame are define in prefix_map
     # 2. All columns in the DataFrame abide by sssom-schema.
     msdf = parse_sssom_table(file_path=input_path)
-    validate(msdf=msdf, validation_types=validation_types)
+    return validate(msdf=msdf, validation_types=validation_types, fail_on_error=fail_on_error)
 
 
 def split_file(input_path: str, output_directory: Union[str, Path]) -> None:

src/sssom/io.py

@@ -1,5 +1,6 @@
 """SSSOM parsers."""
 
+import gzip
 import io
 import itertools as itt
 import json
@@ -96,6 +97,10 @@ def _open_input(input: Union[str, Path, TextIO]) -> io.StringIO:
         elif "\n" in input or "\r" in input:
             # It's string data
             return io.StringIO(input)
+        elif input.endswith(".gz"):
+            with gzip.open(input, "rt") as file:
+                file_content = file.read()
+            return io.StringIO(file_content)
         else:
             # It's a local file path
             with open(input, "r") as file:

src/sssom/parsers.py

@@ -3,6 +3,7 @@
 import logging
 from typing import Any, Dict, List, Optional
 
+import curies
 from linkml_runtime.utils.metamodelcore import URIorCURIE
 from rdflib import Graph, URIRef
 from sssom_schema import EntityReference, Mapping
@@ -26,7 +27,7 @@ def rewire_graph(
     if mdoc.mapping_set.mappings is None:
         raise TypeError
 
-    converter = mdoc.converter
+    converter: curies.Converter = mdoc.converter
     rewire_map: Dict[URIorCURIE, URIorCURIE] = {}
     for m in mdoc.mapping_set.mappings:
         if not isinstance(m, Mapping):
@@ -42,8 +43,8 @@ def rewire_graph(
                 curr_tgt = rewire_map[src]
                 logging.info(f"Ambiguous: {src} -> {tgt} vs {curr_tgt}")
                 if precedence:
-                    curr_pfx, _ = converter.parse_curie(curr_tgt)
-                    tgt_pfx, _ = converter.parse_curie(tgt)
+                    curr_pfx = converter.parse_curie(curr_tgt, strict=True).prefix
+                    tgt_pfx = converter.parse_curie(tgt, strict=True).prefix
                     if tgt_pfx in precedence:
                         if curr_pfx not in precedence or precedence.index(
                             tgt_pfx

src/sssom/rdf_util.py

@@ -17,7 +17,7 @@
 import pandas as pd
 import validators
 import yaml
-from curies import Converter
+from curies import Converter, ReferenceTuple
 from jsonschema import ValidationError
 from linkml_runtime.linkml_model.types import Uriorcurie
 from sssom_schema import Mapping as SSSOM_Mapping
@@ -257,7 +257,7 @@ def clean_prefix_map(self, strict: bool = True) -> None:
                        listed in the 'curie_map'.
         :raises ValueError: If prefixes absent in 'curie_map' and strict flag = True
         """
-        prefixes_in_table = get_prefixes_used_in_table(self.df, converter=self.converter)
+        prefixes_in_table = get_prefixes_used_in_table(self.df)
         if self.metadata:
             prefixes_in_table.update(get_prefixes_used_in_metadata(self.metadata))
 
@@ -1118,20 +1118,20 @@ def _is_iri(string: str) -> bool:
 def get_prefix_from_curie(curie: str) -> str:
     """Get the prefix from a CURIE."""
     if _is_curie(curie):
-        return curie.split(":")[0]
+        return ReferenceTuple.from_curie(curie).prefix
     else:
         return ""
 
 
-def get_prefixes_used_in_table(df: pd.DataFrame, converter: Converter) -> Set[str]:
+def get_prefixes_used_in_table(df: pd.DataFrame) -> Set[str]:
     """Get a list of prefixes used in CURIEs in key feature columns in a dataframe."""
     prefixes = set(SSSOM_BUILT_IN_PREFIXES)
     if df.empty:
         return prefixes
     sssom_schema_object = _get_sssom_schema_object()
     entity_reference_slots = sssom_schema_object.entity_reference_slots & set(df.columns)
     new_prefixes = {
-        converter.parse_curie(row).prefix
+        ReferenceTuple.from_curie(row).prefix
         for col in entity_reference_slots
         for row in df[col]
         if not _is_iri(row) and _is_curie(row)

src/sssom/util.py

@@ -1,7 +1,7 @@
 """Validators."""
 
 import logging
-from typing import Callable, List, Mapping
+from typing import Callable, List, Mapping, Optional
 
 from jsonschema import ValidationError
 from linkml.validator import ValidationReport, Validator
@@ -12,22 +12,29 @@
 from sssom.parsers import to_mapping_set_document
 from sssom.util import MappingSetDataFrame, get_all_prefixes
 
-from .constants import SCHEMA_YAML, SchemaValidationType, _get_sssom_schema_object
+from .constants import (
+    DEFAULT_VALIDATION_TYPES,
+    SCHEMA_YAML,
+    SchemaValidationType,
+    _get_sssom_schema_object,
+)
 
 
 def validate(
     msdf: MappingSetDataFrame,
-    validation_types: List[SchemaValidationType],
+    validation_types: Optional[List[SchemaValidationType]] = None,
     fail_on_error: bool = True,
-) -> None:
+) -> dict[SchemaValidationType, ValidationReport]:
     """Validate SSSOM files against `sssom-schema` using linkML's validator function.
 
     :param msdf: MappingSetDataFrame.
     :param validation_types: SchemaValidationType
     :param fail_on_error: If true, throw an error when execution of a method has failed
+    :returns: A dictionary from validation types to validation reports
     """
-    for vt in validation_types:
-        VALIDATION_METHODS[vt](msdf, fail_on_error)
+    if validation_types is None:
+        validation_types = DEFAULT_VALIDATION_TYPES
+    return {vt: VALIDATION_METHODS[vt](msdf, fail_on_error) for vt in validation_types}
 
 
 def print_linkml_report(report: ValidationReport, fail_on_error: bool = True):
@@ -88,7 +95,7 @@ def _clean_dict(d):
     return cleaned_dict
 
 
-def validate_json_schema(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> None:
+def validate_json_schema(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> ValidationReport:
     """Validate JSON Schema using linkml's JsonSchemaDataValidator.
 
     :param msdf: MappingSetDataFrame to eb validated.
@@ -106,9 +113,10 @@ def validate_json_schema(msdf: MappingSetDataFrame, fail_on_error: bool = True)
 
     report = validator.validate(mapping_set_dict, "mapping set")
     print_linkml_report(report, fail_on_error)
+    return report
 
 
-def validate_shacl(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> None:
+def validate_shacl(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> ValidationReport:
     """Validate SCHACL file.
 
     :param msdf: TODO: https://github.com/linkml/linkml/issues/850 .
@@ -118,7 +126,7 @@ def validate_shacl(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> Non
     raise NotImplementedError
 
 
-def validate_sparql(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> None:
+def validate_sparql(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> ValidationReport:
     """Validate SPARQL file.
 
     :param msdf: MappingSetDataFrame
@@ -132,7 +140,9 @@ def validate_sparql(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> No
     raise NotImplementedError
 
 
-def check_all_prefixes_in_curie_map(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> None:
+def check_all_prefixes_in_curie_map(
+    msdf: MappingSetDataFrame, fail_on_error: bool = True
+) -> ValidationReport:
     """Check all `EntityReference` slots are mentioned in 'curie_map'.
 
     :param msdf: MappingSetDataFrame
@@ -154,9 +164,12 @@ def check_all_prefixes_in_curie_map(msdf: MappingSetDataFrame, fail_on_error: bo
         )
     report = ValidationReport(results=validation_results)
     print_linkml_report(report, fail_on_error)
+    return report
 
 
-def check_strict_curie_format(msdf: MappingSetDataFrame, fail_on_error: bool = True) -> None:
+def check_strict_curie_format(
+    msdf: MappingSetDataFrame, fail_on_error: bool = True
+) -> ValidationReport:
     """Check all `EntityReference` slots are formatted as unambiguous curies.
 
     Implemented rules:
@@ -194,9 +207,10 @@ def check_strict_curie_format(msdf: MappingSetDataFrame, fail_on_error: bool = T
 
     report = ValidationReport(results=validation_results)
     print_linkml_report(report, fail_on_error)
+    return report
 
 
-VALIDATION_METHODS: Mapping[SchemaValidationType, Callable] = {
+VALIDATION_METHODS: Mapping[SchemaValidationType, Callable[..., ValidationReport]] = {
     SchemaValidationType.JsonSchema: validate_json_schema,
     SchemaValidationType.Shacl: validate_shacl,
     SchemaValidationType.Sparql: validate_sparql,

src/sssom/validators.py

@@ -146,6 +146,22 @@ def test_clean_prefix_map_strict(self):
     def test_clean_prefix_map_not_strict(self):
         """Test clean prefix map with 'strict'=False."""
         msdf = parse_sssom_table(f"{data_dir}/test_clean_prefix.tsv")
+        self.assertEqual(
+            {
+                "a": "http://example.org/a/",
+                "b": "http://example.org/b/",
+                "c": "http://example.org/c/",
+                "d": "http://example.org/d/",
+                "orcid": "https://orcid.org/",
+                "owl": "http://www.w3.org/2002/07/owl#",
+                "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+                "rdfs": "http://www.w3.org/2000/01/rdf-schema#",
+                "semapv": "https://w3id.org/semapv/vocab/",
+                "skos": "http://www.w3.org/2004/02/skos/core#",
+                "sssom": "https://w3id.org/sssom/",
+            },
+            msdf.prefix_map,
+        )
         original_curie_map = msdf.prefix_map
         self.assertEqual(
             {"a", "b", "c", "d", "orcid"}.union(SSSOM_BUILT_IN_PREFIXES),
@@ -202,7 +218,7 @@ def test_get_prefixes(self):
         metadata_path = data_dir.joinpath("enm_example.yml")
         metadata = yaml.safe_load(metadata_path.read_text())
         msdf = parse_sssom_table(path, meta=metadata)
-        prefixes = get_prefixes_used_in_table(msdf.df, converter=msdf.converter)
+        prefixes = get_prefixes_used_in_table(msdf.df)
         self.assertNotIn("http", prefixes)
         self.assertNotIn("https", prefixes)
         self.assertEqual(

tests/test_utils.py

@@ -28,7 +28,11 @@ def test_validate_json(self):
         Validate of the incoming file (basic.tsv) abides
         by the rules set by `sssom-schema`.
         """
-        self.assertIsNone(validate(self.correct_msdf1, self.validation_types))
+        rv = validate(self.correct_msdf1, self.validation_types)
+        self.assertIsNotNone(rv)
+        self.assertIn(SchemaValidationType.JsonSchema, rv)
+        json_validation = rv[SchemaValidationType.JsonSchema]
+        self.assertEqual([], json_validation.results)
 
     @unittest.skip(
         reason="""\

tests/test_validate.py

@@ -92,7 +92,10 @@ exclude =
 
 
 [testenv:mypy]
-deps = mypy
+deps =
+    mypy
+    types-PyYAML
+    types-requests
 skip_install = true
 commands = mypy --install-types --non-interactive --ignore-missing-imports --implicit-optional src/sssom tests/
 description = Run the mypy tool to check static typing on the project.