Merge pull request #21 from ImageMarkup/isic-167-skip-emptyish-values

Skip emptyish values in processing metadata

Author: danlamanna

isic_metadata/metadata.py

@@ -4,6 +4,7 @@
 import functools
 from typing import Any, Callable, Optional, Union
 
+import numpy as np
 from pydantic import (
     BaseModel,
     BeforeValidator,
@@ -169,6 +170,16 @@ def strip(cls, v):
             v = v.strip()
         return v
 
+    @model_validator(mode="before")
+    @classmethod
+    def strip_none_and_nan_values(cls, values: dict[str, Any]) -> dict[str, Any]:
+        for field_name, value in list(values.items()):
+            if value is None or value is np.nan:
+                del values[field_name]
+            elif isinstance(value, str) and not value.strip():
+                del values[field_name]
+        return values
+
     @field_validator(
         "anatom_site_general",
         "benign_malignant",

setup.py

@@ -33,6 +33,6 @@
         "Programming Language :: Python",
     ],
     python_requires=">=3.9",
-    install_requires=["pandas", "pydantic>=2"],
+    install_requires=["pandas", "numpy", "pydantic>=2"],
     packages=find_packages(),
 )

tests/test_fields.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from hypothesis import given, strategies as st
+import numpy as np
 from pydantic import ValidationError
 import pytest
 
@@ -13,6 +14,13 @@ def test_unstructured_fields():
     assert metadata.unstructured["hello"] == "world"
 
 
+@pytest.mark.parametrize(("emptyish_value"), ["", " ", "\t", np.nan, None])
+def test_empty_fields_are_omitted(emptyish_value):
+    metadata = MetadataRow(diagnosis="melanoma", mel_type=emptyish_value)
+    assert metadata.diagnosis == "melanoma"
+    assert metadata.mel_thick_mm is None
+
+
 def test_melanoma_fields():
     with pytest.raises(ValidationError) as excinfo:
         # mel_class can only be set if diagnosis is melanoma