Feat 571 Basic metadata validators (#556)

* subject and data description validation

* procedures

* feat: removes unuses imports

* feat: remove comment

* feat: uses by_alias when writing field names

---------

Co-authored-by: jtyoung84 <[email protected]>

Author: mekhlakapoor

src/aind_data_schema/__init__.py

@@ -12,6 +12,7 @@
 from .imaging.acquisition import Acquisition, Axis
 from .imaging.instrument import Instrument
 from .imaging.mri_session import MriSession
+from .metadata import Metadata
 from .ophys.ophys_rig import OphysRig
 from .ophys.ophys_session import OphysSession
 from .procedures import Procedures
@@ -44,4 +45,5 @@
     "MriSession",
     "Rig",
     "Session",
+    "Metadata",
 ]

src/aind_data_schema/base.py

@@ -154,7 +154,7 @@ def write_standard_file(self, output_directory: Optional[Path] = None, prefix=No
             filename = output_directory / filename
 
         with open(filename, "w") as f:
-            f.write(self.json(indent=3))
+            f.write(self.json(indent=3, by_alias=True))
 
 
 class _TypeEnumSubset(object):

src/aind_data_schema/metadata.py

@@ -2,11 +2,19 @@
 
 from datetime import datetime
 from enum import Enum
-from typing import Dict, List
+from typing import Dict, List, Optional
+from uuid import UUID, uuid4
 
-from pydantic import Extra, Field
+from pydantic import Field, root_validator, validate_model
 
 from aind_data_schema.base import AindCoreModel
+from aind_data_schema.data_description import DataDescription
+from aind_data_schema.imaging.acquisition import Acquisition
+from aind_data_schema.imaging.instrument import Instrument
+from aind_data_schema.procedures import Procedures
+from aind_data_schema.processing import Processing
+from aind_data_schema.rig import Rig
+from aind_data_schema.session import Session
 from aind_data_schema.subject import Subject
 
 
@@ -29,8 +37,10 @@ class Metadata(AindCoreModel):
     """The records in the Data Asset Collection needs to contain certain fields
     to easily query and index the data."""
 
-    id: str = Field(
-        ...,
+    schema_version: str = Field("0.0.2", description="schema version", title="Version", const=True)
+
+    id: UUID = Field(
+        default_factory=uuid4,
         alias="_id",
         title="Data Asset ID",
         description="The unique id of the data asset.",
@@ -40,31 +50,90 @@ class Metadata(AindCoreModel):
         description="Name of the data asset.",
         title="Data Asset Name",
     )
+    # We'll set created and last_modified defaults using the root_validator
+    # to ensure they're synced on creation
     created: datetime = Field(
-        ...,
+        default_factory=datetime.utcnow,
         title="Created",
-        description="The data and time the data asset created.",
+        description="The utc date and time the data asset created.",
     )
     last_modified: datetime = Field(
-        ..., title="Last Modified", description="The date and time that the data asset was last modified."
+        default_factory=datetime.utcnow,
+        title="Last Modified",
+        description="The utc date and time that the data asset was last modified.",
     )
     location: str = Field(
         ...,
         title="Location",
         description="Current location of the data asset.",
     )
-    metadata_status: MetadataStatus = Field(..., title=" Metadata Status", description="The status of the metadata.")
-    schema_version: str = Field("0.0.1", title="Schema Version", const=True)
+    metadata_status: MetadataStatus = Field(
+        default=MetadataStatus.UNKNOWN, title=" Metadata Status", description="The status of the metadata."
+    )
     external_links: List[Dict[ExternalPlatforms, str]] = Field(
-        ..., title="External Links", description="Links to the data asset on different platforms."
+        default=[], title="External Links", description="Links to the data asset on different platforms."
     )
-    subject: Subject = Field(
-        ...,
+    # We can make the AindCoreModel fields optional for now and do more
+    # granular validations using validators. We may have some older data
+    # assets in S3 that don't have metadata attached. We'd still like to
+    # index that data, but we can flag those instances as MISSING or UNKNOWN
+    subject: Optional[Subject] = Field(
+        None,
         title="Subject",
-        description="Description of a subject of data collection.",
+        description="Subject of data collection.",
     )
+    data_description: Optional[DataDescription] = Field(
+        None, title="Data Description", description="A logical collection of data files."
+    )
+    procedures: Optional[Procedures] = Field(
+        None, title="Procedures", description="All procedures performed on a subject."
+    )
+    session: Optional[Session] = Field(None, title="Session", description="Description of a session.")
+    rig: Optional[Rig] = Field(None, title="Rig", description="Rig.")
+    processing: Optional[Processing] = Field(None, title="Processing", description="All processes run on data.")
+    acquisition: Optional[Acquisition] = Field(None, title="Acquisition", description="Imaging acquisition session")
+    instrument: Optional[Instrument] = Field(
+        None, title="Instrument", description="Instrument, which is a collection of devices"
+    )
+
+    @root_validator(pre=False)
+    def validate_metadata(cls, values):
+        """Validator for metadata"""
 
-    class Config:
-        """Need to allow for additional fields to append to base model"""
+        # There's a simpler way to do this if we drop support for py37
+        all_model_fields = []
+        for field_name in cls.__fields__:
+            field_to_check = cls.__fields__[field_name]
+            try:
+                if issubclass(field_to_check.type_, AindCoreModel):
+                    all_model_fields.append(field_to_check)
+            except TypeError:
+                # Type errors in python3.7 when using issubclass on type
+                # generics
+                pass
 
-        extra = Extra.allow
+        # For each model field, check that is present and check if the model
+        # is valid. If it isn't valid, still add it, but mark MetadataStatus
+        # as INVALID
+        metadata_status = MetadataStatus.VALID
+        for model_field in all_model_fields:
+            model_class = model_field.type_
+            model_name = model_field.name
+            if values.get(model_name) is not None:
+                model = values[model_name]
+                # Since pre=False, the dictionaries get converted to models
+                # upstream
+                model_contents = model.dict()
+                *_, validation_error = validate_model(model_class, model_contents)
+                if validation_error:
+                    model_instance = model_class.construct(**model_contents)
+                    metadata_status = MetadataStatus.INVALID
+                else:
+                    model_instance = model_class(**model_contents)
+                values[model_name] = model_instance
+        # For certain required fields, like subject, if they are not present,
+        # mark the metadata record as missing
+        if values.get("subject") is None:
+            metadata_status = MetadataStatus.MISSING
+        values["metadata_status"] = metadata_status
+        return values

tests/test_metadata.py

@@ -1,47 +1,91 @@
-"""Tests Metadata model"""
+"""Tests metadata module"""
 
+import json
 import unittest
-from datetime import datetime
 
-from aind_data_schema.metadata import ExternalPlatforms, Metadata, MetadataStatus
+from pydantic import ValidationError
+
+from aind_data_schema.metadata import Metadata, MetadataStatus
 from aind_data_schema.procedures import Procedures
-from aind_data_schema.subject import Subject
+from aind_data_schema.subject import Sex, Species, Subject
 
 
 class TestMetadata(unittest.TestCase):
     """Class to test Metadata model"""
 
-    def test_constructors(self):
-        """test building from component parts"""
+    def test_valid_subject_info(self):
+        """Tests that the record is marked as VALID if a valid subject model
+        is present."""
+        s1 = Subject(
+            species=Species.MUS_MUSCULUS,
+            subject_id="123345",
+            sex=Sex.MALE,
+            date_of_birth="2020-10-10",
+            genotype="Emx1-IRES-Cre;Camk2a-tTA;Ai93(TITL-GCaMP6f)",
+        )
+        d1 = Metadata(name="ecephys_655019_2023-04-03_18-17-09", location="bucket", subject=s1)
+        self.assertEqual("ecephys_655019_2023-04-03_18-17-09", d1.name)
+        self.assertEqual("bucket", d1.location)
+        self.assertEqual(MetadataStatus.VALID, d1.metadata_status)
+        self.assertEqual(s1, d1.subject)
+
+        # Test construction via dictionary
+        d2 = Metadata(name="ecephys_655019_2023-04-03_18-17-09", location="bucket", subject=s1.dict())
+        self.assertEqual(MetadataStatus.VALID, d2.metadata_status)
+        self.assertEqual(s1, d2.subject)
+
+        # Test round-trip serialization
+        # We may want override the default file writer to always use by_alias
+        # when writing the Metadata records. This sets the field in the json
+        # file to _id instead of id, which makes it easier to write to
+        # MongoDB.
+        d3 = Metadata.parse_obj(json.loads(d2.json(by_alias=True)))
+        self.assertEqual(d2, d3)
+
+    def test_missing_subject_info(self):
+        """Marks the metadata status as MISSING if a Subject model is not
+        present"""
 
         d1 = Metadata(
-            _id="00000",
-            name="Test Name",
-            created=datetime(2023, 9, 27, 0, 0, 0),
-            last_modified=datetime(2023, 9, 28, 10, 20, 30),
-            location="Test Location",
-            metadata_status=MetadataStatus.VALID,
-            subject=Subject.construct(),
-            external_links=[{ExternalPlatforms.CODEOCEAN: "abc-1234"}],
+            name="ecephys_655019_2023-04-03_18-17-09",
+            location="bucket",
+        )
+        self.assertEqual(MetadataStatus.MISSING, d1.metadata_status)
+        self.assertEqual("ecephys_655019_2023-04-03_18-17-09", d1.name)
+        self.assertEqual("bucket", d1.location)
+
+        # Assert at least a name and location are required
+        with self.assertRaises(ValidationError) as e:
+            Metadata()
+        expected_exception_message = (
+            "2 validation errors for Metadata\n"
+            "name\n"
+            "  field required (type=value_error.missing)\n"
+            "location\n"
+            "  field required (type=value_error.missing)"
+        )
+        self.assertEqual(expected_exception_message, str(e.exception))
+
+    def test_invalid_core_models(self):
+        """Test that invalid models don't raise an error, but marks the
+        metadata_status as INVALID"""
+
+        # Invalid subject model
+        d1 = Metadata(name="ecephys_655019_2023-04-03_18-17-09", location="bucket", subject=Subject.construct())
+        self.assertEqual(MetadataStatus.INVALID, d1.metadata_status)
+
+        # Valid subject model, but invalid procedures model
+        s2 = Subject(
+            species=Species.MUS_MUSCULUS,
+            subject_id="123345",
+            sex=Sex.MALE,
+            date_of_birth="2020-10-10",
+            genotype="Emx1-IRES-Cre;Camk2a-tTA;Ai93(TITL-GCaMP6f)",
         )
-        self.assertIsNotNone(d1)
-        self.assertEqual(d1.schema_version, "0.0.1")
-        self.assertEqual(d1.location, "Test Location")
-        self.assertTrue(hasattr(d1, "subject"))
         d2 = Metadata(
-            _id="00000",
-            name="Test Name",
-            created=datetime(2023, 9, 27, 0, 0, 0),
-            last_modified=datetime(2023, 9, 28, 10, 20, 30),
-            location="Test Location",
-            metadata_status=MetadataStatus.VALID,
-            subject=Subject.construct(),
-            procedures=Procedures.construct(),
-            external_links=[{ExternalPlatforms.CODEOCEAN: "abc-1234"}],
+            name="ecephys_655019_2023-04-03_18-17-09", location="bucket", subject=s2, procedures=Procedures.construct()
         )
-        self.assertIsNotNone(d2)
-        self.assertTrue(hasattr(d2, "procedures"))
-        self.assertTrue(hasattr(d2, "external_links"))
+        self.assertEqual(MetadataStatus.INVALID, d2.metadata_status)
 
 
 if __name__ == "__main__":