fix: [hotfix] add missing validator for one of required file sets (#1521)

* fix: [hotfix] add missing validator for one of required file sets

* tests: fix tests to handle new validator

* fix: properly construct Metadata when core files are missing, bypassing core file validators

* tests: need coverage on condition with missing required files

* chore: lint

Author: dbirman

src/aind_data_schema/core/metadata.py

@@ -145,6 +145,17 @@ def validate_expected_files_by_modality(self):
 
         return self
 
+    @model_validator(mode="after")
+    def validate_required_files(self):
+        """Validator to ensure that one of the key files from the file sets is present."""
+
+        one_of_required = REQUIRED_FILE_SETS.keys()
+
+        if not any(getattr(self, file) for file in one_of_required):
+            raise ValueError(f"Metadata must contain at least one of the following files: {', '.join(one_of_required)}")
+
+        return self
+
     @model_validator(mode="after")
     def validate_smartspim_metadata(self):
         """Validator for smartspim metadata"""
@@ -394,7 +405,7 @@ def create_metadata_json(
         metadata_json = json.loads(metadata.model_dump_json(by_alias=True))
     except Exception as e:
         logging.warning(f"Issue with metadata construction! {e.args}")
-        metadata = Metadata.model_validate(params)
+        metadata = Metadata.model_construct(**params)
         metadata_json = json.loads(metadata.model_dump_json(by_alias=True))
         for key, value in core_fields.items():
             metadata_json[key] = value

tests/test_metadata.py

@@ -31,6 +31,7 @@
 from aind_data_schema.core.acquisition import StimulusEpoch
 
 from examples.data_description import d as data_description
+from examples.subject import s as subject
 
 
 EXAMPLES_DIR = Path(__file__).parents[1] / "examples"
@@ -146,7 +147,7 @@ def test_injection_material_validator_spim(self):
                     subject_id="655019",
                     data_level="raw",
                 ),
-                subject=Subject.model_construct(),
+                subject=subject,
                 procedures=Procedures.model_construct(subject_procedures=[surgery2]),
                 acquisition=Acquisition.model_construct(subject_details=AcquisitionSubjectDetails.model_construct()),
                 instrument=self.spim_instrument,
@@ -171,7 +172,7 @@ def test_injection_material_validator_ephys(self):
                     subject_id="655019",
                     data_level="raw",
                 ),
-                subject=Subject.model_construct(),
+                subject=subject,
                 procedures=Procedures.model_construct(subject_procedures=[surgery2]),
                 instrument=ephys_inst,
                 processing=Processing.model_construct(),
@@ -201,7 +202,7 @@ def test_validate_instrument_acquisition_compatibility(self):
                     subject_id="655019",
                     data_level="raw",
                 ),
-                subject=Subject.model_construct(),
+                subject=subject,
                 procedures=Procedures.model_construct(),
                 instrument=inst,
                 processing=Processing.model_construct(),
@@ -221,6 +222,7 @@ def test_validate_old_schema_version(self):
             name="name",
             location="location",
             id="1",
+            subject=subject,
         )
 
         m_dict = m.model_dump()
@@ -273,7 +275,7 @@ def test_create_from_core_jsons_invalid(self):
             "data_description": None,
             "procedures": self.procedures_json,
             "instrument": Instrument.model_construct().model_dump(),
-            "processing": Procedures.model_construct(injection_materials=["some materials"]).model_dump(),
+            "processing": Processing.model_construct().model_dump(),
             "acquisition": None,
             "quality_control": None,
         }
@@ -319,6 +321,18 @@ def test_validate_expected_files_by_modality(self):
         self.assertIn("Metadata missing required file: instrument", warning_messages)
         self.assertIn("Metadata missing required file: acquisition", warning_messages)
 
+        # Test case where ALL required file set keys are missing (subject, processing, model)
+        with self.assertRaises(ValueError) as context:
+            Metadata(
+                name="655019_2023-04-03T181709",
+                location="bucket",
+                # No subject, processing, or model - should trigger validation error
+            )
+        self.assertIn(
+            "Metadata must contain at least one of the following files: subject, processing, model",
+            str(context.exception),
+        )
+
     def test_validate_acquisition_connections(self):
         """Tests that acquisition connections are validated correctly."""
         # Case where all connection devices are present in instrument components
@@ -340,6 +354,7 @@ def test_validate_acquisition_connections(self):
         metadata = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             instrument=instrument,
             acquisition=acquisition,
         )
@@ -362,6 +377,7 @@ def test_validate_acquisition_connections(self):
             Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 instrument=instrument,
                 acquisition=acquisition,
             )
@@ -387,6 +403,7 @@ def test_validate_acquisition_connections(self):
             Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 instrument=instrument,
                 acquisition=acquisition_missing_source,
             )
@@ -416,6 +433,7 @@ def test_validate_acquisition_active_devices(self):
         metadata = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             instrument=instrument,
             acquisition=acquisition,
         )
@@ -435,6 +453,7 @@ def test_validate_acquisition_active_devices(self):
             Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 instrument=instrument,
                 acquisition=acquisition,
             )
@@ -461,6 +480,7 @@ def test_validate_training_protocol_references(self):
         metadata = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             procedures=procedures,
             acquisition=acquisition,
         )
@@ -479,6 +499,7 @@ def test_validate_training_protocol_references(self):
             metadata = Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 procedures=procedures,
                 acquisition=acquisition_invalid,
             )
@@ -499,6 +520,7 @@ def test_validate_training_protocol_references(self):
             metadata = Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 procedures=procedures_empty,
                 acquisition=acquisition_invalid,
             )
@@ -525,6 +547,7 @@ def test_validate_training_protocol_references(self):
         metadata_none = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             procedures=procedures,
             acquisition=acquisition_none,
         )
@@ -534,6 +557,7 @@ def test_validate_training_protocol_references(self):
         metadata_no_acquisition = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             procedures=procedures,
         )
         self.assertIsNotNone(metadata_no_acquisition)
@@ -542,6 +566,7 @@ def test_validate_training_protocol_references(self):
         metadata_no_procedures = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             acquisition=acquisition,
         )
         self.assertIsNotNone(metadata_no_procedures)
@@ -578,6 +603,7 @@ def test_validate_data_description_name_time_consistency(self):
         metadata_matching = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             data_description=data_description,
             acquisition=acquisition,
         )
@@ -599,6 +625,7 @@ def test_validate_data_description_name_time_consistency(self):
         metadata_later_same_day = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             data_description=data_description_later,
             acquisition=acquisition,
         )
@@ -620,6 +647,7 @@ def test_validate_data_description_name_time_consistency(self):
         metadata_next_day = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             data_description=data_description_next_day,
             acquisition=acquisition,
         )
@@ -644,20 +672,25 @@ def test_validate_data_description_name_time_consistency(self):
             metadata_with_warning = Metadata(
                 name="Test Metadata",
                 location="Test Location",
+                subject=subject,
                 data_description=data_description_before,
                 acquisition=acquisition,
             )
             # Should have successfully created the metadata object
             self.assertIsNotNone(metadata_with_warning)
             # Should have issued a warning
-            self.assertEqual(len(warning_list), 1)
-            self.assertIn("Creation time from data_description.name", str(warning_list[0].message))
-            self.assertIn("should be close to the acquisition end time", str(warning_list[0].message))
+            print(warning_list)
+            # Filter to only the time consistency warning
+            time_warnings = [w for w in warning_list if "Creation time from data_description" in str(w.message)]
+            self.assertEqual(len(time_warnings), 1)
+            self.assertIn("Creation time from data_description", str(time_warnings[0].message))
+            self.assertIn("should be close to the acquisition end time", str(time_warnings[0].message))
 
         # Test case where data_description is None (should pass)
         metadata_no_data_desc = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             acquisition=acquisition,
         )
         self.assertIsNotNone(metadata_no_data_desc)
@@ -666,6 +699,7 @@ def test_validate_data_description_name_time_consistency(self):
         metadata_no_acquisition = Metadata(
             name="Test Metadata",
             location="Test Location",
+            subject=subject,
             data_description=data_description,
         )
         self.assertIsNotNone(metadata_no_acquisition)