1062 quality control needs versioning for evaluations (#1067)

* refactor: changing references -> reference

* feat: add an overall evaluator name

* feat: adding QCStatus and lists of status for versioning

* chore: lint

* tests: updating tests for QCStatus

Author: dbirman

examples/quality_control.json

@@ -1,8 +1,13 @@
 {
    "describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py",
    "schema_version": "1.0.0",
-   "overall_status": "Pass",
-   "overall_status_date": "2022-11-22",
+   "overall_status": [
+      {
+         "evaluator": "Automated",
+         "status": "Pass",
+         "timestamp": "2022-11-22"
+      }
+   ],
    "evaluations": [
       {
          "evaluation_modality": {
@@ -12,29 +17,33 @@
          "evaluation_stage": "Processing",
          "evaluation_name": "Drift map",
          "evaluation_description": "Qualitative check that drift map shows minimal movement",
-         "evaluator": "Fred Flintstone",
-         "evaluation_date": "2022-11-22",
          "qc_metrics": [
             {
                "name": "Probe A drift",
                "value": "High",
                "description": null,
-               "references": null
+               "reference": "ecephys-drift-map"
             },
             {
                "name": "Probe B drift",
                "value": "Low",
                "description": null,
-               "references": null
+               "reference": "ecephys-drift-map"
             },
             {
                "name": "Probe C drift",
                "value": "Low",
                "description": null,
-               "references": null
+               "reference": "ecephys-drift-map"
+            }
+         ],
+         "evaluation_status": [
+            {
+               "evaluator": "Fred Flintstone",
+               "status": "Fail",
+               "timestamp": "2022-11-22"
             }
          ],
-         "stage_status": "Fail",
          "notes": "Manually annotated: failed due to high drift on probe A"
       },
       {
@@ -45,23 +54,27 @@
          "evaluation_stage": "Raw data",
          "evaluation_name": "Video frame count check",
          "evaluation_description": null,
-         "evaluator": "Fred Flinstone",
-         "evaluation_date": "2022-11-22",
          "qc_metrics": [
             {
                "name": "video_1_num_frames",
                "value": 662,
                "description": null,
-               "references": null
+               "reference": null
             },
             {
                "name": "video_2_num_frames",
                "value": 662,
                "description": null,
-               "references": null
+               "reference": null
+            }
+         ],
+         "evaluation_status": [
+            {
+               "evaluator": "Fred Flintstone",
+               "status": "Fail",
+               "timestamp": "2022-11-22"
             }
          ],
-         "stage_status": "Pass",
          "notes": "Pass when video_1_num_frames==video_2_num_frames"
       },
       {
@@ -72,29 +85,33 @@
          "evaluation_stage": "Raw data",
          "evaluation_name": "Probes present",
          "evaluation_description": null,
-         "evaluator": "Automated",
-         "evaluation_date": "2022-11-22",
          "qc_metrics": [
             {
                "name": "ProbeA_success",
                "value": true,
                "description": null,
-               "references": null
+               "reference": null
             },
             {
                "name": "ProbeB_success",
                "value": true,
                "description": null,
-               "references": null
+               "reference": null
             },
             {
                "name": "ProbeC_success",
                "value": true,
                "description": null,
-               "references": null
+               "reference": null
+            }
+         ],
+         "evaluation_status": [
+            {
+               "evaluator": "Automated",
+               "status": "Pass",
+               "timestamp": "2022-11-22"
             }
          ],
-         "stage_status": "Pass",
          "notes": null
       }
    ],

examples/quality_control.py

@@ -4,7 +4,7 @@
 
 from aind_data_schema_models.modalities import Modality
 
-from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status
+from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status, QCStatus
 
 t = date(2022, 11, 22)
 
@@ -13,32 +13,44 @@
     evaluation_description="Qualitative check that drift map shows minimal movement",
     evaluation_modality=Modality.ECEPHYS,
     evaluation_stage=Stage.PROCESSING,
-    evaluator="Fred Flintstone",
-    evaluation_date=t,
+    evaluation_status=[
+        QCStatus(
+            evaluator="Fred Flintstone",
+            timestamp=t,
+            status=Status.FAIL
+        )
+    ],
     qc_metrics=[
         QCMetric(
             name="Probe A drift",
-            value="High"
+            value="High",
+            reference="ecephys-drift-map"
         ),
         QCMetric(
             name="Probe B drift",
-            value="Low"
+            value="Low",
+            reference="ecephys-drift-map"
         ),
         QCMetric(
             name="Probe C drift",
-            value="Low"
+            value="Low",
+            reference="ecephys-drift-map"
         )
     ],
-    stage_status=Status.FAIL,
     notes="Manually annotated: failed due to high drift on probe A"
 )
 
 eval1 = QCEvaluation(
     evaluation_name="Video frame count check",
     evaluation_modality=Modality.BEHAVIOR_VIDEOS,
     evaluation_stage=Stage.RAW,
-    evaluator="Fred Flinstone",
-    evaluation_date=t,
+    evaluation_status=[
+        QCStatus(
+            evaluator="Fred Flintstone",
+            timestamp=t,
+            status=Status.FAIL
+        )
+    ],
     qc_metrics=[
         QCMetric(
             name="video_1_num_frames",
@@ -49,16 +61,20 @@
             value=662
         )
     ],
-    stage_status=Status.PASS,
     notes="Pass when video_1_num_frames==video_2_num_frames"
 )
 
 eval2 = QCEvaluation(
     evaluation_name="Probes present",
     evaluation_modality=Modality.ECEPHYS,
     evaluation_stage=Stage.RAW,
-    evaluator="Automated",
-    evaluation_date=t,
+    evaluation_status=[
+        QCStatus(
+            evaluator="Automated",
+            timestamp=t,
+            status=Status.PASS
+        )
+    ],
     qc_metrics=[
         QCMetric(
             name="ProbeA_success",
@@ -73,12 +89,16 @@
             value=True
         )
     ],
-    stage_status=Status.PASS,
 )
 
 q = QualityControl(
-    overall_status="Pass",
-    overall_status_date=t,
+    overall_status=[
+        QCStatus(
+            evaluator="Automated",
+            timestamp=t,
+            status=Status.PASS
+        )
+    ],
     evaluations=[eval0, eval1, eval2]
 )
 

src/aind_data_schema/core/quality_control.py

@@ -31,12 +31,21 @@ class Stage(str, Enum):
     ANALYSIS = "Analysis"
 
 
+class QCStatus(BaseModel):
+    """Description of a QC status, set by an evaluator"""
+
+    evaluator: str = Field(..., title="Status evaluator full name")
+    status: Status = Field(..., title="Status")
+    timestamp: date = Field(..., title="Status date")
+
+
 class QCMetric(BaseModel):
     """Description of a single quality control metric"""
+
     name: str = Field(..., title="Metric name")
     value: Any = Field(..., title="Metric value")
     description: Optional[str] = Field(default=None, title="Metric description")
-    references: Optional[List[str]] = Field(default=None, title="Metric reference URLs")
+    reference: Optional[str] = Field(default=None, title="Metric reference image URL or plot type")
 
 
 class QCEvaluation(AindModel):
@@ -46,10 +55,8 @@ class QCEvaluation(AindModel):
     evaluation_stage: Stage = Field(..., title="Evaluation stage")
     evaluation_name: str = Field(..., title="Evaluation name")
     evaluation_description: Optional[str] = Field(default=None, title="Evaluation description")
-    evaluator: str = Field(..., title="Evaluator full name")
-    evaluation_date: date = Field(..., title="Evaluation date")
     qc_metrics: List[QCMetric] = Field(..., title="QC metrics")
-    stage_status: Status = Field(..., title="Stage status")
+    evaluation_status: List[QCStatus] = Field(..., title="Evaluation status")
     notes: Optional[str] = Field(default=None, title="Notes")
 
 
@@ -59,7 +66,6 @@ class QualityControl(AindCoreModel):
     _DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/quality_control.py"
     describedBy: str = Field(_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
     schema_version: Literal["1.0.0"] = Field("1.0.0")
-    overall_status: Status = Field(..., title="Overall status")
-    overall_status_date: date = Field(..., title="Date of status")
+    overall_status: List[QCStatus] = Field(..., title="Overall status")
     evaluations: List[QCEvaluation] = Field(..., title="Evaluations")
     notes: Optional[str] = Field(default=None, title="Notes")

tests/test_quality_control.py

@@ -6,7 +6,7 @@
 from aind_data_schema_models.modalities import Modality
 from pydantic import ValidationError
 
-from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status
+from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status, QCStatus
 
 
 class QualityControlTests(unittest.TestCase):
@@ -20,8 +20,13 @@ def test_constructors(self):
 
         test_eval = QCEvaluation(
                 evaluation_name="Drift map",
-                evaluator="Bob",
-                evaluation_date=date.fromisoformat("2020-10-10"),
+                evaluation_status=[
+                    QCStatus(
+                        evaluator="Fred Flintstone",
+                        timestamp=date.fromisoformat("2020-10-10"),
+                        status=Status.PASS
+                    )
+                ],
                 evaluation_modality=Modality.ECEPHYS,
                 evaluation_stage=Stage.PROCESSING,
                 qc_metrics=[
@@ -36,12 +41,16 @@ def test_constructors(self):
                         references=["s3://some-data-somewhere"]
                     )
                 ],
-                stage_status=Status.PASS,
             )
 
         q = QualityControl(
-            overall_status_date=date.fromisoformat("2020-10-10"),
-            overall_status=Status.PASS,
+            overall_status=[
+                QCStatus(
+                    evaluator="Bob",
+                    timestamp=date.fromisoformat("2020-10-10"),
+                    status=Status.PASS
+                )
+            ],
             evaluations=[
                 test_eval
             ],