Docs improve qc example (#1194)

* refactor: improve example QC to be more realistic * tests: gen examples * chore: lint
AllenNeuralDynamics · Dec 16, 2024 · a0fd74a · a0fd74a
1 parent d38a508
commit a0fd74a
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 23 deletions.
diff --git a/examples/quality_control.json b/examples/quality_control.json
@@ -9,7 +9,7 @@
          },
          "stage": "Raw data",
          "name": "Drift map",
-         "description": "Qualitative check that drift map shows minimal movement",
+         "description": "Check that all probes show minimal drift",
          "metrics": [
             {
                "name": "Probe A drift",
@@ -34,7 +34,7 @@
                      "timestamp": "2022-11-22T00:00:00Z"
                   }
                ],
-               "description": null,
+               "description": "Qualitative check that drift map shows minimal movement",
                "reference": "ecephys-drift-map",
                "evaluated_assets": null
             },
@@ -43,13 +43,15 @@
                "value": {
                   "value": "",
                   "options": [
-                     "Drift visible in entire session",
+                     "No Drift",
                      "Drift visible in part of session",
+                     "Drift visible in entire session",
                      "Sudden movement event"
                   ],
                   "status": [
-                     "Fail",
                      "Pass",
+                     "Pass",
+                     "Fail",
                      "Fail"
                   ],
                   "type": "checkbox"
@@ -61,7 +63,7 @@
                      "timestamp": "2022-11-22T00:00:00Z"
                   }
                ],
-               "description": null,
+               "description": "Qualitative check that drift map shows minimal movement",
                "reference": "ecephys-drift-map",
                "evaluated_assets": null
             },
@@ -95,7 +97,7 @@
          "description": null,
          "metrics": [
             {
-               "name": "video_1_num_frames",
+               "name": "Expected frame count",
                "value": 662,
                "status_history": [
                   {
@@ -104,12 +106,12 @@
                      "timestamp": "2022-11-22T00:00:00Z"
                   }
                ],
-               "description": null,
+               "description": "Expected frame count from experiment length, always pass",
                "reference": null,
                "evaluated_assets": null
             },
             {
-               "name": "video_2_num_frames",
+               "name": "Video 1 frame count",
                "value": 662,
                "status_history": [
                   {
@@ -118,13 +120,27 @@
                      "timestamp": "2022-11-22T00:00:00Z"
                   }
                ],
-               "description": null,
+               "description": "Pass when frame count matches expected",
+               "reference": null,
+               "evaluated_assets": null
+            },
+            {
+               "name": "Video 2 num frames",
+               "value": 662,
+               "status_history": [
+                  {
+                     "evaluator": "Automated",
+                     "status": "Pass",
+                     "timestamp": "2022-11-22T00:00:00Z"
+                  }
+               ],
+               "description": "Pass when frame count matches expected",
                "reference": null,
                "evaluated_assets": null
             }
          ],
          "tags": null,
-         "notes": "Pass when video_1_num_frames==video_2_num_frames",
+         "notes": null,
          "allow_failed_metrics": false,
          "latest_status": "Pass"
       },
@@ -135,10 +151,10 @@
          },
          "stage": "Raw data",
          "name": "Probes present",
-         "description": null,
+         "description": "Pass when data from a probe is present",
          "metrics": [
             {
-               "name": "ProbeA_success",
+               "name": "ProbeA",
                "value": true,
                "status_history": [
                   {
@@ -152,7 +168,7 @@
                "evaluated_assets": null
             },
             {
-               "name": "ProbeB_success",
+               "name": "ProbeB",
                "value": true,
                "status_history": [
                   {
@@ -166,7 +182,7 @@
                "evaluated_assets": null
             },
             {
-               "name": "ProbeC_success",
+               "name": "ProbeC",
                "value": true,
                "status_history": [
                   {

diff --git a/examples/quality_control.py b/examples/quality_control.py
@@ -26,25 +26,32 @@
 # Example of how to use a dictionary to provide multiple checkable flags, some of which will fail the metric
 drift_value_with_flags = {
     "value": "",
-    "options": ["Drift visible in entire session", "Drift visible in part of session", "Sudden movement event"],
-    "status": ["Fail", "Pass", "Fail"],
+    "options": [
+        "No Drift",
+        "Drift visible in part of session",
+        "Drift visible in entire session",
+        "Sudden movement event",
+    ],
+    "status": ["Pass", "Pass", "Fail", "Fail"],
     "type": "checkbox",
 }
 
 eval0 = QCEvaluation(
     name="Drift map",
-    description="Qualitative check that drift map shows minimal movement",
+    description="Check that all probes show minimal drift",
     modality=Modality.ECEPHYS,
     stage=Stage.RAW,
     metrics=[
         QCMetric(
             name="Probe A drift",
+            description="Qualitative check that drift map shows minimal movement",
             value=drift_value_with_options,
             reference="ecephys-drift-map",
             status_history=[sp],
         ),
         QCMetric(
             name="Probe B drift",
+            description="Qualitative check that drift map shows minimal movement",
             value=drift_value_with_flags,
             reference="ecephys-drift-map",
             status_history=[sp],
@@ -59,20 +66,36 @@
     modality=Modality.BEHAVIOR_VIDEOS,
     stage=Stage.RAW,
     metrics=[
-        QCMetric(name="video_1_num_frames", value=662, status_history=[s]),
-        QCMetric(name="video_2_num_frames", value=662, status_history=[s]),
+        QCMetric(
+            name="Expected frame count",
+            description="Expected frame count from experiment length, always pass",
+            value=662,
+            status_history=[s],
+        ),
+        QCMetric(
+            name="Video 1 frame count",
+            description="Pass when frame count matches expected",
+            value=662,
+            status_history=[s],
+        ),
+        QCMetric(
+            name="Video 2 num frames",
+            description="Pass when frame count matches expected",
+            value=662,
+            status_history=[s],
+        ),
     ],
-    notes="Pass when video_1_num_frames==video_2_num_frames",
 )
 
 eval2 = QCEvaluation(
     name="Probes present",
+    description="Pass when data from a probe is present",
     modality=Modality.ECEPHYS,
     stage=Stage.RAW,
     metrics=[
-        QCMetric(name="ProbeA_success", value=True, status_history=[s]),
-        QCMetric(name="ProbeB_success", value=True, status_history=[s]),
-        QCMetric(name="ProbeC_success", value=True, status_history=[s]),
+        QCMetric(name="ProbeA", value=True, status_history=[s]),
+        QCMetric(name="ProbeB", value=True, status_history=[s]),
+        QCMetric(name="ProbeC", value=True, status_history=[s]),
     ],
 )
 

diff --git a/tests/test_base.py b/tests/test_base.py
@@ -130,11 +130,13 @@ def test_schema_bump(self):
 
         class Modelv1(AindCoreModel):
             """test class"""
+
             describedBy: str = "modelv1"
             schema_version: SkipValidation[Literal["1.0.0"]] = "1.0.0"
 
         class Modelv2(AindCoreModel):
             """test class"""
+
             describedBy: str = "modelv2"
             schema_version: SkipValidation[Literal["1.0.1"]] = "1.0.1"
             extra_field: str = "extra_field"