From a0fd74a443b512acb68a4e863a74afdf4cba9b49 Mon Sep 17 00:00:00 2001 From: Dan Birman Date: Mon, 16 Dec 2024 11:01:43 -0800 Subject: [PATCH] Docs improve qc example (#1194) * refactor: improve example QC to be more realistic * tests: gen examples * chore: lint --- examples/quality_control.json | 44 ++++++++++++++++++++++++----------- examples/quality_control.py | 41 +++++++++++++++++++++++++------- tests/test_base.py | 2 ++ 3 files changed, 64 insertions(+), 23 deletions(-) diff --git a/examples/quality_control.json b/examples/quality_control.json index 2b944773b..10aadb632 100644 --- a/examples/quality_control.json +++ b/examples/quality_control.json @@ -9,7 +9,7 @@ }, "stage": "Raw data", "name": "Drift map", - "description": "Qualitative check that drift map shows minimal movement", + "description": "Check that all probes show minimal drift", "metrics": [ { "name": "Probe A drift", @@ -34,7 +34,7 @@ "timestamp": "2022-11-22T00:00:00Z" } ], - "description": null, + "description": "Qualitative check that drift map shows minimal movement", "reference": "ecephys-drift-map", "evaluated_assets": null }, @@ -43,13 +43,15 @@ "value": { "value": "", "options": [ - "Drift visible in entire session", + "No Drift", "Drift visible in part of session", + "Drift visible in entire session", "Sudden movement event" ], "status": [ - "Fail", "Pass", + "Pass", + "Fail", "Fail" ], "type": "checkbox" @@ -61,7 +63,7 @@ "timestamp": "2022-11-22T00:00:00Z" } ], - "description": null, + "description": "Qualitative check that drift map shows minimal movement", "reference": "ecephys-drift-map", "evaluated_assets": null }, @@ -95,7 +97,7 @@ "description": null, "metrics": [ { - "name": "video_1_num_frames", + "name": "Expected frame count", "value": 662, "status_history": [ { @@ -104,12 +106,12 @@ "timestamp": "2022-11-22T00:00:00Z" } ], - "description": null, + "description": "Expected frame count from experiment length, always pass", "reference": null, "evaluated_assets": null }, { - "name": "video_2_num_frames", + "name": "Video 1 frame count", "value": 662, "status_history": [ { @@ -118,13 +120,27 @@ "timestamp": "2022-11-22T00:00:00Z" } ], - "description": null, + "description": "Pass when frame count matches expected", + "reference": null, + "evaluated_assets": null + }, + { + "name": "Video 2 num frames", + "value": 662, + "status_history": [ + { + "evaluator": "Automated", + "status": "Pass", + "timestamp": "2022-11-22T00:00:00Z" + } + ], + "description": "Pass when frame count matches expected", "reference": null, "evaluated_assets": null } ], "tags": null, - "notes": "Pass when video_1_num_frames==video_2_num_frames", + "notes": null, "allow_failed_metrics": false, "latest_status": "Pass" }, @@ -135,10 +151,10 @@ }, "stage": "Raw data", "name": "Probes present", - "description": null, + "description": "Pass when data from a probe is present", "metrics": [ { - "name": "ProbeA_success", + "name": "ProbeA", "value": true, "status_history": [ { @@ -152,7 +168,7 @@ "evaluated_assets": null }, { - "name": "ProbeB_success", + "name": "ProbeB", "value": true, "status_history": [ { @@ -166,7 +182,7 @@ "evaluated_assets": null }, { - "name": "ProbeC_success", + "name": "ProbeC", "value": true, "status_history": [ { diff --git a/examples/quality_control.py b/examples/quality_control.py index df687d4de..daef406f5 100644 --- a/examples/quality_control.py +++ b/examples/quality_control.py @@ -26,25 +26,32 @@ # Example of how to use a dictionary to provide multiple checkable flags, some of which will fail the metric drift_value_with_flags = { "value": "", - "options": ["Drift visible in entire session", "Drift visible in part of session", "Sudden movement event"], - "status": ["Fail", "Pass", "Fail"], + "options": [ + "No Drift", + "Drift visible in part of session", + "Drift visible in entire session", + "Sudden movement event", + ], + "status": ["Pass", "Pass", "Fail", "Fail"], "type": "checkbox", } eval0 = QCEvaluation( name="Drift map", - description="Qualitative check that drift map shows minimal movement", + description="Check that all probes show minimal drift", modality=Modality.ECEPHYS, stage=Stage.RAW, metrics=[ QCMetric( name="Probe A drift", + description="Qualitative check that drift map shows minimal movement", value=drift_value_with_options, reference="ecephys-drift-map", status_history=[sp], ), QCMetric( name="Probe B drift", + description="Qualitative check that drift map shows minimal movement", value=drift_value_with_flags, reference="ecephys-drift-map", status_history=[sp], @@ -59,20 +66,36 @@ modality=Modality.BEHAVIOR_VIDEOS, stage=Stage.RAW, metrics=[ - QCMetric(name="video_1_num_frames", value=662, status_history=[s]), - QCMetric(name="video_2_num_frames", value=662, status_history=[s]), + QCMetric( + name="Expected frame count", + description="Expected frame count from experiment length, always pass", + value=662, + status_history=[s], + ), + QCMetric( + name="Video 1 frame count", + description="Pass when frame count matches expected", + value=662, + status_history=[s], + ), + QCMetric( + name="Video 2 num frames", + description="Pass when frame count matches expected", + value=662, + status_history=[s], + ), ], - notes="Pass when video_1_num_frames==video_2_num_frames", ) eval2 = QCEvaluation( name="Probes present", + description="Pass when data from a probe is present", modality=Modality.ECEPHYS, stage=Stage.RAW, metrics=[ - QCMetric(name="ProbeA_success", value=True, status_history=[s]), - QCMetric(name="ProbeB_success", value=True, status_history=[s]), - QCMetric(name="ProbeC_success", value=True, status_history=[s]), + QCMetric(name="ProbeA", value=True, status_history=[s]), + QCMetric(name="ProbeB", value=True, status_history=[s]), + QCMetric(name="ProbeC", value=True, status_history=[s]), ], ) diff --git a/tests/test_base.py b/tests/test_base.py index a823216a5..9aa22a0c2 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -130,11 +130,13 @@ def test_schema_bump(self): class Modelv1(AindCoreModel): """test class""" + describedBy: str = "modelv1" schema_version: SkipValidation[Literal["1.0.0"]] = "1.0.0" class Modelv2(AindCoreModel): """test class""" + describedBy: str = "modelv2" schema_version: SkipValidation[Literal["1.0.1"]] = "1.0.1" extra_field: str = "extra_field"