Skip to content

Commit

Permalink
Refactor to add QCMetric class and list of URLs for reference images/…
Browse files Browse the repository at this point in the history 
…figures (#1037)

* feat: refactor qc_metrics to a List of metrics, with additional metadata

* fix: replace AindGenericType with typing.Any

* feat/tests: making references optional but name required, fixing typo, trying to fix tests

* tests: fixing broken quality_control example + tests

* fix: add default= for Optional fields

* tests: changing how examples are generated

* refactor: QC.Stage and adding _name and _desc fields

* fix: no shortened names, lint

* chore: missed name changes
  • Loading branch information
@dbirman
dbirman authored Aug 30, 2024
1 parent 9dc0556 commit 8246813
Show file tree
Hide file tree
Showing 5 changed files with 304 additions and 63 deletions.
93 changes: 77 additions & 16 deletions examples/quality_control.json
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,99 @@
{
"describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_metrics.py",
"describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py",
"schema_version": "1.0.0",
"overall_status": "Pass",
"overall_status_date": "2022-11-22",
"evaluations": [
{
"evaluation_modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Preprocessing",
"evaluation_name": "Drift map",
"evaluation_description": "Qualitative check that drift map shows minimal movement",
"evaluator": "Fred Flintstone",
"evaluation_date": "2022-11-22",
"qc_metrics": [
{
"name": "Probe A drift",
"value": "High",
"description": null,
"references": null
},
{
"name": "Probe B drift",
"value": "Low",
"description": null,
"references": null
},
{
"name": "Probe C drift",
"value": "Low",
"description": null,
"references": null
}
],
"stage_status": "Fail",
"notes": "Manually annotated: failed due to high drift on probe A"
},
{
"evaluation_modality": {
"name": "Behavior videos",
"abbreviation": "behavior-videos"
},
"evaluation_stage": "Data acquisition",
"evaluator_full_name": "Fred Flinstone",
"evaluation_stage": "Raw data",
"evaluation_name": "Video frame count check",
"evaluation_description": null,
"evaluator": "Fred Flinstone",
"evaluation_date": "2022-11-22",
"qc_metrics": {
"Video_1_num_frames": 662,
"Video_2_num_frames": 662,
"Frame_match": true
},
"qc_metrics": [
{
"name": "video_1_num_frames",
"value": 662,
"description": null,
"references": null
},
{
"name": "video_2_num_frames",
"value": 662,
"description": null,
"references": null
}
],
"stage_status": "Pass",
"notes": null
"notes": "Pass when video_1_num_frames==video_2_num_frames"
},
{
"evaluation_modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Data acquisition",
"evaluator_full_name": "Fred Flinstone",
"evaluation_stage": "Raw data",
"evaluation_name": "Probes present",
"evaluation_description": null,
"evaluator": "Automated",
"evaluation_date": "2022-11-22",
"qc_metrics": {
"ProbeA_success": true,
"ProbeB_success": true,
"ProbeC_success": false
},
"qc_metrics": [
{
"name": "ProbeA_success",
"value": true,
"description": null,
"references": null
},
{
"name": "ProbeB_success",
"value": true,
"description": null,
"references": null
},
{
"name": "ProbeC_success",
"value": true,
"description": null,
"references": null
}
],
"stage_status": "Pass",
"notes": null
}
Expand Down
97 changes: 70 additions & 27 deletions examples/quality_control.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,39 +4,82 @@

from aind_data_schema_models.modalities import Modality

from aind_data_schema.core.quality_control import QCEvaluation, QualityControl
from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status

t = date(2022, 11, 22)

q = QualityControl(
overall_status="Pass",
overall_status_date=t,
evaluations=[
QCEvaluation(
evaluation_modality=Modality.BEHAVIOR_VIDEOS,
evaluation_stage="Data acquisition",
evaluator_full_name="Fred Flinstone",
evaluation_date=t,
qc_metrics={
"Video_1_num_frames": 662,
"Video_2_num_frames": 662,
"Frame_match": True,
},
stage_status="Pass",
eval0 = QCEvaluation(
evaluation_name="Drift map",
evaluation_description="Qualitative check that drift map shows minimal movement",
evaluation_modality=Modality.ECEPHYS,
evaluation_stage=Stage.PREPROCESSING,
evaluator="Fred Flintstone",
evaluation_date=t,
qc_metrics=[
QCMetric(
name="Probe A drift",
value="High"
),
QCEvaluation(
evaluation_modality=Modality.ECEPHYS,
evaluation_stage="Data acquisition",
evaluator_full_name="Fred Flinstone",
evaluation_date=t,
qc_metrics={
"ProbeA_success": True,
"ProbeB_success": True,
"ProbeC_success": False,
},
stage_status="Pass",
QCMetric(
name="Probe B drift",
value="Low"
),
QCMetric(
name="Probe C drift",
value="Low"
)
],
stage_status=Status.FAIL,
notes="Manually annotated: failed due to high drift on probe A"
)

eval1 = QCEvaluation(
evaluation_name="Video frame count check",
evaluation_modality=Modality.BEHAVIOR_VIDEOS,
evaluation_stage=Stage.RAW,
evaluator="Fred Flinstone",
evaluation_date=t,
qc_metrics=[
QCMetric(
name="video_1_num_frames",
value=662
),
QCMetric(
name="video_2_num_frames",
value=662
)
],
stage_status=Status.PASS,
notes="Pass when video_1_num_frames==video_2_num_frames"
)

eval2 = QCEvaluation(
evaluation_name="Probes present",
evaluation_modality=Modality.ECEPHYS,
evaluation_stage=Stage.RAW,
evaluator="Automated",
evaluation_date=t,
qc_metrics=[
QCMetric(
name="ProbeA_success",
value=True
),
QCMetric(
name="ProbeB_success",
value=True
),
QCMetric(
name="ProbeC_success",
value=True
)
],
stage_status=Status.PASS,
)

q = QualityControl(
overall_status="Pass",
overall_status_date=t,
evaluations=[eval0, eval1, eval2]
)

serialized = q.model_dump_json()
Expand Down
102 changes: 102 additions & 0 deletions quality_control.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
{
"describedBy": "https://raw.githubusercontent.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py",
"schema_version": "1.0.0",
"overall_status": "Pass",
"overall_status_date": "2022-11-22",
"evaluations": [
{
"evaluation_modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Preprocessing",
"evaluation_name": "Drift map",
"evaluation_desc": "Qualitative check that drift map shows minimal movement",
"evaluator": "Fred Flintstone",
"evaluation_date": "2022-11-22",
"qc_metrics": [
{
"name": "Probe A drift",
"value": "High",
"description": null,
"references": null
},
{
"name": "Probe B drift",
"value": "Low",
"description": null,
"references": null
},
{
"name": "Probe C drift",
"value": "Low",
"description": null,
"references": null
}
],
"stage_status": "Fail",
"notes": "Manually annotated: failed due to high drift on probe A"
},
{
"evaluation_modality": {
"name": "Behavior videos",
"abbreviation": "behavior-videos"
},
"evaluation_stage": "Raw data",
"evaluation_name": "Video frame count check",
"evaluation_desc": null,
"evaluator": "Fred Flinstone",
"evaluation_date": "2022-11-22",
"qc_metrics": [
{
"name": "video_1_num_frames",
"value": 662,
"description": null,
"references": null
},
{
"name": "video_2_num_frames",
"value": 662,
"description": null,
"references": null
}
],
"stage_status": "Pass",
"notes": "Pass when video_1_num_frames==video_2_num_frames"
},
{
"evaluation_modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Raw data",
"evaluation_name": "Probes present",
"evaluation_desc": null,
"evaluator": "Automated",
"evaluation_date": "2022-11-22",
"qc_metrics": [
{
"name": "ProbeA_success",
"value": true,
"description": null,
"references": null
},
{
"name": "ProbeB_success",
"value": true,
"description": null,
"references": null
},
{
"name": "ProbeC_success",
"value": true,
"description": null,
"references": null
}
],
"stage_status": "Pass",
"notes": null
}
],
"notes": null
}
41 changes: 31 additions & 10 deletions src/aind_data_schema/core/quality_control.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@

from datetime import date
from enum import Enum
from typing import List, Literal, Optional
from typing import List, Literal, Optional, Any

from aind_data_schema_models.modalities import Modality
from pydantic import Field
from pydantic import Field, BaseModel

from aind_data_schema.base import AindCoreModel, AindGeneric, AindGenericType, AindModel
from aind_data_schema.base import AindCoreModel, AindModel


class Status(str, Enum):
Expand All @@ -19,25 +19,46 @@ class Status(str, Enum):
PASS = "Pass"


class Stage(str, Enum):
"""QCEvaluation Stage
When during data processing the QC metrics were derived.
"""

RAW = "Raw data"
PREPROCESSING = "Preprocessing"
ANALYSIS = "Analysis"


class QCMetric(BaseModel):
"""Description of a single quality control metric"""
name: str = Field(..., title="Metric name")
value: Any = Field(..., title="Metric value")
description: Optional[str] = Field(default=None, title="Metric description")
references: Optional[List[str]] = Field(default=None, title="Metric reference URLs")


class QCEvaluation(AindModel):
"""Description of one evaluation stage"""
"""Description of one evaluation stage, with one or more metrics"""

evaluation_modality: Modality.ONE_OF = Field(..., title="Modality")
evaluation_stage: str = Field(..., title="Evaluation stage")
evaluator_full_name: str = Field(..., title="Evaluator full name")
evaluation_stage: Stage = Field(..., title="Evaluation stage")
evaluation_name: str = Field(..., title="Evaluation name")
evaluation_description: Optional[str] = Field(default=None, title="Evaluation description")
evaluator: str = Field(..., title="Evaluator full name")
evaluation_date: date = Field(..., title="Evaluation date")
qc_metrics: AindGenericType = Field(AindGeneric(), title="QC metrics")
qc_metrics: List[QCMetric] = Field(..., title="QC metrics")
stage_status: Status = Field(..., title="Stage status")
notes: Optional[str] = Field(None, title="Notes")
notes: Optional[str] = Field(default=None, title="Notes")


class QualityControl(AindCoreModel):
"""Description of quality metrics for a data asset"""

_DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/quality_metrics.py"
_DESCRIBED_BY_URL = AindCoreModel._DESCRIBED_BY_BASE_URL.default + "aind_data_schema/core/quality_control.py"
describedBy: str = Field(_DESCRIBED_BY_URL, json_schema_extra={"const": _DESCRIBED_BY_URL})
schema_version: Literal["1.0.0"] = Field("1.0.0")
overall_status: Status = Field(..., title="Overall status")
overall_status_date: date = Field(..., title="Date of status")
evaluations: List[QCEvaluation] = Field(..., title="Evaluations")
notes: Optional[str] = Field(None, title="Notes")
notes: Optional[str] = Field(default=None, title="Notes")
Loading

0 comments on commit 8246813

Please sign in to comment.