Update sds schema validation (#1538)

Author: VirajP1002

app/authentication/auth_payload_versions.py

@@ -3,8 +3,3 @@
 
 class AuthPayloadVersion(Enum):
     V2 = "v2"
-
-
-class SupplementaryDataSchemaVersion(Enum):
-    V2 = "v2"
-    V1 = "v1"

app/routes/session.py

@@ -177,6 +177,7 @@ def _set_questionnaire_supplementary_data(
         dataset_id=new_sds_dataset_id,
         identifier=identifier,  # type: ignore
         survey_id=metadata["survey_id"],  # type: ignore
+        sds_schema_version=schema.json.get("sds_schema_version"),
     )
     logger.info(
         "fetched supplementary data",

app/services/supplementary_data.py

@@ -44,7 +44,11 @@ class InvalidSupplementaryData(Exception):
 
 
 def get_supplementary_data_v1(
-    *, dataset_id: str, identifier: str, survey_id: str
+    *,
+    dataset_id: str,
+    identifier: str,
+    survey_id: str,
+    sds_schema_version: str | None = None,
 ) -> dict:
     # Type ignore: current_app is a singleton in this application and has the key_store key in its eq attribute.
     key_store = current_app.eq["key_store"]  # type: ignore
@@ -93,6 +97,7 @@ def get_supplementary_data_v1(
             dataset_id=dataset_id,
             identifier=identifier,
             survey_id=survey_id,
+            sds_schema_version=sds_schema_version,
         )
 
     logger.error(
@@ -121,14 +126,19 @@ def decrypt_supplementary_data(
 
 
 def validate_supplementary_data(
-    supplementary_data: Mapping, dataset_id: str, identifier: str, survey_id: str
-) -> dict:
+    supplementary_data: Mapping,
+    dataset_id: str,
+    identifier: str,
+    survey_id: str,
+    sds_schema_version: str | None = None,
+) -> dict[str, str | dict | int | list]:
     try:
         return validate_supplementary_data_v1(
             supplementary_data=supplementary_data,
             dataset_id=dataset_id,
             identifier=identifier,
             survey_id=survey_id,
+            sds_schema_version=sds_schema_version,
         )
     except ValidationError as e:
         raise ValidationError("Invalid supplementary data") from e

app/utilities/supplementary_data_parser.py

@@ -10,7 +10,6 @@
     validates_schema,
 )
 
-from app.authentication.auth_payload_versions import SupplementaryDataSchemaVersion
 from app.utilities.metadata_parser_v2 import VALIDATORS, StripWhitespaceMixin
 
 
@@ -35,14 +34,6 @@ class ItemsData(Schema, StripWhitespaceMixin):
 
 class SupplementaryData(Schema, StripWhitespaceMixin):
     identifier = VALIDATORS["string"](validate=validate.Length(min=1))
-    schema_version = VALIDATORS["string"](
-        validate=validate.OneOf(
-            [
-                SupplementaryDataSchemaVersion.V1.value,
-                SupplementaryDataSchemaVersion.V2.value,
-            ]
-        )
-    )
     items = fields.Nested(ItemsData, required=False, unknown=INCLUDE)
 
     @validates_schema()
@@ -66,27 +57,35 @@ class SupplementaryDataMetadataSchema(Schema, StripWhitespaceMixin):
     )
 
     @validates_schema()
-    def validate_dataset_and_survey_id(  # pylint: disable=unused-argument
-        self, data: Mapping, **kwargs: Any
+    def validate_payload(  # pylint: disable=unused-argument
+        self, payload: Mapping, **kwargs: Any
     ) -> None:
-        if data:
-            if data["dataset_id"] != self.context["dataset_id"]:
+        if payload:
+            if payload["dataset_id"] != self.context["dataset_id"]:
                 raise ValidationError(
                     "Supplementary data did not return the specified Dataset ID"
                 )
 
-            if data["survey_id"] != self.context["survey_id"]:
+            if payload["survey_id"] != self.context["survey_id"]:
                 raise ValidationError(
                     "Supplementary data did not return the specified Survey ID"
                 )
 
+            if self.context["sds_schema_version"] and (
+                payload["data"]["schema_version"] != self.context["sds_schema_version"]
+            ):
+                raise ValidationError(
+                    "The Supplementary Dataset Schema Version does not match the version set in the Questionnaire Schema"
+                )
+
 
 def validate_supplementary_data_v1(
     supplementary_data: Mapping,
     dataset_id: str,
     identifier: str,
     survey_id: str,
-) -> dict:
+    sds_schema_version: str | None = None,
+) -> dict[str, str | dict | int | list]:
     """Validate claims required for supplementary data"""
     supplementary_data_metadata_schema = SupplementaryDataMetadataSchema(
         unknown=INCLUDE
@@ -95,6 +94,7 @@ def validate_supplementary_data_v1(
         "dataset_id": dataset_id,
         "identifier": identifier,
         "survey_id": survey_id,
+        "sds_schema_version": sds_schema_version,
     }
     validated_supplementary_data = supplementary_data_metadata_schema.load(
         supplementary_data

tests/app/parser/test_supplementary_data_parser.py

@@ -1,3 +1,4 @@
+from contextlib import contextmanager
 from copy import deepcopy
 
 import pytest
@@ -6,6 +7,15 @@
 from app.services.supplementary_data import validate_supplementary_data
 from app.utilities.supplementary_data_parser import validate_supplementary_data_v1
 
+
+@contextmanager
+def not_raises(exception):
+    try:
+        yield
+    except exception as validation_error:
+        raise pytest.fail(f"{validation_error} RAISED")
+
+
 SUPPLEMENTARY_DATA_PAYLOAD = {
     "dataset_id": "44f1b432-9421-49e5-bd26-e63e18a30b69",
     "survey_id": "123",
@@ -54,6 +64,35 @@ def test_invalid_supplementary_data_payload_raises_error():
     assert str(error.value) == "Invalid supplementary data"
 
 
+def test_invalid_supplementary_dataset_version_raises_error():
+    with pytest.raises(ValidationError) as error:
+        validate_supplementary_data_v1(
+            supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
+            dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
+            identifier="12345678901",
+            survey_id="123",
+            sds_schema_version="v6",
+        )
+
+    assert (
+        str(error.value)
+        == "{'_schema': ['The Supplementary Dataset Schema Version does not match the version set in the Questionnaire Schema']}"
+    )
+
+
+def test_valid_supplementary_dataset_version_does_not_raise_error():
+    with not_raises(ValidationError):
+        validated_payload = validate_supplementary_data_v1(
+            supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
+            dataset_id="44f1b432-9421-49e5-bd26-e63e18a30b69",
+            identifier="12345678901",
+            survey_id="123",
+            sds_schema_version="v1",
+        )
+
+    assert validated_payload == SUPPLEMENTARY_DATA_PAYLOAD
+
+
 def test_validate_supplementary_data_payload():
     validated_payload = validate_supplementary_data_v1(
         supplementary_data=SUPPLEMENTARY_DATA_PAYLOAD,
@@ -171,30 +210,6 @@ def test_validate_supplementary_data_payload_with_unknown_field():
     assert validated_payload == payload
 
 
-def test_validate_supplementary_data_invalid_schema_version():
-    payload = {
-        "dataset_id": "44f1b432-9421-49e5-bd26-e63e18a30b69",
-        "survey_id": "123",
-        "some_field": "value",
-        "data": {
-            "schema_version": "v3",
-            "identifier": "12345678901",
-        },
-    }
-
-    with pytest.raises(ValidationError) as error:
-        validate_supplementary_data_v1(
-            supplementary_data=payload,
-            dataset_id="001",
-            identifier="12345678901",
-            survey_id="123",
-        )
-
-    assert (
-        str(error.value) == "{'data': {'schema_version': ['Must be one of: v1, v2.']}}"
-    )
-
-
 def test_validate_supplementary_data_payload_missing_identifier_in_items():
     payload = deepcopy(SUPPLEMENTARY_DATA_PAYLOAD)
     payload["data"]["items"]["local_units"][0].pop("identifier")