From 04cf36a2fd31407b3be6329c51150a981daac6ef Mon Sep 17 00:00:00 2001 From: David Almeida Date: Tue, 28 Jan 2025 16:03:49 +0100 Subject: [PATCH 1/5] Check if constituents of common regions are in native regions --- nomenclature/error.py | 4 ++++ nomenclature/processor/region.py | 15 +++++++++++++++ .../mappings/mapping_2.yaml | 1 - ...llegal_mapping_constituent_native_missing.yaml | 10 ++++++++++ tests/test_region_aggregation.py | 6 +++++- 5 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/data/region_processing/region_aggregation/illegal_mapping_constituent_native_missing.yaml diff --git a/nomenclature/error.py b/nomenclature/error.py index 4cdca95f..b240277e 100644 --- a/nomenclature/error.py +++ b/nomenclature/error.py @@ -40,6 +40,10 @@ "region_not_defined", "Region(s)\n{regions}\nin {file}\nnot found in RegionCodeList", ), + "ConstituentsNotNativeError": ( + "constituents_not_native", + "Constituent region(s)\n{regions}\nin {file} not found in RegionCodeList", + ), } PydanticCustomErrors = namedtuple("PydanticCustomErrors", pydantic_custom_error_config) diff --git a/nomenclature/processor/region.py b/nomenclature/processor/region.py index e853e648..19e238ae 100644 --- a/nomenclature/processor/region.py +++ b/nomenclature/processor/region.py @@ -232,6 +232,21 @@ def check_exclude_common_region_overlap( ) -> "RegionAggregationMapping": return _check_exclude_region_overlap(v, "common_regions") + @model_validator(mode="after") + @classmethod + def check_constituent_regions_in_native_regions( + cls, v: "RegionAggregationMapping" + ) -> "RegionAggregationMapping": + if v.common_regions and v.native_regions: + if missing := set( + [cr for r in v.common_regions for cr in r.constituent_regions] + ).difference([r.name for r in v.native_regions if v.native_regions]): + raise PydanticCustomError( + *custom_pydantic_errors.ConstituentsNotNativeError, + {"regions": missing, "file": v.file}, + ) + return v + @classmethod def from_file(cls, file: Path | str) -> "RegionAggregationMapping": """Initialize a RegionAggregationMapping from a file. diff --git a/tests/data/cli/structure_validation_fails/mappings/mapping_2.yaml b/tests/data/cli/structure_validation_fails/mappings/mapping_2.yaml index 1c171c6a..be91833e 100644 --- a/tests/data/cli/structure_validation_fails/mappings/mapping_2.yaml +++ b/tests/data/cli/structure_validation_fails/mappings/mapping_2.yaml @@ -4,4 +4,3 @@ native_regions: common_regions: - World: - region_a - - region_b diff --git a/tests/data/region_processing/region_aggregation/illegal_mapping_constituent_native_missing.yaml b/tests/data/region_processing/region_aggregation/illegal_mapping_constituent_native_missing.yaml new file mode 100644 index 00000000..838a5326 --- /dev/null +++ b/tests/data/region_processing/region_aggregation/illegal_mapping_constituent_native_missing.yaml @@ -0,0 +1,10 @@ +# Constituent region is not defined in native region +model: model_a +native_regions: + - region_a: alternative_name_a + - region_b: alternative_name_b +common_regions: + - common_region_1: + - region_a + - region_b + - region_c diff --git a/tests/test_region_aggregation.py b/tests/test_region_aggregation.py index 5640b71f..e0bc49e5 100644 --- a/tests/test_region_aggregation.py +++ b/tests/test_region_aggregation.py @@ -83,6 +83,10 @@ def test_mapping(): "illegal_mapping_model_only.yaml", "one of 'native_regions' and 'common_regions'", ), + ( + "illegal_mapping_constituent_native_missing.yaml", + "Constituent region\(s\)\n.*\n", + ), ], ) def test_illegal_mappings(file, error_msg_pattern): @@ -191,7 +195,7 @@ def test_region_processor_wrong_args(): def test_region_processor_multiple_wrong_mappings(simple_definition): # Read in the entire region_aggregation directory and return **all** errors - msg = "Collected 9 errors" + msg = "Collected 10 errors" with pytest.raises(ValueError, match=msg): RegionProcessor.from_directory( From ebfdf037043eea00818b95275160883869e238e4 Mon Sep 17 00:00:00 2001 From: David Almeida Date: Wed, 29 Jan 2025 11:03:01 +0100 Subject: [PATCH 2/5] Update hash to fix tests --- .../data/region_processing/external_repo_test/nomenclature.yaml | 2 +- .../external_repo_test_missing_region/nomenclature.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/data/region_processing/external_repo_test/nomenclature.yaml b/tests/data/region_processing/external_repo_test/nomenclature.yaml index aae2198a..4cf4f11b 100644 --- a/tests/data/region_processing/external_repo_test/nomenclature.yaml +++ b/tests/data/region_processing/external_repo_test/nomenclature.yaml @@ -4,7 +4,7 @@ dimensions: repositories: common-definitions: url: https://github.com/IAMconsortium/common-definitions.git/ - hash: cb85704 + hash: 091c0fe definitions: region: repository: common-definitions diff --git a/tests/data/region_processing/external_repo_test_missing_region/nomenclature.yaml b/tests/data/region_processing/external_repo_test_missing_region/nomenclature.yaml index 1dacf528..b0cae5c2 100644 --- a/tests/data/region_processing/external_repo_test_missing_region/nomenclature.yaml +++ b/tests/data/region_processing/external_repo_test_missing_region/nomenclature.yaml @@ -4,7 +4,7 @@ dimensions: repositories: common-definitions: url: https://github.com/IAMconsortium/common-definitions.git/ - hash: cb85704 + hash: 091c0fe definitions: region: repository: From 6884f475d0fcb03142b17150a34cbf455eb6c737 Mon Sep 17 00:00:00 2001 From: David Almeida Date: Wed, 29 Jan 2025 11:20:27 +0100 Subject: [PATCH 3/5] Fix error message --- nomenclature/error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomenclature/error.py b/nomenclature/error.py index b240277e..958f3c34 100644 --- a/nomenclature/error.py +++ b/nomenclature/error.py @@ -42,7 +42,7 @@ ), "ConstituentsNotNativeError": ( "constituents_not_native", - "Constituent region(s)\n{regions}\nin {file} not found in RegionCodeList", + "Constituent region(s)\n{regions}\nin {file} not found in native region(s)", ), } From 38d62e8952b5681af0e6d845f073a0b3d3f71bd5 Mon Sep 17 00:00:00 2001 From: David Almeida Date: Wed, 29 Jan 2025 15:49:36 +0100 Subject: [PATCH 4/5] Remove redundant check --- nomenclature/processor/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomenclature/processor/region.py b/nomenclature/processor/region.py index e129411a..e207196c 100644 --- a/nomenclature/processor/region.py +++ b/nomenclature/processor/region.py @@ -228,7 +228,7 @@ def check_constituent_regions_in_native_regions( if v.common_regions and v.native_regions: if missing := set( [cr for r in v.common_regions for cr in r.constituent_regions] - ).difference([r.name for r in v.native_regions if v.native_regions]): + ).difference([r.name for r in v.native_regions]): raise PydanticCustomError( *custom_pydantic_errors.ConstituentsNotNativeError, {"regions": missing, "file": v.file}, From 856a8b4d3969fde8ca264389f851ad3053058cb6 Mon Sep 17 00:00:00 2001 From: David Almeida Date: Wed, 29 Jan 2025 21:00:14 +0100 Subject: [PATCH 5/5] Check excluded regions --- nomenclature/processor/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomenclature/processor/region.py b/nomenclature/processor/region.py index e207196c..552d9787 100644 --- a/nomenclature/processor/region.py +++ b/nomenclature/processor/region.py @@ -228,7 +228,7 @@ def check_constituent_regions_in_native_regions( if v.common_regions and v.native_regions: if missing := set( [cr for r in v.common_regions for cr in r.constituent_regions] - ).difference([r.name for r in v.native_regions]): + ).difference([r.name for r in v.native_regions] + v.exclude_regions): raise PydanticCustomError( *custom_pydantic_errors.ConstituentsNotNativeError, {"regions": missing, "file": v.file},