From c2b08af1e99fce01abc2c94449cbc7cf67fc3e73 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:06:01 +0100 Subject: [PATCH 01/23] Add output_directory option to parse_model_registration --- nomenclature/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nomenclature/__init__.py b/nomenclature/__init__.py index 4733c50d..8815d58f 100644 --- a/nomenclature/__init__.py +++ b/nomenclature/__init__.py @@ -57,8 +57,7 @@ def parse_model_registration( model_registration_file : str, path, file-like object Path to xlsx model registration file. output_directory : str, path, file-like object - Directory where the model mapping and region file will be saved; - defaults to current working directory + Directory, where the model mapping and region file will be saved; default: "." """ if not isinstance(output_directory, Path): output_directory = Path(output_directory) From f9ad30cff00da9340a55c3a084c19007309dadfd Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:06:58 +0100 Subject: [PATCH 02/23] Update config to use multiple repositories --- nomenclature/config.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/nomenclature/config.py b/nomenclature/config.py index 92aea787..09336c03 100644 --- a/nomenclature/config.py +++ b/nomenclature/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, Optional +from typing import Dict, Optional, List, Set import yaml from git import Repo @@ -8,15 +8,16 @@ class CodeListConfig(BaseModel): dimension: str - repository: str | None = None - repository_dimension_path: Path | None = None + repositories: Set[str] | None = Field(None, validation_alias="repository") - @model_validator(mode="after") + @field_validator("repositories", mode="before") @classmethod - def set_repository_dimension_path(cls, v: "CodeListConfig") -> "CodeListConfig": - if v.repository is not None and v.repository_dimension_path is None: - v.repository_dimension_path = f"definitions/{v.dimension}" - return v + def convert_to_set(cls, v) -> Set: + return v if isinstance(v, set) else {v} + + @property + def repository_dimension_path(self) -> str: + return f"definitions/{self.dimension}" class RegionCodeListConfig(CodeListConfig): @@ -86,14 +87,19 @@ def add_dimension(cls, v, info: ValidationInfo): @property def repos(self) -> Dict[str, str]: return { - dimension: getattr(self, dimension).repository + dimension: getattr(self, dimension).repositories for dimension in ("region", "variable") - if getattr(self, dimension) and getattr(self, dimension).repository + if getattr(self, dimension) and getattr(self, dimension).repositories } class RegionMappingConfig(BaseModel): - repository: str + repositories: Set[str] = Field(..., validation_alias="repository") + + @field_validator("repositories", mode="before") + @classmethod + def convert_to_set(cls, v) -> Set: + return v if isinstance(v, set) else {v} class NomenclatureConfig(BaseModel): @@ -107,11 +113,11 @@ def check_definitions_repository( cls, v: "NomenclatureConfig" ) -> "NomenclatureConfig": definitions_repos = v.definitions.repos if v.definitions else {} - mapping_repos = {"mappings": v.mappings.repository} if v.mappings else {} + mapping_repos = {"mappings": v.mappings.repositories} if v.mappings else {} repos = {**definitions_repos, **mapping_repos} - for use, repository in repos.items(): - if repository not in v.repositories: - raise ValueError((f"Unknown repository '{repository}' in {use}.")) + for use, repositories in repos.items(): + if repositories - v.repositories.keys(): + raise ValueError((f"Unknown repository {repositories} in '{use}'.")) return v def fetch_repos(self, target_folder: Path): From f1df18e9681a204fe7e68178adf9ebaaaf19dd69 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:07:20 +0100 Subject: [PATCH 03/23] Update codelist to use multiple external repos --- nomenclature/codelist.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 37da4965..9e33977b 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -202,17 +202,17 @@ def from_directory( with suppress(AttributeError): dimension = path.name codelistconfig = getattr(config.definitions, dimension) - repo_path = ( - config.repositories[codelistconfig.repository].local_path - / codelistconfig.repository_dimension_path - ) - code_list = ( - cls._parse_codelist_dir( - repo_path, - file_glob_pattern, + for repo in codelistconfig.repositories: + repo_path = ( + config.repositories[repo].local_path / "definitions" / dimension + ) + code_list = ( + cls._parse_codelist_dir( + repo_path, + file_glob_pattern, + ) + + code_list ) - + code_list - ) mapping: Dict[str, Code] = {} for code in code_list: @@ -615,17 +615,18 @@ def from_directory( code_list.append(RegionCode(name=c.name, hierarchy="Country")) # importing from an external repository - if config.definitions.region.repository: + for repository in config.definitions.region.repositories: repo_path = ( - config.repositories[config.definitions.region.repository].local_path - / config.definitions.region.repository_dimension_path + config.repositories[repository].local_path + / "definitions" + / "region" ) code_list = cls._parse_region_code_dir( code_list, repo_path, file_glob_pattern, - repository=config.definitions.region.repository, + repository=config.definitions.region.repositories, ) code_list = cls._parse_and_replace_tags( code_list, repo_path, file_glob_pattern @@ -639,7 +640,7 @@ def from_directory( mapping: Dict[str, RegionCode] = {} for code in code_list: if code.name in mapping: - raise DuplicateCodeError(name=name, code=code.name) + raise ValueError(f"Trying to set a duplicate code {code.name}") mapping[code.name] = code return cls(name=name, mapping=mapping) From cb0caa725e652f2a4b5c8a5bed59a8b74181bbea Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:07:32 +0100 Subject: [PATCH 04/23] Update RegionProcessor to use multiple external repos --- nomenclature/processor/region.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nomenclature/processor/region.py b/nomenclature/processor/region.py index 7df675c1..d2964650 100644 --- a/nomenclature/processor/region.py +++ b/nomenclature/processor/region.py @@ -482,14 +482,14 @@ def from_directory(cls, path: DirectoryPath, dsd: DataStructureDefinition): mapping_files = [f for f in path.glob("**/*") if f.suffix in {".yaml", ".yml"}] if dsd.config and dsd.config.mappings: - mapping_files = [ - f - for f in ( - dsd.config.repositories[dsd.config.mappings.repository].local_path - / "mappings" - ).glob("**/*") - if f.suffix in {".yaml", ".yml"} - ] + mapping_files + for repository in dsd.config.mappings.repositories: + mapping_files.extend( + f + for f in ( + dsd.config.repositories[repository].local_path / "mappings" + ).glob("**/*") + if f.suffix in {".yaml", ".yml"} + ) for file in mapping_files: try: From a2619e2da105af9151124a32299fb690fbeac91f Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:08:55 +0100 Subject: [PATCH 05/23] Remove unnecessary repository_dimension_path --- tests/data/general-config-only/nomenclature.yaml | 1 - tests/data/general-config/nomenclature.yaml | 1 - tests/data/nomenclature_configs/hash_and_release.yaml | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/data/general-config-only/nomenclature.yaml b/tests/data/general-config-only/nomenclature.yaml index e9515d9e..1c7e59e9 100644 --- a/tests/data/general-config-only/nomenclature.yaml +++ b/tests/data/general-config-only/nomenclature.yaml @@ -7,4 +7,3 @@ definitions: country: true variable: repository: common-definitions - repository_dimension_path: definitions/variable diff --git a/tests/data/general-config/nomenclature.yaml b/tests/data/general-config/nomenclature.yaml index e9515d9e..1c7e59e9 100644 --- a/tests/data/general-config/nomenclature.yaml +++ b/tests/data/general-config/nomenclature.yaml @@ -7,4 +7,3 @@ definitions: country: true variable: repository: common-definitions - repository_dimension_path: definitions/variable diff --git a/tests/data/nomenclature_configs/hash_and_release.yaml b/tests/data/nomenclature_configs/hash_and_release.yaml index 8e8a0fe5..502d18e4 100644 --- a/tests/data/nomenclature_configs/hash_and_release.yaml +++ b/tests/data/nomenclature_configs/hash_and_release.yaml @@ -9,4 +9,3 @@ definitions: country: true variable: repository: common-definitions - repository_dimension_path: definitions/variable From 8bcba396805b1ac8319e766cf1ffad7ee84efa1f Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:09:08 +0100 Subject: [PATCH 06/23] Update test for multiple external repos --- tests/test_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 3e72cb72..95cb9320 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,9 @@ def test_setting_local_path_raises(): def test_unknown_repo_raises(): - with raises(ValueError, match="Unknown repository 'common-definitions'"): + with raises( + ValueError, match="Unknown repository {'common-definitions'} in 'region'" + ): NomenclatureConfig.from_file( TEST_DATA_DIR / "nomenclature_configs" / "unknown_repo.yaml" ) From f30f37b6b6496a5d7392b6573819e9536ce8bd71 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:24:56 +0100 Subject: [PATCH 07/23] Fix repository set for definitions and mappings --- nomenclature/config.py | 48 ++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/nomenclature/config.py b/nomenclature/config.py index 09336c03..d8fbf2b5 100644 --- a/nomenclature/config.py +++ b/nomenclature/config.py @@ -1,19 +1,37 @@ from pathlib import Path -from typing import Dict, Optional, List, Set +from typing import Annotated, Optional import yaml from git import Repo -from pydantic import BaseModel, Field, ValidationInfo, field_validator, model_validator +from pydantic import ( + BaseModel, + Field, + ValidationInfo, + field_validator, + model_validator, + ConfigDict, + BeforeValidator, +) + + +def convert_to_set(v: str | list[str] | set[str]) -> set[str]: + match v: + case str(v): + return {v} + case list(v): + return set(v) + case set(v): + return v + case _: + raise TypeError("`repositories` must be of type str, list or set.") class CodeListConfig(BaseModel): dimension: str - repositories: Set[str] | None = Field(None, validation_alias="repository") - - @field_validator("repositories", mode="before") - @classmethod - def convert_to_set(cls, v) -> Set: - return v if isinstance(v, set) else {v} + repositories: Annotated[set[str] | None, BeforeValidator(convert_to_set)] = Field( + None, alias="repository" + ) + model_config = ConfigDict(populate_by_name=True) @property def repository_dimension_path(self) -> str: @@ -85,7 +103,7 @@ def add_dimension(cls, v, info: ValidationInfo): return {"dimension": info.field_name, **v} @property - def repos(self) -> Dict[str, str]: + def repos(self) -> dict[str, str]: return { dimension: getattr(self, dimension).repositories for dimension in ("region", "variable") @@ -94,16 +112,14 @@ def repos(self) -> Dict[str, str]: class RegionMappingConfig(BaseModel): - repositories: Set[str] = Field(..., validation_alias="repository") - - @field_validator("repositories", mode="before") - @classmethod - def convert_to_set(cls, v) -> Set: - return v if isinstance(v, set) else {v} + repositories: Annotated[set[str], BeforeValidator(convert_to_set)] = Field( + ..., alias="repository" + ) + model_config = ConfigDict(populate_by_name=True) class NomenclatureConfig(BaseModel): - repositories: Dict[str, Repository] = {} + repositories: dict[str, Repository] = {} definitions: Optional[DataStructureConfig] = None mappings: Optional[RegionMappingConfig] = None From 3f129f6d6918b5506ca4190e005886686acbf564 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:25:12 +0100 Subject: [PATCH 08/23] Add tests for config --- tests/test_config.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 95cb9320..b03fd3e2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ from pathlib import Path from pytest import raises -from nomenclature.config import Repository, NomenclatureConfig +from nomenclature.config import Repository, NomenclatureConfig, CodeListConfig -from conftest import TEST_DATA_DIR +from conftest import TEST_DATA_DIR, clean_up_external_repos def test_hash_and_release_raises(): @@ -25,3 +25,33 @@ def test_unknown_repo_raises(): NomenclatureConfig.from_file( TEST_DATA_DIR / "nomenclature_configs" / "unknown_repo.yaml" ) + + +def test_multiple_definition_repos(): + nomenclature_config = NomenclatureConfig.from_file( + TEST_DATA_DIR / "nomenclature_configs" / "multiple_repos_per_dimension.yaml" + ) + try: + exp_repos = {"common-definitions", "legacy-definitions"} + assert nomenclature_config.repositories.keys() == exp_repos + assert nomenclature_config.definitions.variable.repositories == exp_repos + finally: + clean_up_external_repos(nomenclature_config.repositories) + + +def test_codelist_config_set_input(): + exp_repos = {"repo1", "repo2"} + code_list_config = CodeListConfig(dimension="variable", repositories=exp_repos) + assert code_list_config.repositories == exp_repos + + +def test_multiple_mapping_repos(): + nomenclature_config = NomenclatureConfig.from_file( + TEST_DATA_DIR / "nomenclature_configs" / "multiple_repos_for_mapping.yaml" + ) + try: + exp_repos = {"common-definitions", "legacy-definitions"} + assert nomenclature_config.mappings.repositories == exp_repos + assert nomenclature_config.repositories.keys() == exp_repos + finally: + clean_up_external_repos(nomenclature_config.repositories) From 74025663cc4c7f8c76b68322cb0f1e1aceecf4bd Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:25:30 +0100 Subject: [PATCH 09/23] Add test for multiple external repos --- .../multiple_repos_for_mapping.yaml | 9 +++++++++ .../multiple_repos_per_dimension.yaml | 10 ++++++++++ .../nomenclature_configs/variable/.gitkeep | 0 tests/test_codelist.py | 20 +++++++++++++++++-- 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tests/data/nomenclature_configs/multiple_repos_for_mapping.yaml create mode 100644 tests/data/nomenclature_configs/multiple_repos_per_dimension.yaml create mode 100644 tests/data/nomenclature_configs/variable/.gitkeep diff --git a/tests/data/nomenclature_configs/multiple_repos_for_mapping.yaml b/tests/data/nomenclature_configs/multiple_repos_for_mapping.yaml new file mode 100644 index 00000000..ff2bd784 --- /dev/null +++ b/tests/data/nomenclature_configs/multiple_repos_for_mapping.yaml @@ -0,0 +1,9 @@ +repositories: + common-definitions: + url: https://github.com/IAMconsortium/common-definitions.git/ + legacy-definitions: + url: https://github.com/IAMconsortium/legacy-definitions.git/ +mappings: + repositories: + - common-definitions + - legacy-definitions diff --git a/tests/data/nomenclature_configs/multiple_repos_per_dimension.yaml b/tests/data/nomenclature_configs/multiple_repos_per_dimension.yaml new file mode 100644 index 00000000..6e510cd2 --- /dev/null +++ b/tests/data/nomenclature_configs/multiple_repos_per_dimension.yaml @@ -0,0 +1,10 @@ +repositories: + common-definitions: + url: https://github.com/IAMconsortium/common-definitions.git/ + legacy-definitions: + url: https://github.com/IAMconsortium/legacy-definitions.git/ +definitions: + variable: + repository: + - common-definitions + - legacy-definitions diff --git a/tests/data/nomenclature_configs/variable/.gitkeep b/tests/data/nomenclature_configs/variable/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_codelist.py b/tests/test_codelist.py index 4fd0a5c0..5668bc65 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -3,15 +3,16 @@ import pandas.testing as pdt import logging -from nomenclature.code import Code, RegionCode, MetaCode, VariableCode +from nomenclature.code import Code, RegionCode, MetaCode from nomenclature.codelist import ( CodeList, VariableCodeList, RegionCodeList, MetaCodeList, ) +from nomenclature.config import NomenclatureConfig -from conftest import TEST_DATA_DIR +from conftest import TEST_DATA_DIR, clean_up_external_repos def test_simple_codelist(): @@ -323,3 +324,18 @@ def test_MetaCodeList_from_directory(): } exp = MetaCodeList(name="Meta", mapping=mapping) assert obs == exp + + +def test_multiple_external_repos(): + nomenclature_config = NomenclatureConfig.from_file( + TEST_DATA_DIR / "nomenclature_configs" / "multiple_repos_per_dimension.yaml" + ) + try: + with raises(ValueError, match="Duplicate"): + variable_code_list = VariableCodeList.from_directory( + "variable", + TEST_DATA_DIR / "nomenclature_configs" / "variable", + nomenclature_config, + ) + finally: + clean_up_external_repos(nomenclature_config.repositories) From 25652f946621971626cbb9aa8248d4b7030fad6e Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:46:07 +0100 Subject: [PATCH 10/23] Revert to previous suggestion from @danielhuppmann --- nomenclature/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nomenclature/__init__.py b/nomenclature/__init__.py index 8815d58f..4733c50d 100644 --- a/nomenclature/__init__.py +++ b/nomenclature/__init__.py @@ -57,7 +57,8 @@ def parse_model_registration( model_registration_file : str, path, file-like object Path to xlsx model registration file. output_directory : str, path, file-like object - Directory, where the model mapping and region file will be saved; default: "." + Directory where the model mapping and region file will be saved; + defaults to current working directory """ if not isinstance(output_directory, Path): output_directory = Path(output_directory) From 9216d7dbcb55232752a2490328b1341c435588f0 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:47:00 +0100 Subject: [PATCH 11/23] Apply suggestions from code review Co-authored-by: Daniel Huppmann --- nomenclature/codelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index 9e33977b..d61aef09 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -615,7 +615,7 @@ def from_directory( code_list.append(RegionCode(name=c.name, hierarchy="Country")) # importing from an external repository - for repository in config.definitions.region.repositories: + for repo in config.definitions.region.repositories: repo_path = ( config.repositories[repository].local_path / "definitions" From 7cd2a8267a67e71bc792c5b28fd528607ab555b3 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 13:51:24 +0100 Subject: [PATCH 12/23] Finish renaming in repo in loop --- nomenclature/codelist.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nomenclature/codelist.py b/nomenclature/codelist.py index d61aef09..7cb39948 100644 --- a/nomenclature/codelist.py +++ b/nomenclature/codelist.py @@ -617,9 +617,7 @@ def from_directory( # importing from an external repository for repo in config.definitions.region.repositories: repo_path = ( - config.repositories[repository].local_path - / "definitions" - / "region" + config.repositories[repo].local_path / "definitions" / "region" ) code_list = cls._parse_region_code_dir( From 30710241f36f3873cacdeb4a2fa4137151684647 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:32:17 +0100 Subject: [PATCH 13/23] Switch cast to set order to clarity --- nomenclature/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nomenclature/config.py b/nomenclature/config.py index d8fbf2b5..3971c1d8 100644 --- a/nomenclature/config.py +++ b/nomenclature/config.py @@ -16,12 +16,12 @@ def convert_to_set(v: str | list[str] | set[str]) -> set[str]: match v: - case str(v): - return {v} - case list(v): - return set(v) case set(v): return v + case list(v): + return set(v) + case str(v): + return {v} case _: raise TypeError("`repositories` must be of type str, list or set.") From c3ef3778a3ef1713caffd976fc4cfc0dee83c453 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:32:50 +0100 Subject: [PATCH 14/23] Allow errors for rmtree for windows debugging --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index d3c2e368..05e49cda 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,4 +52,4 @@ def clean_up_external_repos(repos): # clean up the external repo for repository in repos.values(): if repository.local_path.exists(): - shutil.rmtree(repository.local_path, ignore_errors=True) + shutil.rmtree(repository.local_path) # , ignore_errors=True) From 90dbffc987535af800ecc2b4bd4c690673c21f6c Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:42:53 +0100 Subject: [PATCH 15/23] Attempt fix for failing Windows test --- tests/test_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index b03fd3e2..d029a7d1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,7 +36,8 @@ def test_multiple_definition_repos(): assert nomenclature_config.repositories.keys() == exp_repos assert nomenclature_config.definitions.variable.repositories == exp_repos finally: - clean_up_external_repos(nomenclature_config.repositories) + pass + # clean_up_external_repos(nomenclature_config.repositories) def test_codelist_config_set_input(): @@ -54,4 +55,5 @@ def test_multiple_mapping_repos(): assert nomenclature_config.mappings.repositories == exp_repos assert nomenclature_config.repositories.keys() == exp_repos finally: - clean_up_external_repos(nomenclature_config.repositories) + pass + # clean_up_external_repos(nomenclature_config.repositories) From b5dda86ae458052f972fa3d7c50c12cbd72072fb Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 14:56:02 +0100 Subject: [PATCH 16/23] Remove all clean-up for Windows fix --- tests/test_codelist.py | 3 ++- tests/test_definition.py | 3 ++- tests/test_region_aggregation.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_codelist.py b/tests/test_codelist.py index 5668bc65..c9e6733e 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -338,4 +338,5 @@ def test_multiple_external_repos(): nomenclature_config, ) finally: - clean_up_external_repos(nomenclature_config.repositories) + pass + # clean_up_external_repos(nomenclature_config.repositories) diff --git a/tests/test_definition.py b/tests/test_definition.py index 2de7187e..377282a9 100644 --- a/tests/test_definition.py +++ b/tests/test_definition.py @@ -61,7 +61,8 @@ def test_definition_from_general_config(workflow_folder): # imported from https://github.com/IAMconsortium/common-definitions repo assert "Primary Energy" in obs.variable finally: - clean_up_external_repos(obs.config.repositories) + pass + # clean_up_external_repos(obs.config.repositories) def test_to_excel(simple_definition, tmpdir): diff --git a/tests/test_region_aggregation.py b/tests/test_region_aggregation.py index f10c7d89..9f8342ab 100644 --- a/tests/test_region_aggregation.py +++ b/tests/test_region_aggregation.py @@ -243,7 +243,8 @@ def test_mapping_from_external_repository(): for model in ("REMIND 3.1", "REMIND-MAgPIE 3.1-4.6") ) finally: - clean_up_external_repos(dsd.config.repositories) + pass + # clean_up_external_repos(dsd.config.repositories) def test_reverse_region_aggregation(): From 5a10dd9c9e67a1d60370648d8113de28f0d8fd78 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:08:17 +0100 Subject: [PATCH 17/23] Change repo permissions on Windows to delete --- tests/conftest.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 05e49cda..3e2410ae 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,15 @@ -from pathlib import Path -import pytest import shutil +import sys +import os +import stat +from pathlib import Path + import pandas as pd +import pytest from pyam import IamDataFrame from pyam.utils import IAMC_IDX -from nomenclature import DataStructureDefinition +from nomenclature import DataStructureDefinition here = Path(__file__).parent TEST_DATA_DIR = here / "data" @@ -52,4 +56,8 @@ def clean_up_external_repos(repos): # clean up the external repo for repository in repos.values(): if repository.local_path.exists(): - shutil.rmtree(repository.local_path) # , ignore_errors=True) + if sys.platform.startswith("win") and not os.access( + repository.local_path, os.W_OK + ): + os.chmod(repository.local_path, stat.S_IWUSR) + shutil.rmtree(repository.local_path, ignore_errors=True) From 0f95818216dfd7774e84a9580de28587db88c7b8 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:08:41 +0100 Subject: [PATCH 18/23] Re-introduce external repo cleanup --- tests/test_codelist.py | 3 +-- tests/test_config.py | 6 ++---- tests/test_definition.py | 3 +-- tests/test_region_aggregation.py | 3 +-- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/tests/test_codelist.py b/tests/test_codelist.py index c9e6733e..5668bc65 100644 --- a/tests/test_codelist.py +++ b/tests/test_codelist.py @@ -338,5 +338,4 @@ def test_multiple_external_repos(): nomenclature_config, ) finally: - pass - # clean_up_external_repos(nomenclature_config.repositories) + clean_up_external_repos(nomenclature_config.repositories) diff --git a/tests/test_config.py b/tests/test_config.py index d029a7d1..b03fd3e2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,8 +36,7 @@ def test_multiple_definition_repos(): assert nomenclature_config.repositories.keys() == exp_repos assert nomenclature_config.definitions.variable.repositories == exp_repos finally: - pass - # clean_up_external_repos(nomenclature_config.repositories) + clean_up_external_repos(nomenclature_config.repositories) def test_codelist_config_set_input(): @@ -55,5 +54,4 @@ def test_multiple_mapping_repos(): assert nomenclature_config.mappings.repositories == exp_repos assert nomenclature_config.repositories.keys() == exp_repos finally: - pass - # clean_up_external_repos(nomenclature_config.repositories) + clean_up_external_repos(nomenclature_config.repositories) diff --git a/tests/test_definition.py b/tests/test_definition.py index 377282a9..2de7187e 100644 --- a/tests/test_definition.py +++ b/tests/test_definition.py @@ -61,8 +61,7 @@ def test_definition_from_general_config(workflow_folder): # imported from https://github.com/IAMconsortium/common-definitions repo assert "Primary Energy" in obs.variable finally: - pass - # clean_up_external_repos(obs.config.repositories) + clean_up_external_repos(obs.config.repositories) def test_to_excel(simple_definition, tmpdir): diff --git a/tests/test_region_aggregation.py b/tests/test_region_aggregation.py index 9f8342ab..f10c7d89 100644 --- a/tests/test_region_aggregation.py +++ b/tests/test_region_aggregation.py @@ -243,8 +243,7 @@ def test_mapping_from_external_repository(): for model in ("REMIND 3.1", "REMIND-MAgPIE 3.1-4.6") ) finally: - pass - # clean_up_external_repos(dsd.config.repositories) + clean_up_external_repos(dsd.config.repositories) def test_reverse_region_aggregation(): From 5a105a2c91756f97f9e2a69f022cc1d8f470e925 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:11:56 +0100 Subject: [PATCH 19/23] Add explicit errors for debugging --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3e2410ae..0791c55a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,4 +60,4 @@ def clean_up_external_repos(repos): repository.local_path, os.W_OK ): os.chmod(repository.local_path, stat.S_IWUSR) - shutil.rmtree(repository.local_path, ignore_errors=True) + shutil.rmtree(repository.local_path) # , ignore_errors=True) From cca22b1df471e22233145e9bd8d346822c7ad66a Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:46:06 +0100 Subject: [PATCH 20/23] Try IWRITE for Windows permission change --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0791c55a..2f938cb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,5 +59,5 @@ def clean_up_external_repos(repos): if sys.platform.startswith("win") and not os.access( repository.local_path, os.W_OK ): - os.chmod(repository.local_path, stat.S_IWUSR) + os.chmod(repository.local_path, stat.S_IWRITE) shutil.rmtree(repository.local_path) # , ignore_errors=True) From 8f027e52d1f6cceaa20d49fa07c25c9ba344a187 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:50:32 +0100 Subject: [PATCH 21/23] Try onerror for rmtree --- tests/conftest.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2f938cb0..360638d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,12 +52,15 @@ def add_meta(df): df.set_meta(["foo", "bar"], "string") +def remove_readonly(func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + + def clean_up_external_repos(repos): # clean up the external repo for repository in repos.values(): if repository.local_path.exists(): - if sys.platform.startswith("win") and not os.access( - repository.local_path, os.W_OK - ): - os.chmod(repository.local_path, stat.S_IWRITE) - shutil.rmtree(repository.local_path) # , ignore_errors=True) + shutil.rmtree( + repository.local_path, onerror=remove_readonly, ignore_errors=True + ) From 21d6f47ac953473e513e471bd47aa94f15125266 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:53:35 +0100 Subject: [PATCH 22/23] Get explicit removing errors --- tests/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 360638d8..ecc90ce4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,4 @@ def clean_up_external_repos(repos): # clean up the external repo for repository in repos.values(): if repository.local_path.exists(): - shutil.rmtree( - repository.local_path, onerror=remove_readonly, ignore_errors=True - ) + shutil.rmtree(repository.local_path, onerror=remove_readonly) From f0933b654345c5f0ce6d6c4ddef426a90d9a9179 Mon Sep 17 00:00:00 2001 From: Philip Hackstock <20710924+phackstock@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:01:42 +0100 Subject: [PATCH 23/23] Update docs for usage of multiple repos --- docs/user_guide/config.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/user_guide/config.rst b/docs/user_guide/config.rst index 2bce39d8..9da5cd70 100644 --- a/docs/user_guide/config.rst +++ b/docs/user_guide/config.rst @@ -8,7 +8,7 @@ General project configuration The following features can be accessed via a general project configuration file: * Import codelists and mappings from public GitHub repositories -* Add all countries to the region codelist (details: :ref:`countries`) +* Add all countries to the region codelist (details: :ref:`countries`) Configuration file format ------------------------- @@ -24,19 +24,23 @@ Importing from an external repository In order to import from an external repository, the configuration file must define the repository and the repositories key. -The repository has a **name** (in the example below *common-definitions*) and a **url**: +The repository has a **name** (in the example below *common-definitions*) and a **url**. Multiple repositories can be used in a single configuration file: .. code:: yaml repositories: common-definitions: url: https://github.com/IAMconsortium/common-definitions.git/ + legacy-definitions: + url: https://github.com/IAMconsortium/legacy-definitions.git/ In order for the import to work the url must be given in the above format, i.e. with the leading *https://* and the trailing *.git/*. Information from an external repository can either be used for codelists ("definitions") -or model mappings, or both. +or model mappings, or both. For each definition dimension, i.e. *region* or *variable* +multiple external repositories can be used as the example below illustrates for +*variable*: .. code:: yaml @@ -47,7 +51,9 @@ or model mappings, or both. region: repository: common-definitions variable: - repository: common-definitions + repositories: + - common-definitions + - legacy-definitions mappings: repository: common-definitions