Merge branch 'main' into illegal-characters

README.md

@@ -63,7 +63,7 @@ pip install nomenclature-iamc
 Alternatively, it can also be installed directly from source:
 
 ```bash
-pip install -e git+https://github.com/IAMconsortium/nomenclature#egg=nomenclature
+pip install -e git+https://github.com/IAMconsortium/nomenclature#egg=nomenclature-iamc
 ```
 
 See the [User Guide](https://nomenclature-iamc.readthedocs.io/en/latest/user_guide.html)

docs/installation.rst

@@ -22,4 +22,4 @@ of the main branch.
 
 .. code-block:: bash
 
-    pip install -e git+https://github.com/IAMconsortium/nomenclature@main#egg=nomenclature
+    pip install -e git+https://github.com/IAMconsortium/nomenclature@main#egg=nomenclature-iamc

docs/user_guide/config.rst

@@ -57,11 +57,47 @@ multiple external repositories can be used as the example below illustrates for
   mappings:
     repository: common-definitions
 
-The value in *definitions.region.repository* needs to reference the repository in the
-*repositories* section.
+The value in *definitions.region.repository* can be a list or a single value.
 
 For model mappings the process is analogous using *mappings.repository*.
 
+Filter code lists imported from external repositories
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Since importing the entirety of, for example, common-definitions is too much for most
+projects, the list can be filtered using ``include`` and ``exclude`` keywords. Under
+these keywords, lists of filters can be given that will be applied to the code list from
+the given repository.
+
+The filtering can be done by any attribute:
+
+.. code:: yaml
+
+  repositories:
+    common-definitions:
+      url: https://github.com/IAMconsortium/common-definitions.git/
+  definitions:
+    variable:
+      repository:
+        name: common-definitions
+        include:
+          - name: [Primary Energy*, Final Energy*]
+          - name: "Population*"
+            tier: 1
+        exclude:
+          - name: "Final Energy|Industry*"
+            depth: 2
+
+If a filter is being used for repositories, the *name* attribute **must be used**
+for the repository.
+
+In the example above we are including:
+1. All variables starting with *Primary Energy* or *Final Energy*
+2. All variables starting with *Population* **and** with the tier attribute equal to 1
+
+From this list we are then **excluding** all variables that match "Final
+Energy|Industry\*" and have a depth of 2 (meaning that they contain two pipe "|"
+characters).
 
 Adding countries to the region codelist
 ---------------------------------------
@@ -90,12 +126,12 @@ By setting *definitions.region.nuts* (optional) in the configuration file:
       nuts:
         nuts-1: [ AT, BE, CZ ]
         nuts-2: [ AT ]
-        nuts-3: [ AT, BE ]
+        nuts-3: true
 
 the nomenclature package will add the selected NUTS regions to the *region* codelist.
 
-In the example above, the package will add all NUTS (1, 2, and 3) for Austria,
-NUTS 1 and 3 for Belgium, and NUTS 1 for Czechia.
+In the example above, the package will add: NUTS 1 regions for Austria, Belgium
+and Czechia, NUTS 2 regions for Austria, NUTS 3 regions for all EU countries.
 
 More details on the list of NUTS regions can be found here: :ref:`nuts`.
 

nomenclature/cli.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-from typing import List, Optional
 import importlib.util
 import sys
 
@@ -55,10 +54,10 @@ def cli_valid_yaml(path: Path):
 def cli_valid_project(
     path: Path,
     definitions: str,
-    mappings: Optional[str],
-    required_data: Optional[str],
-    validate_data: Optional[str],
-    dimensions: Optional[List[str]],
+    mappings: str | None,
+    required_data: str | None,
+    validate_data: str | None,
+    dimensions: list[str] | None,
 ):
     """Assert that `path` is a valid project nomenclature
 
@@ -74,7 +73,7 @@ def cli_valid_project(
         Name of folder for 'required data' criteria, default to "required_data"
     validate_data: str, optional
         Name of folder for data validation criteria, default to "validate_data"
-    dimensions : List[str], optional
+    dimensions : list[str], optional
         Dimensions to be checked, defaults to all sub-folders of `definitions`
 
     Example
@@ -125,8 +124,8 @@ def check_region_aggregation(
     workflow_directory: Path,
     definitions: str,
     mappings: str,
-    processed_data: Optional[Path],
-    differences: Optional[Path],
+    processed_data: Path | None,
+    differences: Path | None,
 ):
     """Perform region processing and compare aggregated and original data
 
@@ -141,10 +140,10 @@ def check_region_aggregation(
         Definitions folder inside workflow_directory, by default "definitions"
     mappings : str
         Model mapping folder inside workflow_directory, by default "mappings"
-    processed_data : Optional[Path]
+    processed_data : Path, optional
         If given, exports the results from region processing to a file called
         `processed_data`, by default "results.xlsx"
-    differences : Optional[Path]
+    differences : Path, optional
         If given, exports the differences between aggregated and model native data to a
         file called `differences`, by default None
 
@@ -295,7 +294,7 @@ def cli_run_workflow(
     multiple=True,
     default=None,
 )
-def cli_validate_scenarios(input_file: Path, definitions: Path, dimensions: List[str]):
+def cli_validate_scenarios(input_file: Path, definitions: Path, dimensions: list[str]):
     """Validate a scenario file against the codelists of a project
 
     Example
@@ -312,7 +311,7 @@ def cli_validate_scenarios(input_file: Path, definitions: Path, dimensions: List
         Input data file, must be IAMC format, .xlsx or .csv
     definitions : Path
         Definitions folder with codelists, by default "definitions"
-    dimensions : List[str], optional
+    dimensions : list[str], optional
         Dimensions to be checked, defaults to all sub-folders of `definitions`
 
     Raises

nomenclature/code.py

@@ -2,7 +2,7 @@
 import re
 from keyword import iskeyword
 from pathlib import Path
-from typing import Any, Dict, List, Set, Union, Optional
+from typing import Any
 from pydantic import (
     field_validator,
     field_serializer,
@@ -24,8 +24,8 @@ class Code(BaseModel):
 
     name: str
     description: str | None = None
-    file: Union[str, Path] | None = None
-    extra_attributes: Dict[str, Any] = {}
+    file: str | Path | None = None
+    extra_attributes: dict[str, Any] = {}
     repository: str | None = None
 
     def __eq__(self, other) -> bool:
@@ -34,8 +34,8 @@ def __eq__(self, other) -> bool:
     @field_validator("extra_attributes")
     @classmethod
     def check_attribute_names(
-        cls, v: Dict[str, Any], info: ValidationInfo
-    ) -> Dict[str, Any]:
+        cls, v: dict[str, Any], info: ValidationInfo
+    ) -> dict[str, Any]:
         # Check that attributes only contains keys which are valid identifiers
         if illegal_keys := [
             key for key in v.keys() if not key.isidentifier() or iskeyword(key)
@@ -79,7 +79,7 @@ def from_dict(cls, mapping) -> "Code":
         )
 
     @classmethod
-    def named_attributes(cls) -> Set[str]:
+    def named_attributes(cls) -> set[str]:
         return {a for a in cls.model_fields if a != "extra_attributes"}
 
     @property
@@ -106,6 +106,10 @@ def flattened_dict_serialized(self):
             for key, value in self.flattened_dict.items()
         }
 
+    @property
+    def depth(self) -> int:
+        return self.name.count("|")
+
     def replace_tag(self, tag: str, target: "Code") -> "Code":
         """Return a new instance with tag applied
 
@@ -177,18 +181,18 @@ def __setattr__(self, name, value):
 
 
 class VariableCode(Code):
-    unit: Union[str, List[str]] = Field(...)
+    unit: str | list[str] = Field(...)
     tier: int | str | None = None
     weight: str | None = None
-    region_aggregation: List[Dict[str, Dict]] | None = Field(
+    region_aggregation: list[dict[str, dict]] | None = Field(
         default=None, alias="region-aggregation"
     )
     skip_region_aggregation: bool | None = Field(
         default=False, alias="skip-region-aggregation"
     )
     method: str | None = None
     check_aggregate: bool | None = Field(default=False, alias="check-aggregate")
-    components: Union[List[str], List[Dict[str, List[str]]]] | None = None
+    components: list[str] | dict[str, list[str]] | None = None
     drop_negative_weights: bool | None = None
     model_config = ConfigDict(populate_by_name=True)
 
@@ -204,22 +208,34 @@ def deserialize_json(cls, v):
     def convert_none_to_empty_string(cls, v):
         return v if v is not None else ""
 
+    @field_validator("components", mode="before")
+    def cast_variable_components_args(cls, v):
+        """Cast "components" list of dicts to a codelist"""
+
+        # translate a list of single-key dictionaries to a simple dictionary
+        if v is not None and isinstance(v, list) and isinstance(v[0], dict):
+            comp = {}
+            for val in v:
+                comp.update(val)
+            return comp
+        return v
+
     @field_serializer("unit")
     def convert_str_to_none_for_writing(self, v):
         return v if v != "" else None
 
     @property
-    def units(self) -> List[str]:
+    def units(self) -> list[str]:
         return self.unit if isinstance(self.unit, list) else [self.unit]
 
     @classmethod
-    def named_attributes(cls) -> Set[str]:
+    def named_attributes(cls) -> set[str]:
         return (
             super().named_attributes().union(f.alias for f in cls.model_fields.values())
         )
 
     @property
-    def pyam_agg_kwargs(self) -> Dict[str, Any]:
+    def pyam_agg_kwargs(self) -> dict[str, Any]:
         # return a dict of all not None pyam aggregation properties
         return {
             field: getattr(self, field)
@@ -233,7 +249,7 @@ def pyam_agg_kwargs(self) -> Dict[str, Any]:
         }
 
     @property
-    def agg_kwargs(self) -> Dict[str, Any]:
+    def agg_kwargs(self) -> dict[str, Any]:
         return (
             {**self.pyam_agg_kwargs, **{"region_aggregation": self.region_aggregation}}
             if self.region_aggregation is not None
@@ -258,11 +274,11 @@ class RegionCode(Code):
     """
 
     hierarchy: str = None
-    countries: Optional[List[str]] = None
-    iso3_codes: Optional[Union[List[str], str]] = None
+    countries: list[str] | None = None
+    iso3_codes: list[str] | str | None = None
 
     @field_validator("countries", mode="before")
-    def check_countries(cls, v: List[str], info: ValidationInfo) -> List[str]:
+    def check_countries(cls, v: list[str], info: ValidationInfo) -> list[str]:
         """Verifies that each country name is defined in `nomenclature.countries`."""
         v = to_list(v)
         if invalid_country_names := set(v) - set(countries.names):
@@ -275,7 +291,7 @@ def check_countries(cls, v: List[str], info: ValidationInfo) -> List[str]:
         return v
 
     @field_validator("iso3_codes")
-    def check_iso3_codes(cls, v: List[str], info: ValidationInfo) -> List[str]:
+    def check_iso3_codes(cls, v: list[str], info: ValidationInfo) -> list[str]:
         """Verifies that each ISO3 code is valid according to pycountry library."""
         errors = ErrorCollector()
         if invalid_iso3_codes := [
@@ -299,9 +315,9 @@ class MetaCode(Code):
 
     Attributes
     ----------
-    allowed_values : Optional(list[any])
+    allowed_values : list[Any], optional
         An optional list of allowed values
 
     """
 
-    allowed_values: List[Any] | None = None
+    allowed_values: list[Any] | None = None