Add uc_t topology check (#268)

Address #154 and convert comemi and comagg to fi_t for processing
etsap-TIMES · Jan 5, 2025 · 2aedff0 · 2aedff0
1 parent 5302aa5
commit 2aedff0
Show file tree

Hide file tree

Showing 6 changed files with 220 additions and 73 deletions.
diff --git a/xl2times/__main__.py b/xl2times/__main__.py
@@ -118,6 +118,7 @@ def convert_xl_to_times(
         transforms.capitalise_table_values,
         transforms.process_regions,
         transforms.process_commodities,
+        transforms.convert_com_tables,
         transforms.process_time_periods,
         transforms.remove_exreg_cols,
         transforms.generate_dummy_processes,
@@ -131,7 +132,6 @@ def convert_xl_to_times(
         transforms.process_topology,
         transforms.process_flexible_import_tables,  # slow
         transforms.process_user_constraint_tables,
-        transforms.process_commodity_emissions,
         transforms.fill_in_missing_values,
         transforms.generate_uc_properties,
         transforms.expand_rows_parallel,  # slow
@@ -148,11 +148,13 @@ def convert_xl_to_times(
         transforms.complete_commodity_groups,
         transforms.process_wildcards,
         transforms.convert_aliases,
+        transforms.fix_topology,
         transforms.apply_transform_tables,
+        transforms.generate_implied_topology,
+        transforms.verify_uc_topology,
         transforms.explode_process_commodity_cols,
         transforms.apply_final_fixup,
         transforms.assign_model_attributes,
-        transforms.fix_topology,
         transforms.resolve_remaining_cgs,
         transforms.complete_dictionary,
         transforms.convert_to_string,

diff --git a/xl2times/config/times-info.json b/xl2times/config/times-info.json
@@ -775,13 +775,13 @@
       "REG",
       "YEAR",
       "COM",
-      "COM"
+      "COM2"
     ],
     "mapping": [
       "region",
       "year",
       "commodity",
-      "commodity"
+      "other_indexes"
     ]
   },
   {
@@ -5104,11 +5104,10 @@
   {
     "name": "VDA_EMCB",
     "gams-cat": "parameter",
-    "type": "special",
     "indexes": [
       "REG",
       "YEAR",
-      "COM",
+      "COM2",
       "COM"
     ],
     "mapping": [

diff --git a/xl2times/config/times_mapping.txt b/xl2times/config/times_mapping.txt
@@ -39,4 +39,3 @@ UNITS_ACT[UNITS] = Units(Unit,Type:activity)
 UNITS_CAP[UNITS] = Units(Unit,Type:capacity)
 UNITS_COM[UNITS] = Units(Unit,Type:commodity)
 UNITS_MONY[CUR] = Units(Unit,Type:currency)
-VDA_EMCB[REG,DATAYEAR,Commodity2,COM,VALUE] = ~COMEMI(Region,Year,Other_Indexes,Commodity,EMCB)
diff --git a/xl2times/config/veda-tags.json b/xl2times/config/veda-tags.json
@@ -33,6 +33,30 @@
       }
     ]
   },
+  {
+    "tag_name": "comagg",
+    "tag_allowed_in": [
+      "BY",
+      "SubRES"
+    ],
+    "valid_fields": [
+      {
+        "name": "commname",
+        "aliases": [
+          "commodity"
+        ],
+        "use_name": "other_indexes",
+        "row_ignore_symbol": [
+          "\\I:",
+          "*"
+        ],
+        "query_field": false,
+        "inherit_above": false,
+        "remove_first_row_if_absent": false,
+        "remove_any_row_if_absent": true
+      }
+    ]
+  },
   {
     "tag_name": "comemi",
     "tag_allowed_in": [
@@ -53,7 +77,7 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": true
       }
     ]
   },
@@ -2160,7 +2184,12 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": false,
+        "valid_values": [
+          "A",
+          "I",
+          "O"
+        ]
       },
       {
         "name": "ts_filter",
@@ -3016,7 +3045,12 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": false,
+        "valid_values": [
+          "A",
+          "I",
+          "O"
+        ]
       },
       {
         "name": "uc_n",
@@ -4031,7 +4065,12 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": false,
+        "valid_values": [
+          "A",
+          "I",
+          "O"
+        ]
       },
       {
         "name": "uc_n",
@@ -4644,7 +4683,14 @@
         "query_field": false,
         "inherit_above": false,
         "remove_first_row_if_absent": false,
-        "remove_any_row_if_absent": false
+        "remove_any_row_if_absent": false,
+        "valid_values": [
+          "A",
+          "I",
+          "O",
+          "NO"
+        ],
+        "default_to": "A"
       },
       {
         "name": "uc_attr",

diff --git a/xl2times/datatypes.py b/xl2times/datatypes.py
@@ -228,6 +228,7 @@ class TimesModel:
     commodities: DataFrame = field(default_factory=DataFrame)
     commodity_groups: DataFrame = field(default_factory=DataFrame)
     topology: DataFrame = field(default_factory=DataFrame)
+    implied_topology: DataFrame = field(default_factory=DataFrame)
     trade: DataFrame = field(default_factory=DataFrame)
     attributes: DataFrame = field(default_factory=DataFrame)
     user_constraints: DataFrame = field(default_factory=DataFrame)
@@ -293,6 +294,8 @@ class Config:
     dd_table_order: Iterable[str]
     all_attributes: set[str]
     attr_aliases: set[str]
+    # Attribute by type
+    attr_by_type: dict[str, set[str]]
     # For each tag, this dictionary maps each column alias to the normalized name
     column_aliases: dict[Tag, dict[str, str]]
     # For each tag, this dictionary maps each column name to its default value
@@ -329,6 +332,7 @@ def __init__(
         (
             self.dd_table_order,
             self.all_attributes,
+            self.attr_by_type,
             param_mappings,
         ) = Config._process_times_info(times_info_file)
         self.times_sets = Config._read_times_sets(times_sets_file)
@@ -366,7 +370,7 @@ def _read_times_sets(
     @staticmethod
     def _process_times_info(
         times_info_file: str,
-    ) -> tuple[Iterable[str], set[str], list[TimesXlMap]]:
+    ) -> tuple[Iterable[str], set[str], dict[str, set[str]], list[TimesXlMap]]:
         # Read times_info_file and compute dd_table_order:
         # We output tables in order by categories: set, subset, subsubset, md-set, and parameter
         with resources.open_text("xl2times.config", times_info_file) as f:
@@ -389,6 +393,25 @@ def _process_times_info(
             if item["gams-cat"] == "parameter"
         }
 
+        # Determine the attributes by type
+        attr_by_type = dict()
+
+        attr_type_conditions = {
+            "commodity": {"commodity": True, "process": False},
+            "process": {"process": True, "commodity": False},
+            "flow": {"process": True, "commodity": True},
+        }
+
+        for attr_type, conditions in attr_type_conditions.items():
+            attr_by_type[attr_type] = {
+                attr["name"]
+                for attr in table_info
+                if all(
+                    (index in attr["mapping"]) is is_present
+                    for index, is_present in conditions.items()
+                )
+            }
+
         # Compute the mapping for attributes / parameters:
         def create_mapping(entity):
             assert entity["gams-cat"] == "parameter"
@@ -417,7 +440,7 @@ def create_mapping(entity):
             and "type" not in x  # TODO Generalise derived parameters?
         ]
 
-        return dd_table_order, attributes, param_mappings
+        return dd_table_order, attributes, attr_by_type, param_mappings
 
     @staticmethod
     def _read_mappings(filename: str) -> list[TimesXlMap]: