Various fixes

xl2times/config/times-info.json

@@ -1619,15 +1619,15 @@
       "YEAR",
       "PRC",
       "CG",
-      "CG",
+      "CG2",
       "TS"
     ],
     "mapping": [
       "region",
       "year",
       "process",
       "other_indexes",
-      "other_indexes",
+      "commodity",
       "timeslice"
     ]
   },
@@ -2017,14 +2017,14 @@
     "indexes": [
       "ALL_R",
       "COM",
-      "ALL_R",
-      "COM"
+      "REG2",
+      "COM2"
     ],
     "mapping": [
       "region",
       "commodity",
-      "region",
-      "commodity"
+      "region2",
+      "commodity2"
     ]
   },
   {
@@ -2035,18 +2035,18 @@
       "YEAR",
       "PRC",
       "COM",
-      "ALL_R",
-      "COM",
-      "TS"
+      "REG2",
+      "COM2",
+      "TS2"
     ],
     "mapping": [
       "region",
       "year",
       "process",
       "commodity",
-      "region",
-      "commodity",
-      "timeslice"
+      "region2",
+      "commodity2",
+      "timeslice2"
     ]
   },
   {
@@ -2059,7 +2059,7 @@
       "COM",
       "TS",
       "IE",
-      "COM",
+      "COM2",
       "IO"
     ],
     "mapping": [
@@ -2069,7 +2069,7 @@
       "commodity",
       "timeslice",
       "other_indexes",
-      "commodity",
+      "commodity2",
       "other_indexes"
     ]
   },
@@ -3295,11 +3295,11 @@
     "indexes": [
       "REG",
       "PRC",
-      "PRC"
+      "PRC2"
     ],
     "mapping": [
       "region",
-      "process",
+      "other_indexes",
       "process"
     ]
   },

xl2times/config/veda-attr-defaults.json

@@ -646,6 +646,12 @@
   },
   "FLO_FUNC": {
     "defaults": {
+      "commodity": [
+        "commodity-in",
+        "commodity-out",
+        "commodity-in-aux",
+        "commodity-out-aux"
+      ],
       "ts-level": "ANNUAL"
     }
   },
@@ -883,7 +889,16 @@
     },
     "times-attribute": "NCAP_PKCNT"
   },
+  "PEAK(CON)": {
+    "defaults": {
+      "ts-level": "ANNUAL"
+    },
+    "times-attribute": "NCAP_PKCNT"
+  },
   "PKCNT": {
+    "defaults": {
+      "ts-level": "ANNUAL"
+    },
     "times-attribute": "NCAP_PKCNT"
   },
   "PKCOI": {

xl2times/datatypes.py

@@ -318,6 +318,8 @@ class Config:
     times_sets: dict[str, list[str]]
     # Switch to prevent overwriting of I/E settings in BASE and SubRES
     ie_override_in_syssettings: bool = False
+    # Switch to include dummy imports in the model
+    include_dummy_imports: bool = True
 
     def __init__(
         self,

xl2times/transforms.py

@@ -885,6 +885,7 @@ def fill_in_missing_values(
     def fill_in_missing_values_table(table):
         df = table.dataframe.copy()
         default_values = config.column_default_value.get(table.tag, {})
+        mapping_to_defaults = {"limtype": "limtype", "timeslice": "tslvl"}
 
         for colname in df.columns:
             # TODO make this more declarative
@@ -900,30 +901,18 @@ def fill_in_missing_values_table(table):
                 ismat = df["csets"] == "MAT"
                 df.loc[isna & ismat, colname] = "FX"
                 df.loc[isna & ~ismat, colname] = "LO"
-            elif (
-                colname == "limtype"
-                and (table.tag == Tag.fi_t or table.tag.startswith("~TFM"))
-                and len(df) > 0
-            ):
-                isna = df[colname].isna()
-                for lim in config.veda_attr_defaults["limtype"].keys():
-                    df.loc[
-                        isna
-                        & df["attribute"]
-                        .str.upper()
-                        .isin(config.veda_attr_defaults["limtype"][lim]),
-                        colname,
-                    ] = lim
-            elif colname == "timeslice" and len(df) > 0 and "attribute" in df.columns:
+            elif colname in {"limtype", "timeslice"} and "attribute" in df.columns:
                 isna = df[colname].isna()
-                for timeslice in config.veda_attr_defaults["tslvl"].keys():
-                    df.loc[
-                        isna
-                        & df["attribute"]
-                        .str.upper()
-                        .isin(config.veda_attr_defaults["tslvl"][timeslice]),
-                        colname,
-                    ] = timeslice
+                if any(isna):
+                    key = mapping_to_defaults[colname]
+                    for value in config.veda_attr_defaults[key].keys():
+                        df.loc[
+                            isna
+                            & df["attribute"].isin(
+                                config.veda_attr_defaults[key][value]
+                            ),
+                            colname,
+                        ] = value
             elif (
                 colname == "tslvl" and table.tag == Tag.fi_process
             ):  # or colname == "CTSLvl" or colname == "PeakTS":
@@ -1829,7 +1818,6 @@ def generate_dummy_processes(
     config: Config,
     tables: list[EmbeddedXlTable],
     model: TimesModel,
-    include_dummy_processes=True,
 ) -> list[EmbeddedXlTable]:
     """Define dummy processes and specify default cost data for them to ensure that a
     TIMES model can always be solved.
@@ -1838,7 +1826,7 @@ def generate_dummy_processes(
     Significant cost is usually associated with the activity of these processes to
     ensure that they are used as a last resort
     """
-    if include_dummy_processes:
+    if config.include_dummy_imports:
         # TODO: Activity units below are arbitrary. Suggest Veda devs not to have any.
         dummy_processes = [
             ["IMP", "IMPNRGZ", "Dummy Import of NRG", "PJ", "", "NRG"],
@@ -1863,9 +1851,8 @@ def generate_dummy_processes(
         )
 
         process_data_specs = process_declarations[["process", "description"]].copy()
-        # Use this as default activity cost for dummy processes
-        # TODO: Should this be included in settings instead?
-        process_data_specs["ACTCOST"] = 1111
+        # Provide an empty value in case an upd table is used to provide data
+        process_data_specs["ACTCOST"] = ""
 
         tables.append(
             EmbeddedXlTable(
@@ -2030,6 +2017,8 @@ def is_year(col_name):
                 value_name="value",
                 ignore_index=False,
             )
+            # Convert the attribute column to uppercase
+            df["attribute"] = df["attribute"].str.upper()
             result.append(
                 replace(table, dataframe=df, tag=Tag(tag.value.split("-")[0]))
             )
@@ -2165,7 +2154,7 @@ def process_transform_availability(
     return result
 
 
-def filter_by_pattern(df: pd.DataFrame, pattern: str) -> pd.DataFrame:
+def filter_by_pattern(df: pd.Series, pattern: str) -> pd.Series:
     """Filter dataframe index by a regex pattern."""
     # Duplicates can be created when a process has multiple commodities that match the pattern
     df = df.filter(regex=utils.create_regexp(pattern), axis="index").drop_duplicates()
@@ -2174,18 +2163,18 @@ def filter_by_pattern(df: pd.DataFrame, pattern: str) -> pd.DataFrame:
     return df.drop(exclude)
 
 
-def intersect(acc, df):
+def intersect(acc: pd.Series | None, df: pd.Series) -> pd.Series | None:
     if acc is None:
         return df
     return acc.merge(df)
 
 
 def get_matching_processes(
-    row: pd.Series, topology: dict[str, DataFrame]
+    row: pd.Series, topology: dict[str, pd.Series]
 ) -> pd.Series | None:
     matching_processes = None
     for col, key in process_map.items():
-        if col in row.index and row[col] not in {None, ""}:
+        if col in row.index and pd.notna(row[col]):
             proc_set = topology[key]
             pattern = row[col].upper()
             filtered = filter_by_pattern(proc_set, pattern)
@@ -2197,10 +2186,12 @@ def get_matching_processes(
     return matching_processes
 
 
-def get_matching_commodities(row: pd.Series, topology: dict[str, DataFrame]):
+def get_matching_commodities(
+    row: pd.Series, topology: dict[str, pd.Series]
+) -> pd.Series | None:
     matching_commodities = None
     for col, key in commodity_map.items():
-        if col in row.index and row[col] not in {None, ""}:
+        if col in row.index and pd.notna(row[col]):
             matching_commodities = intersect(
                 matching_commodities,
                 filter_by_pattern(topology[key], row[col].upper()),
@@ -2221,7 +2212,7 @@ def df_indexed_by_col(df, col):
 
 def generate_topology_dictionary(
     tables: dict[str, DataFrame], model: TimesModel
-) -> dict[str, DataFrame]:
+) -> dict[str, pd.Series]:
     # We need to be able to fetch processes based on any combination of name, description, set, comm-in, or comm-out
     # So we construct tables whose indices are names, etc. and use pd.filter
 
@@ -2326,7 +2317,7 @@ def process_wildcards(
 def _match_wildcards(
     df: pd.DataFrame,
     col_map: dict[str, str],
-    dictionary: dict[str, pd.DataFrame],
+    dictionary: dict[str, pd.Series],
     matcher: Callable,
     result_col: str,
     explode: bool = False,
@@ -2362,12 +2353,7 @@ def _match_wildcards(
     # match all the wildcards columns against the dictionary names
     matches = unique_filters.apply(lambda row: matcher(row, dictionary), axis=1)
 
-    # we occasionally get a Dataframe back from  the matchers.  convert these to Series.
-    matches = (
-        matches.iloc[:, 0].to_list()
-        if isinstance(matches, pd.DataFrame)
-        else matches.to_list()
-    )
+    matches = matches.to_list()
     matches = [
         df.iloc[:, 0].to_list() if df is not None and len(df) != 0 else None
         for df in matches
@@ -2438,11 +2424,11 @@ def query(
     def is_missing(field):
         return pd.isna(field) if not isinstance(field, list) else pd.isna(field).all()
 
-    qs = []
-
-    for k, v in query_fields.items():
-        if not is_missing(v):
-            qs.append(f"{k} in {v if isinstance(v, list) else [v]}")
+    qs = [
+        f"{k} in {v if isinstance(v, list) else [v]}"
+        for k, v in query_fields.items()
+        if not is_missing(v)
+    ]
 
     query_str = " and ".join(qs)
     row_idx = table.query(query_str).index
@@ -2454,6 +2440,11 @@ def eval_and_update(table: DataFrame, rows_to_update: pd.Index, new_value: str)
     which can be a update formula like `*2.3`.
     """
     if isinstance(new_value, str) and new_value[0] in {"*", "+", "-", "/"}:
+        # Do not perform arithmetic operations on rows with i/e options
+        if "year" in table.columns:
+            rows_to_update = rows_to_update.intersection(
+                table.index[table["year"] != 0]
+            )
         old_values = table.loc[rows_to_update, "value"]
         updated = old_values.astype(float).map(lambda x: eval("x" + new_value))
         table.loc[rows_to_update, "value"] = updated
@@ -2694,8 +2685,8 @@ def apply_transform_tables(
                 new_rows = table.loc[rows_to_update].copy()
                 # Modify values in all '*2' columns
                 for c, v in row.items():
-                    if c.endswith("2") and v is not None:
-                        new_rows.loc[:, c[:-1]] = v
+                    if str(c).endswith("2") and v is not None:
+                        new_rows.loc[:, str(c)[:-1]] = v
                 # Evaluate 'value' column based on existing values
                 eval_and_update(new_rows, rows_to_update, row["value"])
                 # In case more than one data module is present in the table, select the one with the highest index
@@ -2794,7 +2785,7 @@ def timeslices_table(
 
         # Ensure that all timeslice levels are uppercase
         timeslices = {
-            col.upper(): list(values.unique())
+            str(col).upper(): list(values.unique())
             for col, values in table.dataframe.items()
         }
 

xl2times/utils.py

@@ -261,8 +261,10 @@ def create_regexp(pattern: str, combined: bool = True) -> str:
     pattern = pattern.replace(",", r"$|^")
     if len(pattern) == 0:
         return r".*"  # matches everything
-    # Handle substite VEDA wildcards with regex patterns
-    for substition in (("*", ".*"), ("?", ".")):
+    # Substite VEDA wildcards with regex patterns; escape metacharacters.
+    # ("_", ".") and ("[.]", "_") are meant to apply one after another to handle
+    # the usage of "_" equivalent to "?" and "[_]" as literal "_".
+    for substition in ((".", "\\."), ("_", "."), ("[.]", "_"), ("*", ".*"), ("?", ".")):
         old, new = substition
         pattern = pattern.replace(old, new)
     # Do not match substrings