diff --git a/xl2times/__main__.py b/xl2times/__main__.py index 66d2c96..2bd8345 100644 --- a/xl2times/__main__.py +++ b/xl2times/__main__.py @@ -129,14 +129,14 @@ def convert_xl_to_times( transforms.process_transform_availability, transforms.process_flexible_import_tables, # slow transforms.process_user_constraint_tables, - transforms.process_tradelinks, + transforms.harmonise_tradelinks, transforms.include_tables_source, transforms.process_processes, transforms.create_model_topology, transforms.fill_in_column_defaults, transforms.generate_uc_properties, transforms.expand_rows_parallel, # slow - transforms.process_trade_links, + transforms.process_tradelinks, transforms.merge_tables, transforms.remove_invalid_values, transforms.include_cgs_in_topology, @@ -148,7 +148,7 @@ def convert_xl_to_times( transforms.enforce_availability, transforms.complete_model_trade, transforms.create_model_cgs, - transforms.apply_fixups, + transforms.prepare_for_querying, transforms.apply_transform_tables, transforms.generate_implied_topology, transforms.verify_uc_topology, diff --git a/xl2times/transforms.py b/xl2times/transforms.py index 10cc198..09aeb6a 100644 --- a/xl2times/transforms.py +++ b/xl2times/transforms.py @@ -1202,12 +1202,18 @@ def _populate_calculated_defaults(df: DataFrame, model: TimesModel): ) -def apply_fixups( +def prepare_for_querying( config: Config, tables: dict[str, DataFrame], model: TimesModel, ) -> dict[str, DataFrame]: - """Apply fixups to the tables.""" + """Prepare tables for querying by harmonising them and filling in missing values. + This includes + - Adding expected columns with NA if they are missing. + - Expanding year columns if they contain ranges. + - Applying attribute-specific default values (incl. if specified under a different column). + - Filling in missing values in year and currency columns with defaults. + """ exclude_tags = {Tag.tfm_mig, Tag.tfm_upd} attribute_tags = [Tag.fi_t, Tag.uc_t] currency = tables[Tag.currencies]["currency"][0] @@ -1443,12 +1449,12 @@ def create_model_cgs( return tables -def process_trade_links( +def process_tradelinks( config: Config, tables: list[EmbeddedXlTable], model: TimesModel, ) -> list[EmbeddedXlTable]: - """Process trade links and create model trade (between internal regions).""" + """Process tradelinks and create model trade (between internal regions).""" cols_list = ["origin", "in", "destination", "out", "process"] cols_list.extend(dm_cols) result = [] @@ -1811,7 +1817,7 @@ def generate_dummy_processes( return tables -def process_tradelinks( +def harmonise_tradelinks( config: Config, tables: list[EmbeddedXlTable], model: TimesModel, @@ -3183,7 +3189,7 @@ def apply_final_fixup( tables: dict[str, DataFrame], model: TimesModel, ) -> dict[str, DataFrame]: - + """Clean up remaining issues. Includes handling of aliases that could not be generalised.""" veda_ire_sets = model.custom_sets reg_com_flows = model.topology[["region", "process", "commodity"]].drop_duplicates( ignore_index=True