From 0b49681f0b833381c7c3d5715e80e166e0e77361 Mon Sep 17 00:00:00 2001 From: Eleftherios Zisis Date: Mon, 11 Mar 2024 15:54:55 +0100 Subject: [PATCH] Feat: Migrate to simplified emodel recipe (#44) --- .../model/nexus_converter.py | 165 +++++++----------- emodel_generalisation/utils.py | 10 ++ tests/data/nexus_recipe.json | 20 ++- .../{config.json => emodelconfiguration.json} | 0 tests/test_access_point.py | 27 ++- 5 files changed, 106 insertions(+), 116 deletions(-) rename tests/data/nexus_recipes/{config.json => emodelconfiguration.json} (100%) diff --git a/emodel_generalisation/model/nexus_converter.py b/emodel_generalisation/model/nexus_converter.py index 66c6221..e257ec2 100644 --- a/emodel_generalisation/model/nexus_converter.py +++ b/emodel_generalisation/model/nexus_converter.py @@ -1,6 +1,5 @@ """Convert nexus generate models to local AccessPoint config folder.""" import filecmp -import json import logging import shutil from copy import copy @@ -8,6 +7,9 @@ import pandas as pd +from emodel_generalisation.utils import load_json +from emodel_generalisation.utils import write_json + L = logging.getLogger(__name__) @@ -26,43 +28,36 @@ def _get_emodel_name(region, mtype, etype, i=None): return base_name -def _make_recipe_entry(config, emodel_name): - with open(config["EModelPipelineSettings"]) as emodelsettings_file: - emodelsettings = json.load(emodelsettings_file) - - with open(config["EModelConfiguration"]) as emodelconfig_file: - emodelconfig = json.load(emodelconfig_file) +def _make_recipe_entry(config): + morph_file = Path(config["morphology"]) - path = Path(emodelconfig["morphology"].get("path")) - return { - "morph_path": str(path.parent), - "morphology": str(path.name), - "params": f"parameters/{emodel_name}.json", - "features": f"features/{emodel_name}.json", + recipe = { + "morph_path": str(morph_file.parent), + "morphology": morph_file.name, + "params": config["params"]["bounds"], + "features": config.get("features", None), "morph_modifiers": [], # to update - "pipeline_settings": { + } + + if config["pipeline_settings"] is not None: + emodelsettings = load_json(config["pipeline_settings"]) + recipe["pipeline_settings"] = { "efel_settings": emodelsettings["efel_settings"], "name_rmp_protocol": emodelsettings["name_rmp_protocol"], "name_Rin_protocol": emodelsettings["name_Rin_protocol"], - }, - } - + } + else: + recipe["pipeline_settings"] = None -def _prepare(out_folder): - """Prepare folder structure.""" - out_folder.mkdir(exist_ok=True) - (out_folder / "parameters").mkdir(exist_ok=True) - (out_folder / "features").mkdir(exist_ok=True) + return recipe -def _make_mechanism(config, mech_path="mechanisms", base_path="."): +def _make_mechanisms(mechanisms: list, mech_path="mechanisms", base_path="."): """Copy mechanisms locally in a mechanisms folder.""" mech_path = Path(mech_path) mech_path.mkdir(exist_ok=True, parents=True) - with open(config["EModelConfiguration"]) as emodelconfig_file: - emodelconfig = json.load(emodelconfig_file) - for mech in emodelconfig["mechanisms"]: + for mech in mechanisms: if mech["path"] is not None: local_mech_path = mech_path / Path(mech["path"]).name @@ -82,107 +77,59 @@ def _make_mechanism(config, mech_path="mechanisms", base_path="."): shutil.copy(path, mech_path / Path(mech["path"]).name) -def _make_parameters(config): - """Convert parameter entry.""" - entry = {"mechanisms": {}, "distributions": {}, "parameters": {}} - with open(config["EModelConfiguration"]) as emodelconfig_file: - emodelconfig = json.load(emodelconfig_file) - - for mech in emodelconfig["mechanisms"]: - if mech["location"] not in entry["mechanisms"]: - entry["mechanisms"][mech["location"]] = {"mech": []} - entry["mechanisms"][mech["location"]]["mech"].append(mech["name"]) - - for distr in emodelconfig["distributions"]: - entry["distributions"][distr["name"]] = {"fun": distr["function"]} - if "parameters" in distr: - entry["distributions"][distr["name"]]["parameters"] = distr["parameters"] - - for param in emodelconfig["parameters"]: - if param["location"] not in entry["parameters"]: - entry["parameters"][param["location"]] = [] - entry["parameters"][param["location"]].append( - {"name": param["name"], "val": param["value"]} - ) - return entry - - -def _make_features(config): - """Convert features entry.""" - with open(config["FitnessCalculatorConfiguration"]) as fitnessconf_file: - fitnessconf = json.load(fitnessconf_file) - return fitnessconf - - -def _make_parameter_entry(config): +def _make_parameter_entry(emodel_values): """Convert model parameters data. Only the parameters are considered, the others are not needed. """ - with open(config["EModel"]) as emodel_file: - emodel = json.load(emodel_file) - - entry = {"params": {}} - for param in emodel["parameter"]: - entry["params"][param["name"]] = param["value"] - return entry + return {"params": {param["name"]: param["value"] for param in emodel_values["parameter"]}} def _add_emodel( - recipes, - final, - emodel_name, config, - out_config_folder, mech_path="mechanisms", base_path=".", ): - """Add a single emodel.""" - recipes[emodel_name] = _make_recipe_entry(config, emodel_name) - final[emodel_name] = _make_parameter_entry(config) + param_values_path = config["params"]["values"] + param_bounds_path = config["params"]["bounds"] - params = _make_parameters(config) - _make_mechanism(config, mech_path, base_path) + final = _make_parameter_entry(load_json(param_values_path)) + recipe = _make_recipe_entry(config) - with open(out_config_folder / recipes[emodel_name]["params"], "w") as param_file: - json.dump(params, param_file, indent=4) - - features = _make_features(config) - with open(out_config_folder / recipes[emodel_name]["features"], "w") as feat_file: - json.dump(features, feat_file, indent=4) + emodel_configuration = load_json(param_bounds_path) + _make_mechanisms( + mechanisms=emodel_configuration["mechanisms"], + mech_path=mech_path, + base_path=base_path, + ) + return final, recipe def convert_all_config(config_path, out_config_folder="config", mech_path="mechanisms"): """Convert a nexus config_json file into a local config folder loadable via AccessPoint.""" - with open(config_path) as config_file: - config = json.load(config_file) + configuration = load_json(config_path) out_config_folder = Path(out_config_folder) - _prepare(out_config_folder) - recipes = {} + out_config_folder.mkdir(exist_ok=True) + + base_config_dir = Path(config_path).parent + final = {} - for emodel_name, _config in config["library"]["eModel"].items(): - for entry, path in _config.items(): - if not Path(path).is_absolute(): - _config[entry] = Path(config_path).parent / path - _add_emodel( - recipes, - final, - emodel_name, - _config, - out_config_folder, - mech_path, - Path(config_path).parent, - ) + recipes = {} + for name, emodel_config in configuration["library"]["eModel"].items(): + emodel_config = _resolve_paths(emodel_config, base_config_dir) - with open(out_config_folder / "recipes.json", "w") as recipes_file: - json.dump(recipes, recipes_file, indent=4) + final[name], recipes[name] = _add_emodel( + emodel_config, + mech_path=mech_path, + base_path=base_config_dir, + ) - with open(out_config_folder / "final.json", "w") as final_file: - json.dump(final, final_file, indent=4) + write_json(filepath=out_config_folder / "recipes.json", data=recipes, indent=4) + write_json(filepath=out_config_folder / "final.json", data=final, indent=4) data = {"region": [], "etype": [], "mtype": [], "emodel": []} - for region, data_region in config["configuration"].items(): + for region, data_region in configuration["configuration"].items(): for mtype, data_mtype in data_region.items(): for etype, data_etype in data_mtype.items(): data["region"].append(region) @@ -192,3 +139,15 @@ def convert_all_config(config_path, out_config_folder="config", mech_path="mecha etype_emodel_map_df = pd.DataFrame.from_dict(data) etype_emodel_map_df.to_csv(out_config_folder / "etype_emodel_map.csv", index=False) + + +def _resolve_paths(emodel_config: dict, base_dir: Path) -> dict: + """Resolve relative paths with respect to the base dir.""" + new_config = {} + for k, v in emodel_config.items(): + if isinstance(v, dict): + new_config[k] = _resolve_paths(v, base_dir) + else: + # if v is absolute, concatenation drops the prefix + new_config[k] = str(base_dir / v) + return new_config diff --git a/emodel_generalisation/utils.py b/emodel_generalisation/utils.py index b01d9d4..4233013 100644 --- a/emodel_generalisation/utils.py +++ b/emodel_generalisation/utils.py @@ -204,3 +204,13 @@ def plot_traces(trace_df, trace_path="traces", pdf_filename="traces.pdf"): plt.suptitle(fig.get_label()) pdf.savefig() plt.close() + + +def load_json(filepath, **kwargs): + """Load from JSON file.""" + return json.loads(Path(filepath).read_bytes(), **kwargs) + + +def write_json(filepath, data, **kwargs): + """Write json file.""" + Path(filepath).write_text(json.dumps(data, **kwargs), encoding="utf-8") diff --git a/tests/data/nexus_recipe.json b/tests/data/nexus_recipe.json index 1255bdc..888cc9b 100644 --- a/tests/data/nexus_recipe.json +++ b/tests/data/nexus_recipe.json @@ -1,14 +1,16 @@ { "library": { - "eModel": { - "c4de21": { - "EModel": "nexus_recipes/emodel.json", - "ExtractionTargetsConfiguration": "nexus_recipes/extraction.json", - "EModelPipelineSettings": "nexus_recipes/pipeline.json", - "FitnessCalculatorConfiguration": "nexus_recipes/fitness.json", - "EModelConfiguration": "nexus_recipes/config.json" - } - } + "eModel": { + "c4de21": { + "morphology": "nexus_recipes/foo.swc", + "params": { + "values": "nexus_recipes/emodel.json", + "bounds": "nexus_recipes/emodelconfiguration.json" + }, + "features": "nexus_recipes/fitness.json", + "pipeline_settings": "nexus_recipes/pipeline.json" + } + } }, "configuration": { "AAA": { diff --git a/tests/data/nexus_recipes/config.json b/tests/data/nexus_recipes/emodelconfiguration.json similarity index 100% rename from tests/data/nexus_recipes/config.json rename to tests/data/nexus_recipes/emodelconfiguration.json diff --git a/tests/test_access_point.py b/tests/test_access_point.py index 1065433..b98da99 100644 --- a/tests/test_access_point.py +++ b/tests/test_access_point.py @@ -522,6 +522,7 @@ def test_load_nexus_recipe(tmpdir): access_point = AccessPoint( nexus_config=DATA / "nexus_recipe.json", emodel_dir=tmpdir / "config", + mech_path=tmpdir / "mechanisms", ) assert access_point.emodels == ["c4de21"] @@ -531,7 +532,25 @@ def test_load_nexus_recipe(tmpdir): ) assert access_point.emodels == ["c4de21"] - assert (tmpdir / "config" / "recipes.json").exists() - assert (tmpdir / "config" / "final.json").exists() - assert (tmpdir / "config" / "parameters" / "c4de21.json").exists() - assert (tmpdir / "config" / "features" / "c4de21.json").exists() + assert access_point.recipes == { + "c4de21": { + "morph_path": str(DATA / "nexus_recipes"), + "morphology": "foo.swc", + "params": str(DATA / "nexus_recipes" / "emodelconfiguration.json"), + "features": str(DATA / "nexus_recipes" / "fitness.json"), + "morph_modifiers": [], + "pipeline_settings": { + "efel_settings": { + "strict_stiminterval": True, + "Threshold": -20.0, + "interp_step": 0.025, + }, + "name_rmp_protocol": "IV_0", + "name_Rin_protocol": "IV_-40", + }, + } + } + assert access_point.get_morphologies("c4de21") == { + "name": "foo", + "path": str(DATA / "nexus_recipes" / "foo.swc"), + }