From 745353a49d78b283c99f19c0312598feb93a0eed Mon Sep 17 00:00:00 2001 From: matsvanes Date: Fri, 19 Jan 2024 14:33:36 +0000 Subject: [PATCH 01/11] save extra_funcs from preproc chain to raw.info['description'] and display in summary report --- osl/preprocessing/batch.py | 11 +++++-- osl/report/preproc_report.py | 36 +++++++++++++++++++++ osl/report/templates/raw_summary_panel.html | 1 + 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/osl/preprocessing/batch.py b/osl/preprocessing/batch.py index e51d1498..7ff94075 100644 --- a/osl/preprocessing/batch.py +++ b/osl/preprocessing/batch.py @@ -22,6 +22,7 @@ from functools import partial, wraps from time import localtime, strftime from datetime import datetime +import inspect import mne import numpy as np @@ -299,7 +300,7 @@ def get_config_from_fif(inst): return config -def append_preproc_info(dataset, config): +def append_preproc_info(dataset, config, extra_funcs=None): """Add to the config of already preprocessed data to ``inst.info['description']``. Parameters @@ -325,6 +326,12 @@ def append_preproc_info(dataset, config): + f"VERSION: {__version__}\n" + f"%% config start %% \n{config} \n%% config end %%" ) + + if extra_funcs is not None: + preproc_info += "\n\nCUSTOM FUNCTIONS USED:\n" + for func in extra_funcs: + preproc_info += f"%% extra_funcs start %% \n{inspect.getsource(func)}\n%% extra_funcs end %%" + dataset["raw"].info["description"] = ( dataset["raw"].info["description"] + preproc_info ) @@ -745,7 +752,7 @@ def run_proc_chain( dataset = func(dataset, userargs) # Add preprocessing info to dataset dict - dataset = append_preproc_info(dataset, config) + dataset = append_preproc_info(dataset, config, extra_funcs) fif_outname = None if outdir is not None: diff --git a/osl/report/preproc_report.py b/osl/report/preproc_report.py index 6926017e..0a44d9de 100644 --- a/osl/report/preproc_report.py +++ b/osl/report/preproc_report.py @@ -15,6 +15,7 @@ import tempfile import pickle import pathlib +import re import numpy as np import matplotlib.pyplot as plt @@ -192,6 +193,7 @@ def gen_html_data(raw, outdir, ica=None, preproc_fif_filename=None): # Generate plots for the report data["plt_config"] = plot_flowchart(raw, savebase) + data["txt_extra_funcs"] = save_extra_funcs(raw, savebase.replace('.png', '.txt')) data['plt_temporalsumsq'] = plot_channel_time_series(raw, savebase, exclude_bads=False) data['plt_temporalsumsq_exclude_bads'] = plot_channel_time_series(raw, savebase, exclude_bads=True) data['plt_badchans'] = plot_sensors(raw, savebase) @@ -317,6 +319,7 @@ def gen_html_summary(reportdir): os.makedirs(f"{reportdir}/summary", exist_ok=True) data["plt_config"] = subject_data[0]["plt_config"] + data["txt_extra_funcs"] = subject_data[0]["txt_extra_funcs"] data["plt_summary_bad_segs"] = plot_summary_bad_segs(subject_data, reportdir) data["plt_summary_bad_chans"] = plot_summary_bad_chans(subject_data, reportdir) @@ -419,6 +422,39 @@ def plot_flowchart(raw, savebase=None): fpath = None return fpath + +def save_extra_funcs(raw, savebase=None): + """ Saves extra functions from the raw.info['description'] to a text file. + + Parameters + ---------- + raw : :py:class:`mne.io.Raw ` + MNE Raw object. + savebase : str + Base string for saving figures. + + Returns + ------- + fpath : str + Path to saved text file. + + """ + extra_funcs = re.findall( + "%% extra_funcs start %%(.*?)%% extra_funcs end %%", + raw.info["description"], + flags=re.DOTALL, + ) + + if savebase is not None: + fpath = savebase.format(f"extra_funcs") + with(open(fpath, 'w')) as file: + [print(func, file=file) for func in extra_funcs] + return fpath + else: + return None + + + def plot_channel_time_series(raw, savebase=None, exclude_bads=False): """Plots sum-square time courses. diff --git a/osl/report/templates/raw_summary_panel.html b/osl/report/templates/raw_summary_panel.html index 67f33dd2..8b651edb 100644 --- a/osl/report/templates/raw_summary_panel.html +++ b/osl/report/templates/raw_summary_panel.html @@ -14,6 +14,7 @@

Config

+
From 682e44c940dd7321b588b888d046ecf39ea522c1 Mon Sep 17 00:00:00 2001 From: matsvanes Date: Fri, 19 Jan 2024 19:24:24 +0000 Subject: [PATCH 02/11] add title --- osl/report/templates/raw_summary_panel.html | 1 + 1 file changed, 1 insertion(+) diff --git a/osl/report/templates/raw_summary_panel.html b/osl/report/templates/raw_summary_panel.html index 8b651edb..15096372 100644 --- a/osl/report/templates/raw_summary_panel.html +++ b/osl/report/templates/raw_summary_panel.html @@ -14,6 +14,7 @@

Config

+

Extra functions

From 483810a75df53247daf9e6b88d955bb3807e0ac6 Mon Sep 17 00:00:00 2001 From: matsvanes Date: Sat, 20 Jan 2024 11:07:52 +0000 Subject: [PATCH 03/11] add extra_funcs to source report --- osl/report/src_report.py | 34 ++++++++++++++++++++- osl/report/templates/src_summary_panel.html | 2 ++ osl/source_recon/batch.py | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/osl/report/src_report.py b/osl/report/src_report.py index 03e5ca68..cf43a1c4 100644 --- a/osl/report/src_report.py +++ b/osl/report/src_report.py @@ -3,6 +3,7 @@ """ # Authors: Chetan Gohil +# Mats van Es import os import os.path as op @@ -13,12 +14,13 @@ import numpy as np import matplotlib.pyplot as plt from tabulate import tabulate +import inspect from . import preproc_report from ..source_recon import parcellation -def gen_html_data(config, src_dir, subject, reportdir, logger=None): +def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs=None): """Generate data for HTML report. Parameters @@ -33,6 +35,8 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None): Report directory. logger : logging.getLogger Logger. + extra_funcs : list + List of extra functions to run """ src_dir = Path(src_dir) reportdir = Path(reportdir) @@ -63,6 +67,8 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None): data = {} data["config"] = config + data['extra_funcs'] = save_extra_funcs(extra_funcs, reportdir / subject) + data["fif_id"] = subject data["filename"] = subject @@ -116,6 +122,31 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None): pickle.dump(data, open(f"{reportdir}/{subject}/data.pkl", "wb")) +def save_extra_funcs(extra_funcs, reportdir): + """ Saves extra functions to a text file. + + Parameters + ---------- + extra_funcs : list + List of extra functions to save. + reportdir : str + Subject report directory. + + Returns + ------- + fpath : str + Path to saved text file. + """ + + if reportdir is not None: + fpath = reportdir / 'extra_funcs.txt' + with(open(fpath, 'w')) as file: + [print(f"{inspect.getsource(func)}\n\n", file=file) for func in extra_funcs] + return fpath + else: + return None + + def gen_html_page(reportdir): """Generate an HTML page from a report directory. @@ -215,6 +246,7 @@ def gen_html_summary(reportdir): data = {} data["total"] = total data["config"] = subject_data[0]["config"] + data["extra_funcs"] = subject_data[0]["extra_funcs"] data["coregister"] = subject_data[0]["coregister"] data["beamform"] = subject_data[0]["beamform"] data["beamform_and_parcellate"] = subject_data[0]["beamform_and_parcellate"] diff --git a/osl/report/templates/src_summary_panel.html b/osl/report/templates/src_summary_panel.html index f12cadc8..270b4575 100644 --- a/osl/report/templates/src_summary_panel.html +++ b/osl/report/templates/src_summary_panel.html @@ -21,6 +21,8 @@

Config

+

Extra functions

+
{% if data.coregister %} diff --git a/osl/source_recon/batch.py b/osl/source_recon/batch.py index 5b7d4668..ff36eee6 100644 --- a/osl/source_recon/batch.py +++ b/osl/source_recon/batch.py @@ -204,7 +204,7 @@ def run_src_chain( return False # Generate HTML data for the report - src_report.gen_html_data(config, src_dir, subject, reportdir) + src_report.gen_html_data(config, src_dir, subject, reportdir, extra_funcs=extra_funcs) return True From a61027bfaee5f6298397a814a9331fe6e7c03eff Mon Sep 17 00:00:00 2001 From: matsvanes Date: Sat, 20 Jan 2024 11:38:28 +0000 Subject: [PATCH 04/11] add raw data plot to time series panel --- osl/report/preproc_report.py | 34 +++++++++++++++++++++ osl/report/templates/raw_subject_panel.html | 4 +++ 2 files changed, 38 insertions(+) diff --git a/osl/report/preproc_report.py b/osl/report/preproc_report.py index 0a44d9de..29f64ae6 100644 --- a/osl/report/preproc_report.py +++ b/osl/report/preproc_report.py @@ -194,6 +194,7 @@ def gen_html_data(raw, outdir, ica=None, preproc_fif_filename=None): # Generate plots for the report data["plt_config"] = plot_flowchart(raw, savebase) data["txt_extra_funcs"] = save_extra_funcs(raw, savebase.replace('.png', '.txt')) + data["plt_rawdata"] = plot_rawdata(raw, savebase) data['plt_temporalsumsq'] = plot_channel_time_series(raw, savebase, exclude_bads=False) data['plt_temporalsumsq_exclude_bads'] = plot_channel_time_series(raw, savebase, exclude_bads=True) data['plt_badchans'] = plot_sensors(raw, savebase) @@ -455,6 +456,39 @@ def save_extra_funcs(raw, savebase=None): +def plot_rawdata(raw, savebase): + """Plots raw data. + + Parameters + ---------- + raw : :py:class:`mne.io.Raw ` + MNE Raw object. + savebase : str + Base string for saving figures. + + Returns + ------- + fpath : str + Path to saved figure. + + """ + + fig = raw.pick(['meg', 'eeg']).plot(n_channels=np.inf, duration=raw.times[-1]) + + if savebase is not None: + figname = savebase.format('rawdata') + fig.savefig(figname, dpi=150, transparent=True) + plt.close(fig) + + # Return the filename + savebase = pathlib.Path(savebase) + filebase = savebase.parent.name + "/" + savebase.name + fpath = filebase.format('rawdata') + else: + fpath = None + return fpath + + def plot_channel_time_series(raw, savebase=None, exclude_bads=False): """Plots sum-square time courses. diff --git a/osl/report/templates/raw_subject_panel.html b/osl/report/templates/raw_subject_panel.html index 901923e1..62e43380 100644 --- a/osl/report/templates/raw_subject_panel.html +++ b/osl/report/templates/raw_subject_panel.html @@ -49,6 +49,10 @@

{{ data.fif_id }}   ({{ data.num }} of {{ data.total }})