Skip to content

Commit

Permalink
Merge pull request #337 from OHBA-analysis/logs
Browse files Browse the repository at this point in the history 
preproc batch and ICA label improvements
  • Loading branch information
@matsvanes
matsvanes authored Sep 4, 2024
2 parents 1b325db + 1719f2a commit 0792b0f
Show file tree
Hide file tree
Showing 4 changed files with 190 additions and 52 deletions.
2 changes: 1 addition & 1 deletion osl/preprocessing/batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -860,7 +860,7 @@ def run_proc_batch(
config,
files,
subjects=None,
ftype=None,
ftype='preproc-raw',
outdir=None,
logsdir=None,
reportdir=None,
Expand Down
218 changes: 175 additions & 43 deletions osl/preprocessing/ica_label.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@
import mne
import numpy as np
import pickle
import logging
import traceback
from glob import glob
from copy import deepcopy
from time import localtime, strftime
from matplotlib import pyplot as plt
from osl.preprocessing.plot_ica import plot_ica
from osl.report import plot_bad_ica
from osl.report.preproc_report import gen_html_page, gen_html_summary
from ..utils import logger as osl_logger

logger = logging.getLogger(__name__)

def ica_label(data_dir, subject, reject=None):

def ica_label(data_dir, subject, reject=None, interactive=True):
"""Data bookkeeping and wrapping plot_ica.
Parameters
Expand All @@ -35,54 +41,74 @@ def ica_label(data_dir, subject, reject=None):
None (default), only save the ICA data; don't reject any
components from the M/EEG data.
"""
# global drive, savedir
if isinstance(subject, list):
for sub in subject:
ica_label(data_dir, sub, reject=reject, interactive=interactive)
return

plt.ion()

# define data paths based on OSL data structure
preproc_file = os.path.join(data_dir, subject, subject + '_preproc-raw.fif')
ica_file = os.path.join(data_dir, subject, subject + '_ica.fif')
report_dir_base = os.path.join(data_dir, 'preproc_report')
report_dir = os.path.join(report_dir_base, subject + '_preproc-raw')
logs_dir = os.path.join(data_dir, 'logs')
logfile = os.path.join(logs_dir, subject + '_preproc-raw.log')

print('LOADING DATA')
raw = mne.io.read_raw(preproc_file, preload=True)
ica = mne.preprocessing.read_ica(ica_file)

# keep these for later
if reject=='manual':
exclude_old = deepcopy(ica.exclude)

# interactive components plot
print('INTERACTIVE ICA LABELING')
plot_ica(ica, raw, block=True, stop=30)
plt.pause(0.1)

if reject == 'all' or reject == 'manual':
print("REMOVING COMPONENTS FROM THE DATA")
if reject == 'all':
ica.apply(raw)
elif reject == 'manual':
# we need to make sure we don't remove components that
# were already removed before
new_ica = deepcopy(ica)
new_ica.exclude = np.setdiff1d(ica.exclude, exclude_old)
new_ica.apply(raw)
# setup loggers
mne.utils._logging.set_log_file(logfile, overwrite=False)
osl_logger.set_up(prefix=subject, log_file=logfile, level="INFO")
mne.set_log_level("INFO")
logger = logging.getLogger(__name__)
now = strftime("%Y-%m-%d %H:%M:%S", localtime())
logger.info("{0} : Starting OSL Processing".format(now))
try:
logger.info('Importing {0}'.format(preproc_file))
raw = mne.io.read_raw(preproc_file, preload=True)
logger.info('Importing {0}'.format(ica_file))
ica = mne.preprocessing.read_ica(ica_file)

# keep these for later
if reject=='manual':
exclude_old = deepcopy(ica.exclude)

# interactive components plot
if interactive:
logger.info('INTERACTIVE ICA LABELING')
plot_ica(ica, raw, block=True, stop=30)
plt.pause(0.1)

if reject == 'all' or reject == 'manual':
logger.info("Removing {0} labelled components from the data".format(reject))
if reject == 'all' or interactive is False:
ica.apply(raw)
elif reject == 'manual':
# we need to make sure we don't remove components that
# were already removed before
new_ica = deepcopy(ica)
new_ica.exclude = np.setdiff1d(ica.exclude, exclude_old)
new_ica.apply(raw)

logger.info("Saving preprocessed data")
raw.save(preproc_file, overwrite=True)
else:
logger.info("Not removing any components from the data")

logger.info("Saving ICA data")
ica.save(ica_file, overwrite=True)

if reject is not None:
logger.info("Attempting to update report")

print("SAVING PREPROCESSED DATA")
raw.save(preproc_file, overwrite=True)

print("SAVING ICA DATA")
ica.save(ica_file, overwrite=True)

if reject is not None:
print("ATTEMPTING TO UPDATE REPORT")
try:
savebase = os.path.join(report_dir, "{0}.png")
print(report_dir)
logger.info("Assuming report directory: {0}".format(report_dir))
if os.path.exists(os.path.join(report_dir, "ica.png")) or os.path.exists(os.path.join(report_dir, "data.pkl")):
logger.info("Generating ICA plot")
_ = plot_bad_ica(raw, ica, savebase)

# try updating the report data
logger.info("Updating data.pkl")
data = pickle.load(open(os.path.join(report_dir, "data.pkl"), 'rb'))
if 'plt_ica' not in data.keys():
data['plt_ica'] = os.path.join(report_dir.split("/")[-1], "ica.png")
Expand All @@ -96,13 +122,33 @@ def ica_label(data_dir, subject, reject=None):
pickle.dump(data, open(os.path.join(report_dir, "data.pkl"), 'wb'))

# gen html pages
logger.info("Generating subject_report.html")
gen_html_page(report_dir_base)
logger.info("Generating summary_report.html")
gen_html_summary(report_dir_base)
logger.info("Successfully updated report")

except Exception as e:
logger.critical("**********************")
logger.critical("* PROCESSING FAILED! *")
logger.critical("**********************")
ex_type, ex_value, ex_traceback = sys.exc_info()
logger.error("osl_ica_label")
logger.error(ex_type)
logger.error(ex_value)
logger.error(traceback.print_tb(ex_traceback))
with open(logfile.replace(".log", ".error.log"), "w") as f:
f.write("OSL PREPROCESSING CHAIN failed at: {0}".format(now))
f.write("\n")
f.write('Processing filed during stage : "{0}"'.format('osl_ica_label'))
f.write(str(ex_type))
f.write("\n")
f.write(str(ex_value))
f.write("\n")
traceback.print_tb(ex_traceback, file=f)

print("REPORT UPDATED")
except:
print("FAILED TO UPDATE REPORT")
print(f'LABELING DATASET {subject} COMPLETE')
now = strftime("%Y-%m-%d %H:%M:%S", localtime())
logger.info("{0} : Processing Complete".format(now))


def main(argv=None):
Expand All @@ -123,18 +169,24 @@ def main(argv=None):
The `reject_argument` specifies whether to reject 'all' selected components from the data, only
the 'manual' rejected, or None (and only save the ICA object, without rejecting components).
If the last two optional arguments are not specified, the function will assume their paths from
the usual OSL structure.
The `subject_name` should be the name of the subject directory in the processed data directory.
The /path/to/processed_data can be omitted when the command is run from the processed data directory.
If both the subject_name and directory are omitted, the script will attempt to process all subjects in the
processed data directory.
For example:
osl_ica_label manual /path/to/proc_dir sub-001_run01
or:
osl_ica_label all sub-001_run01
Then use the GUI to label components (click on the time course to mark, use
number keys to label marked components as specific artefacts, and use
the arrow keys to navigate. Close the plot.
all/manual/None components will be removed from the M/EEG data and saved. The
ICA data will be saved with the new labels. If the report directory is specified
or in the assumed OSL directory structure, the subject report is updated.
or in the assumed OSL directory structure, the subject report and log file is updated.
"""

Expand All @@ -144,8 +196,88 @@ def main(argv=None):
reject = argv[0]
if reject == 'None':
reject = None

if len(argv)<3:
data_dir = os.getcwd()
if len(argv)==2:
subject = argv[1]
else:
g = sorted(glob(os.path.join(f"{data_dir}", '*', '*_ica.fif')))
subject = [f.split('/')[-2] for f in g]
# batch log
logs_dir = os.path.join(data_dir, 'logs')
logfile = os.path.join(logs_dir, 'osl_batch.log')
osl_logger.set_up(log_file=logfile, level="INFO", startup=False)
logger.info('Starting OSL-ICA Batch Processing')
logger.info('Running osl_ica_label on {0} subjects with reject={1}'.format(len(subject), str(reject)))
else:
data_dir = argv[1]
subject = argv[2]

ica_label(data_dir=data_dir, subject=subject, reject=reject)


def apply(argv=None):
"""
Command-line function for removing all labeled components from the data.
Parameters
----------
argv : list
List of strings to be parsed as command-line arguments. If None,
sys.argv will be used.
ica_label(data_dir=argv[1], subject=argv[2], reject=argv[0])
Example
-------
From the command line (in the OSL environment), use as follows:
osl_ica_apply /path/to/processed_data subject_name
The `subject_name` should be the name of the subject directory in the processed data directory. If omitted,
the script will attempt to process all subjects in the processed data directory. The /path/to/processed_data
can also be omitted when the command is run from the processed data directory (only when processing all subjects).
For example:
osl_ica_apply /path/to/proc_dir sub-001_run01
or:
osl_ica_apply
Then use the GUI to label components (click on the time course to mark, use
number keys to label marked components as specific artefacts, and use
the arrow keys to navigate. Close the plot.
all/manual/None components will be removed from the M/EEG data and saved. The
ICA data will be saved with the new labels. If the report/logs directories are
in the assumed OSL directory structure, the subject report and log file are updated.
"""

if argv is None and len(sys.argv)>1:
argv = sys.argv[1:]

subject = None
if argv is None:
data_dir = os.getcwd()
else:
data_dir = argv[0]
if len(argv)==2:
subject = argv[1]

if subject is None:
g = sorted(glob(os.path.join(f"{data_dir}", '*', '*_ica.fif')))
subject = [f.split('/')[-2] for f in g]

# batch log
logs_dir = os.path.join(data_dir, 'logs')
logfile = os.path.join(logs_dir, 'osl_batch.log')
osl_logger.set_up(log_file=logfile, level="INFO", startup=False)
logger.info('Starting OSL-ICA Batch Processing')
logger.info('Running osl_ica_apply on {0} subjects'.format(len(subject)))

ica_label(data_dir=data_dir, subject=subject, reject='all', interactive=False)



if __name__ == '__main__':
Expand Down
21 changes: 13 additions & 8 deletions osl/preprocessing/plot_ica.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
"""

# Authors: Mats van Es <mats.vanes@psych.ox.ac.uk>
import logging
import matplotlib.pyplot as plt

# Configure logging
logger = logging.getLogger(__name__)

def plot_ica(
ica,
Expand Down Expand Up @@ -1105,9 +1108,6 @@ def _keypress(self, event):
self.mne.ica.labels_[tmp_label].append(last_bad_component)
else:
self.mne.ica.labels_[tmp_label] = [last_bad_component]
print(
f'Component {last_bad_component} labeled as "{tmp_label}"'
)
self._draw_traces() # This makes sure the traces are given the corresponding color right away
else: # check for close key / fullscreen toggle
super()._keypress(event)
Expand All @@ -1117,7 +1117,7 @@ def _close(self, event):
# OSL VERSION - SIMILAR TO OLD MNE VERSION TODO: Check if we need to adopt this
"""Handle close events (via keypress or window [x])."""
from matplotlib.pyplot import close
from mne.utils import logger, set_config
from mne.utils import set_config
import numpy as np

# write out bad epochs (after converting epoch numbers to indices)
Expand All @@ -1128,7 +1128,6 @@ def _close(self, event):
# proj checkboxes are for viz only and shouldn't modify the instance)
if self.mne.instance_type in ("raw", "epochs"):
self.mne.inst.info["bads"] = self.mne.info["bads"]
logger.info(f"Channels marked as bad: {self.mne.info['bads'] or 'none'}")

# OSL ADDITION
# ICA excludes
Expand Down Expand Up @@ -1156,7 +1155,7 @@ def _close(self, event):
for ix in allix:
self.mne.ica.labels_[list(self.mne.ica.labels_.keys())[ix]] = \
np.setdiff1d(self.mne.ica.labels_[list(self.mne.ica.labels_.keys())[ix]], ch)

# label bad components without a manual label as "unknown"
for ch in self.mne.ica.exclude:
tmp = list(self.mne.ica.labels_.values())
Expand All @@ -1170,7 +1169,7 @@ def _close(self, event):
self.mne.ica.labels_["unknown"].append(ch)
if type(self.mne.ica.labels_["unknown"]) is np.ndarray:
self.mne.ica.labels_["unknown"] = self.mne.ica.labels_["unknown"].tolist()

# Add to labels_ a generic eog/ecg field
if len(list(self.mne.ica.labels_.keys())) > 0:
if "ecg" not in self.mne.ica.labels_:
Expand All @@ -1192,7 +1191,13 @@ def _close(self, event):
self.mne.ica.labels_["eog"] = [v for v in self.mne.ica.labels_["eog"] if v!= []]
self.mne.ica.labels_["ecg"] = np.unique(self.mne.ica.labels_["ecg"]).tolist()
self.mne.ica.labels_["eog"] = np.unique(self.mne.ica.labels_["eog"]).tolist()


# write logs
logger.info(f"Components marked as bad: {sorted(self.mne.ica.exclude) or 'none'}")
for lb in self.mne.ica.labels_.keys():
if 'manual' in lb or lb=='unknown':
logger.info(f"Components manually labeled as '{lb.split('/')[0]}': {sorted(self.mne.ica.labels_[lb])}")

# write window size to config
size = ",".join(self.get_size_inches().astype(str))
set_config("MNE_BROWSE_RAW_SIZE", size, set_env=False)
Expand Down
1 change: 1 addition & 0 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
'console_scripts': [
'osl_maxfilter = osl.maxfilter.maxfilter:main',
'osl_ica_label = osl.preprocessing.ica_label:main',
'osl_ica_apply = osl.preprocessing.ica_label:apply',
'osl_preproc = osl.preprocessing.batch:main',
'osl_func = osl.utils.run_func:main',
]},
Expand Down

0 comments on commit 0792b0f

Please sign in to comment.