diff --git a/examples/ctf/ctf_with_headshape_pos/1_preprocess.py b/examples/ctf/ctf_with_headshape_pos/1_preprocess.py index 23f6717d..f3db02ef 100644 --- a/examples/ctf/ctf_with_headshape_pos/1_preprocess.py +++ b/examples/ctf/ctf_with_headshape_pos/1_preprocess.py @@ -24,13 +24,20 @@ "data/raw/Nottingham/sub-not002/meg/sub-not002_task-resteyesopen_meg.ds", ] +# Subject IDs +subjects = [ + "sub-not001_task-resteyesopen", + "sub-not002_task-resteyesopen", +] + # Directory to save output to -outdir = "data/preproc" +outdir = "data" # Do preprocessing preprocessing.run_proc_batch( config, inputs, + subjects=subjects, outdir=outdir, overwrite=True, ) diff --git a/examples/ctf/ctf_with_headshape_pos/2_coregister.py b/examples/ctf/ctf_with_headshape_pos/2_coregister.py index 483502b9..f596255f 100644 --- a/examples/ctf/ctf_with_headshape_pos/2_coregister.py +++ b/examples/ctf/ctf_with_headshape_pos/2_coregister.py @@ -9,7 +9,7 @@ from osl import source_recon, utils -def save_polhemus_from_pos(src_dir, subject, preproc_file, smri_file, epoch_file): +def save_polhemus_from_pos(outdir, subject): """Saves fiducials/headshape from a pos file.""" # Get path to pos file @@ -17,7 +17,7 @@ def save_polhemus_from_pos(src_dir, subject, preproc_file, smri_file, epoch_file utils.logger.log_or_print(f"Saving polhemus info from {pos_file}") # Get coreg filenames - filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject) + filenames = source_recon.rhino.get_coreg_filenames(outdir, subject) # Load in txt file, these values are in cm in polhemus space: num_headshape_pnts = int(pd.read_csv(pos_file, header=None).to_numpy()[0]) @@ -53,11 +53,11 @@ def save_polhemus_from_pos(src_dir, subject, preproc_file, smri_file, epoch_file np.savetxt(filenames["polhemus_lpa_file"], polhemus_lpa) np.savetxt(filenames["polhemus_headshape_file"], polhemus_headshape) -def fix_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file): +def fix_headshape_points(outdir, subject, preproc_file, smri_file, epoch_file): """Remove headshape points on the nose and neck.""" # Load saved headshape and nasion files - filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject) + filenames = source_recon.rhino.get_coreg_filenames(outdir, subject) hs = np.loadtxt(filenames["polhemus_headshape_file"]) nas = np.loadtxt(filenames["polhemus_nasion_file"]) lpa = np.loadtxt(filenames["polhemus_lpa_file"]) @@ -100,27 +100,30 @@ def fix_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file): """ # Subject IDs -subjects = ["sub-not001", "sub-not002"] +subjects = [ + "sub-not001_task-resteyesopen", + "sub-not002_task-resteyesopen", +] # Fif files containing the sensor-level preprocessed data for each subject preproc_files = [ - "data/preproc/sub-not001_task-resteyesopen_meg/sub-not001_task-resteyesopen_meg_preproc_raw.fif", - "data/preproc/sub-not002_task-resteyesopen_meg/sub-not002_task-resteyesopen_meg_preproc_raw.fif", + "data/sub-not001_task-resteyesopen/sub-not001_task-resteyesopen_preproc-raw.fif", + "data/sub-not002_task-resteyesopen/sub-not002_task-resteyesopen_preproc-raw.fif", ] # The corresponding structurals for each subject smri_files = [ - "data/smri/sub-not001_T1w.nii.gz", - "data/smri/sub-not002_T1w.nii.gz", + "smri/sub-not001_T1w.nii.gz", + "smri/sub-not002_T1w.nii.gz", ] # Directory to save output to -coreg_dir = "data/coreg" +outdir = "data" # Source reconstruction source_recon.run_src_batch( config, - src_dir=coreg_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, diff --git a/examples/ctf/ctf_with_headshape_pos/3_source_reconstruct.py b/examples/ctf/ctf_with_headshape_pos/3_source_reconstruct.py index 7b984cda..b2b70ca7 100644 --- a/examples/ctf/ctf_with_headshape_pos/3_source_reconstruct.py +++ b/examples/ctf/ctf_with_headshape_pos/3_source_reconstruct.py @@ -4,22 +4,8 @@ # Authors: Chetan Gohil -import os - from osl import source_recon -# Directories -coreg_dir = "data/coreg" -src_dir = "data/src" - -# First copy the coregistration directory -if os.path.exists(src_dir): - print(f"Please first delete: {src_dir}") - exit() -cmd = f"cp -r {coreg_dir} {src_dir}" -print(cmd) -os.system(cmd) - # Settings config = """ source_recon: @@ -33,18 +19,24 @@ """ # Subject IDs -subjects = ["sub-not001", "sub-not002"] +subjects = [ + "sub-not001_task-resteyesopen", + "sub-not002_task-resteyesopen", +] # Fif files containing the sensor-level preprocessed data for each subject preproc_files = [ - "data/preproc/sub-not001_task-resteyesopen_meg/sub-not001_task-resteyesopen_meg_preproc_raw.fif", - "data/preproc/sub-not002_task-resteyesopen_meg/sub-not002_task-resteyesopen_meg_preproc_raw.fif", + "data/sub-not001_task-resteyesopen/sub-not001_task-resteyesopen_preproc-raw.fif", + "data/sub-not002_task-resteyesopen/sub-not002_task-resteyesopen_preproc-raw.fif", ] +# Directory to save output to +outdir = "data" + # Source reconstruction source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, ) diff --git a/examples/ctf/ctf_with_headshape_pos/4_sign_flip.py b/examples/ctf/ctf_with_headshape_pos/4_sign_flip.py index b85a5394..0c9d82a0 100644 --- a/examples/ctf/ctf_with_headshape_pos/4_sign_flip.py +++ b/examples/ctf/ctf_with_headshape_pos/4_sign_flip.py @@ -13,12 +13,15 @@ from osl import source_recon # Source directory and subjects to sign flip -src_dir = "data/src" -subjects = ["sub-not001", "sub-not002"] +outdir = "data" +subjects = [ + "sub-not001_task-resteyesopen", + "sub-not002_task-resteyesopen", +] # Find a good template subject to align other subjects to template = source_recon.find_template_subject( - src_dir, subjects, n_embeddings=15, standardize=True + outdir, subjects, n_embeddings=15, standardize=True ) # Settings @@ -34,4 +37,4 @@ """ # Do the sign flipping -source_recon.run_src_batch(config, src_dir, subjects) +source_recon.run_src_batch(config, outdir, subjects) diff --git a/examples/ctf/ctf_with_headshape_pos/5_save_npy.py b/examples/ctf/ctf_with_headshape_pos/5_save_npy.py index 62030a88..ae7f1760 100644 --- a/examples/ctf/ctf_with_headshape_pos/5_save_npy.py +++ b/examples/ctf/ctf_with_headshape_pos/5_save_npy.py @@ -8,7 +8,7 @@ from osl_dynamics.data import Data -files = sorted(glob("data/src/*/sflip_parc-raw.fif")) +files = sorted(glob("data/*/*_sflip_parc-raw.fif")) data = Data( files, picks="misc", diff --git a/examples/ctf/ctf_with_smri_fid/1_preprocess.py b/examples/ctf/ctf_with_smri_fid/1_preprocess.py index 6d00eb23..f30050a8 100644 --- a/examples/ctf/ctf_with_smri_fid/1_preprocess.py +++ b/examples/ctf/ctf_with_smri_fid/1_preprocess.py @@ -17,13 +17,17 @@ # Create a list of paths to files to preprocess inputs = ["data/raw/mg04938_BrainampDBS_20170504_01_raw.fif"] +# Subject IDs +subjects = ["LN_VTA2"] + # Directory to save output to -outdir = "data/preproc" +outdir = "data" # Do preprocessing preprocessing.run_proc_batch( config, inputs, + subjects=subjects, outdir=outdir, overwrite=True, ) diff --git a/examples/ctf/ctf_with_smri_fid/2_coregister.py b/examples/ctf/ctf_with_smri_fid/2_coregister.py index b0c9ac1f..82adf8ab 100644 --- a/examples/ctf/ctf_with_smri_fid/2_coregister.py +++ b/examples/ctf/ctf_with_smri_fid/2_coregister.py @@ -25,18 +25,17 @@ subjects = ["LN_VTA2"] # Lists for input files -preproc_files = ["data/preproc/mg04938_BrainampDBS_20170504_01_preproc_raw.fif"] -smri_files = ["data/smri/LN_VTA2.nii"] +preproc_files = ["data/LN_VTA2/mg04938_BrainampDBS_20170504_01_preproc-raw.fif"] +smri_files = ["smri/LN_VTA2.nii"] # Output directory -coreg_dir = "data/coreg" +outdir = "data" # Do coregistration source_recon.run_src_batch( config, - src_dir=coreg_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, - extra_funcs=[save_mni_fids], ) diff --git a/examples/ctf/ctf_with_smri_fid/3_source_reconstruct.py b/examples/ctf/ctf_with_smri_fid/3_source_reconstruct.py index 20ccf110..79ee9011 100644 --- a/examples/ctf/ctf_with_smri_fid/3_source_reconstruct.py +++ b/examples/ctf/ctf_with_smri_fid/3_source_reconstruct.py @@ -4,22 +4,8 @@ # Authors: Chetan Gohil -import os - from osl import source_recon -# Directories -coreg_dir = "data/coreg" -src_dir = "data/src" - -# First copy the coregistration directory -if os.path.exists(src_dir): - print(f"Please first delete: {src_dir}") - exit() -cmd = f"cp -r {coreg_dir} {src_dir}" -print(cmd) -os.system(cmd) - # Settings config = """ source_recon: @@ -38,12 +24,15 @@ subjects = ["LN_VTA2"] # Fif files containing the sensor-level preprocessed data for each subject -preproc_files = ["data/preproc/mg04938_BrainampDBS_20170504_01_preproc_raw.fif"] +preproc_files = ["data/LN_VTA2/mg04938_BrainampDBS_20170504_01_preproc-raw.fif"] + +# Directories +outdir = "data" # Source reconstruction source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, ) diff --git a/examples/ctf/ctf_with_smri_fid/4_sign_flip.py b/examples/ctf/ctf_with_smri_fid/4_sign_flip.py index 371d1a3e..491a1631 100644 --- a/examples/ctf/ctf_with_smri_fid/4_sign_flip.py +++ b/examples/ctf/ctf_with_smri_fid/4_sign_flip.py @@ -26,7 +26,7 @@ def load(filename): # Files to sign flip -files = sorted(glob("data/src/*/parc/parc-raw.fif")) +files = sorted(glob("data/*/parc/parc-raw.fif")) # Get covariance matrices covs = load_covariances( diff --git a/examples/elekta/1_maxfilter.py b/examples/elekta/1_maxfilter.py index 78f4ab92..58787530 100644 --- a/examples/elekta/1_maxfilter.py +++ b/examples/elekta/1_maxfilter.py @@ -9,12 +9,12 @@ # Setup paths to raw (pre-maxfiltered) fif files input_files = [ - "data/raw/file1.fif", - "data/raw/file2.fif", + "raw/file1.fif", + "raw/file2.fif", ] # Directory to save the maxfiltered data to -output_directory = "data/maxfilter" +output_directory = "maxfilter" # Run MaxFiltering # diff --git a/examples/elekta/2_preprocess.py b/examples/elekta/2_preprocess.py index 1f77f339..dea8cdfb 100644 --- a/examples/elekta/2_preprocess.py +++ b/examples/elekta/2_preprocess.py @@ -10,8 +10,8 @@ from osl import preprocessing, utils # Files and directories -raw_file = "data/maxfilter/{subject}_tsss.fif" # {subject} will be replace by the name for the subject -preproc_dir = "data/preproc" # output directory containing the preprocess files +raw_file = "maxfilter/{subject}_tsss.fif" # {subject} will be replace by the name for the subject +outdir = "data" subjects = ["sub-001", "sub-002"] @@ -50,7 +50,8 @@ preprocessing.run_proc_batch( config, inputs, - outdir=preproc_dir, + subjects=subjects, + outdir=outdir, overwrite=True, dask_client=True, ) diff --git a/examples/elekta/3_coregister.py b/examples/elekta/3_coregister.py index 10ffba20..1d60f3aa 100644 --- a/examples/elekta/3_coregister.py +++ b/examples/elekta/3_coregister.py @@ -17,13 +17,12 @@ from osl import source_recon, utils # Directories -preproc_dir = "data/preproc" -anat_dir = "data/smri" -coreg_dir = "data/coreg" +outdir = "data" +anatdir = "smri" # Files ({subject} will be replaced by the name for the subject) -preproc_file = preproc_dir + "/{subject}_tsss_preproc_raw.fif" -smri_file = anat_dir + "/{subject}/anat/{subject}_T1w.nii" +preproc_file = outdir + "{subject}/{subject}_tsss_preproc-raw.fif" +smri_file = anatdir + "/{subject}/anat/{subject}_T1w.nii" # Subjects to coregister subjects = ["sub-001", "sub-002"] @@ -60,7 +59,7 @@ # Run coregistration source_recon.run_src_batch( config, - src_dir=coreg_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, diff --git a/examples/elekta/4_source_reconstruct.py b/examples/elekta/4_source_reconstruct.py index e79450ad..1064db95 100644 --- a/examples/elekta/4_source_reconstruct.py +++ b/examples/elekta/4_source_reconstruct.py @@ -13,12 +13,10 @@ from osl import source_recon, utils # Directories -preproc_dir = "data/preproc" -coreg_dir = "data/coreg" -src_dir = "data/src" +outdir = "data" # Files -preproc_file = preproc_dir + "/{subject}_tsss_preproc_raw.fif" # {subject} will be replaced by the subject name +preproc_file = outdir + "/{subject}_tsss_preproc-raw.fif" # {subject} will be replaced by the subject name # Subjects to do subjects = ["sub-001", "sub-002"] @@ -40,12 +38,6 @@ if __name__ == "__main__": utils.logger.set_up(level="INFO") - # Copy directory containing the coregistration - if not os.path.exists(src_dir): - cmd = f"cp -r {coreg_dir} {src_dir}" - print(cmd) - os.system(cmd) - # Get paths to files preproc_files = [] for subject in subjects: @@ -60,7 +52,7 @@ # Source reconstruction source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, dask_client=True, diff --git a/examples/elekta/5_sign_flip.py b/examples/elekta/5_sign_flip.py index 870e4af6..8b6a2bf8 100644 --- a/examples/elekta/5_sign_flip.py +++ b/examples/elekta/5_sign_flip.py @@ -11,7 +11,7 @@ from osl.source_recon import find_template_subject, run_src_batch # Directories -src_dir = "data/src" +outdir = "data" if __name__ == "__main__": utils.logger.set_up(level="INFO") @@ -19,7 +19,7 @@ # Subjects to sign flip # We create a list by looking for subjects that have a parc/parc-raw.fif file subjects = [] - for path in sorted(glob(src_dir + "/*/parc/parc-raw.fif")): + for path in sorted(glob(f"{outdir}/*/parc/parc-raw.fif")): subject = path.split("/")[-3] subjects.append(subject) @@ -49,7 +49,7 @@ # Do the sign flipping run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, dask_client=True, ) diff --git a/examples/opm/1_preprocess.py b/examples/opm/1_preprocess.py index 23da9bf5..6f4f05d1 100644 --- a/examples/opm/1_preprocess.py +++ b/examples/opm/1_preprocess.py @@ -16,15 +16,19 @@ """ # List of fif files to preprocess -inputs = ["data/raw/13703-braille_test-meg.fif"] +inputs = ["raw/13703-braille_test-meg.fif"] + +# Subject IDs +subjects = ["13703"] # Directory to save output to -outdir = "data/preproc" +outdir = "data" # Do preprocessing dataset = preprocessing.run_proc_batch( config, inputs, + subjects=subjects, outdir=outdir, overwrite=True, ) diff --git a/examples/opm/2_coregister.py b/examples/opm/2_coregister.py index 76864080..1fd4bb7f 100644 --- a/examples/opm/2_coregister.py +++ b/examples/opm/2_coregister.py @@ -8,13 +8,13 @@ from osl import source_recon, utils -def coregister(src_dir, subject, preproc_file, smri_file, epoch_file): +def coregister(outdir, subject, preproc_file): """Coregister OPM data.""" # Create dummy coregistration files source_recon.rhino.coreg( fif_file=preproc_file, - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, use_headshape=False, use_nose=False, @@ -22,7 +22,7 @@ def coregister(src_dir, subject, preproc_file, smri_file, epoch_file): ) # Copy head to MRI transformation (needed for the forward model) - filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject) + filenames = source_recon.rhino.get_coreg_filenames(outdir, subject) in_file = f"data/raw/{subject}-head_scaledmri-trans.fif" out_file = filenames["head_scaledmri_t_file"] cmd = f"cp {in_file} {out_file}" @@ -44,19 +44,19 @@ def coregister(src_dir, subject, preproc_file, smri_file, epoch_file): # Fif files containing the sensor-level preprocessed data for each subject preproc_files = [ - "data/preproc/13703-braille_test-meg/13703-braille_test-meg_preproc_raw.fif", + "data/13703/13703-braille_test-meg_preproc-raw.fif", ] # The corresponding structurals for each subject -smri_files = ["data/raw/13703.nii"] +smri_files = ["raw/13703.nii"] # Directory to save output to -outdir = "data/coreg" +outdir = "data" # Perform coregistration source_recon.run_src_batch( config, - src_dir=outdir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, diff --git a/examples/opm/3_source_reconstruct.py b/examples/opm/3_source_reconstruct.py index b944b57e..6cb3d35e 100644 --- a/examples/opm/3_source_reconstruct.py +++ b/examples/opm/3_source_reconstruct.py @@ -9,16 +9,7 @@ from osl import source_recon # Directories -coreg_dir = "data/coreg" -src_dir = "data/src" - -# First copy the coregistration directory -if os.path.exists(src_dir): - print(f"Please first delete: {src_dir}") - exit() -cmd = f"cp -r {coreg_dir} {src_dir}" -print(cmd) -os.system(cmd) +outdir = "data" # Settings config = """ @@ -38,13 +29,13 @@ # Fif files containing the sensor-level preprocessed data for each subject preproc_files = [ - "data/preproc/13703-braille_test-meg/13703-braille_test-meg_preproc_raw.fif", + "data/13703/13703-braille_test-meg_preproc-raw.fif", ] # Do source reconstruction source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, ) diff --git a/examples/opm/4_sign_flip.py b/examples/opm/4_sign_flip.py index feaed527..9b37943e 100644 --- a/examples/opm/4_sign_flip.py +++ b/examples/opm/4_sign_flip.py @@ -13,12 +13,12 @@ from osl import source_recon # Source directory and subjects to sign flip -src_dir = "data/src" +outdir = "data" subjects = ["13703"] # Find a good template subject to align other subjects to template = source_recon.find_template_subject( - src_dir, subjects, n_embeddings=15, standardize=True + outdir, subjects, n_embeddings=15, standardize=True ) # Settings @@ -34,4 +34,4 @@ """ # Do the sign flipping -source_recon.run_src_batch(config, src_dir, subjects) +source_recon.run_src_batch(config, outdir, subjects) diff --git a/examples/opm/5_save_npy.py b/examples/opm/5_save_npy.py index 62030a88..ae7f1760 100644 --- a/examples/opm/5_save_npy.py +++ b/examples/opm/5_save_npy.py @@ -8,7 +8,7 @@ from osl_dynamics.data import Data -files = sorted(glob("data/src/*/sflip_parc-raw.fif")) +files = sorted(glob("data/*/*_sflip_parc-raw.fif")) data = Data( files, picks="misc", diff --git a/examples/parallelisation/parallel_preprocess.py b/examples/parallelisation/parallel_preprocess.py index 21f4dbd8..de7ada1c 100644 --- a/examples/parallelisation/parallel_preprocess.py +++ b/examples/parallelisation/parallel_preprocess.py @@ -14,8 +14,8 @@ if __name__ == "__main__": utils.logger.set_up(level="INFO") - raw_dir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/meg/pipeline/release005/BIDSsep/rest" - preproc_dir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" + rawdir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/meg/pipeline/release005/BIDSsep/rest" + outdir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" config = """ preproc: @@ -30,9 +30,9 @@ # Get input files inputs = [] - for subject in glob(raw_dir + "/sub-*"): + for subject in sorted(glob(f"{rawdir}/sub-*")): subject = pathlib.Path(subject).stem - inputs.append(raw_dir + f"/{subject}/ses-rest/meg/{subject}_ses-rest_task-rest_meg.fif") + inputs.append(f"{rawdir}/{subject}/ses-rest/meg/{subject}_ses-rest_task-rest_meg.fif") inputs = inputs[:2] # Setup a Dask client for parallel processing @@ -52,7 +52,7 @@ preprocessing.run_proc_batch( config, inputs, - outdir=preproc_dir, + outdir=outdir, overwrite=True, dask_client=True, ) diff --git a/examples/parallelisation/parallel_source_reconstruct.py b/examples/parallelisation/parallel_source_reconstruct.py index e6ca9208..6d742452 100644 --- a/examples/parallelisation/parallel_source_reconstruct.py +++ b/examples/parallelisation/parallel_source_reconstruct.py @@ -23,13 +23,12 @@ utils.logger.set_up(level="INFO") # Directories - anat_dir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/mri/pipeline/release004/BIDS_20190411/anat" - preproc_dir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" - src_dir = "/ohba/pi/mwoolrich/cgohil/camcan/src" + anatdir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/mri/pipeline/release004/BIDS_20190411/anat" + outdir = "/ohba/pi/mwoolrich/cgohil/camcan/src" # Files - SMRI_FILE = anat_dir + "/{0}/anat/{0}_T1w.nii" - PREPROC_FILE = preproc_dir + "{0}_ses-rest_task-rest_meg/{0}_ses-rest_task-rest_meg_preproc_raw.fif" + smri_file = anatdir + "/{0}/anat/{0}_T1w.nii" + preproc_file = outdir + "{0}_ses-rest_task-rest_meg/{0}_ses-rest_task-rest_meg_preproc-raw.fif" # Settings config = """ @@ -52,11 +51,11 @@ orthogonalisation: symmetric """ - def remove_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file): + def remove_headshape_points(outdir, subject): """Removes headshape points near the nose.""" # Get coreg filenames - filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject) + filenames = source_recon.rhino.get_coreg_filenames(outdir, subject) # Load saved headshape and nasion files hs = np.loadtxt(filenames["polhemus_headshape_file"]) @@ -104,7 +103,7 @@ def remove_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_fil # Beamforming and parcellation source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, diff --git a/examples/parallelisation/serial_preprocess.py b/examples/parallelisation/serial_preprocess.py index 89c47525..b4d81553 100644 --- a/examples/parallelisation/serial_preprocess.py +++ b/examples/parallelisation/serial_preprocess.py @@ -10,8 +10,8 @@ from osl import preprocessing -raw_dir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/meg/pipeline/release005/BIDSsep/rest" -preproc_dir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" +rawdir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/meg/pipeline/release005/BIDSsep/rest" +outdir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" config = """ preproc: @@ -26,15 +26,15 @@ # Get input files inputs = [] -for subject in glob(raw_dir + "/sub-*"): +for subject in sorted(glob(f"{rawdir}/sub-*")): subject = pathlib.Path(subject).stem - inputs.append(raw_dir + f"/{subject}/ses-rest/meg/{subject}_ses-rest_task-rest_meg.fif") + inputs.append(f"{rawdir}/{subject}/ses-rest/meg/{subject}_ses-rest_task-rest_meg.fif") inputs = inputs[:2] # Main preprocessing preprocessing.run_proc_batch( config, inputs, - outdir=preproc_dir, + outdir=outdir, overwrite=True, ) diff --git a/examples/parallelisation/serial_source_reconstruct.py b/examples/parallelisation/serial_source_reconstruct.py index 5bffa415..4731cbc2 100644 --- a/examples/parallelisation/serial_source_reconstruct.py +++ b/examples/parallelisation/serial_source_reconstruct.py @@ -19,13 +19,12 @@ logger = logging.getLogger("osl") # Directories -anat_dir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/mri/pipeline/release004/BIDS_20190411/anat" -preproc_dir = "/ohba/pi/mwoolrich/cgohil/camcan/preproc" -src_dir = "/ohba/pi/mwoolrich/cgohil/camcan/src" +anatdir = "/ohba/pi/mwoolrich/datasets/CamCan_2021/cc700/mri/pipeline/release004/BIDS_20190411/anat" +outdir = "/ohba/pi/mwoolrich/cgohil/camcan/src" # Files -smri_file = anat_dir + "/{0}/anat/{0}_T1w.nii" -preproc_file = preproc_dir + "{0}_ses-rest_task-rest_meg/{0}_ses-rest_task-rest_meg_preproc_raw.fif" +smri_file = anatdir + "/{0}/anat/{0}_T1w.nii" +preproc_file = outdir + "{0}_ses-rest_task-rest_meg/{0}_ses-rest_task-rest_meg_preproc-raw.fif" # Settings config = """ @@ -48,11 +47,11 @@ orthogonalisation: symmetric """ -def remove_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file): +def remove_headshape_points(outdir, subject): """Removes headshape points near the nose.""" # Get coreg filenames - filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject) + filenames = source_recon.rhino.get_coreg_filenames(outdir, subject) # Load saved headshape and nasion files hs = np.loadtxt(filenames["polhemus_headshape_file"]) @@ -74,7 +73,7 @@ def remove_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_fil # Get subjects subjects = [] -for subject in glob(preproc_dir + "/sub-*"): +for subject in sorted(glob(f"{outdir}/sub-*")): subjects.append(pathlib.Path(subject).stem.split("_")[0]) # Setup files @@ -87,7 +86,7 @@ def remove_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_fil # Beamforming and parcellation source_recon.run_src_batch( config, - src_dir=src_dir, + outdir=outdir, subjects=subjects, preproc_files=preproc_files, smri_files=smri_files, diff --git a/osl/preprocessing/README.md b/osl/preprocessing/README.md index 2ce82d84..2b005c45 100644 --- a/osl/preprocessing/README.md +++ b/osl/preprocessing/README.md @@ -35,7 +35,7 @@ outdir = '/where/do/i/want/my/output_dir' # Process a single file raw_file = '/path/to/file.fif' -osl.preprocessing.run_proc_chain(config, raw_file, outdir) # creates /path/to/file_preproc_raw.fif +osl.preprocessing.run_proc_chain(config, raw_file, outdir) # creates /path/to/file_preproc-raw.fif # Process a list of files list_of_raw_files = ['/path/to/file1.fif','/path/to/file2.fif','/path/to/file3.fif'] @@ -74,7 +74,7 @@ The following code runs the chain on a file: ``` fname = '/path/to/my/dataset.fif' -osl.preprocessing.run_proc_chain(config, fname) # creates dataset_preproc_raw.fif and dataset_epo.fif +osl.preprocessing.run_proc_chain(config, fname) # creates dataset_preproc-raw.fif and dataset_epo.fif # Average the epochs object and visualise a response epochs = mne.io.read_raw_fif('/path/to/my/dataset_epo.fif') diff --git a/osl/preprocessing/batch.py b/osl/preprocessing/batch.py index 2ebeaf7c..18497bc7 100644 --- a/osl/preprocessing/batch.py +++ b/osl/preprocessing/batch.py @@ -29,7 +29,7 @@ import yaml from . import mne_wrappers, osl_wrappers -from ..utils import find_run_id, validate_outdir, process_file_inputs, add_subdir +from ..utils import find_run_id, validate_outdir, process_file_inputs from ..utils import logger as osl_logger from ..utils.parallel import dask_parallel_bag from ..utils.version_utils import check_version @@ -105,14 +105,17 @@ def import_data(infile, preload=True): elif os.path.splitext(infile)[1] == ".fif": logger.info("Detected fif file format, using: mne.io.read_raw_fif") raw = mne.io.read_raw_fif(infile, preload=preload) + # EDF file elif os.path.splitext(infile)[1].lower() == ".edf": logger.info("Detected edf file format, using: mne.io.read_raw_edf") raw = mne.io.read_raw_edf(infile, preload=preload) + # CTF data in ds directory elif os.path.splitext(infile)[1] == ".ds": logger.info("Detected CTF file format, using: mne.io.read_raw_ctf") raw = mne.io.read_raw_ctf(infile, preload=preload) + elif os.path.splitext(infile)[1] == ".meg4": logger.info("Detected CTF file format, using: mne.io.read_raw_ctf") raw = mne.io.read_raw_ctf(os.path.dirname(infile), preload=preload) @@ -134,6 +137,10 @@ def import_data(infile, preload=True): elif os.path.splitext(infile)[1] == ".bdf": logger.info("Detected BDF file format, using: mne.io.read_raw_bdf") raw = mne.io.read_raw_bdf(infile, preload=preload) + + elif os.path.splitext(infile)[1] == ".mff": + logger.info("Detected EGI file format, using mne.io.read_raw_egi") + raw = mne.io.read_raw_egi(infile, preload=preload) # Other formats not accepted else: @@ -289,10 +296,8 @@ def check_config_versions(config): ------ AssertionError Raised if package version mismatch found in 'version_assert' - - WARNING + Warning Raised if package version mismatch found in 'version_warn' - """ config = load_config(config) @@ -380,7 +385,7 @@ def append_preproc_info(dataset, config, extra_funcs=None): return dataset -def write_dataset(dataset, outbase, run_id, ftype='preproc_raw', overwrite=False): +def write_dataset(dataset, outbase, run_id, ftype='preproc-raw', overwrite=False): """Write preprocessed data to a file. Will write all keys in the dataset dict to disk with corresponding extensions. @@ -394,7 +399,7 @@ def write_dataset(dataset, outbase, run_id, ftype='preproc_raw', overwrite=False run_id : str ID for the output file. ftype: str - Extension for the fif file (default ``preproc_raw``) + Extension for the fif file (default ``preproc-raw``) overwrite : bool Should we overwrite if the file already exists? @@ -404,8 +409,8 @@ def write_dataset(dataset, outbase, run_id, ftype='preproc_raw', overwrite=False The saved fif file name """ - # Strip "_preproc_raw" or "_raw" from the run id - for string in ["_preproc_raw", "_raw"]: + # Strip "_preproc-raw" or "_raw" from the run id + for string in ["_preproc-raw", "_raw"]: if string in run_id: run_id = run_id.replace(string, "") @@ -450,7 +455,7 @@ def read_dataset(fif, preload=False, ftype=None): ftype : str Extension for the fif file (will be replaced for e.g. ``'_events.npy'`` or ``'_ica.fif'``). If ``None``, we assume the fif file is preprocessed with - OSL and has the extension ``'_preproc_raw'``. If this fails, we guess + OSL and has the extension ``'_preproc-raw'``. If this fails, we guess the extension as whatever comes after the last ``'_'``. Returns @@ -466,9 +471,9 @@ def read_dataset(fif, preload=False, ftype=None): # Guess extension if ftype is None: logger.info("Guessing the preproc extension") - if "preproc_raw" in fif: - logger.info('Assuming fif file type is "preproc_raw"') - ftype = "preproc_raw" + if "preproc-raw" in fif: + logger.info('Assuming fif file type is "preproc-raw"') + ftype = "preproc-raw" else: if len(fif.split("_"))<2: logger.error("Unable to guess the fif file extension") @@ -478,7 +483,6 @@ def read_dataset(fif, preload=False, ftype=None): # add extension to fif file name ftype = ftype + ".fif" - events = Path(fif.replace(ftype, "events.npy")) if events.exists(): @@ -628,8 +632,8 @@ def plot_preproc_flowchart( def run_proc_chain( config, infile, - outname=None, - ftype='preproc_raw', + subject=None, + ftype='preproc-raw', outdir=None, logsdir=None, reportdir=None, @@ -648,19 +652,12 @@ def run_proc_chain( Preprocessing config. infile : str Path to input file. - outname : str - Output filename. + subject : str + Subject ID. This will be the sub-directory in outdir. ftype: str - Extension for the fif file (default ``preproc_raw``) + Extension for the fif file (default ``preproc-raw``) outdir : str - Output directory. If processing multiple files, they can - be put in unique sub directories by including ``{x:0}`` at - the end of the outdir, where ``x`` is the pattern which - precedes the unique identifier and ``0`` is the length of - the unique identifier. For example: if the outdir is - ``../../{sub-:3}`` and each is like - ``/sub-001_task-rest.fif``, the outdir for the file will be - ``../../sub-001/``. + Output directory. logsdir : str Directory to save log files to. reportdir : str @@ -687,11 +684,8 @@ def run_proc_chain( An empty dict is returned if preprocessing fails. If ``ret_dataset=False``, we return a flag indicating whether preprocessing was successful. """ - # Generate a run ID - if outname is None: - run_id = find_run_id(infile) - else: - run_id = os.path.splitext(outname)[0] + # Get run (subject) ID + run_id = subject or find_run_id(infile) name_base = "{run_id}_{ftype}.{fext}" if not ret_dataset: @@ -706,10 +700,9 @@ def run_proc_chain( gen_report = True if gen_report is None else gen_report # Create output directories if they don't exist - outdir = add_subdir(infile, outdir, run_id) - outdir = validate_outdir(outdir) + outdir = validate_outdir(f"{outdir}/{run_id}") logsdir = validate_outdir(logsdir or outdir / "logs") - reportdir = validate_outdir(reportdir or outdir / "report") + reportdir = validate_outdir(reportdir or outdir / "preproc_report") else: # We're not saving the output to disk @@ -719,7 +712,7 @@ def run_proc_chain( gen_report = gen_report or reportdir is not None or False if gen_report: # Make sure we have a directory to write the report to - reportdir = validate_outdir(reportdir or os.getcwd() + "/report") + reportdir = validate_outdir(reportdir or os.getcwd() + "/preproc_report") # Allow the user to create a log if they pass logsdir if logsdir is not None: @@ -809,6 +802,7 @@ def run_proc_chain( ica=dataset["ica"], preproc_fif_filename=fif_outname, logsdir=logsdir, + run_id=run_id, ) gen_html_page(reportdir) @@ -865,7 +859,7 @@ def run_proc_chain( def run_proc_batch( config, files, - outnames=None, + subjects=None, ftype=None, outdir=None, logsdir=None, @@ -890,19 +884,12 @@ def run_proc_batch( files : str or list or mne.Raw Can be a list of Raw objects or a list of filenames (or ``.ds`` dir names if CTF data) or a path to a textfile list of filenames (or ``.ds`` dir names if CTF data). - outnames: None or list of str - output names of the preprocessed files. If None (default), they will be automatically - determined. + subjects : list of str + Subject directory names. These are sub-directories in outdir. ftype: None or str - Extension of the preprocessed fif files. Default option is `_preproc_raw`. + Extension of the preprocessed fif files. Default option is `_preproc-raw`. outdir : str - Output directory. If processing multiple files, they can - be put in unique sub directories by including ``{x:0}`` at - the end of the outdir, where ``x`` is the pattern which - precedes the unique identifier and ``0`` is the length of - the unique identifier. For example: if the outdir is - ``../../{sub-:3}`` and each is like ``/sub-001_task-rest.fif``, - the outdir for the file will be ``../../sub-001/``. + Output directory. logsdir : str Directory to save log files to. reportdir : str @@ -945,7 +932,7 @@ def run_proc_batch( # Validate the parent outdir - later do so for each subdirectory tmpoutdir = validate_outdir(outdir.split('{')[0]) logsdir = validate_outdir(logsdir or tmpoutdir / "logs") - reportdir = validate_outdir(reportdir or tmpoutdir / "report") + reportdir = validate_outdir(reportdir or tmpoutdir / "preproc_report") # Initialise Loggers mne.set_log_level(mneverbose) @@ -961,14 +948,14 @@ def run_proc_batch( infiles, good_files_outnames, good_files = process_file_inputs(files) # Specify filenames for the output data - if outnames is None: - outnames = good_files_outnames + if subjects is None: + subjects = good_files_outnames else: - if len(outnames) != len(good_files_outnames): + if len(subjects) != len(good_files_outnames): logger.critical( - f"Number of outnames ({len(outnames)}) does not match " + f"Number of subjects ({len(subjects)}) does not match " f"number of good files {len(good_files_outnames)}. " - "Fix outnames or use outnames=None." + "Please fix the subjects list or pass subjects=None." ) if strictrun and click.confirm('Is this correct set of inputs?') is False: @@ -1011,8 +998,8 @@ def run_proc_batch( # Loop through input files to generate arguments for run_proc_chain args = [] - for infile, outname in zip(infiles, outnames): - args.append((config, infile, outname)) + for infile, subject in zip(infiles, subjects): + args.append((config, infile, subject)) # Actually run the processes if dask_client: diff --git a/osl/preprocessing/ica_label.py b/osl/preprocessing/ica_label.py index d853890b..0faa266c 100644 --- a/osl/preprocessing/ica_label.py +++ b/osl/preprocessing/ica_label.py @@ -20,15 +20,15 @@ from ..utils import logger as osl_logger -def ica_label(preproc_dir, ica_dir, reject=None, report_dir=None): +def ica_label(data_dir, subject, reject=None): """Data bookkeeping and wrapping plot_ica. Parameters ---------- - preproc_dir : str - Path to preprocessed (M/EEG) data. - ica_dir : str - Path to ICA data. + data_dir : str + Path to processed (M/EEG) data. + subject : str + Subject/session specific data directory name reject : bool or str If 'all', reject all components (previously labeled and newly labeled). If 'manual', reject only manually labeled components. If @@ -37,11 +37,16 @@ def ica_label(preproc_dir, ica_dir, reject=None, report_dir=None): """ # global drive, savedir plt.ion() - - + + # define data paths based on OSL data structure + preproc_file = os.path.join(data_dir, subject, subject + '_preproc-raw.fif') + ica_file = os.path.join(data_dir, subject, subject + '_ica.fif') + report_dir_base = os.path.join(data_dir, 'preproc_report') + report_dir = os.path.join(report_dir_base, subject + '_preproc-raw') + print('LOADING DATA') - raw = mne.io.read_raw(preproc_dir, preload=True) - ica = mne.preprocessing.read_ica(ica_dir) + raw = mne.io.read_raw(preproc_file, preload=True) + ica = mne.preprocessing.read_ica(ica_file) # keep these for later if reject=='manual': @@ -64,25 +69,16 @@ def ica_label(preproc_dir, ica_dir, reject=None, report_dir=None): new_ica.apply(raw) print("SAVING PREPROCESSED DATA") - raw.save(preproc_dir, overwrite=True) + raw.save(preproc_file, overwrite=True) print("SAVING ICA DATA") - ica.save(ica_dir, overwrite=True) + ica.save(ica_file, overwrite=True) if reject is not None: print("ATTEMPTING TO UPDATE REPORT") - try: - if report_dir is None: - report_dir = os.path.join("/".join(preproc_dir.split("/")[:-2]), "report") - - report_dir_base = deepcopy(report_dir) - if os.path.exists(os.path.join(report_dir, preproc_dir.split("/")[-2])): - report_dir = os.path.join(report_dir, preproc_dir.split("/")[-2]) - elif os.path.exists(os.path.join(report_dir, preproc_dir.split("/")[-2]).replace("_raw", "") + "_preproc_raw"): - report_dir = os.path.join(report_dir, preproc_dir.split("/")[-2]).replace("_raw", "") + "_preproc_raw" - print(report_dir) - + try: savebase = os.path.join(report_dir, "{0}.png") + print(report_dir) if os.path.exists(os.path.join(report_dir, "ica.png")) or os.path.exists(os.path.join(report_dir, "data.pkl")): _ = plot_bad_ica(raw, ica, savebase) @@ -106,7 +102,7 @@ def ica_label(preproc_dir, ica_dir, reject=None, report_dir=None): print("REPORT UPDATED") except: print("FAILED TO UPDATE REPORT") - print(f'LABELING DATASET {preproc_dir.split("/")[-1]} COMPLETE') + print(f'LABELING DATASET {subject} COMPLETE') def main(argv=None): @@ -123,7 +119,7 @@ def main(argv=None): ------- From the command line (in the OSL environment), use as follows: - osl_ica_label reject_argument /path/to/preproc_raw.fif [/path/to/ica.fif] [/path/to/reports_dir] + osl_ica_label reject_argument /path/to/processed_data subject_name The `reject_argument` specifies whether to reject 'all' selected components from the data, only the 'manual' rejected, or None (and only save the ICA object, without rejecting components). @@ -131,7 +127,7 @@ def main(argv=None): the usual OSL structure. For example: - osl_ica_label manual /path/to/sub-001_preproc_raw.fif + osl_ica_label manual /path/to/proc_dir sub-001_run01 Then use the GUI to label components (click on the time course to mark, use number keys to label marked components as specific artefacts, and use @@ -149,21 +145,7 @@ def main(argv=None): if reject == 'None': reject = None - preproc_dir = argv[1] - if len(argv)>2 or (len(argv)==3 and "report" in argv[2]): - ica_dir = argv[2] - else: - ica_dir = preproc_dir.replace('preproc_raw.fif', 'ica.fif') - - if (len(argv)==3 and "report" in argv[2]): - report_dir = argv[2] - elif (len(argv)==4 and "report" in argv[3]): - report_dir = argv[3] - else: - # try to find it in the directory structure - report_dir = None - - ica_label(preproc_dir, ica_dir, reject, report_dir) + ica_label(data_dir=argv[1], subject=argv[2], reject=argv[0]) if __name__ == '__main__': diff --git a/osl/report/preproc_report.py b/osl/report/preproc_report.py index 0442e5d3..c82cdf71 100644 --- a/osl/report/preproc_report.py +++ b/osl/report/preproc_report.py @@ -47,7 +47,7 @@ # Report generation -def gen_report_from_fif(infiles, outdir, ftype=None, logsdir=None): +def gen_report_from_fif(infiles, outdir, ftype=None, logsdir=None, run_id=None): """Generate web-report for a set of MNE data objects. Parameters @@ -57,10 +57,12 @@ def gen_report_from_fif(infiles, outdir, ftype=None, logsdir=None): outdir : str Directory to save HTML report and figures to. ftype : str - Type of fif file, e.g., ``'raw'`` or ``'preproc_raw'``. + Type of fif file, e.g., ``'raw'`` or ``'preproc-raw'``. logsdir : str Directory the log files were saved into. If None, log files are assumed to be in outdir.replace('report', 'logs') + run_id : str + Run ID. """ # Validate input files and directory to save html file and plots to @@ -73,7 +75,13 @@ def gen_report_from_fif(infiles, outdir, ftype=None, logsdir=None): dataset = read_dataset(infile, ftype=ftype) run_id = get_header_id(dataset['raw']) htmldatadir = validate_outdir(outdir / run_id) - gen_html_data(dataset['raw'], htmldatadir, ica=dataset['ica'], logsdir=logsdir) + gen_html_data( + dataset['raw'], + htmldatadir, + ica=dataset['ica'], + logsdir=logsdir, + run_id=run_id, + ) # Create report gen_html_page(outdir) @@ -100,7 +108,9 @@ def get_header_id(raw): return raw.filenames[0].split('/')[-1].strip('.fif') -def gen_html_data(raw, outdir, ica=None, preproc_fif_filename=None, logsdir=None): +def gen_html_data( + raw, outdir, ica=None, preproc_fif_filename=None, logsdir=None, run_id=None +): """Generate HTML web-report for an MNE data object. Parameters @@ -116,12 +126,14 @@ def gen_html_data(raw, outdir, ica=None, preproc_fif_filename=None, logsdir=None logsdir : str Directory the log files were saved into. If None, log files are assumed to be in reportdir.replace('report', 'logs') + run_id : str + Run ID. """ data = {} data['filename'] = raw.filenames[0] data['preproc_fif_filename'] = preproc_fif_filename - data['fif_id'] = get_header_id(raw) + data['fif_id'] = run_id or get_header_id(raw) # Scan info data['projname'] = raw.info['proj_name'] @@ -254,7 +266,7 @@ def gen_html_data(raw, outdir, ica=None, preproc_fif_filename=None, logsdir=None # add log files if possible if logsdir is None: - logsdir = outdir._str.replace('/report/', '/logs/') + logsdir = outdir._str.replace('preproc_report', 'logs') elif type(logsdir)==pathlib.PosixPath: logsdir = os.path.join(logsdir._str, outdir._str.split('/')[-1]) @@ -383,7 +395,7 @@ def gen_html_summary(reportdir, logsdir=None): # log files if logsdir is None: - logsdir = reportdir._str.replace('report', 'logs') + logsdir = reportdir._str.replace('preproc_report', 'logs') elif type(logsdir)==pathlib.PosixPath: logsdir = logsdir._str diff --git a/osl/report/src_report.py b/osl/report/src_report.py index 27c960dc..f6baab5e 100644 --- a/osl/report/src_report.py +++ b/osl/report/src_report.py @@ -22,14 +22,14 @@ from ..source_recon import parcellation -def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs=None, logsdir=None): +def gen_html_data(config, outdir, subject, reportdir, logger=None, extra_funcs=None, logsdir=None): """Generate data for HTML report. Parameters ---------- config : dict Source reconstruction config. - src_dir : str + outdir : str Source reconstruction directory. subject : str Subject name. @@ -43,20 +43,13 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs= Directory the log files were saved into. If None, log files are assumed to be in reportdir.replace('report', 'logs') """ - src_dir = Path(src_dir) + outdir = Path(outdir) reportdir = Path(reportdir) # Make directory for plots contained in the report os.makedirs(reportdir, exist_ok=True) os.makedirs(reportdir / subject, exist_ok=True) - # Open data saved by the source_recon.wrappers - data_file = f"{src_dir}/{subject}/report_data.pkl" - if Path(data_file).exists(): - subject_data = pickle.load(open(data_file, "rb")) - else: - return - # Check if this function has been called before if Path(f"{reportdir}/{subject}/data.pkl").exists(): # Load the data object from last time @@ -65,12 +58,8 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs= if "config" in data: # Update the config based on this run data["config"] = update_config(data["config"], config) - - # Otherwise, this is the first time this has been called - else: - # Data for the report - data = {} - data["config"] = config + else: + data["config"] = config # add extra funcs if they exist if extra_funcs is not None: @@ -83,54 +72,54 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs= data["filename"] = subject # What have we done for this subject? - data["compute_surfaces"] = subject_data.pop("compute_surfaces", False) - data["coregister"] = subject_data.pop("coregister", False) - data["beamform"] = subject_data.pop("beamform", False) - data["beamform_and_parcellate"] = subject_data.pop("beamform_and_parcellate", False) - data["fix_sign_ambiguity"] = subject_data.pop("fix_sign_ambiguity", False) + data["compute_surfaces"] = data.pop("compute_surfaces", False) + data["coregister"] = data.pop("coregister", False) + data["beamform"] = data.pop("beamform", False) + data["beamform_and_parcellate"] = data.pop("beamform_and_parcellate", False) + data["fix_sign_ambiguity"] = data.pop("fix_sign_ambiguity", False) # Save info if data["beamform_and_parcellate"]: - data["n_samples"] = subject_data["n_samples"] + data["n_samples"] = data["n_samples"] if data["coregister"]: - data["fid_err"] = subject_data["fid_err"] + data["fid_err"] = data["fid_err"] if data["beamform_and_parcellate"]: - data["parcellation_file"] = subject_data["parcellation_file"] - data["parcellation_filename"] = Path(subject_data["parcellation_file"]).name + data["parcellation_file"] = data["parcellation_file"] + data["parcellation_filename"] = Path(data["parcellation_file"]).name if data["fix_sign_ambiguity"]: - data["template"] = subject_data["template"] - data["metrics"] = subject_data["metrics"] + data["template"] = data["template"] + data["metrics"] = data["metrics"] # Copy plots - if "surface_plots" in subject_data: - for plot in subject_data["surface_plots"]: + if "surface_plots" in data: + for plot in data["surface_plots"]: surface = "surfaces_" + Path(plot).stem data[f"plt_{surface}"] = f"{subject}/{surface}.png" - copy("{}/{}".format(src_dir, plot), "{}/{}/{}.png".format(reportdir, subject, surface)) + copy("{}/{}".format(outdir, plot), "{}/{}/{}.png".format(reportdir, subject, surface)) - if "coreg_plot" in subject_data: + if "coreg_plot" in data: data["plt_coreg"] = f"{subject}/coreg.html" - copy("{}/{}".format(src_dir, subject_data["coreg_plot"]), "{}/{}/coreg.html".format(reportdir, subject)) + copy("{}/{}".format(outdir, data["coreg_plot"]), "{}/{}/coreg.html".format(reportdir, subject)) - if "filters_cov_plot" in subject_data: + if "filters_cov_plot" in data: data["plt_filters_cov"] = f"{subject}/filters_cov.png" - copy("{}/{}".format(src_dir, subject_data["filters_cov_plot"]), "{}/{}/filters_cov.png".format(reportdir, subject)) + copy("{}/{}".format(outdir, data["filters_cov_plot"]), "{}/{}/filters_cov.png".format(reportdir, subject)) - if "filters_svd_plot" in subject_data: + if "filters_svd_plot" in data: data["plt_filters_svd"] = f"{subject}/filters_svd.png" - copy("{}/{}".format(src_dir, subject_data["filters_svd_plot"]), "{}/{}/filters_svd.png".format(reportdir, subject)) + copy("{}/{}".format(outdir, data["filters_svd_plot"]), "{}/{}/filters_svd.png".format(reportdir, subject)) - if "parc_psd_plot" in subject_data: + if "parc_psd_plot" in data: data["plt_parc_psd"] = f"{subject}/parc_psd.png" - copy("{}/{}".format(src_dir, subject_data["parc_psd_plot"]), "{}/{}/parc_psd.png".format(reportdir, subject)) + copy("{}/{}".format(outdir, data["parc_psd_plot"]), "{}/{}/parc_psd.png".format(reportdir, subject)) - if "parc_corr_plot" in subject_data: + if "parc_corr_plot" in data: data["plt_parc_corr"] = f"{subject}/parc_corr.png" - copy("{}/{}".format(src_dir, subject_data["parc_corr_plot"]), "{}/{}/parc_corr.png".format(reportdir, subject)) + copy("{}/{}".format(outdir, data["parc_corr_plot"]), "{}/{}/parc_corr.png".format(reportdir, subject)) # Logs if logsdir is None: - logsdir = os.path.join(src_dir._str, 'logs') + logsdir = os.path.join(outdir, 'logs') # guess the log file name g = glob(os.path.join(logsdir, f'{subject}*.log')) @@ -142,7 +131,6 @@ def gen_html_data(config, src_dir, subject, reportdir, logger=None, extra_funcs= else: data['log'] = log_file.read() - # Save data in the report directory pickle.dump(data, open(f"{reportdir}/{subject}/data.pkl", "wb")) @@ -293,7 +281,7 @@ def gen_html_summary(reportdir, logsdir=None): # log files if logsdir is None: - logsdir = reportdir._str.replace('report', 'logs') + logsdir = reportdir._str.replace('src_report', 'logs') if os.path.exists(os.path.join(logsdir, 'osl_batch.log')): with open(os.path.join(logsdir, 'osl_batch.log'), 'r') as log_file: @@ -466,9 +454,11 @@ def add_to_data(data_file, info): info : dict Info to add. """ - if Path(data_file).exists(): + data_file = Path(data_file) + if data_file.exists(): data = pickle.load(open(data_file, "rb")) else: + data_file.parent.mkdir(parents=True) data = {} data.update(info) pickle.dump(data, open(data_file, "wb")) diff --git a/osl/source_recon/batch.py b/osl/source_recon/batch.py index 4ea641c3..8a4ad588 100644 --- a/osl/source_recon/batch.py +++ b/osl/source_recon/batch.py @@ -90,7 +90,7 @@ def find_func(method, extra_funcs): def run_src_chain( config, - src_dir, + outdir, subject, preproc_file=None, smri_file=None, @@ -108,7 +108,7 @@ def run_src_chain( ---------- config : str or dict Source reconstruction config. - src_dir : str + outdir : str Source reconstruction directory. subject : str Subject name. @@ -139,15 +139,12 @@ def run_src_chain( rhino.fsl_utils.check_fsl() # Directories - src_dir = validate_outdir(src_dir) - logsdir = validate_outdir(logsdir or src_dir / "logs") - reportdir = validate_outdir(reportdir or src_dir / "report") + outdir = validate_outdir(outdir) + logsdir = validate_outdir(logsdir or outdir / "logs") + reportdir = validate_outdir(reportdir or outdir / "src_report") - # Get run ID - if preproc_file is None: - run_id = subject - else: - run_id = find_run_id(preproc_file) + # Use the subject ID for the run ID + run_id = subject # Generate log filename name_base = "{run_id}_{ftype}.{fext}" @@ -161,7 +158,7 @@ def run_src_chain( logger = logging.getLogger(__name__) now = strftime("%Y-%m-%d %H:%M:%S", localtime()) logger.info("{0} : Starting OSL Processing".format(now)) - logger.info("input : {0}".format(src_dir / subject)) + logger.info("input : {0}".format(outdir / subject)) # Load config if not isinstance(config, dict): @@ -189,7 +186,26 @@ def run_src_chain( + "Please pass via extra_funcs " + f"or use available functions: {avail_names}." ) - func(src_dir, subject, preproc_file, smri_file, epoch_file, **userargs) + def wrapped_func(**kwargs): + args, _, _, defaults = inspect.getargspec(func) + args_with_defaults = args[-len(defaults):] + kwargs_to_pass = {} + for a in args: + if a in kwargs: + kwargs_to_pass[a] = kwargs[a] + elif a not in args_with_defaults: + raise ValueError(f"{a} needs to be passed to {func.__name__}") + return func(**kwargs_to_pass) + wrapped_func( + outdir=outdir, + subject=subject, + preproc_file=preproc_file, + smri_file=smri_file, + epoch_file=epoch_file, + reportdir=reportdir, + logsdir=logsdir, + **userargs, + ) except Exception as e: logger.critical("*********************************") @@ -215,16 +231,15 @@ def run_src_chain( return False if gen_report: - # Generate data and individual HTML for the report - src_report.gen_html_data(config, src_dir, subject, reportdir, extra_funcs=extra_funcs) - src_report.gen_html_page(reportdir) + # Generate data and individual HTML data for the report + src_report.gen_html_data(config, outdir, subject, reportdir, extra_funcs=extra_funcs) return True def run_src_batch( config, - src_dir, + outdir, subjects, preproc_files=None, smri_files=None, @@ -243,7 +258,7 @@ def run_src_batch( ---------- config : str or dict Source reconstruction config. - src_dir : str + outdir : str Source reconstruction directory. subjects : list of str Subject names. @@ -277,9 +292,9 @@ def run_src_batch( rhino.fsl_utils.check_fsl() # Directories - src_dir = validate_outdir(src_dir) - logsdir = validate_outdir(logsdir or src_dir / "logs") - reportdir = validate_outdir(reportdir or src_dir / "report") + outdir = validate_outdir(outdir) + logsdir = validate_outdir(logsdir or outdir / "logs") + reportdir = validate_outdir(reportdir or outdir / "src_report") # Initialise Loggers mne.set_log_level(mneverbose) @@ -292,12 +307,50 @@ def run_src_batch( config_str = pprint.PrettyPrinter().pformat(config) logger.info('Running config\n {0}'.format(config_str)) - # Validation + # Number of files (subjects) to process n_subjects = len(subjects) - if preproc_files is not None: - n_preproc_files = len(preproc_files) - if n_subjects != n_preproc_files: - raise ValueError(f"Got {n_subjects} subjects and {n_preproc_files} preproc_files.") + + # Validation + if preproc_files is not None and epoch_files is not None: + raise ValueError("Please pass either preproc_file or epoch_files, not both.") + + if preproc_files and epoch_files: + raise ValueError( + "Cannot pass both preproc_files=True and epoch_files=True. " + "Please only pass one of these." + ) + + if isinstance(preproc_files, list): + n_files = len(preproc_files) + if n_subjects != n_files: + raise ValueError(f"Got {n_subjects} subjects and {n_files} preproc_files.") + + elif isinstance(epoch_files, list): + n_files = len(epoch_files) + if n_subjects != n_files: + raise ValueError(f"Got {n_subjects} subjects and {n_files} epoch_files.") + + else: + # Check what files are in the output directory + preproc_files_list = [] + epoch_files_list = [] + for subject in subjects: + preproc_file = f"{outdir}/{subject}/{subject}_preproc-raw.fif" + epoch_file = f"{outdir}/{subject}/{subject}-epo.fif" + if os.path.exists(preproc_file) and os.path.exists(epoch_file): + if preproc_files is None and epoch_files is None: + raise ValueError( + "Both preproc and epoch fif files found. " + "Please pass preproc_files=True or epoch_files=True." + ) + elif os.path.exists(preproc_file): + preproc_files_list.append(preproc_file) + elif os.path.exists(epoch_file): + epoch_files_list.append(epoch_file) + if len(preproc_files_list) > 0: + preproc_files = preproc_files_list + elif len(epoch_files_list) > 0: + epoch_files = epochs_file_list doing_coreg = ( any(["compute_surfaces" in method for method in config["source_recon"]]) or @@ -329,7 +382,7 @@ def run_src_batch( # Loop through input files to generate arguments for run_coreg_chain args = [] for subject, preproc_file, smri_file, epoch_file, in zip(subjects, preproc_files, smri_files, epoch_files): - args.append((config, src_dir, subject, preproc_file, smri_file, epoch_file)) + args.append((config, outdir, subject, preproc_file, smri_file, epoch_file)) # Actually run the processes if dask_client: diff --git a/osl/source_recon/beamforming.py b/osl/source_recon/beamforming.py index e7d2f942..2e2d2a10 100644 --- a/osl/source_recon/beamforming.py +++ b/osl/source_recon/beamforming.py @@ -70,7 +70,7 @@ def get_beamforming_filenames(subjects_dir, subject): """ basedir = op.join(subjects_dir, subject, "beamform") if " " in basedir: - raise ValueError("subjects_dir/src_dir cannot contain spaces.") + raise ValueError("subjects_dir cannot contain spaces.") os.makedirs(basedir, exist_ok=True) filenames = { diff --git a/osl/source_recon/rhino/polhemus.py b/osl/source_recon/rhino/polhemus.py index f2a19696..d039ae12 100644 --- a/osl/source_recon/rhino/polhemus.py +++ b/osl/source_recon/rhino/polhemus.py @@ -310,14 +310,14 @@ def on_press(event): plt.show() -def remove_stray_headshape_points(src_dir, subject, nose=True): +def remove_stray_headshape_points(outdir, subject, nose=True): """Remove stray headshape points. Removes headshape points near the nose, on the neck or far away from the head. Parameters ---------- - src_dir : str + outdir : str Path to subjects directory. subject : str Subject directory name. @@ -326,7 +326,7 @@ def remove_stray_headshape_points(src_dir, subject, nose=True): Useful to remove these if we have defaced structurals or aren't extracting the nose from the structural. """ - filenames = get_coreg_filenames(src_dir, subject) + filenames = get_coreg_filenames(outdir, subject) # Load saved headshape and nasion files hs = np.loadtxt(filenames["polhemus_headshape_file"]) @@ -352,12 +352,12 @@ def remove_stray_headshape_points(src_dir, subject, nose=True): np.savetxt(filenames["polhemus_headshape_file"], hs) -def extract_polhemus_from_pos(src_dir, subject, filepath): +def extract_polhemus_from_pos(outdir, subject, filepath): """Saves fiducials/headshape from a pos file. Parameters ---------- - src_dir : str + outdir : str Subjects directory. subject : str Subject subdirectory/ID. @@ -369,7 +369,7 @@ def extract_polhemus_from_pos(src_dir, subject, filepath): """ # Get coreg filenames - filenames = get_coreg_filenames(src_dir, subject) + filenames = get_coreg_filenames(outdir, subject) # Load file if "{0}" in filepath: @@ -405,12 +405,12 @@ def extract_polhemus_from_pos(src_dir, subject, filepath): np.savetxt(filenames["polhemus_headshape_file"], polhemus_headshape) -def extract_polhemus_from_elc(src_dir, subject, filepath, remove_headshape_near_nose=False): +def extract_polhemus_from_elc(outdir, subject, filepath, remove_headshape_near_nose=False): """Saves fiducials/headshape from an elc file. Parameters ---------- - src_dir : str + outdir : str Subjects directory. subject : str Subject subdirectory/ID. @@ -424,7 +424,7 @@ def extract_polhemus_from_elc(src_dir, subject, filepath, remove_headshape_near_ """ # Get coreg filenames - filenames = get_coreg_filenames(src_dir, subject) + filenames = get_coreg_filenames(outdir, subject) # Load elc file if "{0}" in filepath: diff --git a/osl/source_recon/rhino/utils.py b/osl/source_recon/rhino/utils.py index 638f4af2..3d51ba6c 100644 --- a/osl/source_recon/rhino/utils.py +++ b/osl/source_recon/rhino/utils.py @@ -63,7 +63,7 @@ def get_rhino_files(subjects_dir, subject): # Base RHINO directory rhino_dir = op.join(subjects_dir, subject, "rhino") if " " in rhino_dir: - raise ValueError("subjects_dir/src_dir cannot contain spaces.") + raise ValueError("subjects_dir cannot contain spaces.") # Surfaces files surfaces_dir = op.join(rhino_dir, "surfaces") diff --git a/osl/source_recon/sign_flipping.py b/osl/source_recon/sign_flipping.py index efb8e585..865dfccc 100644 --- a/osl/source_recon/sign_flipping.py +++ b/osl/source_recon/sign_flipping.py @@ -291,12 +291,12 @@ def apply_flips_to_covariance(cov, flips, n_embeddings=1): return cov * flips -def apply_flips(src_dir, subject, flips, epoched=False): +def apply_flips(outdir, subject, flips, epoched=False): """Saves the sign flipped data. Parameters ---------- - src_dir : str + outdir : str Path to source reconstruction directory. subject : str Subject name/id. @@ -306,7 +306,7 @@ def apply_flips(src_dir, subject, flips, epoched=False): Are we performing sign flipping on parc-raw.fif (epoched=False) or parc-epo.fif files (epoched=True)? """ if epoched: - parc_file = op.join(src_dir, str(subject), "parc", "parc-epo.fif") + parc_file = op.join(outdir, str(subject), "parc", "parc-epo.fif") epochs = mne.read_epochs(parc_file, verbose=False) sflip_epochs = epochs.copy() sflip_epochs.load_data() @@ -318,13 +318,13 @@ def flip(data): sflip_epochs.apply_function(flip, picks=_get_parc_chans(epochs), channel_wise=False) # Save - outfile = op.join(src_dir, str(subject), "sflip_parc-epo.fif") + outfile = op.join(outdir, str(subject), str(subject) + "_sflip_parc-epo.fif") log_or_print(f"saving: {outfile}") sflip_epochs.save(outfile, overwrite=True) else: # Load parcellated data - parc_file = op.join(src_dir, str(subject), "parc", "parc-raw.fif") + parc_file = op.join(outdir, str(subject), "parc", "parc-raw.fif") raw = mne.io.read_raw_fif(parc_file, verbose=False) sflip_raw = raw.copy() sflip_raw.load_data() @@ -336,7 +336,7 @@ def flip(data): sflip_raw.apply_function(flip, picks=_get_parc_chans(raw), channel_wise=False) # Save - outfile = op.join(src_dir, str(subject), "sflip_parc-raw.fif") + outfile = op.join(outdir, str(subject), str(subject) + "_sflip_parc-raw.fif") log_or_print(f"saving: {outfile}") sflip_raw.save(outfile, overwrite=True) diff --git a/osl/source_recon/wrappers.py b/osl/source_recon/wrappers.py index 089dbece..44ce5d67 100644 --- a/osl/source_recon/wrappers.py +++ b/osl/source_recon/wrappers.py @@ -1,14 +1,7 @@ """Wrappers for source reconstruction. -This module contains the functions callable using a 'source_recon' section -of a config. - -All functions in this module accept the following arguments for consistency: - - func(src_dir, subject, preproc_file, smri_file, epoch_file, *userargs) - -Custom functions (i.e. functions passed via the extra_funcs argument) must -also conform to this. +This module contains the functions callable using a 'source_recon' +section of a config. """ # Authors: Chetan Gohil @@ -34,40 +27,36 @@ def extract_polhemus_from_info( - src_dir, + outdir, subject, preproc_file, - smri_file, - epoch_file, - **userargs, + include_eeg_as_headshape=False, + include_hpi_as_headshape=True, ): """Wrapper function to extract fiducials/headshape points. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. preproc_file : str Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. - userargs : keyword arguments - Keyword arguments to pass to - osl.source_recon.rhino.extract_polhemus_from_info. + include_eeg_as_headshape : bool, optional + Should we include EEG locations as headshape points? + include_hpi_as_headshape : bool, optional + Should we include HPI locations as headshape points? """ - filenames = rhino.get_coreg_filenames(src_dir, subject) + filenames = rhino.get_coreg_filenames(outdir, subject) rhino.extract_polhemus_from_info( fif_file=preproc_file, headshape_outfile=filenames["polhemus_headshape_file"], nasion_outfile=filenames["polhemus_nasion_file"], rpa_outfile=filenames["polhemus_rpa_file"], lpa_outfile=filenames["polhemus_lpa_file"], - **userargs, + include_eeg_as_headshape=include_eeg_as_headshape, + include_hpi_as_headshape=include_hpi_as_headshape, ) @@ -77,62 +66,34 @@ def extract_fiducials_from_fif(*args, **kwargs): extract_polhemus_from_info(*args, **kwargs) -def remove_stray_headshape_points( - src_dir, - subject, - preproc_file, - smri_file, - epoch_file, - nose=True, -): +def remove_stray_headshape_points(outdir, subject, nose=True): """Remove stray headshape points. This function removes headshape points on the nose, neck and far from the head. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. Not used. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. noise : bool, optional Should we remove headshape points near the nose? Useful to remove these if we have defaced structurals or aren't extracting the nose from the structural. """ - rhino.remove_stray_headshape_points(src_dir, subject, nose=nose) + rhino.remove_stray_headshape_points(outdir, subject, nose=nose) -def save_mni_fiducials( - src_dir, - subject, - preproc_file, - smri_file, - epoch_file, - filepath, -): +def save_mni_fiducials(outdir, subject, filepath): """Wrapper to save MNI fiducials. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. Not used. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. filepath : str Full path to the text file containing the fiducials. @@ -151,7 +112,7 @@ def save_mni_fiducials( The order of the coordinates is the same as given in FSLeyes. """ - filenames = rhino.get_coreg_filenames(src_dir, subject) + filenames = rhino.get_coreg_filenames(outdir, subject) if "{0}" in filepath: fiducials_file = filepath.format(subject) else: @@ -164,44 +125,27 @@ def save_mni_fiducials( ) -def extract_polhemus_from_pos( - src_dir, - subject, - preproc_file, - smri_file, - epoch_file, - filepath, -): +def extract_polhemus_from_pos(outdir, subject, filepath): """Wrapper to save polhemus data from a .pos file. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. Not used. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. filepath : str Full path to the pos file for this subject. Any reference to '{subject}' (or '{0}') is replaced by the subject ID. E.g. 'data/{subject}/meg/{subject}_headshape.pos' with subject='sub-001' becomes 'data/sub-001/meg/sub-001_headshape.pos'. """ - rhino.extract_polhemus_from_pos(src_dir, subject, filepath) + rhino.extract_polhemus_from_pos(outdir, subject, filepath) def extract_polhemus_from_elc( - src_dir, + outdir, subject, - preproc_file, - smri_file, - epoch_file, filepath, remove_headshape_near_nose=False, ): @@ -209,17 +153,10 @@ def extract_polhemus_from_elc( Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. Not used. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. filepath : str Full path to the elc file for this subject. Any reference to '{subject}' (or '{0}') is replaced by the subject ID. @@ -229,36 +166,31 @@ def extract_polhemus_from_elc( Should we remove any headshape points near the nose? """ rhino.extract_polhemus_from_elc( - src_dir, subject, filepath, remove_headshape_near_nose + outdir, subject, filepath, remove_headshape_near_nose ) def compute_surfaces( - src_dir, + outdir, subject, - preproc_file, smri_file, - epoch_file, include_nose=True, recompute_surfaces=False, do_mri2mniaxes_xform=True, use_qform=False, + reportdir=None, ): """Wrapper for computing surfaces. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. smri_file : str Path to the T1 weighted structural MRI file to use in source reconstruction. - epoch_file : str - Path to epoched preprocessed fif file. include_nose : bool, optional Should we include the nose when we're extracting the surfaces? recompute_surfaces : bool, optional @@ -271,6 +203,8 @@ def compute_surfaces( use_qform : bool, optional Should we replace the sform with the qform? Useful if the sform code is incompatible with OSL, but the qform is compatible. + reportdir : str, optional + Path to report directory. """ if smri_file == "standard": std_struct = "MNI152_T1_2mm.nii.gz" @@ -280,7 +214,7 @@ def compute_surfaces( # Compute surfaces already_computed = rhino.compute_surfaces( smri_file=smri_file, - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, include_nose=include_nose, recompute_surfaces=recompute_surfaces, @@ -290,40 +224,41 @@ def compute_surfaces( # Plot surfaces surface_plots = rhino.plot_surfaces( - src_dir, subject, include_nose, already_computed - ) - surface_plots = [s.replace(f"{src_dir}/", "") for s in surface_plots] - - # Save info for the report - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "compute_surfaces": True, - "include_nose": include_nose, - "do_mri2mniaxes_xform": do_mri2mniaxes_xform, - "use_qform": use_qform, - "surface_plots": surface_plots, - }, + outdir, subject, include_nose, already_computed ) + surface_plots = [s.replace(f"{outdir}/", "") for s in surface_plots] + + if reportdir is not None: + # Save info for the report + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "compute_surfaces": True, + "include_nose": include_nose, + "do_mri2mniaxes_xform": do_mri2mniaxes_xform, + "use_qform": use_qform, + "surface_plots": surface_plots, + }, + ) def coregister( - src_dir, + outdir, subject, preproc_file, smri_file, - epoch_file, use_nose=True, use_headshape=True, already_coregistered=False, allow_smri_scaling=False, n_init=1, + reportdir=None, ): """Wrapper for coregistration. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. @@ -332,8 +267,6 @@ def coregister( smri_file : str Path to the T1 weighted structural MRI file to use in source reconstruction. - epoch_file : str - Path to epoched preprocessed fif file. use_nose : bool, optional Should we use the nose in the coregistration? use_headshape : bool, optional @@ -350,11 +283,13 @@ def coregister( using a template sMRI that has not come from this subject. n_init : int, optional Number of initialisations for coregistration. + reportdir : str, optional + Path to report directory. """ # Run coregistration rhino.coreg( fif_file=preproc_file, - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, use_headshape=use_headshape, use_nose=use_nose, @@ -367,59 +302,51 @@ def coregister( if already_coregistered: fid_err = None else: - fid_err = rhino.coreg_metrics(subjects_dir=src_dir, subject=subject) + fid_err = rhino.coreg_metrics(subjects_dir=outdir, subject=subject) # Save plots - coreg_dir = rhino.get_coreg_filenames(src_dir, subject)["basedir"] + coreg_dir = rhino.get_coreg_filenames(outdir, subject)["basedir"] rhino.coreg_display( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, display_outskin_with_nose=False, filename=f"{coreg_dir}/coreg.html", ) - coreg_filename = f"{coreg_dir}/coreg.html".replace(f"{src_dir}/", "") - - # Save info for the report - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "coregister": True, - "use_headshape": use_headshape, - "use_nose": use_nose, - "already_coregistered": already_coregistered, - "allow_smri_scaling": allow_smri_scaling, - "n_init_coreg": n_init, - "fid_err": fid_err, - "coreg_plot": coreg_filename, - }, - ) + coreg_filename = f"{coreg_dir}/coreg.html".replace(f"{outdir}/", "") + + if reportdir is not None: + # Save info for the report + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "coregister": True, + "use_headshape": use_headshape, + "use_nose": use_nose, + "already_coregistered": already_coregistered, + "allow_smri_scaling": allow_smri_scaling, + "n_init_coreg": n_init, + "fid_err": fid_err, + "coreg_plot": coreg_filename, + }, + ) def forward_model( - src_dir, + outdir, subject, - preproc_file, - smri_file, - epoch_file, gridstep=8, model="Single Layer", eeg=False, + reportdir=None, ): """Wrapper for computing the forward model. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. - epoch_file : str - Path to epoched preprocessed fif file. gridstep : int, optional A grid will be constructed with the spacing given by ``gridstep`` in mm, generating a volume source space. @@ -430,26 +357,29 @@ def forward_model( 'Triple Layer' uses three layers (scalp, inner skull, brain/cortex) eeg : bool, optional Are we using EEG channels in the source reconstruction? + reportdir : str, optional + Path to report directory. """ # Compute forward model rhino.forward_model( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, model=model, gridstep=gridstep, eeg=eeg, ) - # Save info for the report - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "forward_model": True, - "model": model, - "gridstep": gridstep, - "eeg": eeg, - }, - ) + if reportdir is not None: + # Save info for the report + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "forward_model": True, + "model": model, + "gridstep": gridstep, + "eeg": eeg, + }, + ) # ------------------------------------- @@ -457,10 +387,9 @@ def forward_model( def beamform( - src_dir, + outdir, subject, preproc_file, - smri_file, epoch_file, chantypes, rank, @@ -468,20 +397,18 @@ def beamform( weight_norm="nai", pick_ori="max-power-pre-weight-norm", reg=0, + reportdir=None, ): """Wrapper function for beamforming. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. preproc_file : str Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. epoch_file : str Path to epoched preprocessed fif file. chantypes : str or list of str @@ -498,6 +425,8 @@ def beamform( Orientation of the dipoles. reg : float, optional The regularization for the whitened data covariance. + reportdir : str, optional + Path to report directory """ logger.info("beamforming") @@ -529,7 +458,7 @@ def beamform( logger.info(f"chantypes: {chantypes}") logger.info(f"rank: {rank}") filters = beamforming.make_lcmv( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, data=data, chantypes=chantypes, @@ -542,31 +471,31 @@ def beamform( # Make plots filters_cov_plot, filters_svd_plot = beamforming.make_plots( - src_dir, subject, filters, data - ) - filters_cov_plot = filters_cov_plot.replace(f"{src_dir}/", "") - filters_svd_plot = filters_svd_plot.replace(f"{src_dir}/", "") - - # Save info for the report - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "beamform": True, - "chantypes": chantypes, - "rank": rank, - "reg": reg, - "freq_range": freq_range, - "filters_cov_plot": filters_cov_plot, - "filters_svd_plot": filters_svd_plot, - }, + outdir, subject, filters, data ) + filters_cov_plot = filters_cov_plot.replace(f"{outdir}/", "") + filters_svd_plot = filters_svd_plot.replace(f"{outdir}/", "") + + if reportdir is not None: + # Save info for the report + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "beamform": True, + "chantypes": chantypes, + "rank": rank, + "reg": reg, + "freq_range": freq_range, + "filters_cov_plot": filters_cov_plot, + "filters_svd_plot": filters_svd_plot, + }, + ) def parcellate( - src_dir, + outdir, subject, preproc_file, - smri_file, epoch_file, parcellation_file, method, @@ -574,20 +503,18 @@ def parcellate( spatial_resolution=None, reference_brain="mni", extra_chans="stim", + reportdir=None, ): """Wrapper function for parcellation. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. preproc_file : str Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. epoch_file : str Path to epoched preprocessed fif file. parcellation_file : str @@ -613,11 +540,19 @@ def parcellate( Extra channels to include in the parc-raw.fif file. Defaults to 'stim'. Stim channels are always added to parc-raw.fif in addition to extra_chans. + reportdir : str, optional + Path to report directory. """ logger.info("parcellate") + if reportdir is None: + raise ValueError( + "This function can only be used then a report was generated " + "when beamforming. Please use beamform_and_parcellate." + ) + # Get settings passed to the beamform wrapper - report_data = pickle.load(open(f"{src_dir}/{subject}/report_data.pkl", "rb")) + report_data = pickle.load(open(f"{reportdir}/{subject}/data.pkl", "rb")) freq_range = report_data.pop("freq_range") chantypes = report_data.pop("chantypes") if isinstance(chantypes, str): @@ -644,7 +579,7 @@ def parcellate( chantype_data = data.copy().pick(chantypes) # Load beamforming filter and apply - filters = beamforming.load_lcmv(src_dir, subject) + filters = beamforming.load_lcmv(outdir, subject) bf_data = beamforming.apply_lcmv(chantype_data, filters) if epoch_file is not None: @@ -652,7 +587,7 @@ def parcellate( else: bf_data = bf_data.data bf_data_mni, _, coords_mni, _ = beamforming.transform_recon_timeseries( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, recon_timeseries=bf_data, spatial_resolution=spatial_resolution, @@ -666,7 +601,7 @@ def parcellate( voxel_timeseries=bf_data_mni, voxel_coords=coords_mni, method=method, - working_dir=f"{src_dir}/{subject}/parc", + working_dir=f"{outdir}/{subject}/parc", ) # Orthogonalisation @@ -681,7 +616,7 @@ def parcellate( if epoch_file is None: # Save parcellated data as a MNE Raw object - parc_fif_file = f"{src_dir}/{subject}/parc/parc-raw.fif" + parc_fif_file = f"{outdir}/{subject}/parc/parc-raw.fif" logger.info(f"saving {parc_fif_file}") parc_raw = parcellation.convert2mne_raw( parcel_data, data, extra_chans=extra_chans @@ -689,7 +624,7 @@ def parcellate( parc_raw.save(parc_fif_file, overwrite=True) else: # Save parcellated data as a MNE Epochs object - parc_fif_file = f"{src_dir}/{subject}/parc/parc-epo.fif" + parc_fif_file = f"{outdir}/{subject}/parc/parc-epo.fif" logger.info(f"saving {parc_fif_file}") parc_epo = parcellation.convert2mne_epochs(parcel_data, data) parc_epo.save(parc_fif_file, overwrite=True) @@ -701,10 +636,10 @@ def parcellate( fs=data.info["sfreq"], freq_range=freq_range, parcellation_file=parcellation_file, - filename=f"{src_dir}/{parc_psd_plot}", + filename=f"{outdir}/{parc_psd_plot}", ) parc_corr_plot = f"{subject}/parc/corr.png" - parcellation.plot_correlation(parcel_data, filename=f"{src_dir}/{parc_corr_plot}") + parcellation.plot_correlation(parcel_data, filename=f"{outdir}/{parc_corr_plot}") # Save info for the report n_parcels = parcel_data.shape[0] @@ -714,7 +649,7 @@ def parcellate( else: n_epochs = None src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", + f"{reportdir}/{subject}/data.pkl", { "parcellate": True, "parcellation_file": parcellation_file, @@ -731,10 +666,9 @@ def parcellate( def beamform_and_parcellate( - src_dir, + outdir, subject, preproc_file, - smri_file, epoch_file, chantypes, rank, @@ -748,20 +682,18 @@ def beamform_and_parcellate( spatial_resolution=None, reference_brain="mni", extra_chans="stim", + reportdir=None, ): """Wrapper function for beamforming and parcellation. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. preproc_file : str Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. epoch_file : str Path to epoched preprocessed fif file. chantypes : str or list of str @@ -802,6 +734,8 @@ def beamform_and_parcellate( Extra channels to include in the parc-raw.fif file. Defaults to 'stim'. Stim channels are always added to parc-raw.fif in addition to extra_chans. + reportdir : str, optional + Path to report directory. """ logger.info("beamform_and_parcellate") @@ -833,7 +767,7 @@ def beamform_and_parcellate( logger.info(f"chantypes: {chantypes}") logger.info(f"rank: {rank}") filters = beamforming.make_lcmv( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, data=data, chantypes=chantypes, @@ -846,10 +780,10 @@ def beamform_and_parcellate( # Make plots filters_cov_plot, filters_svd_plot = beamforming.make_plots( - src_dir, subject, filters, chantype_data + outdir, subject, filters, chantype_data ) - filters_cov_plot = filters_cov_plot.replace(f"{src_dir}/", "") - filters_svd_plot = filters_svd_plot.replace(f"{src_dir}/", "") + filters_cov_plot = filters_cov_plot.replace(f"{outdir}/", "") + filters_svd_plot = filters_svd_plot.replace(f"{outdir}/", "") # Apply beamforming bf_data = beamforming.apply_lcmv(chantype_data, filters) @@ -859,7 +793,7 @@ def beamform_and_parcellate( else: bf_data = bf_data.data bf_data_mni, _, coords_mni, _ = beamforming.transform_recon_timeseries( - subjects_dir=src_dir, + subjects_dir=outdir, subject=subject, recon_timeseries=bf_data, spatial_resolution=spatial_resolution, @@ -874,7 +808,7 @@ def beamform_and_parcellate( voxel_timeseries=bf_data_mni, voxel_coords=coords_mni, method=method, - working_dir=f"{src_dir}/{subject}/parc", + working_dir=f"{outdir}/{subject}/parc", ) # Orthogonalisation @@ -889,7 +823,7 @@ def beamform_and_parcellate( if epoch_file is None: # Save parcellated data as a MNE Raw object - parc_fif_file = f"{src_dir}/{subject}/parc/parc-raw.fif" + parc_fif_file = f"{outdir}/{subject}/parc/parc-raw.fif" logger.info(f"saving {parc_fif_file}") parc_raw = parcellation.convert2mne_raw( parcel_data, data, extra_chans=extra_chans @@ -897,7 +831,7 @@ def beamform_and_parcellate( parc_raw.save(parc_fif_file, overwrite=True) else: # Save parcellated data as a MNE Epochs object - parc_fif_file = f"{src_dir}/{subject}/parc/parc-epo.fif" + parc_fif_file = f"{outdir}/{subject}/parc/parc-epo.fif" logger.info(f"saving {parc_fif_file}") parc_epo = parcellation.convert2mne_epochs(parcel_data, data) parc_epo.save(parc_fif_file, overwrite=True) @@ -909,41 +843,42 @@ def beamform_and_parcellate( fs=data.info["sfreq"], freq_range=freq_range, parcellation_file=parcellation_file, - filename=f"{src_dir}/{parc_psd_plot}", + filename=f"{outdir}/{parc_psd_plot}", ) parc_corr_plot = f"{subject}/parc/corr.png" - parcellation.plot_correlation(parcel_data, filename=f"{src_dir}/{parc_corr_plot}") - - # Save info for the report - n_parcels = parcel_data.shape[0] - n_samples = parcel_data.shape[1] - if parcel_data.ndim == 3: - n_epochs = parcel_data.shape[2] - else: - n_epochs = None - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "beamform_and_parcellate": True, - "beamform": True, - "parcellate": True, - "chantypes": chantypes, - "rank": rank, - "reg": reg, - "freq_range": freq_range, - "filters_cov_plot": filters_cov_plot, - "filters_svd_plot": filters_svd_plot, - "parcellation_file": parcellation_file, - "method": method, - "orthogonalisation": orthogonalisation, - "parc_fif_file": str(parc_fif_file), - "n_samples": n_samples, - "n_parcels": n_parcels, - "n_epochs": n_epochs, - "parc_psd_plot": parc_psd_plot, - "parc_corr_plot": parc_corr_plot, - }, - ) + parcellation.plot_correlation(parcel_data, filename=f"{outdir}/{parc_corr_plot}") + + if reportdir is not None: + # Save info for the report + n_parcels = parcel_data.shape[0] + n_samples = parcel_data.shape[1] + if parcel_data.ndim == 3: + n_epochs = parcel_data.shape[2] + else: + n_epochs = None + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "beamform_and_parcellate": True, + "beamform": True, + "parcellate": True, + "chantypes": chantypes, + "rank": rank, + "reg": reg, + "freq_range": freq_range, + "filters_cov_plot": filters_cov_plot, + "filters_svd_plot": filters_svd_plot, + "parcellation_file": parcellation_file, + "method": method, + "orthogonalisation": orthogonalisation, + "parc_fif_file": str(parc_fif_file), + "n_samples": n_samples, + "n_parcels": n_parcels, + "n_epochs": n_epochs, + "parc_psd_plot": parc_psd_plot, + "parc_corr_plot": parc_corr_plot, + }, + ) # ---------------------- @@ -951,7 +886,7 @@ def beamform_and_parcellate( def find_template_subject( - src_dir, + outdir, subjects, n_embeddings=1, standardize=True, @@ -960,12 +895,12 @@ def find_template_subject( """Function to find a good subject to align other subjects to in the sign flipping. Note, this function expects parcellated data to exist in the following - location: src_dir/*/parc/parc-*.fif, the * here represents subject + location: outdir/*/parc/parc-*.fif, the * here represents subject directories or 'raw' vs 'epo'. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subjects : str Subjects to include. @@ -988,9 +923,9 @@ def find_template_subject( parc_files = [] for subject in subjects: if epoched: - parc_file = f"{src_dir}/{subject}/parc/parc-epo.fif" + parc_file = f"{outdir}/{subject}/parc/parc-epo.fif" else: - parc_file = f"{src_dir}/{subject}/parc/parc-raw.fif" + parc_file = f"{outdir}/{subject}/parc/parc-raw.fif" if Path(parc_file).exists(): parc_files.append(parc_file) else: @@ -1016,11 +951,9 @@ def find_template_subject( def fix_sign_ambiguity( - src_dir, + outdir, subject, preproc_file, - smri_file, - epoch_file, template, n_embeddings, standardize, @@ -1028,22 +961,18 @@ def fix_sign_ambiguity( n_iter, max_flips, epoched=False, + reportdir=None, ): """Wrapper function for fixing the dipole sign ambiguity. Parameters ---------- - src_dir : str + outdir : str Path to where to output the source reconstruction files. subject : str Subject name/id. preproc_file : str Path to the preprocessed fif file. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. - epoch_file : str - Path to epoched preprocessed fif file. template : str Template subject. n_embeddings : int @@ -1059,6 +988,8 @@ def fix_sign_ambiguity( epoched : bool, optional Are we performing sign flipping on parc-raw.fif (epoched=False) or parc-epo.fif files (epoched=True)? + reportdir : str, optional + Path to report directory. """ logger.info("fix_sign_ambiguity") logger.info(f"using template: {template}") @@ -1067,9 +998,9 @@ def fix_sign_ambiguity( parc_files = [] for sub in [subject, template]: if epoched: - parc_file = f"{src_dir}/{sub}/parc/parc-epo.fif" + parc_file = f"{outdir}/{sub}/parc/parc-epo.fif" else: - parc_file = f"{src_dir}/{sub}/parc/parc-raw.fif" + parc_file = f"{outdir}/{sub}/parc/parc-raw.fif" if not Path(parc_file).exists(): raise ValueError(f"{parc_file} not found") parc_files.append(parc_file) @@ -1091,54 +1022,41 @@ def fix_sign_ambiguity( ) # Apply flips to the parcellated data - sign_flipping.apply_flips(src_dir, subject, flips, epoched=epoched) - - # Save info for the report - src_report.add_to_data( - f"{src_dir}/{subject}/report_data.pkl", - { - "fix_sign_ambiguity": True, - "template": template, - "n_embeddings": n_embeddings, - "standardize": standardize, - "n_init": n_init, - "n_iter": n_iter, - "max_flips": max_flips, - "metrics": metrics, - }, - ) + sign_flipping.apply_flips(outdir, subject, flips, epoched=epoched) + + if reportdir is not None: + # Save info for the report + src_report.add_to_data( + f"{reportdir}/{subject}/data.pkl", + { + "fix_sign_ambiguity": True, + "template": template, + "n_embeddings": n_embeddings, + "standardize": standardize, + "n_init": n_init, + "n_iter": n_iter, + "max_flips": max_flips, + "metrics": metrics, + }, + ) # -------------- # Other wrappers -def extract_rhino_files( - src_dir, - subject, - preproc_file, - smri_file, - epoch_file, - old_src_dir, -): +def extract_rhino_files(outdir, subject, old_outdir): """Wrapper function for extracting RHINO files from a previous run. Parameters ---------- - src_dir : str + outdir : str Path to the NEW source reconstruction directory. subject : str Subject name/id. - preproc_file : str - Path to the preprocessed fif file. Not used. - smri_file : str - Path to the T1 weighted structural MRI file to use in source - reconstruction. Not used. - epoch_file : str - Path to epoched preprocessed fif file. Not used. - old_src_dir : str + old_outdir : str OLD source reconstruction directory to copy RHINO files to. """ rhino.utils.extract_rhino_files( - old_src_dir, src_dir, subjects=subject, gen_report=False + old_outdir, outdir, subjects=subject, gen_report=False ) diff --git a/osl/utils/file_handling.py b/osl/utils/file_handling.py index ea810dd4..96bed6ba 100644 --- a/osl/utils/file_handling.py +++ b/osl/utils/file_handling.py @@ -84,7 +84,7 @@ def process_file_inputs(inputs): if check_paths: #infiles = [sanitise_filepath(f) for f in infiles] for idx, fif in enumerate(infiles): - if fif.endswith('.ds'): + if fif.endswith('.ds') or fif.endswith('.mff'): good_files[idx] = int(os.path.isdir(fif)) else: good_files[idx] = int(os.path.isfile(fif)) @@ -189,6 +189,8 @@ def get_rawdir(files): def add_subdir(file, outdir, run_id=None): + """Add sub-directory.""" + if not type(outdir) == str: outdir = str(outdir) if '{' in outdir and '}' in outdir: