New output directory structure (#320)

* Updated output directory structure for preprocessing.
* Updated variable names in source recon.
* Make ica label compatible with new data structure
* Added option to load new EEG file type.
* Fixed file not found bug with .mff directories.
* Passing preproc_files in run_src_batch now optional.
* More flexible function definitions.
* Also pass logsdir to custom source recon functions.
* Refacted source recon wrapper (possible due to the more flexible
args/kwargs)
* Rename: _preproc_raw -> _preproc-raw.
* Updated examples.

---------

Co-authored-by: matsvanes <mats.vanes@psych.ox.ac.uk>

examples/ctf/ctf_with_headshape_pos/1_preprocess.py

@@ -24,13 +24,20 @@
     "data/raw/Nottingham/sub-not002/meg/sub-not002_task-resteyesopen_meg.ds",
 ]
 
+# Subject IDs
+subjects = [
+    "sub-not001_task-resteyesopen",
+    "sub-not002_task-resteyesopen",
+]
+
 # Directory to save output to
-outdir = "data/preproc"
+outdir = "data"
 
 # Do preprocessing
 preprocessing.run_proc_batch(
     config,
     inputs,
+    subjects=subjects,
     outdir=outdir,
     overwrite=True,
 )

examples/ctf/ctf_with_headshape_pos/2_coregister.py

@@ -9,15 +9,15 @@
 
 from osl import source_recon, utils
 
-def save_polhemus_from_pos(src_dir, subject, preproc_file, smri_file, epoch_file):
+def save_polhemus_from_pos(outdir, subject):
     """Saves fiducials/headshape from a pos file."""
 
     # Get path to pos file
     pos_file = f"data/raw/Nottingham/{subject}/meg/{subject}_headshape.pos"
     utils.logger.log_or_print(f"Saving polhemus info from {pos_file}")
 
     # Get coreg filenames
-    filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject)
+    filenames = source_recon.rhino.get_coreg_filenames(outdir, subject)
 
     # Load in txt file, these values are in cm in polhemus space:
     num_headshape_pnts = int(pd.read_csv(pos_file, header=None).to_numpy()[0])
@@ -53,11 +53,11 @@ def save_polhemus_from_pos(src_dir, subject, preproc_file, smri_file, epoch_file
     np.savetxt(filenames["polhemus_lpa_file"], polhemus_lpa)
     np.savetxt(filenames["polhemus_headshape_file"], polhemus_headshape)
 
-def fix_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file):
+def fix_headshape_points(outdir, subject, preproc_file, smri_file, epoch_file):
     """Remove headshape points on the nose and neck."""
 
     # Load saved headshape and nasion files
-    filenames = source_recon.rhino.get_coreg_filenames(src_dir, subject)
+    filenames = source_recon.rhino.get_coreg_filenames(outdir, subject)
     hs = np.loadtxt(filenames["polhemus_headshape_file"])
     nas = np.loadtxt(filenames["polhemus_nasion_file"])
     lpa = np.loadtxt(filenames["polhemus_lpa_file"])
@@ -100,27 +100,30 @@ def fix_headshape_points(src_dir, subject, preproc_file, smri_file, epoch_file):
 """
 
 # Subject IDs
-subjects = ["sub-not001", "sub-not002"]
+subjects = [
+    "sub-not001_task-resteyesopen",
+    "sub-not002_task-resteyesopen",
+]
 
 # Fif files containing the sensor-level preprocessed data for each subject
 preproc_files = [
-    "data/preproc/sub-not001_task-resteyesopen_meg/sub-not001_task-resteyesopen_meg_preproc_raw.fif",
-    "data/preproc/sub-not002_task-resteyesopen_meg/sub-not002_task-resteyesopen_meg_preproc_raw.fif",
+    "data/sub-not001_task-resteyesopen/sub-not001_task-resteyesopen_preproc-raw.fif",
+    "data/sub-not002_task-resteyesopen/sub-not002_task-resteyesopen_preproc-raw.fif",
 ]
 
 # The corresponding structurals for each subject
 smri_files = [
-    "data/smri/sub-not001_T1w.nii.gz",
-    "data/smri/sub-not002_T1w.nii.gz",
+    "smri/sub-not001_T1w.nii.gz",
+    "smri/sub-not002_T1w.nii.gz",
 ]
 
 # Directory to save output to
-coreg_dir = "data/coreg"
+outdir = "data"
 
 # Source reconstruction
 source_recon.run_src_batch(
     config,
-    src_dir=coreg_dir,
+    outdir=outdir,
     subjects=subjects,
     preproc_files=preproc_files,
     smri_files=smri_files,

examples/ctf/ctf_with_headshape_pos/3_source_reconstruct.py

@@ -4,22 +4,8 @@
 
 # Authors: Chetan Gohil <chetan.gohil@psych.ox.ac.uk>
 
-import os
-
 from osl import source_recon
 
-# Directories
-coreg_dir = "data/coreg"
-src_dir = "data/src"
-
-# First copy the coregistration directory
-if os.path.exists(src_dir):
-    print(f"Please first delete: {src_dir}")
-    exit()
-cmd = f"cp -r {coreg_dir} {src_dir}"
-print(cmd)
-os.system(cmd)
-
 # Settings
 config = """
     source_recon:
@@ -33,18 +19,24 @@
 """
 
 # Subject IDs
-subjects = ["sub-not001", "sub-not002"]
+subjects = [
+    "sub-not001_task-resteyesopen",
+    "sub-not002_task-resteyesopen",
+]
 
 # Fif files containing the sensor-level preprocessed data for each subject
 preproc_files = [
-    "data/preproc/sub-not001_task-resteyesopen_meg/sub-not001_task-resteyesopen_meg_preproc_raw.fif",
-    "data/preproc/sub-not002_task-resteyesopen_meg/sub-not002_task-resteyesopen_meg_preproc_raw.fif",
+    "data/sub-not001_task-resteyesopen/sub-not001_task-resteyesopen_preproc-raw.fif",
+    "data/sub-not002_task-resteyesopen/sub-not002_task-resteyesopen_preproc-raw.fif",
 ]
 
+# Directory to save output to
+outdir = "data"
+
 # Source reconstruction
 source_recon.run_src_batch(
     config,
-    src_dir=src_dir,
+    outdir=outdir,
     subjects=subjects,
     preproc_files=preproc_files,
 )

examples/ctf/ctf_with_headshape_pos/4_sign_flip.py

@@ -13,12 +13,15 @@
 from osl import source_recon
 
 # Source directory and subjects to sign flip
-src_dir = "data/src"
-subjects = ["sub-not001", "sub-not002"]
+outdir = "data"
+subjects = [
+    "sub-not001_task-resteyesopen",
+    "sub-not002_task-resteyesopen",
+]
 
 # Find a good template subject to align other subjects to
 template = source_recon.find_template_subject(
-    src_dir, subjects, n_embeddings=15, standardize=True
+    outdir, subjects, n_embeddings=15, standardize=True
 )
 
 # Settings
@@ -34,4 +37,4 @@
 """
 
 # Do the sign flipping
-source_recon.run_src_batch(config, src_dir, subjects)
+source_recon.run_src_batch(config, outdir, subjects)

examples/ctf/ctf_with_headshape_pos/5_save_npy.py

@@ -8,7 +8,7 @@
 
 from osl_dynamics.data import Data
 
-files = sorted(glob("data/src/*/sflip_parc-raw.fif"))
+files = sorted(glob("data/*/*_sflip_parc-raw.fif"))
 data = Data(
     files,
     picks="misc",

examples/ctf/ctf_with_smri_fid/1_preprocess.py

@@ -17,13 +17,17 @@
 # Create a list of paths to files to preprocess
 inputs = ["data/raw/mg04938_BrainampDBS_20170504_01_raw.fif"]
 
+# Subject IDs
+subjects = ["LN_VTA2"]
+
 # Directory to save output to
-outdir = "data/preproc"
+outdir = "data"
 
 # Do preprocessing
 preprocessing.run_proc_batch(
     config,
     inputs,
+    subjects=subjects,
     outdir=outdir,
     overwrite=True,
 )

examples/ctf/ctf_with_smri_fid/2_coregister.py

@@ -25,18 +25,17 @@
 subjects = ["LN_VTA2"]
 
 # Lists for input files
-preproc_files = ["data/preproc/mg04938_BrainampDBS_20170504_01_preproc_raw.fif"]
-smri_files = ["data/smri/LN_VTA2.nii"]
+preproc_files = ["data/LN_VTA2/mg04938_BrainampDBS_20170504_01_preproc-raw.fif"]
+smri_files = ["smri/LN_VTA2.nii"]
 
 # Output directory
-coreg_dir = "data/coreg"
+outdir = "data"
 
 # Do coregistration
 source_recon.run_src_batch(
     config,
-    src_dir=coreg_dir,
+    outdir=outdir,
     subjects=subjects,
     preproc_files=preproc_files,
     smri_files=smri_files,
-    extra_funcs=[save_mni_fids],
 )

examples/ctf/ctf_with_smri_fid/3_source_reconstruct.py

@@ -4,22 +4,8 @@
 
 # Authors: Chetan Gohil <chetan.gohil@psych.ox.ac.uk>
 
-import os
-
 from osl import source_recon
 
-# Directories
-coreg_dir = "data/coreg"
-src_dir = "data/src"
-
-# First copy the coregistration directory
-if os.path.exists(src_dir):
-    print(f"Please first delete: {src_dir}")
-    exit()
-cmd = f"cp -r {coreg_dir} {src_dir}"
-print(cmd)
-os.system(cmd)
-
 # Settings
 config = """
     source_recon:
@@ -38,12 +24,15 @@
 subjects = ["LN_VTA2"]
 
 # Fif files containing the sensor-level preprocessed data for each subject
-preproc_files = ["data/preproc/mg04938_BrainampDBS_20170504_01_preproc_raw.fif"]
+preproc_files = ["data/LN_VTA2/mg04938_BrainampDBS_20170504_01_preproc-raw.fif"]
+
+# Directories
+outdir = "data"
 
 # Source reconstruction
 source_recon.run_src_batch(
     config,
-    src_dir=src_dir,
+    outdir=outdir,
     subjects=subjects,
     preproc_files=preproc_files,
 )

examples/ctf/ctf_with_smri_fid/4_sign_flip.py

@@ -26,7 +26,7 @@ def load(filename):
 
 
 # Files to sign flip
-files = sorted(glob("data/src/*/parc/parc-raw.fif"))
+files = sorted(glob("data/*/parc/parc-raw.fif"))
 
 # Get covariance matrices
 covs = load_covariances(

examples/elekta/1_maxfilter.py

@@ -9,12 +9,12 @@
 
 # Setup paths to raw (pre-maxfiltered) fif files
 input_files = [
-    "data/raw/file1.fif",
-    "data/raw/file2.fif",
+    "raw/file1.fif",
+    "raw/file2.fif",
 ]
 
 # Directory to save the maxfiltered data to
-output_directory = "data/maxfilter"
+output_directory = "maxfilter"
 
 # Run MaxFiltering
 #

examples/elekta/2_preprocess.py

@@ -10,8 +10,8 @@
 from osl import preprocessing, utils
 
 # Files and directories
-raw_file = "data/maxfilter/{subject}_tsss.fif"  # {subject} will be replace by the name for the subject
-preproc_dir = "data/preproc"  # output directory containing the preprocess files
+raw_file = "maxfilter/{subject}_tsss.fif"  # {subject} will be replace by the name for the subject
+outdir = "data"
 
 subjects = ["sub-001", "sub-002"]
 
@@ -50,7 +50,8 @@
     preprocessing.run_proc_batch(
         config,
         inputs,
-        outdir=preproc_dir,
+        subjects=subjects,
+        outdir=outdir,
         overwrite=True,
         dask_client=True,
     )

examples/elekta/3_coregister.py

@@ -17,13 +17,12 @@
 from osl import source_recon, utils
 
 # Directories
-preproc_dir = "data/preproc"
-anat_dir = "data/smri"
-coreg_dir = "data/coreg"
+outdir = "data"
+anatdir = "smri"
 
 # Files ({subject} will be replaced by the name for the subject)
-preproc_file = preproc_dir + "/{subject}_tsss_preproc_raw.fif"
-smri_file = anat_dir + "/{subject}/anat/{subject}_T1w.nii"
+preproc_file = outdir + "{subject}/{subject}_tsss_preproc-raw.fif"
+smri_file = anatdir + "/{subject}/anat/{subject}_T1w.nii"
 
 # Subjects to coregister
 subjects = ["sub-001", "sub-002"]
@@ -60,7 +59,7 @@
     # Run coregistration
     source_recon.run_src_batch(
         config,
-        src_dir=coreg_dir,
+        outdir=outdir,
         subjects=subjects,
         preproc_files=preproc_files,
         smri_files=smri_files,

examples/elekta/4_source_reconstruct.py

@@ -13,12 +13,10 @@
 from osl import source_recon, utils
 
 # Directories
-preproc_dir = "data/preproc"
-coreg_dir = "data/coreg"
-src_dir = "data/src"
+outdir = "data"
 
 # Files
-preproc_file = preproc_dir + "/{subject}_tsss_preproc_raw.fif"  # {subject} will be replaced by the subject name
+preproc_file = outdir + "/{subject}_tsss_preproc-raw.fif"  # {subject} will be replaced by the subject name
 
 # Subjects to do
 subjects = ["sub-001", "sub-002"]
@@ -40,12 +38,6 @@
 if __name__ == "__main__":
     utils.logger.set_up(level="INFO")
 
-    # Copy directory containing the coregistration
-    if not os.path.exists(src_dir):
-        cmd = f"cp -r {coreg_dir} {src_dir}"
-        print(cmd)
-        os.system(cmd)
-
     # Get paths to files
     preproc_files = []
     for subject in subjects:
@@ -60,7 +52,7 @@
     # Source reconstruction
     source_recon.run_src_batch(
         config,
-        src_dir=src_dir,
+        outdir=outdir,
         subjects=subjects,
         preproc_files=preproc_files,
         dask_client=True,

examples/elekta/5_sign_flip.py

@@ -11,15 +11,15 @@
 from osl.source_recon import find_template_subject, run_src_batch
 
 # Directories
-src_dir = "data/src"
+outdir = "data"
 
 if __name__ == "__main__":
     utils.logger.set_up(level="INFO")
 
     # Subjects to sign flip
     # We create a list by looking for subjects that have a parc/parc-raw.fif file
     subjects = []
-    for path in sorted(glob(src_dir + "/*/parc/parc-raw.fif")):
+    for path in sorted(glob(f"{outdir}/*/parc/parc-raw.fif")):
         subject = path.split("/")[-3]
         subjects.append(subject)
 
@@ -49,7 +49,7 @@
     # Do the sign flipping
     run_src_batch(
         config,
-        src_dir=src_dir,
+        outdir=outdir,
         subjects=subjects,
         dask_client=True,
     )