From 37b11ec298c4d05b625b03b380a0bbda3ea96dee Mon Sep 17 00:00:00 2001
From: Taylor Salo <tsalo006@fiu.edu>
Date: Tue, 16 Apr 2024 16:43:09 -0400
Subject: [PATCH 01/13] Add `descriptions.tsv` to the schema (#1707)

* Add descriptions.tsv to schema.

* Fix up column names.

* Keep working on schema.

* Fix mistakes and add template.

* Update src/derivatives/common-data-types.md

Co-authored-by: Chris Markiewicz <effigies@gmail.com>

* Make description column generic.

* Remove duplicate file definition.

descriptions is a suffix, rather than a file.

---------

Co-authored-by: Chris Markiewicz <effigies@gmail.com>
---
 src/derivatives/common-data-types.md          | 31 ++++++++++++-------
 src/schema/objects/columns.yaml               | 21 +++++++++++++
 src/schema/objects/suffixes.yaml              | 12 +++++++
 src/schema/rules/files/deriv/tables.yaml      | 11 +++++++
 .../derivatives/common_derivatives.yaml       | 16 ++++++++++
 5 files changed, 80 insertions(+), 11 deletions(-)
 create mode 100644 src/schema/rules/files/deriv/tables.yaml

diff --git a/src/derivatives/common-data-types.md b/src/derivatives/common-data-types.md
index 55ad6e3bf1..5a2e725e91 100644
--- a/src/derivatives/common-data-types.md
+++ b/src/derivatives/common-data-types.md
@@ -256,24 +256,33 @@ static volume, a `RepetitionTime` property would no longer be relevant).
 
 ## descriptions.tsv
 
+Template:
+
+```Text
+[sub-<label>/]
+    [ses-<label>/]
+        [sub-<label>_][ses-<label>_]descriptions.tsv
+        [sub-<label>_][ses-<label>_]descriptions.json
+```
+
+Optional: Yes
+
 To keep a record of processing steps applied to the data, a `descriptions.tsv` file MAY be used.
-The `descriptions.tsv` file MUST contain at least the following two columns:
+The `descriptions.tsv` file consists of one row for each unique `desc-<label>`
+entity used in the dataset and a set of REQUIRED and OPTIONAL columns:
 
--   `desc_id`
--   `description`
+<!-- This block generates a columns table.
+The definitions of these fields can be found in
+  src/schema/rules/tabular_data/*.yaml
+and a guide for using macros can be found at
+ https://github.com/bids-standard/bids-specification/blob/master/macros_doc.md
+-->
+{{ MACROS___make_columns_table("derivatives.common_derivatives.Descriptions") }}
 
 This file MAY be located at the root of the derivative dataset,
 or at the subject or session level
 ([Inheritance Principle](../common-principles.md#the-inheritance-principle)).
 
-The `desc_id` column contains the labels used with the [`desc entity`](../appendices/entities.md#desc),
-within the particular nesting that the `descriptions.tsv` file is placed.
-For example, if the `descriptions.tsv` file is placed at the root of the derivative dataset,
-its `desc_id` column SHOULD contain all labels of the [`desc entity`](../appendices/entities.md#desc)
-used across the entire derivative dataset.
-
-The `description` column contains human-readable descriptions of the processing steps.
-
 The use of `descriptions.tsv` files together with the [`desc entity`](../appendices/entities.md#desc)
 are helpful to document how files are generated, even if their use may not be sufficient
 to provide full computational reproducibility.
diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml
index bc11e7def7..3ccb32d324 100644
--- a/src/schema/objects/columns.yaml
+++ b/src/schema/objects/columns.yaml
@@ -110,6 +110,21 @@ derived_from:
     for example a slice of tissue (`sample-02`) derived from a block of tissue (`sample-01`).
   type: string
   pattern: ^sample-[0-9a-zA-Z]+$
+desc_id:
+  name: desc_id
+  display_name: Description Label
+  description: |
+    A `desc-<label>` entity present in the derivatives dataset.
+
+    The `desc_id` column contains the labels used with the
+    [`desc` entity](SPEC_ROOT/appendices/entities.md#desc),
+    within the particular nesting that the `descriptions.tsv` file is placed.
+
+    For example, if the `descriptions.tsv` file is placed at the root of the derivative dataset,
+    its `desc_id` column SHOULD contain all labels of the `desc` entity)
+    used across the entire derivative dataset.
+  type: string
+  pattern: ^desc-[0-9a-zA-Z]+$
 description:
   name: description
   display_name: Description
@@ -122,6 +137,12 @@ description__optode:
   description: |
     Free-form text description of the optode, or other information of interest.
   type: string
+description__entities:
+  name: description
+  display_name: Description
+  description: |
+    Free-form text description of the entity's label (defined in `<entity>_id` column).
+  type: string
 dimension:
   name: dimension
   display_name: Dimension
diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml
index 13ee57ebc5..0ddc8647d2 100644
--- a/src/schema/objects/suffixes.yaml
+++ b/src/schema/objects/suffixes.yaml
@@ -549,6 +549,18 @@ defacemask:
   description: |
     A binary mask that was used to remove facial features from an anatomical MRI
     image.
+descriptions:
+  value: descriptions
+  display_name: Description Entity Definitions
+  description: |
+    A TSV file describing labels found for the `desc` entity in a Derivatives dataset.
+
+    This file MAY be located at the root of the derivative dataset,
+    or at the subject or session level.
+
+    The use of `descriptions.tsv` files together with the desc entity are helpful to
+    document how files are generated,
+    even if their use may not be sufficient to provide full computational reproducibility.
 dseg:
   value: dseg
   display_name: Discrete Segmentation
diff --git a/src/schema/rules/files/deriv/tables.yaml b/src/schema/rules/files/deriv/tables.yaml
new file mode 100644
index 0000000000..107be3fa7a
--- /dev/null
+++ b/src/schema/rules/files/deriv/tables.yaml
@@ -0,0 +1,11 @@
+---
+descriptions:
+  level: optional
+  suffixes:
+    - descriptions
+  extensions:
+    - .tsv
+    - .json
+  entities:
+    subject: optional
+    session: optional
diff --git a/src/schema/rules/tabular_data/derivatives/common_derivatives.yaml b/src/schema/rules/tabular_data/derivatives/common_derivatives.yaml
index 3fef5e88fa..01b8bb04ea 100644
--- a/src/schema/rules/tabular_data/derivatives/common_derivatives.yaml
+++ b/src/schema/rules/tabular_data/derivatives/common_derivatives.yaml
@@ -10,3 +10,19 @@ SegmentationLookup:
     color: optional
     mapping: optional
   index_columns: [index]
+
+Descriptions:
+  selectors:
+    - suffix == "descriptions"
+    - extension == ".tsv"
+  initial_columns:
+    - desc_id
+    - description__entities
+  columns:
+    desc_id: required
+    description__entities:
+      level: required
+      description_addendum: |
+        The corresponding label column is `desc_id`.
+  index_columns: [desc_id]
+  additional_columns: allowed

From bff4f0c30c8c5aa2525b200e67fb87e2a3189afa Mon Sep 17 00:00:00 2001
From: Ben Dichter <ben.dichter@gmail.com>
Date: Wed, 17 Apr 2024 14:24:22 -0400
Subject: [PATCH 02/13] FIX: Make `namespace.to_dict()` recurse fully (#1790)

* fix namespace.to_dict

* add test searching for namespaces

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
---
 .../bidsschematools/tests/test_schema.py        | 13 +++++++++++++
 .../bidsschematools/types/namespace.py          | 17 +++++++++++------
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/tools/schemacode/bidsschematools/tests/test_schema.py b/tools/schemacode/bidsschematools/tests/test_schema.py
index 5e7b0e3575..bd5fc28942 100644
--- a/tools/schemacode/bidsschematools/tests/test_schema.py
+++ b/tools/schemacode/bidsschematools/tests/test_schema.py
@@ -343,3 +343,16 @@ def test_dereferencing():
             },
         },
     }
+
+
+def test_namespace_to_dict():
+
+    def check_for_namespaces(obj):
+        if isinstance(obj, dict):
+            [check_for_namespaces(val) for val in obj.values()]
+        elif isinstance(obj, list):
+            [check_for_namespaces(val) for val in obj]
+        elif isinstance(obj, types.Namespace):
+            raise ValueError("Namespace object found in dict")
+
+    check_for_namespaces(schema.load_schema().to_dict())
diff --git a/tools/schemacode/bidsschematools/types/namespace.py b/tools/schemacode/bidsschematools/types/namespace.py
index 39943ac3f0..e2c37eee40 100644
--- a/tools/schemacode/bidsschematools/types/namespace.py
+++ b/tools/schemacode/bidsschematools/types/namespace.py
@@ -168,12 +168,17 @@ def __init__(self, *args, **kwargs):
         self._properties = dict(*args, **kwargs)
 
     def to_dict(self) -> dict:
-        ret = {}
-        for key, val in self._properties.items():
-            if isinstance(val, Namespace):
-                val = val.to_dict()
-            ret[key] = val
-        return ret
+
+        def _to_dict(obj):
+            if isinstance(obj, Namespace):
+                return {k: _to_dict(v) for k, v in obj._properties.items()}
+            if isinstance(obj, list):
+                return [_to_dict(v) for v in obj]
+            if isinstance(obj, dict):
+                return {k: _to_dict(v) for k, v in obj.items()}
+            return obj
+
+        return _to_dict(self)
 
     def __deepcopy__(self, memo):
         return self.build(self.to_dict())

From d7b37931c3e3754904611b6e6f64b7da8932443c Mon Sep 17 00:00:00 2001
From: Ben Dichter <ben.dichter@gmail.com>
Date: Wed, 17 Apr 2024 21:46:20 -0400
Subject: [PATCH 03/13] code cleanup: (#1793)

* rmv "infra": ":metro:" as it is a repeat
* fix broken docstrings
* shield mutable default args
---
 pdf_build_src/process_markdowns.py               |  2 +-
 tools/add_contributors.py                        |  3 +--
 tools/mkdocs_macros_bids/macros.py               |  2 +-
 .../schemacode/bidsschematools/render/tables.py  |  2 +-
 tools/schemacode/bidsschematools/render/text.py  |  4 ----
 tools/schemacode/bidsschematools/schema.py       |  2 +-
 tools/schemacode/bidsschematools/validator.py    | 16 ++++++++++++----
 7 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py
index 705daa8b22..42e984fa98 100644
--- a/pdf_build_src/process_markdowns.py
+++ b/pdf_build_src/process_markdowns.py
@@ -559,7 +559,7 @@ def correct_tables(root_path, debug=False):
                             for i, new_line in enumerate(content):
                                 if i == start_line:
                                     new_content.pop()
-                                if i >= start_line and i < end_line:
+                                if start_line <= i < end_line:
                                     new_content.append("|".join(table[count]) + " \n")
                                     count += 1
                                 elif i == end_line:
diff --git a/tools/add_contributors.py b/tools/add_contributors.py
index 1fe86f6abf..c6cc537dcd 100644
--- a/tools/add_contributors.py
+++ b/tools/add_contributors.py
@@ -115,7 +115,6 @@ def emoji_map(reverse=False) -> dict[str, str]:
         "tutorial": ":check_mark_button:",
         "maintenance": ":construction:",
         "financial": ":dollar_banknote:",
-        "infra": ":metro:",
     }
 
 
@@ -333,7 +332,7 @@ def update_tributors(
 """ALLCONTRIB"""
 
 
-def load_allcontrib(allcontrib_file: Path) -> None:
+def load_allcontrib(allcontrib_file: Path) -> dict:
     """Load `.all-contributorsrc` file."""
     with open(allcontrib_file, "r", encoding="utf8") as input_file:
         return json.load(input_file)
diff --git a/tools/mkdocs_macros_bids/macros.py b/tools/mkdocs_macros_bids/macros.py
index f68453c6b8..e43e8449d3 100644
--- a/tools/mkdocs_macros_bids/macros.py
+++ b/tools/mkdocs_macros_bids/macros.py
@@ -208,7 +208,7 @@ def make_metadata_table(field_info, src_path=None):
 
     Parameters
     ----------
-    field_names : dict
+    field_info : dict
         A list of the field names.
         Field names correspond to filenames in the "metadata" directory of the
         schema.
diff --git a/tools/schemacode/bidsschematools/render/tables.py b/tools/schemacode/bidsschematools/render/tables.py
index cf8645ffeb..b32e982e52 100644
--- a/tools/schemacode/bidsschematools/render/tables.py
+++ b/tools/schemacode/bidsschematools/render/tables.py
@@ -284,7 +284,7 @@ def make_entity_table(schema, tablefmt="github", src_path=None, **kwargs):
 
     Parameters
     ----------
-    schema_path : str
+    schema : str
         Directory containing schema, which is stored in yaml files.
     tablefmt : string, optional
         The target table format. The default is "github" (GitHub format).
diff --git a/tools/schemacode/bidsschematools/render/text.py b/tools/schemacode/bidsschematools/render/text.py
index 1106e17185..c9fb6f2298 100644
--- a/tools/schemacode/bidsschematools/render/text.py
+++ b/tools/schemacode/bidsschematools/render/text.py
@@ -559,10 +559,6 @@ def render_text(schema, key: str, src_path=None):
     schema : dict
         The BIDS schema.
 
-    object : str
-        The object to render the description for:
-        possible values correspond to the keys in schema["objects"].
-
     key : str
         The key of the object to render the description for:
         possible values correspond to the keys in schema["objects"][object]
diff --git a/tools/schemacode/bidsschematools/schema.py b/tools/schemacode/bidsschematools/schema.py
index 8a51725f1b..cae15fece1 100644
--- a/tools/schemacode/bidsschematools/schema.py
+++ b/tools/schemacode/bidsschematools/schema.py
@@ -130,7 +130,7 @@ def flatten_enums(namespace, inplace=True):
 
     Parameters
     ----------
-    schema : dict
+    namespace : dict
         Schema in dictionary form to be flattened.
 
     Returns
diff --git a/tools/schemacode/bidsschematools/validator.py b/tools/schemacode/bidsschematools/validator.py
index 34621a8c60..4ec04f6df6 100644
--- a/tools/schemacode/bidsschematools/validator.py
+++ b/tools/schemacode/bidsschematools/validator.py
@@ -133,9 +133,9 @@ def _get_directory_suffixes(my_schema):
 
 def _get_paths(
     bids_paths,
-    pseudofile_suffixes=[],
+    pseudofile_suffixes=None,
     dummy_paths=False,
-    exclude_files=[],
+    exclude_files=None,
 ):
     """
     Get all paths from a list of directories, excluding hidden subdirectories from distribution.
@@ -166,6 +166,11 @@ def _get_paths(
         first.
     """
 
+    if exclude_files is None:
+        exclude_files = []
+    if pseudofile_suffixes is None:
+        pseudofile_suffixes = []
+
     path_list = []
     bidsignore_list = []
     bids_root_found = False
@@ -387,7 +392,7 @@ def validate_all(
 
     Parameters
     ----------
-    bids_paths : list or str
+    paths_list : list or str
         A string pointing to a BIDS directory for which paths should be validated, or a list
         of strings pointing to individual files or subdirectories which *all* reside within
         one and only one BIDS directory root (i.e. nested datasets should be validated
@@ -548,7 +553,7 @@ def validate_bids(
     report_path=False,
     suppress_errors=False,
     accept_non_bids_dir=False,
-    exclude_files=[],
+    exclude_files=None,
 ):
     """
     Validate paths according to BIDS schema.
@@ -604,6 +609,9 @@ def validate_bids(
         https://github.com/bids-standard/bids-specification/pull/969#issuecomment-1132119492
     """
 
+    if exclude_files is None:
+        exclude_files = []
+
     if isinstance(in_paths, str):
         in_paths = [in_paths]
 

From 53eafdab0b7641cc28c4a947b509679fcf45072f Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Thu, 18 Apr 2024 18:43:27 +0200
Subject: [PATCH 04/13] [ENH] use full width entity page (#1784)

* full width entity page

* lint front matter

* lint front matter

* Update src/appendices/entity-table.md
---
 src/appendices/entity-table.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/appendices/entity-table.md b/src/appendices/entity-table.md
index 4dff2dcf93..4ee4857d00 100644
--- a/src/appendices/entity-table.md
+++ b/src/appendices/entity-table.md
@@ -1,3 +1,8 @@
+---
+hide:
+-   toc
+---
+
 # Entity table
 
 This section compiles the entities (key-value pairs within filenames)

From 735bf2339c673c449586bb9b4c971de00b134322 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Sat, 23 Mar 2024 14:47:25 -0400
Subject: [PATCH 05/13] SCHEMA: Update existence checks to consider empty lists

---
 src/schema/rules/checks/references.yaml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/schema/rules/checks/references.yaml b/src/schema/rules/checks/references.yaml
index cbbc8f5454..54957cb959 100644
--- a/src/schema/rules/checks/references.yaml
+++ b/src/schema/rules/checks/references.yaml
@@ -3,6 +3,7 @@ SubjectRelativeIntendedFor:
   selectors:
     - datatype != "ieeg"
     - type(sidecar.IntendedFor) != "null"
+    - length(sidecar.IntendedFor) > 0
   checks:
     - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "subject")
 
@@ -10,6 +11,7 @@ DatasetRelativeIntendedFor:
   selectors:
     - datatype == "ieeg"
     - type(sidecar.IntendedFor) != "null"
+    - length(sidecar.IntendedFor) > 0
   checks:
     - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "dataset")
 
@@ -17,21 +19,15 @@ AssociatedEmptyRoom:
   selectors:
     - suffix == "meg"
     - type(sidecar.AssociatedEmptyRoom) != "null"
+    - length(sidecar.AssociatedEmptyRoom) > 0
   checks:
     - exists(sidecar.AssociatedEmptyRoom, "bids-uri") || exists(sidecar.AssociatedEmptyRoom, "dataset")
 
-Stimuli:
-  selectors:
-    - suffix == "events"
-    - extension == ".tsv"
-    - type(columns.stim_file) != "null"
-  checks:
-    - exists(columns.stim_file, "stimuli")
-
 Sources:
   selectors:
     - dataset.dataset_description.DatasetType == "derivatives"
     - type(sidecar.Sources) != "null"
+    - length(sidecar.Sources) > 0
   checks:
     - exists(sidecar.Sources, "bids-uri") || exists(sidecar.Sources, "dataset")
 

From 61c11c7dc4ad31097a196049915622bcb80d4d37 Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Thu, 18 Apr 2024 15:10:39 -0400
Subject: [PATCH 06/13] RF: Split rules by array or string

---
 src/schema/rules/checks/references.yaml | 62 ++++++++++++++++++-------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/src/schema/rules/checks/references.yaml b/src/schema/rules/checks/references.yaml
index 54957cb959..099e913f0f 100644
--- a/src/schema/rules/checks/references.yaml
+++ b/src/schema/rules/checks/references.yaml
@@ -1,39 +1,65 @@
 ---
-SubjectRelativeIntendedFor:
+SubjectRelativeIntendedForString:
   selectors:
     - datatype != "ieeg"
-    - type(sidecar.IntendedFor) != "null"
-    - length(sidecar.IntendedFor) > 0
+    - type(sidecar.IntendedFor) == "string"
   checks:
-    - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "subject")
+    - exists(sidecar.IntendedFor, "bids-uri") + exists(sidecar.IntendedFor, "subject") == 1
 
-DatasetRelativeIntendedFor:
+SubjectRelativeIntendedForArray:
+  selectors:
+    - datatype != "ieeg"
+    - type(sidecar.IntendedFor) == "array"
+  checks:
+    - exists(sidecar.IntendedFor, "bids-uri") + exists(sidecar.IntendedFor, "subject") == length(sidecar.IntendedFor)
+
+DatasetRelativeIntendedForString:
+  selectors:
+    - datatype == "ieeg"
+    - type(sidecar.IntendedFor) == "string"
+  checks:
+    - exists(sidecar.IntendedFor, "bids-uri") + exists(sidecar.IntendedFor, "dataset") == 1
+
+DatasetRelativeIntendedForArray:
   selectors:
     - datatype == "ieeg"
-    - type(sidecar.IntendedFor) != "null"
-    - length(sidecar.IntendedFor) > 0
+    - type(sidecar.IntendedFor) == "array"
   checks:
-    - exists(sidecar.IntendedFor, "bids-uri") || exists(sidecar.IntendedFor, "dataset")
+    - exists(sidecar.IntendedFor, "bids-uri") + exists(sidecar.IntendedFor, "dataset") == length(sidecar.IntendedFor)
 
-AssociatedEmptyRoom:
+AssociatedEmptyRoomString:
   selectors:
-    - suffix == "meg"
-    - type(sidecar.AssociatedEmptyRoom) != "null"
-    - length(sidecar.AssociatedEmptyRoom) > 0
+    - datatype == "meg"
+    - type(sidecar.AssociatedEmptyRoom) == "string"
   checks:
-    - exists(sidecar.AssociatedEmptyRoom, "bids-uri") || exists(sidecar.AssociatedEmptyRoom, "dataset")
+    - exists(sidecar.AssociatedEmptyRoom, "bids-uri") + exists(sidecar.AssociatedEmptyRoom, "dataset") == 1
+
+AssociatedEmptyRoomArray:
+  selectors:
+    - datatype == "meg"
+    - type(sidecar.AssociatedEmptyRoom) == "array"
+  checks:
+    - |
+      exists(sidecar.AssociatedEmptyRoom, "bids-uri") + exists(sidecar.AssociatedEmptyRoom, "dataset")
+      == length(sidecar.AssociatedEmptyRoom)
 
 Sources:
   selectors:
     - dataset.dataset_description.DatasetType == "derivatives"
     - type(sidecar.Sources) != "null"
-    - length(sidecar.Sources) > 0
   checks:
-    - exists(sidecar.Sources, "bids-uri") || exists(sidecar.Sources, "dataset")
+    - exists(sidecar.Sources, "bids-uri") + exists(sidecar.Sources, "dataset") == length(sidecar.Sources)
+
+SpatialReferencesString:
+  selectors:
+    - dataset.dataset_description.DatasetType == "derivatives"
+    - type(sidecar.SpatialReference.URI) == "string"
+  checks:
+    - exists(sidecar.SpatialReference.URI, "bids-uri") + exists(sidecar.SpatialReference.URI, "dataset") == 1
 
-SpatialReferences:
+SpatialReferencesArray:
   selectors:
     - dataset.dataset_description.DatasetType == "derivatives"
-    - type(sidecar.SpatialReference.URI) != "null"
+    - type(sidecar.SpatialReference.URI) == "array"
   checks:
-    - exists(sidecar.SpatialReference.URI, "bids-uri") || exists(sidecar.SpatialReference.URI, "dataset")
+    - exists(sidecar.SpatialReference.URI, "bids-uri") + exists(sidecar.SpatialReference.URI, "dataset") == length(sidecar.SpatialReference.URI)

From de4bb1a9bbdba35039274809ba61350c52f4355f Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <effigies@gmail.com>
Date: Thu, 18 Apr 2024 15:28:12 -0400
Subject: [PATCH 07/13] SpatialReferences considered out-of-scope

---
 src/schema/rules/checks/references.yaml | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/schema/rules/checks/references.yaml b/src/schema/rules/checks/references.yaml
index 099e913f0f..b0f1dd6e9d 100644
--- a/src/schema/rules/checks/references.yaml
+++ b/src/schema/rules/checks/references.yaml
@@ -50,16 +50,15 @@ Sources:
   checks:
     - exists(sidecar.Sources, "bids-uri") + exists(sidecar.Sources, "dataset") == length(sidecar.Sources)
 
-SpatialReferencesString:
-  selectors:
-    - dataset.dataset_description.DatasetType == "derivatives"
-    - type(sidecar.SpatialReference.URI) == "string"
-  checks:
-    - exists(sidecar.SpatialReference.URI, "bids-uri") + exists(sidecar.SpatialReference.URI, "dataset") == 1
-
-SpatialReferencesArray:
-  selectors:
-    - dataset.dataset_description.DatasetType == "derivatives"
-    - type(sidecar.SpatialReference.URI) == "array"
-  checks:
-    - exists(sidecar.SpatialReference.URI, "bids-uri") + exists(sidecar.SpatialReference.URI, "dataset") == length(sidecar.SpatialReference.URI)
+## Verifying the existence of SpatialReferences is out-of-scope for schema validation
+## The ability to use any URI leaves this field too open-ended for effective validation, and the
+## object of key/URI pairs is worse.
+##
+## 2024.04.18 CJM & RWB
+#
+# SpatialReferencesString:
+#   selectors:
+#     - dataset.dataset_description.DatasetType == "derivatives"
+#     - type(sidecar.SpatialReference) == "string"
+#   checks:
+#     - exists(sidecar.SpatialReference, "bids-uri") + exists(sidecar.SpatialReference, "dataset") == 1

From e21aa37052ef7a881bb3e9e37a299d46c252923c Mon Sep 17 00:00:00 2001
From: Chris Markiewicz <markiewicz@stanford.edu>
Date: Thu, 18 Apr 2024 15:56:30 -0400
Subject: [PATCH 08/13] MNT: Add precommit for yamllint (#1795)

---
 .pre-commit-config.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9451a1cb54..0d2c0ed0fa 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -43,6 +43,12 @@ repos:
       - id: prettier
         entry: env PRETTIER_LEGACY_CLI=1 prettier # temporary fix for https://github.com/prettier/prettier/issues/15742
         files: src/schema/.*/.*\.yaml
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args: [-f=standard, -c=.yamllint.yml]
+        files: src/schema/.*/.*\.yaml
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:

From 3896ba58864adf6efee0598e1423b2b303cc8aa8 Mon Sep 17 00:00:00 2001
From: Ben Dichter <ben.dichter@gmail.com>
Date: Fri, 19 Apr 2024 09:11:24 -0400
Subject: [PATCH 09/13] [INFRA] Introduce metaschema (#1787)

* add metaschema and code to validate it.

* WIP metaschema and code to validate

* update schema

* objects finished

* add rules

* add checks

* add rules.files.common

* add files.common.core and files.common.tables

* add rules.files.deriv

* dereferencing working

* checkpoint

* checkpoint

* add rules.sidecars

* sidecar derivatives checkpoint

* tabular_data checkpoint

* DONE!

* DONE!

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix typo

* blackify

* include references to draft7 metaschema

* move metaschema

* add tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* move things around

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update tools/schemacode/bidsschematools/validate_schema.py

Co-authored-by: Taylor Salo <tsalo90@gmail.com>

* fix typo

* add jsonschema to deps

* fixing up checks

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* install jsonschema types

* fix pathing

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* flake8

* try changing schema path

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix paths

* use existing functions

* convert from namespace

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update fields for existing loader

* fix required props

* ENH: Link metaschema into package data

* RF: Use package data instead of relying on CWD

* STY: Import from top-level jsonschema

* PY38: importlib_resources

* use "deprecated" instead of "DEPRECATED"

* move to schema.py and test_schema.py

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update src/schema/objects/common_principles.yaml

* Update README.md

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Taylor Salo <tsalo90@gmail.com>
Co-authored-by: Chris Markiewicz <effigies@gmail.com>
---
 .pre-commit-config.yaml                       |   1 +
 src/metaschema.json                           | 727 ++++++++++++++++++
 src/schema/README.md                          |   8 +
 src/schema/rules/sidecars/mri.yaml            |   2 +-
 .../bidsschematools/data/metaschema.json      |   1 +
 tools/schemacode/bidsschematools/schema.py    |  28 +
 .../bidsschematools/tests/test_schema.py      |  28 +
 tools/schemacode/setup.cfg                    |   2 +
 8 files changed, 796 insertions(+), 1 deletion(-)
 create mode 100644 src/metaschema.json
 create mode 120000 tools/schemacode/bidsschematools/data/metaschema.json

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0d2c0ed0fa..b2f40321ca 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -68,5 +68,6 @@ repos:
           - pytest
           - types-PyYAML
           - types-tabulate
+          - types-jsonschema
         args: ["tools/schemacode/bidsschematools"]
         pass_filenames: false
diff --git a/src/metaschema.json b/src/metaschema.json
new file mode 100644
index 0000000000..129e3f1c5d
--- /dev/null
+++ b/src/metaschema.json
@@ -0,0 +1,727 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "meta": {
+      "type": "object",
+      "properties": {
+        "associations": {
+          "type": "object",
+          "patternProperties":
+          {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "selectors": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                },
+                "target": {
+                  "type": "object",
+                  "properties": {
+                    "suffix": {
+                      "type": "string"
+                    },
+                    "extension": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        }, {
+                          "type": "array",
+                          "items": {
+                            "type": "string"
+                          }
+                        }
+                      ]
+                    }
+                  },
+                  "required": ["extension"],
+                  "additionalProperties": false
+                },
+                "inherit": {
+                  "type": "boolean"
+                }
+              },
+              "required": ["target"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "context": {
+          "type": "object"
+        },
+        "expression_tests": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "expression": {
+                "type": "string"
+              },
+              "result": {}
+            },
+            "required": ["expression", "result"],
+            "additionalProperties": false
+          }
+        },
+        "versions": {
+          "type": "array",
+          "items": {
+            "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+          }
+        }
+      },
+      "required": ["associations", "context", "expression_tests", "versions"]
+    },
+    "objects": {
+      "type": "object",
+      "properties": {
+        "columns": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                },
+                "type": {
+                  "$ref": "http://json-schema.org/draft-07/schema#/properties/type"
+                },
+                "format": {
+                  "type": "string"
+                },
+                "pattern": {
+                  "type": "string"
+                },
+                "unit": {
+                  "type": "string"
+                },
+                "enum": {
+                  "type": "array"
+                },
+                "anyOf": {
+                  "$ref": "http://json-schema.org/draft-07/schema#/properties/anyOf"
+                },
+                "maximum": {
+                  "type": "number"
+                },
+                "minimum": {
+                  "type": "number"
+                }
+              },
+              "required": ["name", "display_name"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "common_principles": {
+         "type": "object",
+         "patternProperties": {
+           "^[a-zA-Z0-9_]+$": {
+             "type": "object",
+             "properties": {
+               "display_name": {
+                 "type": "string"
+               },
+               "description": {
+                 "type": "string"
+               }
+             },
+             "required": ["display_name", "description"],
+             "additionalProperties": false
+           },
+           "additionalProperties": false
+         }
+        },
+        "datatypes": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "value": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                }
+              },
+              "required": ["value", "display_name", "description"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "entities": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                },
+                "type": {
+                  "$ref": "http://json-schema.org/draft-07/schema#/properties/type"
+                },
+                "format": {
+                  "type": "string"
+                },
+                "enum": {
+                  "type": "array"
+                }
+              },
+              "required": ["name", "display_name", "description", "type", "format"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "enums": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_-]+$": {
+              "type": "object",
+              "properties": {
+                "type": {
+                  "$ref": "http://json-schema.org/draft-07/schema#/properties/type"
+                },
+                "enum": {
+                  "type": "array"
+                },
+                "value": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                },
+                "tags": {
+                  "type": "array"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "extensions": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "value": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                }
+              },
+              "required": ["display_name", "description", "value"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "files": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "display_name": {
+                  "type": "string"
+                },
+                "file_type": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                }
+              },
+              "required": ["display_name", "file_type", "description"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "formats": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                },
+                "pattern": {
+                  "format": "regex"
+                }
+              },
+              "required": ["display_name", "description", "pattern"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "metadata": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                },
+                "type": {
+                  "$ref": "http://json-schema.org/draft-07/schema#/properties/type"
+                }
+              },
+              "required": ["name","display_name", "description"]
+            }
+          },
+          "additionalProperties": false
+        },
+        "modalities": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "type": "object",
+              "properties": {
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                }
+              },
+              "required": ["display_name", "description"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "suffixes": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "value": {
+                  "type": "string"
+                },
+                "display_name": {
+                  "type": "string"
+                },
+                "description": {
+                  "type": "string"
+                }
+              },
+              "required": ["value", "display_name", "description"]
+            },
+            "additionalProperties": false
+          }
+        }
+      },
+        "required": ["columns", "common_principles", "datatypes", "entities", "enums", "extensions", "files", "formats", "metadata", "modalities", "suffixes"],
+        "additionalProperties": false
+    },
+    "rules": {
+      "type": "object",
+      "properties": {
+        "checks": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                  "type": "object",
+                  "properties": {
+                    "issues": {
+                      "type": "object",
+                      "properties": {
+                        "code": {"type": "string"},
+                        "message": {"type": "string"},
+                        "level": {"enum": ["error", "warning"]}
+                      },
+                      "required": ["code", "message", "level"],
+                      "additionalProperties": false
+                    },
+                    "selectors": {
+                      "type": "array",
+                      "items": {"type": "string"}
+                    },
+                    "checks": {
+                      "type": "array",
+                      "items": {"type": "string"}
+                    }
+                  },
+                  "required": ["checks", "selectors"]
+                }
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        "files": {
+          "type": "object",
+          "properties": {
+            "common": {
+              "type": "object",
+              "properties": {
+                "core": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^[a-zA-Z0-9_]+$": {
+                      "type": "object",
+                      "properties": {
+                        "level": {"type": "string"},
+                        "path": {"type": "string"},
+                        "extensions": {"type": "array"},
+                        "stem": {"type": "string"}
+                      },
+                      "required": ["level"],
+                      "additionalProperties": false
+                    }
+                  },
+                  "additionalProperties": false
+                },
+                "tables": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^[a-zA-Z0-9_]+$": {
+                      "type": "object",
+                      "properties": {
+                        "level": {"type": "string"},
+                        "path": {"type": "string"},
+                        "extensions": {"type": "array"},
+                        "stem": {"type": "string"},
+                        "entities": {"$ref": "#/definitions/entities"},
+                        "suffixes": {"type": "array"}
+                      },
+                      "required": ["level", "extensions"],
+                      "additionalProperties": false
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              },
+              "required": ["core", "tables"],
+              "additionalProperties": false
+            },
+            "deriv": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^[a-zA-Z0-9_]+$": {
+                      "type": "object",
+                      "properties": {
+                        "entities": {
+                          "$ref": "#/definitions/entities"
+                        }
+                      }
+                    },
+                    "additionalProperties": false
+                  }
+                }
+              },
+              "additionalProperties": false
+            },
+            "raw": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-z]+$": {
+                  "type": "object",
+                  "patternProperties": {
+                    "^[a-zA-Z0-9_]+$": {
+                      "type": "object",
+                      "properties": {
+                        "suffixes": {
+                          "type": "array",
+                          "items": {"type": "string"}
+                        },
+                        "extensions": {
+                          "type": "array",
+                          "items": {"pattern": "^[./][a-z.]+|"}
+                        },
+                        "datatypes": {
+                          "type": "array",
+                          "items": {"pattern": "^[a-z]+$"}
+                        },
+                        "entities": {"$ref": "#/definitions/entities"}
+                      }
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "required": ["common", "deriv", "raw"],
+          "additionalProperties": false
+        },
+        "sidecars": {
+          "type": "object",
+          "patternProperties": {
+            "^derivatives$": {
+              "type": "object",
+              "properties": {
+                "common_derivatives": {"$ref": "#/definitions/sidecar"}
+              },
+              "required": ["common_derivatives"],
+              "additionalProperties": false
+            },
+            "^(?!derivatives$)[a-z_]+$": {
+              "$ref": "#/definitions/sidecar"
+            },
+            "additionalProperties": false
+          },
+          "additionalProperties": false
+        },
+        "tabular_data": {
+          "type": "object",
+          "patternProperties": {
+            "^derivatives$": {
+              "type": "object",
+              "properties": {
+                "common_derivatives": {
+                  "$ref": "#/definitions/tabular_data"
+                }
+              },
+              "required": [
+                "common_derivatives"
+              ],
+              "additionalProperties": false
+            },
+            "^(?!derivatives$)[a-z_]+$": {
+              "$ref": "#/definitions/tabular_data"
+            }
+          },
+          "additionalProperties": false
+        },
+        "common_principles": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "dataset_metadata": {
+          "type": "object"
+        },
+        "directories": {
+          "type": "object"
+        },
+        "entities": {
+          "type": "array",
+          "items": {"type": "string"}
+        },
+        "errors": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-zA-Z0-9_]+$": {
+              "type": "object",
+              "properties": {
+                "code": {"type": "string"},
+                "message": {"type": "string"},
+                "level": {"enum": ["error", "warning"]},
+                "selectors": {
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              },
+              "required": ["message", "level"],
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "modalities": {
+          "type": "object",
+          "patternProperties": {
+            "^[a-z]+$": {
+              "type": "object",
+              "properties": {
+                "datatypes": {
+                  "type": "array",
+                  "items": {"pattern": "^[a-z]+$"}
+                }
+              },
+              "required": ["datatypes"],
+              "additionalProperties": false
+            }
+          }
+        }
+      },
+      "required": [
+          "entities",
+          "files",
+          "sidecars",
+          "tabular_data",
+          "common_principles",
+          "dataset_metadata",
+          "directories",
+          "errors",
+          "modalities"
+      ],
+      "additionalProperties": false
+    },
+    "bids_version": {
+      "type": "string"
+    },
+    "schema_version": {
+      "type": "string"
+    },
+    "README": {
+      "type": "string"
+    }
+  },
+  "required": ["meta", "objects", "rules", "bids_version", "schema_version"],
+  "additionalProperties": false,
+  "definitions": {
+    "entities": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-z]+$": {
+          "anyOf": [
+            {"enum": ["optional", "required"]},
+            {
+              "type": "object",
+              "properties": {
+                "level": {"enum": ["optional", "required"]},
+                "enum": {
+                  "type": "array",
+                  "items": {"type": "string"}
+                }
+              },
+              "required": ["level", "enum"]
+            }
+          ]
+        }
+      },
+      "additionalProperties": false
+    },
+    "sidecar": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9_]+$": {
+          "type": "object",
+          "properties": {
+            "selectors": {
+              "type": "array",
+              "items": {"type": "string"}
+            },
+            "fields": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                  "anyOf": [
+                    {"enum": ["recommended", "optional", "required", "deprecated"]},
+                    {
+                      "type":  "object",
+                      "properties": {
+                        "level": {"enum": ["recommended", "optional", "required", "deprecated"]},
+                        "level_addendum": {"type": "string"}
+                      },
+                      "required": ["level", "level_addendum"],
+                      "additionalProperties": false
+                    },
+                    {"pattern": "recommended.*"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "required": ["selectors", "fields"],
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "tabular_data": {
+      "type": "object",
+      "patternProperties": {
+        "^[a-zA-Z0-9_]+$": {
+          "type": "object",
+          "properties": {
+            "selectors": {
+              "type": "array",
+              "items": {"type": "string"}
+            },
+            "columns": {
+              "type": "object",
+              "patternProperties": {
+                "^[a-zA-Z0-9_]+$": {
+                  "anyOf": [
+                    {"enum": ["recommended", "optional", "required", "deprecated"]},
+                    {
+                      "type":  "object",
+                      "properties": {
+                        "level": {"enum": ["recommended", "optional", "required", "deprecated"]},
+                        "level_addendum": {"type": "string"},
+                        "description_addendum": {"type": "string"}
+                      },
+                      "required": ["level"],
+                      "additionalProperties": false
+                    },
+                    {"pattern": "recommended.*"}
+                  ]
+                }
+              },
+              "additionalProperties": false
+            },
+            "additional_columns": {
+              "type": "string"
+            },
+            "index_columns": {"type": "array", "items": {"type": "string"}},
+            "initial_columns": {
+              "type": "array",
+              "items": {"type": "string"}
+            }
+          },
+          "required": ["selectors", "columns"],
+          "additionalProperties": false
+        }
+      }
+    }
+  }
+}
diff --git a/src/schema/README.md b/src/schema/README.md
index d801314595..e8cb197757 100644
--- a/src/schema/README.md
+++ b/src/schema/README.md
@@ -1023,3 +1023,11 @@ be found at <https://bids-specification.readthedocs.io/en/latest/schema.json>.
 The JSON version of the schema contains `schema_version` and `bids_version` keys
 that identify the state of both the schema and the specification at the time it was
 compiled.
+
+## Metaschema
+
+The `metaschema.json` file is a meta-schema that uses the JSON Schema language to
+formalize the allowable directories, files, fields and values of the BIDS schema,
+ensuring consistency across the entire schema directory. Validation of the schema is
+incorporated into the CI, so any changes that are inconsistent will be flagged before
+inclusion.
diff --git a/src/schema/rules/sidecars/mri.yaml b/src/schema/rules/sidecars/mri.yaml
index b937898b3d..cc27d676be 100644
--- a/src/schema/rules/sidecars/mri.yaml
+++ b/src/schema/rules/sidecars/mri.yaml
@@ -25,7 +25,7 @@ MRIHardware:
     SoftwareVersions:
       level: recommended
       description_addendum: Corresponds to DICOM Tag 0018, 1020 `Software Versions`.
-    HardcopyDeviceSoftwareVersion: DEPRECATED
+    HardcopyDeviceSoftwareVersion: deprecated
     MagneticFieldStrength:
       level: recommended, but required for Arterial Spin Labeling
     ReceiveCoilName: recommended
diff --git a/tools/schemacode/bidsschematools/data/metaschema.json b/tools/schemacode/bidsschematools/data/metaschema.json
new file mode 120000
index 0000000000..ae3cfc38f9
--- /dev/null
+++ b/tools/schemacode/bidsschematools/data/metaschema.json
@@ -0,0 +1 @@
+../../../../src/metaschema.json
\ No newline at end of file
diff --git a/tools/schemacode/bidsschematools/schema.py b/tools/schemacode/bidsschematools/schema.py
index cae15fece1..ed6b5da5f3 100644
--- a/tools/schemacode/bidsschematools/schema.py
+++ b/tools/schemacode/bidsschematools/schema.py
@@ -1,12 +1,22 @@
 """Schema loading- and processing-related functions."""
 
+import json
 import logging
 import os
 import re
+import sys
+import tempfile
 from collections.abc import Iterable, Mapping
 from copy import deepcopy
 from functools import lru_cache
 
+from jsonschema import ValidationError, validate
+
+if sys.version_info < (3, 9):
+    from importlib_resources import files
+else:
+    from importlib.resources import files
+
 from . import __bids_version__, __version__, utils
 from .types import Namespace
 
@@ -274,3 +284,21 @@ def filter_schema(schema, **kwargs):
             if isinstance(item, dict):
                 new_schema[i] = filter_schema(item, **kwargs)
     return new_schema
+
+
+def validate_schema(schema: Namespace):
+    """Validate a schema against the BIDS metaschema."""
+    metaschema = json.loads(files("bidsschematools.data").joinpath("metaschema.json").read_text())
+
+    # validate is put in this try/except clause because the error is sometimes too long to
+    # print in the terminal
+    try:
+        validate(instance=schema.to_dict(), schema=metaschema)
+    except ValidationError as e:
+        with tempfile.NamedTemporaryFile(
+            prefix="schema_error_", suffix=".txt", delete=False, mode="w+"
+        ) as file:
+            file.write(str(e))
+            # ValidationError does not have an add_note method yet
+            # e.add_note(f"See {file.name} for full error log.")
+            raise e
diff --git a/tools/schemacode/bidsschematools/tests/test_schema.py b/tools/schemacode/bidsschematools/tests/test_schema.py
index bd5fc28942..6c52f0c2c1 100644
--- a/tools/schemacode/bidsschematools/tests/test_schema.py
+++ b/tools/schemacode/bidsschematools/tests/test_schema.py
@@ -4,6 +4,7 @@
 from collections.abc import Mapping
 
 import pytest
+from jsonschema.exceptions import ValidationError
 
 from bidsschematools import __bids_version__, schema, types
 
@@ -356,3 +357,30 @@ def check_for_namespaces(obj):
             raise ValueError("Namespace object found in dict")
 
     check_for_namespaces(schema.load_schema().to_dict())
+
+
+def test_valid_schema():
+    """Test that a valid schema does not raise an error."""
+    namespace = schema.load_schema()
+    schema.validate_schema(namespace)
+
+
+def test_add_legal_field():
+    """Test that adding a legal field does not raise an error."""
+    namespace = schema.load_schema()
+    namespace["rules"]["files"]["deriv"]["preprocessed_data"]["anat_nonparametric_common"][
+        "entities"
+    ]["density"] = "optional"
+    schema.validate_schema(namespace)
+
+
+def test_invalid_value():
+    """Test that an invalid value raises an error."""
+    namespace = schema.load_schema()
+    namespace["rules"]["files"]["deriv"]["preprocessed_data"]["anat_nonparametric_common"][
+        "entities"
+    ]["density"] = "invalid"
+    with pytest.raises(ValidationError) as e:
+        schema.validate_schema(namespace)
+    print(e.value)
+    assert "invalid" in str(e.value)
diff --git a/tools/schemacode/setup.cfg b/tools/schemacode/setup.cfg
index 27159dee84..f75732b593 100644
--- a/tools/schemacode/setup.cfg
+++ b/tools/schemacode/setup.cfg
@@ -24,6 +24,7 @@ install_requires =
     click
     pyyaml
     importlib_resources; python_version < "3.9"
+    jsonschema
 packages = find:
 include_package_data = false
 zip_safe = false
@@ -54,6 +55,7 @@ all =
 
 [options.package_data]
 bidsschematools =
+    data/metaschema.json
     data/schema/BIDS_VERSION
     data/schema/SCHEMA_VERSION
     data/schema/**/*.yaml

From bbe2283dffb5c7d001a0fc3de8c9f5f978b743b4 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 19 Apr 2024 06:16:40 -0700
Subject: [PATCH 10/13] ENH: Use "Tag XXXX, YYYY" not "Tag (XXXX,YYYY)" for
 DICOM Tags (#1758)

In general I like more (XXXX,YYYY) form but the other form more prevalent and
thus what I also suggested in https://github.com/bids-standard/bids-specification/pull/1450/files
---
 src/schema/rules/sidecars/pet.yaml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/schema/rules/sidecars/pet.yaml b/src/schema/rules/sidecars/pet.yaml
index dc4ff9a572..2b2264a59b 100644
--- a/src/schema/rules/sidecars/pet.yaml
+++ b/src/schema/rules/sidecars/pet.yaml
@@ -114,10 +114,10 @@ PETPharmaceuticals:
   fields:
     PharmaceuticalName:
       level: recommended
-      description_addendum: Corresponds to DICOM Tag (0008,0034) `Intervention Drug Name`.
+      description_addendum: Corresponds to DICOM Tag 0008, 0034 `Intervention Drug Name`.
     PharmaceuticalDoseAmount:
       level: recommended
-      description_addendum: Corresponds to DICOM Tag (0008,0028) `Intervention Drug Dose`.
+      description_addendum: Corresponds to DICOM Tag 0008, 0028 `Intervention Drug Dose`.
     PharmaceuticalDoseUnits: recommended
     PharmaceuticalDoseRegimen: recommended
     PharmaceuticalDoseTime:
@@ -136,17 +136,17 @@ PETTime:
     ScanStart: required
     InjectionStart:
       level: required
-      description_addendum: Corresponds to DICOM Tag (0018,1072) `Radiopharmaceutical Start Time`.
+      description_addendum: Corresponds to DICOM Tag 0018, 1072 `Radiopharmaceutical Start Time`.
     FrameTimesStart: required
     FrameDuration: required
     InjectionEnd:
       level: recommended
       description_addendum: |
-        Corresponds to DICOM Tag (0018,1073) `Radiopharmaceutical Stop Time`
+        Corresponds to DICOM Tag 0018, 1073 `Radiopharmaceutical Stop Time`
         converted to seconds relative to TimeZero.
     ScanDate:
       level: deprecated
-      description_addendum: Corresponds to DICOM Tag (0008,0022) `Acquisition Date`.
+      description_addendum: Corresponds to DICOM Tag 0008, 0022 `Acquisition Date`.
 
 PETReconstruction:
   selectors:
@@ -158,40 +158,40 @@ PETReconstruction:
     ImageDecayCorrectionTime: required
     ReconMethodName:
       level: required
-      description_addendum: This partly matches the DICOM Tag (0054,1103) `Reconstruction Method`.
+      description_addendum: This partly matches the DICOM Tag 0054, 1103 `Reconstruction Method`.
     ReconMethodParameterLabels:
       level: required
-      description_addendum: This partly matches the DICOM Tag (0054,1103) `Reconstruction Method`.
+      description_addendum: This partly matches the DICOM Tag 0054, 1103 `Reconstruction Method`.
     ReconMethodParameterUnits:
       level: recommended
       level_addendum: required if `ReconMethodParameterLabels` does not contain `"none"`
-      description_addendum: This partly matches the DICOM Tag (0054,1103) `Reconstruction Method`.
+      description_addendum: This partly matches the DICOM Tag 0054, 1103 `Reconstruction Method`.
     ReconMethodParameterValues:
       level: recommended
       level_addendum: required if `ReconMethodParameterLabels` does not contain `"none"`
-      description_addendum: This partly matches the DICOM Tag (0054,1103) `Reconstruction Method`.
+      description_addendum: This partly matches the DICOM Tag 0054, 1103 `Reconstruction Method`.
     ReconFilterType:
       level: required
-      description_addendum: This partly matches the DICOM Tag (0018,1210) `Convolution Kernel`.
+      description_addendum: This partly matches the DICOM Tag 0018, 1210 `Convolution Kernel`.
     ReconFilterSize:
       level: recommended
       level_addendum: required if `ReconFilterType` is not `"none"`
-      description_addendum: This partly matches the DICOM Tag (0018,1210) `Convolution Kernel`.
+      description_addendum: This partly matches the DICOM Tag 0018, 1210 `Convolution Kernel`.
     AttenuationCorrection:
       level: required
-      description_addendum: This corresponds to DICOM Tag (0054,1101) `Attenuation Correction Method`.
+      description_addendum: This corresponds to DICOM Tag 0054, 1101 `Attenuation Correction Method`.
     ReconMethodImplementationVersion: recommended
     AttenuationCorrectionMethodReference: recommended
     ScaleFactor: recommended
     ScatterFraction:
       level: recommended
-      description_addendum: Corresponds to DICOM Tag (0054,1323) `Scatter Fraction Factor`.
+      description_addendum: Corresponds to DICOM Tag 0054, 1323 `Scatter Fraction Factor`.
     DecayCorrectionFactor:
       level: recommended
-      description_addendum: Corresponds to DICOM Tag (0054,1321) `Decay Factor`.
+      description_addendum: Corresponds to DICOM Tag 0054, 1321 `Decay Factor`.
     DoseCalibrationFactor:
       level: recommended
-      description_addendum: Corresponds to DICOM Tag (0054,1322) `Dose Calibration Factor`.
+      description_addendum: Corresponds to DICOM Tag 0054, 1322 `Dose Calibration Factor`.
     PromptRate: recommended
     SinglesRate: recommended
     RandomRate: recommended

From 1afbfe8f67f2d6975c2f401b9dcde78110ed1e15 Mon Sep 17 00:00:00 2001
From: Yaroslav Halchenko <debian@onerussian.com>
Date: Fri, 19 Apr 2024 12:56:07 -0700
Subject: [PATCH 11/13] Make extensions identifiers contain . to separate
 extensions where multiple used (#1755)

* Make extensions identifiers contain . to separate extensions where multiple used

* Use underscore since "." is used in path to any object

* [DATALAD RUNCMD] sed -i'' -e 's/tsvgz/tsv_gz/g' src/*.md ...

=== Do not change lines below ===
{
 "chain": [],
 "cmd": "sed -i'' -e 's/tsvgz/tsv_gz/g' src/*.md src/*/*.md",
 "exit": 0,
 "extra_inputs": [],
 "inputs": [],
 "outputs": [],
 "pwd": "."
}
^^^ Do not change lines above ^^^

---------

Co-authored-by: Chris Markiewicz <effigies@gmail.com>
---
 src/common-principles.md                                | 2 +-
 src/modality-specific-files/physiological-recordings.md | 2 +-
 src/modality-specific-files/task-events.md              | 2 +-
 src/schema/objects/extensions.yaml                      | 4 ++--
 tools/schemacode/bidsschematools/render/text.py         | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/common-principles.md b/src/common-principles.md
index 461842ddfc..ac05174e59 100644
--- a/src/common-principles.md
+++ b/src/common-principles.md
@@ -563,7 +563,7 @@ like in the example below.
 ### Compressed tabular files
 
 Large tabular information, such as physiological recordings, MUST be stored with
-[compressed tab-delineated (TSV.GZ) files](glossary.md#tsvgz-extensions) when
+[compressed tab-delineated (TSV.GZ) files](glossary.md#tsv_gz-extensions) when
 so established by the specifications.
 Rules for formatting plain-text tabular files apply to TSVGZ files with three exceptions:
 
diff --git a/src/modality-specific-files/physiological-recordings.md b/src/modality-specific-files/physiological-recordings.md
index 897deb3d1b..8db4a03a02 100644
--- a/src/modality-specific-files/physiological-recordings.md
+++ b/src/modality-specific-files/physiological-recordings.md
@@ -2,7 +2,7 @@
 
 Physiological recordings such as cardiac and respiratory signals MAY be
 specified using a [compressed tabular file](../common-principles.md#compressed-tabular-files)
-([TSV.GZ file](../glossary.md#tsvgz-extensions)) and a corresponding
+([TSV.GZ file](../glossary.md#tsv_gz-extensions)) and a corresponding
 JSON file for storing metadata fields (see below).
 
 !!! example "Example datasets"
diff --git a/src/modality-specific-files/task-events.md b/src/modality-specific-files/task-events.md
index c3822094a5..ca5925909d 100644
--- a/src/modality-specific-files/task-events.md
+++ b/src/modality-specific-files/task-events.md
@@ -325,7 +325,7 @@ in the accompanying JSON sidecar as follows (based on the example of the previou
 Signals related to stimuli (such as parameters of a film or audio stimuli) that are
 evenly recorded at a constant sampling frequency MUST be specified using a
 [compressed tabular file](../common-principles.md#compressed-tabular-files)
-([TSV.GZ file](../glossary.md#tsvgz-extensions)) and a corresponding
+([TSV.GZ file](../glossary.md#tsv_gz-extensions)) and a corresponding
 JSON file for storing metadata fields (see below).
 
 Template:
diff --git a/src/schema/objects/extensions.yaml b/src/schema/objects/extensions.yaml
index 96056e497d..db38cf55b2 100644
--- a/src/schema/objects/extensions.yaml
+++ b/src/schema/objects/extensions.yaml
@@ -184,7 +184,7 @@ nii:
   display_name: NIfTI
   description: |
     A Neuroimaging Informatics Technology Initiative (NIfTI) data file.
-niigz:
+nii_gz:
   value: .nii.gz
   display_name: Compressed NIfTI
   description: |
@@ -274,7 +274,7 @@ tsv:
   display_name: Tab-Delimited
   description: |
     A tab-delimited file.
-tsvgz:
+tsv_gz:
   value: .tsv.gz
   display_name: Compressed Tab-Delimited
   description: |
diff --git a/tools/schemacode/bidsschematools/render/text.py b/tools/schemacode/bidsschematools/render/text.py
index c9fb6f2298..f26ba73204 100644
--- a/tools/schemacode/bidsschematools/render/text.py
+++ b/tools/schemacode/bidsschematools/render/text.py
@@ -401,8 +401,8 @@ def make_filename_template(
 
             ext_headings = []
             for extension in extensions:
-                # The glossary indexes by the extension identifier (niigz instead of .nii.gz),
-                # but the rules reference the actual suffix string (.nii.gz instead of niigz),
+                # The glossary indexes by the extension identifier (nii_gz instead of .nii.gz),
+                # but the rules reference the actual suffix string (.nii.gz instead of nii_gz),
                 # so we need to look it up.
                 key = ext_key_table.get(extension)
                 if key:

From 3e6d1a78b9a4e38c28791f91dcfe3909b2403952 Mon Sep 17 00:00:00 2001
From: Dimitri Papadopoulos Orfanos
 <3234522+DimitriPapadopoulos@users.noreply.github.com>
Date: Mon, 22 Apr 2024 16:49:32 +0200
Subject: [PATCH 12/13] Fix typo (#1802)

---
 src/appendices/qmri.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/appendices/qmri.md b/src/appendices/qmri.md
index 162ba798cd..31e3b3ef7d 100644
--- a/src/appendices/qmri.md
+++ b/src/appendices/qmri.md
@@ -211,7 +211,7 @@ but also by which metadata fields are provided in accompanying json files.
 |----------------------|------------------------------------------------------------------------------------------------------------------------------|----------------------------|
 | VFA                  | `FlipAngle`, `PulseSequenceType`, `RepetitionTimeExcitation`                                                                 | `SpoilingRFPhaseIncrement` |
 | IRT1                 | `InversionTime`                                                                                                              |                            |
-| MP2RAGE<sup>\*</sup> | `FlipAngle`, `InversionTime`, `RepetitionTimeExcitation`, `RepetitionTimePreperation`, `NumberShots`,`MagneticFieldStrength` | `EchoTime`                 |
+| MP2RAGE<sup>\*</sup> | `FlipAngle`, `InversionTime`, `RepetitionTimeExcitation`, `RepetitionTimePreparation`, `NumberShots`,`MagneticFieldStrength` | `EchoTime`                 |
 | MESE                 | `EchoTime`                                                                                                                   |                            |
 | MEGRE                | `EchoTime`                                                                                                                   |                            |
 | MTR                  | `MTState`                                                                                                                    |                            |

From 01025da63297468a3801f8fe43f97a140f2f9551 Mon Sep 17 00:00:00 2001
From: Remi Gau <remi_gau@hotmail.com>
Date: Mon, 22 Apr 2024 16:51:31 +0200
Subject: [PATCH 13/13] [ENH] Add warning about deidentification when sharing
 sourcedata (#1769)

* reset

* Apply suggestions from code review

Co-authored-by: Oscar Esteban <code@oscaresteban.es>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* semantic line break

* Update src/common-principles.md

---------

Co-authored-by: Oscar Esteban <code@oscaresteban.es>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Chris Markiewicz <effigies@gmail.com>
---
 src/common-principles.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/common-principles.md b/src/common-principles.md
index ac05174e59..9b10ba19f2 100644
--- a/src/common-principles.md
+++ b/src/common-principles.md
@@ -309,6 +309,28 @@ field in `dataset_description.json` of each subdirectory of `derivatives` to:
 }
 ```
 
+!!! danger "Caution"
+
+    Sharing source data may help amend errors and missing data discovered
+    only with the reuse of the raw dataset in practice.
+    Therefore, from an Open Science perspective, it is RECOMMENDED to share
+    the source data whenever it is possible.
+
+    However, more stringent sharing limitations may apply to the source data
+    than those applicable to the raw data.
+    For example, human data almost always requires deidentification
+    before they can be redistributed,
+    or the subjects' consent form did not explicitly state that the source files
+    would be shared after deidentification.
+    Further examples in which sharing source data may not be possible
+    include original data formats that are not redistributable
+    as per the acquisition device's license.
+
+    As for raw data, all regulatory, ethical, and legal aspects SHOULD
+    be carefully considered before sharing data
+    through the `sourcedata/` directory mechanism.
+    In the case of source data, these aspects are likely more stringent.
+
 ### Storage of derived datasets
 
 Derivatives can be stored/distributed in two ways: