diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9e6b41fa62..ee82d5b003 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2.1
jobs:
build_docs:
docker:
- - image: cimg/python:3.8
+ - image: cimg/python:3.12-node
steps:
# checkout code to default ~/project
- checkout
@@ -21,14 +21,15 @@ jobs:
- persist_to_workspace:
# the mkdocs build outputs are in ~/project/site
root: ~/project
- paths: site
+ paths:
+ - site
- store_artifacts:
path: ~/project/site/
destination: dev_docs
check_links:
docker:
- - image: cimg/python:3.8
+ - image: cimg/python:3.12
steps:
# checkout code to default ~/project
- checkout
@@ -131,7 +132,8 @@ jobs:
- persist_to_workspace:
# raw generated changelog in ~/changelog_build/CHANGES.md
root: ~/.
- paths: changelog_build
+ paths:
+ - changelog_build
# Lint and fix the auto generated changes.md file
lint_generated_changelog:
@@ -164,7 +166,8 @@ jobs:
- persist_to_workspace:
# linted and fixed changelog in ~/changelog_build/CHANGES.md
root: ~/.
- paths: changelog_build
+ paths:
+ - changelog_build
# Push built changelog to repo
commit_generated_changelog:
@@ -178,7 +181,7 @@ jobs:
- attach_workspace:
# fixed+linted changelog in ~/changelog_build/CHANGES.md
at: ~/.
- - deploy:
+ - run:
name: Changelog deployment
# $CHANGE_TOKEN is generated via the GitHub web UI, and then securely stored within CircleCI web UI
command: |
@@ -197,7 +200,6 @@ jobs:
fi
workflows:
- version: 2
search_build:
jobs:
- build_docs
diff --git a/.github/workflows/schemacode_ci.yml b/.github/workflows/schemacode_ci.yml
index c5809f72b2..6cf4dbcd4c 100644
--- a/.github/workflows/schemacode_ci.yml
+++ b/.github/workflows/schemacode_ci.yml
@@ -44,7 +44,7 @@ jobs:
run: python -m build tools/schemacode
- name: "Check distribution metadata"
run: twine check tools/schemacode/dist/*
- - uses: actions/upload-artifact@v3
+ - uses: actions/upload-artifact@v4
with:
name: dist
path: tools/schemacode/dist/
@@ -76,7 +76,7 @@ jobs:
run: python -c "import sys; print(sys.version)"
- name: "Fetch packages"
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
with:
name: dist
path: dist/
@@ -91,7 +91,7 @@ jobs:
--cov-append --cov-report=xml --cov=bidsschematools --doctest-modules
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: unit_${{ matrix.os }}_${{ matrix.python-version }}
path: coverage.xml
@@ -108,7 +108,7 @@ jobs:
python-version: ["3.11"]
steps:
- name: "Fetch packages"
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
with:
name: dist
path: dist/
@@ -146,7 +146,7 @@ jobs:
run: python -m pytest --pyargs bidsschematools -m "validate_schema" --cov-append --cov-report=xml --cov=bidsschematools
- name: Upload artifacts
- uses: actions/upload-artifact@v3
+ uses: actions/upload-artifact@v4
with:
name: schema_validation
path: coverage.xml
@@ -161,10 +161,10 @@ jobs:
uses: actions/checkout@v4
- name: Download artifacts
- uses: actions/download-artifact@v3
+ uses: actions/download-artifact@v4
- name: Upload to CodeCov
- uses: codecov/codecov-action@v3
+ uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }} # not required but might help API rate limits
fail_ci_if_error: true
diff --git a/.gitignore b/.gitignore
index 59fa1a1de3..9e8cf66a91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,7 @@ venvs
pdf_build_src/bids-spec.pdf
pdf_build_src/bids-spec_pandoc_log.json
pdf_build_src/src_copy
+pdf_build_src/tests/data/output
# JS/NPM
package-lock.json
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3780ff968a..b2f40321ca 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,7 +3,7 @@
exclude: 'tools/schemacode/bidsschematools/tests/data/broken_dataset_description.json'
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.5.0
+ rev: v4.6.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
@@ -13,7 +13,7 @@ repos:
- id: check-added-large-files
- id: check-case-conflict
- repo: https://github.com/psf/black
- rev: 23.12.1
+ rev: 24.4.0
hooks:
- id: black
files: ^tools/(?!schemacode)
@@ -43,13 +43,19 @@ repos:
- id: prettier
entry: env PRETTIER_LEGACY_CLI=1 prettier # temporary fix for https://github.com/prettier/prettier/issues/15742
files: src/schema/.*/.*\.yaml
+ - repo: https://github.com/adrienverge/yamllint
+ rev: v1.35.1
+ hooks:
+ - id: yamllint
+ args: [-f=standard, -c=.yamllint.yml]
+ files: src/schema/.*/.*\.yaml
- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
args: ["--config=.codespellrc", "--dictionary=-", "--dictionary=.codespell_dict"]
- repo: https://github.com/pre-commit/mirrors-mypy
- rev: v1.8.0
+ rev: v1.9.0
hooks:
- id: mypy
# Sync with project.optional-dependencies.typing
@@ -62,5 +68,6 @@ repos:
- pytest
- types-PyYAML
- types-tabulate
+ - types-jsonschema
args: ["tools/schemacode/bidsschematools"]
pass_filenames: false
diff --git a/.remarkrc b/.remarkrc
index fc065759c2..8d6c26487a 100644
--- a/.remarkrc
+++ b/.remarkrc
@@ -10,6 +10,7 @@
["lint-maximum-heading-length", false],
["lint-no-shortcut-reference-link", false],
["lint-no-trailing-spaces"],
+ ["remark-lint-code-block-style", false],
["lint-no-undefined-references", false]
]
}
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 96b8a53c0a..c34f9cb514 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -226,6 +226,37 @@ That would look like this:
|--------------|----------------------------------------------------------|
| Manufacturer | Manufacturer of the equipment, for example (`"Siemens"`) |
+
+#### MkDocs admonitions
+
+It is possible to use [Mkdocs admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#inline-blocks-inline-end)
+to highlight certain aspect of the specification.
+
+Admonitions are written like this:
+
+````
+!!! note "displayed heading is preceded by a keyword and 3 `!`"
+
+ Body of the admonition
+ can be written on several lines,
+ but must be always preceded by 4 spaces.
+````
+
+The keyword for the heading must be one of the following:
+
+- note
+- abstract
+- info
+- tip
+- success
+- question
+- warning
+- failure: octicons
+- danger
+- bug
+- example
+- quote
+
## Using macros
We use [mkdocs-macros](https://mkdocs-macros-plugin.readthedocs.io/en/latest/)
diff --git a/LICENSE b/LICENSE
index d591978380..22091e01bb 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
Attribution 4.0 International
-Copyright (c) 2018-2022, BIDS Contributors.
+Copyright (c) 2018, BIDS Contributors.
=======================================================================
diff --git a/README.md b/README.md
index 574659f292..6474f470a1 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
[](https://github.com/bids-standard/bids-specification/actions/workflows/validation.yml)
[](https://circleci.com/gh/bids-standard/bids-specification)
[](https://fosstodon.org/@bidsstandard)
-[](https://twitter.com/BIDSstandard)
+[](https://x.com/BIDSstandard)
[](https://doi.org/10.5281/zenodo.3686061)
@@ -13,7 +13,6 @@ organization of neuroimaging data.
In this repository, we develop the
[BIDS specification](https://bids-specification.readthedocs.io/en/latest/).
-
# When to use BIDS
To organize your data in BIDS, all you need is neuro data, a computer, and the
@@ -44,10 +43,7 @@ The specification is provided in the form of a webpage, built using
1. Read some introductory material, most likely the very basic problems have already been addressed!
- [BIDS Starter Kit](https://github.com/bids-standard/bids-starter-kit) for tutorials, wikis, templates, ...
2. Post your question in one of several channels where BIDS members are active
- - the [NeuroStars](https://neurostars.org/tags/bids) discourse forum
- - the [BrainHack Mattermost](https://mattermost.brainhack.org), for instant messaging (see also this [news item](https://bids.neuroimaging.io/2020/06/24/Join-the-BIDS-community-on-the-BrainHack-Mattermost.html))
- - the [Google group](https://groups.google.com/forum/#!forum/bids-discussion), for broader discussions surrounding BIDS
- - the [specification repository issue page](https://github.com/bids-standard/bids-specification/issues), if you found inconsistencies, typos, or other issues with the BIDS specification itself
+ - see: [BIDS communication channels](#bids-communication-channels)
# Contributing to BIDS
@@ -58,11 +54,34 @@ For a current list of our contributors, please see our [Contributors appendix](h
When you're ready to get started, check out [our contributing guidelines](https://github.com/bids-standard/bids-specification/blob/master/CONTRIBUTING.md).
-We ask that all contributions to BIDS across all project-related spaces (including but not limited to:
-[GitHub](https://github.com/bids-standard),
-the [Google group](https://groups.google.com/forum/#!forum/bids-discussion), and newsletter emails),
+We ask that all contributions to BIDS across all project-related spaces
+(including but not limited to:
+[GitHub](https://github.com/bids-standard), and the
+[Google group](https://groups.google.com/forum/#!forum/bids-discussion); see
+[BIDS communication channels](#bids-communication-channels))
adhere to our [code of conduct](https://github.com/bids-standard/bids-specification/blob/master/CODE_OF_CONDUCT.md).
+# BIDS communication channels
+
+## Main communication channels
+
+ - "Issue" pages on the different GitHub repositories of the [`bids-standard` GitHub organization](https://github.com/bids-standard),
+ such as the [BIDS specification repository](https://github.com/bids-standard/bids-specification/issues),
+ for reporting problems or making suggestions
+ - The [NeuroStars Discourse forum](https://neurostars.org/tags/bids), for asking usage questions
+ - The [BrainHack Mattermost](https://mattermost.brainhack.org), for instant messaging
+ (see also this [news item](https://bids.neuroimaging.io/2020/06/24/Join-the-BIDS-community-on-the-BrainHack-Mattermost.html))
+ - The [Google group](https://groups.google.com/forum/#!forum/bids-discussion), for broader discussions and announcements surrounding BIDS
+ - The [BIDS website "news"](https://bids.neuroimaging.io/news.html), similar to the Google group, for broader discussions and announcements
+
+## Social media channels
+
+- [X](https://x.com/BIDSstandard)
+- [Mastodon](https://fosstodon.org/@bidsstandard)
+- [Bluesky](https://bsky.app/profile/bidsstandard.bsky.social)
+- [Youtube](https://www.youtube.com/channel/UCxZUcYfd_nvIVWAbzRB1tlw)
+- [Instagram](https://www.instagram.com/bidsstandard/)
+
## Contributors
Thanks goes to these wonderful people.
diff --git a/Release_Protocol.md b/Release_Protocol.md
index f3163cf403..c93c039a48 100644
--- a/Release_Protocol.md
+++ b/Release_Protocol.md
@@ -278,5 +278,5 @@ Update the following files in the BIDS website repository (https://github.com/bi
### 12. Sharing news of the release
-Please share news of the release on the [identified platforms](https://docs.google.com/spreadsheets/d/16SAGK3zG93WM2EWuoZDcRIC7ygPc5b7PDNGpFyC3obA/edit#gid=0).
+Please share news of the release on the [identified platforms](https://github.com/bids-standard/bids-specification?tab=readme-ov-file#BIDS-communication-channels).
Please use our previous release posts as a guide.
diff --git a/mkdocs.yml b/mkdocs.yml
index 82badc4288..114e05c135 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -12,7 +12,7 @@ nav:
- Electroencephalography: modality-specific-files/electroencephalography.md
- Intracranial Electroencephalography: modality-specific-files/intracranial-electroencephalography.md
- Task events: modality-specific-files/task-events.md
- - Physiological and other continuous recordings: modality-specific-files/physiological-and-other-continuous-recordings.md
+ - Physiological recordings: modality-specific-files/physiological-recordings.md
- Behavioral experiments (with no neural recordings): modality-specific-files/behavioral-experiments.md
- Genetic Descriptor: modality-specific-files/genetic-descriptor.md
- Positron Emission Tomography: modality-specific-files/positron-emission-tomography.md
@@ -84,12 +84,16 @@ extra:
social:
- icon: fontawesome/brands/github
link: https://github.com/bids-standard/bids-specification/
- - icon: fontawesome/brands/twitter
- link: https://twitter.com/BIDSstandard/
+ - icon: fontawesome/brands/x-twitter
+ link: https://x.com/BIDSstandard/
- icon: fontawesome/brands/mastodon
link: https://fosstodon.org/@bidsstandard
- icon: fontawesome/brands/google
link: https://groups.google.com/g/bids-discussion
+ - icon: fontawesome/brands/instagram
+ link: https://www.instagram.com/bidsstandard/
+ - icon: fontawesome/brands/youtube
+ link: https://www.youtube.com/channel/UCxZUcYfd_nvIVWAbzRB1tlw
extra_javascript:
- js/jquery-3.6.0.min.js
@@ -97,6 +101,8 @@ markdown_extensions:
- toc:
anchorlink: true
- pymdownx.superfences
+ - admonition
+ - pymdownx.details
plugins:
- search
- branchcustomization:
@@ -116,7 +122,7 @@ plugins:
"04-modality-specific-files/03-electroencephalography.md": "modality-specific-files/electroencephalography.md"
"04-modality-specific-files/04-intracranial-electroencephalography.md": "modality-specific-files/intracranial-electroencephalography.md"
"04-modality-specific-files/05-task-events.md": "modality-specific-files/task-events.md"
- "04-modality-specific-files/06-physiological-and-other-continuous-recordings.md": "modality-specific-files/physiological-and-other-continuous-recordings.md"
+ "04-modality-specific-files/06-physiological-and-other-continuous-recordings.md": "modality-specific-files/physiological-recordings.md"
"04-modality-specific-files/07-behavioral-experiments.md": "modality-specific-files/behavioral-experiments.md"
"04-modality-specific-files/08-genetic-descriptor.md": "modality-specific-files/genetic-descriptor.md"
"04-modality-specific-files/09-positron-emission-tomography.md": "modality-specific-files/positron-emission-tomography.md"
diff --git a/pdf_build_src/pandoc_script.py b/pdf_build_src/pandoc_script.py
index b7b1842b96..e9710482ab 100644
--- a/pdf_build_src/pandoc_script.py
+++ b/pdf_build_src/pandoc_script.py
@@ -2,6 +2,7 @@
This is done once the duplicate src directory is processed.
"""
+
import subprocess
import yaml
from pathlib import Path
@@ -11,9 +12,7 @@
def _find(path, filename):
return next(
- parent / filename
- for parent in path.parents
- if Path.is_file(parent / filename)
+ parent / filename for parent in path.parents if Path.is_file(parent / filename)
)
diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py
index 5d30aabc21..42e984fa98 100644
--- a/pdf_build_src/process_markdowns.py
+++ b/pdf_build_src/process_markdowns.py
@@ -17,6 +17,8 @@
import numpy as np
+from remove_admonitions import remove_admonitions
+
sys.path.append("../tools/")
# functions from module macros are called by eval() later on
from mkdocs_macros_bids import macros # noqa: F401
@@ -557,7 +559,7 @@ def correct_tables(root_path, debug=False):
for i, new_line in enumerate(content):
if i == start_line:
new_content.pop()
- if i >= start_line and i < end_line:
+ if start_line <= i < end_line:
new_content.append("|".join(table[count]) + " \n")
count += 1
elif i == end_line:
@@ -585,9 +587,9 @@ def edit_titlepage():
data = file.readlines()
data[-1] = (
- fr"\textsc{{\large {version_number}}}"
+ rf"\textsc{{\large {version_number}}}"
r"\\[0.5cm]"
- fr"{{\large {build_date}}}"
+ rf"{{\large {build_date}}}"
r"\\[2cm]"
r"\vfill"
r"\end{titlepage}"
@@ -679,33 +681,38 @@ def process_macros(duplicated_src_dir_path):
duplicated_src_dir_path = "src_copy/src"
- # Step 1: make a copy of the src directory in the current directory
+ # make a copy of the src directory in the current directory
copy_src()
- # Step 2: run mkdocs macros embedded in markdown files
+ # run mkdocs macros embedded in markdown files
process_macros(duplicated_src_dir_path)
- # Step 3: copy BIDS_logo to images directory of the src_copy directory
+ # remove mkdocs admonition
+ remove_admonitions(
+ input_folder=duplicated_src_dir_path, output_folder=duplicated_src_dir_path
+ )
+
+ # copy BIDS_logo to images directory of the src_copy directory
copy_bids_logo()
- # Step 4: copy images from subdirectories of src_copy directory
+ # copy images from subdirectories of src_copy directory
copy_images(duplicated_src_dir_path)
subprocess.call("mv src_copy/src/images/images/* src_copy/src/images/", shell=True)
- # Step 5: extract the latest version number, date and title
+ # extract the latest version number, date and title
extract_header_string()
add_header()
edit_titlepage()
- # Step 6: modify changelog to be a level 1 heading to facilitate section
+ # modify changelog to be a level 1 heading to facilitate section
# separation
modify_changelog()
- # Step 7: remove all internal links
+ # remove all internal links
assert_no_multiline_links(duplicated_src_dir_path)
remove_internal_links_inline(duplicated_src_dir_path)
remove_internal_links_reference(duplicated_src_dir_path)
- # Step 8: correct number of dashes and fences alignment for rendering tables in PDF
+ # correct number of dashes and fences alignment for rendering tables in PDF
correct_tables(duplicated_src_dir_path)
diff --git a/pdf_build_src/remove_admonitions.py b/pdf_build_src/remove_admonitions.py
new file mode 100644
index 0000000000..1942f8b5b0
--- /dev/null
+++ b/pdf_build_src/remove_admonitions.py
@@ -0,0 +1,64 @@
+"""Script to remove all mkdocs admonition from the markdown files in a directory.
+
+See the pdf_build_src/tests/data/input directory to see what admonitions look like.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+INDENT = " "
+
+ADMONITION_DELIMITERS = ["!!!", "???", "???+"]
+
+
+def remove_admonitions(
+ input_folder: str | Path, output_folder: str | Path, indent: str = None
+):
+
+ if indent is None:
+ indent = INDENT
+
+ md_files = Path(input_folder).glob("**/*.md")
+
+ for file in md_files:
+
+ with open(file, "r", encoding="utf8") as f:
+ content = f.readlines()
+
+ output_file = Path(output_folder) / file.relative_to(input_folder)
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+ print(f"processing: {file}\n to: {output_file}")
+
+ with open(output_file, "w", encoding="utf8") as f:
+
+ is_admonition = False
+ counter = 0
+ for line in content:
+
+ if any(line.startswith(x) for x in ADMONITION_DELIMITERS):
+ is_admonition = True
+ counter = 0
+ continue
+
+ # skip first line after admonition
+ if is_admonition and counter == 0:
+ counter += 1
+ continue
+
+ if line != "\n" and not line.startswith(indent):
+ is_admonition = False
+
+ if is_admonition:
+ line = line.lstrip(indent)
+
+ f.write(line)
+
+
+if __name__ == "__main__":
+ """If run as a script this will just run the main function on test data."""
+ input_folder = Path(__file__).parent / "tests" / "data" / "input"
+ output_folder = Path(__file__).parent / "tests" / "data" / "output"
+ shutil.rmtree(output_folder)
+ remove_admonitions(input_folder, output_folder)
diff --git a/pdf_build_src/test_remove_admonitions.py b/pdf_build_src/test_remove_admonitions.py
new file mode 100644
index 0000000000..e995e42246
--- /dev/null
+++ b/pdf_build_src/test_remove_admonitions.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from remove_admonitions import remove_admonitions
+
+
+def test_remove_admonitions(tmp_path):
+ input_folder = Path(__file__).parent / "tests" / "data" / "input"
+ expected_folder = Path(__file__).parent / "tests" / "data" / "expected"
+
+ remove_admonitions(input_folder, tmp_path)
+
+ generated_files = list(tmp_path.glob("**/*.md"))
+
+ for file in generated_files:
+
+ expected = expected_folder / file.relative_to(tmp_path)
+
+ with open(expected, "r", encoding="utf8") as f:
+ expected_content = f.readlines()
+
+ with open(file, "r", encoding="utf8") as f:
+ generated_content = f.readlines()
+
+ for expected_line, generated_line in zip(expected_content, generated_content):
+ assert generated_line == expected_line
diff --git a/pdf_build_src/tests/data/expected/README.md b/pdf_build_src/tests/data/expected/README.md
new file mode 100644
index 0000000000..37fadc295d
--- /dev/null
+++ b/pdf_build_src/tests/data/expected/README.md
@@ -0,0 +1,32 @@
+# Test inputs
+
+This input directory contains data to use for testing the pdf build code of the BIDS specification.
+
+For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Nulla et euismod nulla.
+Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+nec semper lorem quam in massa.
+
+The `expected` directory should contain the documents
+as they should look like after processing.
+
+[Mkdocs admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#inline-blocks-inline-end)
+come in different type. In aaddtion of the classical admonitions show above you have also:
+
+Collapsible admonitions start with 3 questions marks (`???`).
+
+Collapsible admonitions that will be shown as expanded
+start with 3 questions marks and a plus sign (`???+`).
+
+
+
+Let's see
+
+- [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+- foo bar [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+
+More of the admonition
+
+And here we resume normal thing.
diff --git a/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md
new file mode 100644
index 0000000000..d70af64a49
--- /dev/null
+++ b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md
@@ -0,0 +1,8 @@
+# Magnetic Resonance Imaging
+
+## Common metadata fields
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Nulla et euismod nulla.
+Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+nec semper lorem quam in massa.
diff --git a/pdf_build_src/tests/data/input/README.md b/pdf_build_src/tests/data/input/README.md
new file mode 100644
index 0000000000..cab4817282
--- /dev/null
+++ b/pdf_build_src/tests/data/input/README.md
@@ -0,0 +1,40 @@
+# Test inputs
+
+This input directory contains data to use for testing the pdf build code of the BIDS specification.
+
+For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`.
+
+!!! note
+
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Nulla et euismod nulla.
+ Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+ nec semper lorem quam in massa.
+
+The `expected` directory should contain the documents
+as they should look like after processing.
+
+[Mkdocs admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#inline-blocks-inline-end)
+come in different type. In aaddtion of the classical admonitions show above you have also:
+
+??? note "Collapsible admonitions"
+
+ Collapsible admonitions start with 3 questions marks (`???`).
+
+???+ note "Expanded collapsible admonitions"
+
+ Collapsible admonitions that will be shown as expanded
+ start with 3 questions marks and a plus sign (`???+`).
+
+
+
+!!! example "non ordered list should be handle propeler"
+
+ Let's see
+
+ - [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+ - foo bar [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+
+ More of the admonition
+
+And here we resume normal thing.
diff --git a/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md
new file mode 100644
index 0000000000..bfc1b8509a
--- /dev/null
+++ b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md
@@ -0,0 +1,10 @@
+# Magnetic Resonance Imaging
+
+## Common metadata fields
+
+!!! warning "foo bar"
+
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Nulla et euismod nulla.
+ Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+ nec semper lorem quam in massa.
diff --git a/readthedocs.yml b/readthedocs.yml
index cc3f461fc7..5611c59505 100644
--- a/readthedocs.yml
+++ b/readthedocs.yml
@@ -2,11 +2,14 @@ version: 2
build:
os: ubuntu-22.04
+ apt_packages:
+ - jq
tools:
python: "3.11"
jobs:
pre_build:
- bst -v export --output src/schema.json
+ - tools/no-bad-schema-paths.sh src/schema.json # README.md might need fixing
mkdocs:
configuration: mkdocs.yml
diff --git a/src/appendices/entity-table.md b/src/appendices/entity-table.md
index 4dff2dcf93..4ee4857d00 100644
--- a/src/appendices/entity-table.md
+++ b/src/appendices/entity-table.md
@@ -1,3 +1,8 @@
+---
+hide:
+- toc
+---
+
# Entity table
This section compiles the entities (key-value pairs within filenames)
diff --git a/src/appendices/schema.md b/src/appendices/schema.md
index 1616a6d5c9..fc0855881d 100644
--- a/src/appendices/schema.md
+++ b/src/appendices/schema.md
@@ -9,7 +9,7 @@ to the BIDS standard.
The BIDS schema is available in two machine readable formats:
-- as a set of [YAML](https://en.wikipedia.org/wiki/YAML) files in the [BIDS specification repository](https://github.com/bids-standard/bids-specification/src/schema)
+- as a set of [YAML](https://en.wikipedia.org/wiki/YAML) files in the [BIDS specification repository](https://github.com/bids-standard/bids-specification/tree/master/src/schema)
- as a [single dereferenced json file](https://bids-specification.readthedocs.io/en/stable/schema.json)
A didactic walkthrough of the schema can be found in the [BEP Guide](https://bids-extensions.readthedocs.io/en/latest/schema/),
diff --git a/src/common-principles.md b/src/common-principles.md
index d3ff223dcb..ac05174e59 100644
--- a/src/common-principles.md
+++ b/src/common-principles.md
@@ -238,10 +238,14 @@ distinguish partial results from the raw data and share the latter.
See [Storage of derived datasets](#storage-of-derived-datasets) for more on
organizing derivatives.
-Similar rules apply to source data, which is defined as data before
-harmonization, reconstruction, and/or file format conversion (for example, E-Prime event logs or DICOM files).
-Storing actual source files with the data is preferred over links to
-external source repositories to maximize long term preservation,
+Similar rules apply to source data, which is defined as data
+before harmonization, reconstruction, and/or file format conversion
+(for example, E-Prime event logs or DICOM files).
+Retaining the source data is especially valuable
+in a case when conversion fails to preserve crucial metadata
+unique to specific acquisition setup.
+Storing actual source files with the data is preferred over links
+to external source repositories to maximize long term preservation,
which would suffer if an external repository would not be available anymore.
This specification currently does not go into the details of
recommending a particular naming scheme for including different types of
@@ -426,35 +430,54 @@ NIfTI header.
### Tabular files
-Tabular data MUST be saved as tab delimited values (`.tsv`) files, that is, CSV
-files where commas are replaced by tabs. Tabs MUST be true tab characters and
-MUST NOT be a series of space characters. Each TSV file MUST start with a header
-line listing the names of all columns (with the exception of
-[physiological and other continuous recordings](modality-specific-files/physiological-and-other-continuous-recordings.md)).
+Tabular data MUST be saved as plain-text, tab-delimited values (TSV) files
+(with [extension `.tsv`](glossary.md#tsv-extensions)),
+that is, [CSV files](https://en.wikipedia.org/wiki/Comma-separated_values) where commas are replaced by tab characters.
+Tabs MUST be true tab characters and MUST NOT be a series of space characters.
+Tabular data such as continuous physiology recordings typically containing
+large numbers of rows MAY be saved as
+[compressed tabular files (with extension `.tsv.gz`)](#compressed-tabular-files),
+which are introduced below.
+Plain-text TSV and compressed TSV are not interchangeable, that is, each section
+of the specification prescribes which one MUST be used for the data type at
+hand.
+Each TSV file MUST start with a header line listing the names of all columns
+with two exceptions:
+
+1. [compressed tabular files](#compressed-tabular-files),
+ for which column names are defined in a sidecar metadata
+ [JSON object](https://www.json.org/json-en.html) described below; and
+1. [motion recording data](modality-specific-files/motion.md),
+ which use plain-text TSV and columns are defined as described
+ in its corresponding section of the specifications.
+
It is RECOMMENDED that the column names in the header of the TSV file are
written in [`snake_case`](https://en.wikipedia.org/wiki/Snake_case) with the
first letter in lower case (for example, `variable_name`, not `Variable_name`).
-As for all other data in the TSV files, column names MUST be separated with tabs.
+Column names defined in the header MUST be separated with tabs as for the data contents.
Furthermore, column names MUST NOT be blank (that is, an empty string) and MUST NOT
be duplicated within a single TSV file.
-String values containing tabs MUST be escaped using double
-quotes. Missing and non-applicable values MUST be coded as `n/a`. Numerical
-values MUST employ the dot (`.`) as decimal separator and MAY be specified
+String values containing tabs MUST be escaped using double quotes.
+Missing and non-applicable values MUST be coded as `n/a`.
+Numerical values MUST employ the dot (`.`) as decimal separator and MAY be specified
in scientific notation, using `e` or `E` to separate the significand from the
-exponent. TSV files MUST be in UTF-8 encoding.
+exponent.
+TSV files MUST be in UTF-8 encoding.
Example:
```Text
-onset duration response_time correct stop_trial go_trial
-200 200 0 n/a n/a n/a
+onset duration response_time trial_type trial_extra
+200 20.0 15.8 word 中国人
+240 5.0 17.34e-1 visual n/a
```
-**Note**: The TSV examples in this document (like the one above this note)
-are occasionally formatted using space characters instead of tabs to improve
-human readability.
-Directly copying and then pasting these examples from the specification
-for use in new BIDS datasets can lead to errors and is discouraged.
+!!! warning "Attention"
+
+ The TSV examples in this document (like the one above this note) are occasionally
+ formatted using space characters instead of tabs to improve human readability.
+ Directly copying and then pasting these examples from the specification
+ for use in new BIDS datasets can lead to errors and is discouraged.
Tabular files MAY be optionally accompanied by a simple data dictionary
in the form of a JSON [object](https://www.json.org/json-en.html)
@@ -531,12 +554,38 @@ like in the example below.
"F": {
"Description": "Female",
"TermURL": "https://www.ncbi.nlm.nih.gov/mesh/68005260"
- },
+ }
}
}
}
```
+### Compressed tabular files
+
+Large tabular information, such as physiological recordings, MUST be stored with
+[compressed tab-delineated (TSV.GZ) files](glossary.md#tsv_gz-extensions) when
+so established by the specifications.
+Rules for formatting plain-text tabular files apply to TSVGZ files with three exceptions:
+
+1. The contents of TSVGZ files MUST be compressed with
+ [gzip](https://datatracker.ietf.org/doc/html/rfc1952).
+1. Compressed tabular files MUST NOT contain a header in the first row
+ indicating the column names.
+1. TSVGZ files MUST have an associated JSON file that defines the columns in the tabular file.
+
+!!! warning "Attention"
+
+ In contrast to plain-text TSV files,
+ compressed tabular files files MUST NOT include a header line.
+ Column names MUST be provided in the JSON file with the
+ [`Columns`](glossary.md#columns-metadata) field.
+ Each column MAY additionally be described with a column description,
+ as described in [Tabular files](#tabular-files).
+
+ TSVGZ are header-less to improve compatibility with existing software
+ (for example, FSL, or PNM), and to facilitate the support for other file formats
+ in the future.
+
### Key-value files (dictionaries)
JavaScript Object Notation (JSON) files MUST be used for storing key-value
diff --git a/src/derivatives/common-data-types.md b/src/derivatives/common-data-types.md
index 55ad6e3bf1..5a2e725e91 100644
--- a/src/derivatives/common-data-types.md
+++ b/src/derivatives/common-data-types.md
@@ -256,24 +256,33 @@ static volume, a `RepetitionTime` property would no longer be relevant).
## descriptions.tsv
+Template:
+
+```Text
+[sub-
@@ -13,7 +13,6 @@ organization of neuroimaging data.
In this repository, we develop the
[BIDS specification](https://bids-specification.readthedocs.io/en/latest/).
-
# When to use BIDS
To organize your data in BIDS, all you need is neuro data, a computer, and the
@@ -44,10 +43,7 @@ The specification is provided in the form of a webpage, built using
1. Read some introductory material, most likely the very basic problems have already been addressed!
- [BIDS Starter Kit](https://github.com/bids-standard/bids-starter-kit) for tutorials, wikis, templates, ...
2. Post your question in one of several channels where BIDS members are active
- - the [NeuroStars](https://neurostars.org/tags/bids) discourse forum
- - the [BrainHack Mattermost](https://mattermost.brainhack.org), for instant messaging (see also this [news item](https://bids.neuroimaging.io/2020/06/24/Join-the-BIDS-community-on-the-BrainHack-Mattermost.html))
- - the [Google group](https://groups.google.com/forum/#!forum/bids-discussion), for broader discussions surrounding BIDS
- - the [specification repository issue page](https://github.com/bids-standard/bids-specification/issues), if you found inconsistencies, typos, or other issues with the BIDS specification itself
+ - see: [BIDS communication channels](#bids-communication-channels)
# Contributing to BIDS
@@ -58,11 +54,34 @@ For a current list of our contributors, please see our [Contributors appendix](h
When you're ready to get started, check out [our contributing guidelines](https://github.com/bids-standard/bids-specification/blob/master/CONTRIBUTING.md).
-We ask that all contributions to BIDS across all project-related spaces (including but not limited to:
-[GitHub](https://github.com/bids-standard),
-the [Google group](https://groups.google.com/forum/#!forum/bids-discussion), and newsletter emails),
+We ask that all contributions to BIDS across all project-related spaces
+(including but not limited to:
+[GitHub](https://github.com/bids-standard), and the
+[Google group](https://groups.google.com/forum/#!forum/bids-discussion); see
+[BIDS communication channels](#bids-communication-channels))
adhere to our [code of conduct](https://github.com/bids-standard/bids-specification/blob/master/CODE_OF_CONDUCT.md).
+# BIDS communication channels
+
+## Main communication channels
+
+ - "Issue" pages on the different GitHub repositories of the [`bids-standard` GitHub organization](https://github.com/bids-standard),
+ such as the [BIDS specification repository](https://github.com/bids-standard/bids-specification/issues),
+ for reporting problems or making suggestions
+ - The [NeuroStars Discourse forum](https://neurostars.org/tags/bids), for asking usage questions
+ - The [BrainHack Mattermost](https://mattermost.brainhack.org), for instant messaging
+ (see also this [news item](https://bids.neuroimaging.io/2020/06/24/Join-the-BIDS-community-on-the-BrainHack-Mattermost.html))
+ - The [Google group](https://groups.google.com/forum/#!forum/bids-discussion), for broader discussions and announcements surrounding BIDS
+ - The [BIDS website "news"](https://bids.neuroimaging.io/news.html), similar to the Google group, for broader discussions and announcements
+
+## Social media channels
+
+- [X](https://x.com/BIDSstandard)
+- [Mastodon](https://fosstodon.org/@bidsstandard)
+- [Bluesky](https://bsky.app/profile/bidsstandard.bsky.social)
+- [Youtube](https://www.youtube.com/channel/UCxZUcYfd_nvIVWAbzRB1tlw)
+- [Instagram](https://www.instagram.com/bidsstandard/)
+
## Contributors
Thanks goes to these wonderful people.
diff --git a/Release_Protocol.md b/Release_Protocol.md
index f3163cf403..c93c039a48 100644
--- a/Release_Protocol.md
+++ b/Release_Protocol.md
@@ -278,5 +278,5 @@ Update the following files in the BIDS website repository (https://github.com/bi
### 12. Sharing news of the release
-Please share news of the release on the [identified platforms](https://docs.google.com/spreadsheets/d/16SAGK3zG93WM2EWuoZDcRIC7ygPc5b7PDNGpFyC3obA/edit#gid=0).
+Please share news of the release on the [identified platforms](https://github.com/bids-standard/bids-specification?tab=readme-ov-file#BIDS-communication-channels).
Please use our previous release posts as a guide.
diff --git a/mkdocs.yml b/mkdocs.yml
index 82badc4288..114e05c135 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -12,7 +12,7 @@ nav:
- Electroencephalography: modality-specific-files/electroencephalography.md
- Intracranial Electroencephalography: modality-specific-files/intracranial-electroencephalography.md
- Task events: modality-specific-files/task-events.md
- - Physiological and other continuous recordings: modality-specific-files/physiological-and-other-continuous-recordings.md
+ - Physiological recordings: modality-specific-files/physiological-recordings.md
- Behavioral experiments (with no neural recordings): modality-specific-files/behavioral-experiments.md
- Genetic Descriptor: modality-specific-files/genetic-descriptor.md
- Positron Emission Tomography: modality-specific-files/positron-emission-tomography.md
@@ -84,12 +84,16 @@ extra:
social:
- icon: fontawesome/brands/github
link: https://github.com/bids-standard/bids-specification/
- - icon: fontawesome/brands/twitter
- link: https://twitter.com/BIDSstandard/
+ - icon: fontawesome/brands/x-twitter
+ link: https://x.com/BIDSstandard/
- icon: fontawesome/brands/mastodon
link: https://fosstodon.org/@bidsstandard
- icon: fontawesome/brands/google
link: https://groups.google.com/g/bids-discussion
+ - icon: fontawesome/brands/instagram
+ link: https://www.instagram.com/bidsstandard/
+ - icon: fontawesome/brands/youtube
+ link: https://www.youtube.com/channel/UCxZUcYfd_nvIVWAbzRB1tlw
extra_javascript:
- js/jquery-3.6.0.min.js
@@ -97,6 +101,8 @@ markdown_extensions:
- toc:
anchorlink: true
- pymdownx.superfences
+ - admonition
+ - pymdownx.details
plugins:
- search
- branchcustomization:
@@ -116,7 +122,7 @@ plugins:
"04-modality-specific-files/03-electroencephalography.md": "modality-specific-files/electroencephalography.md"
"04-modality-specific-files/04-intracranial-electroencephalography.md": "modality-specific-files/intracranial-electroencephalography.md"
"04-modality-specific-files/05-task-events.md": "modality-specific-files/task-events.md"
- "04-modality-specific-files/06-physiological-and-other-continuous-recordings.md": "modality-specific-files/physiological-and-other-continuous-recordings.md"
+ "04-modality-specific-files/06-physiological-and-other-continuous-recordings.md": "modality-specific-files/physiological-recordings.md"
"04-modality-specific-files/07-behavioral-experiments.md": "modality-specific-files/behavioral-experiments.md"
"04-modality-specific-files/08-genetic-descriptor.md": "modality-specific-files/genetic-descriptor.md"
"04-modality-specific-files/09-positron-emission-tomography.md": "modality-specific-files/positron-emission-tomography.md"
diff --git a/pdf_build_src/pandoc_script.py b/pdf_build_src/pandoc_script.py
index b7b1842b96..e9710482ab 100644
--- a/pdf_build_src/pandoc_script.py
+++ b/pdf_build_src/pandoc_script.py
@@ -2,6 +2,7 @@
This is done once the duplicate src directory is processed.
"""
+
import subprocess
import yaml
from pathlib import Path
@@ -11,9 +12,7 @@
def _find(path, filename):
return next(
- parent / filename
- for parent in path.parents
- if Path.is_file(parent / filename)
+ parent / filename for parent in path.parents if Path.is_file(parent / filename)
)
diff --git a/pdf_build_src/process_markdowns.py b/pdf_build_src/process_markdowns.py
index 5d30aabc21..42e984fa98 100644
--- a/pdf_build_src/process_markdowns.py
+++ b/pdf_build_src/process_markdowns.py
@@ -17,6 +17,8 @@
import numpy as np
+from remove_admonitions import remove_admonitions
+
sys.path.append("../tools/")
# functions from module macros are called by eval() later on
from mkdocs_macros_bids import macros # noqa: F401
@@ -557,7 +559,7 @@ def correct_tables(root_path, debug=False):
for i, new_line in enumerate(content):
if i == start_line:
new_content.pop()
- if i >= start_line and i < end_line:
+ if start_line <= i < end_line:
new_content.append("|".join(table[count]) + " \n")
count += 1
elif i == end_line:
@@ -585,9 +587,9 @@ def edit_titlepage():
data = file.readlines()
data[-1] = (
- fr"\textsc{{\large {version_number}}}"
+ rf"\textsc{{\large {version_number}}}"
r"\\[0.5cm]"
- fr"{{\large {build_date}}}"
+ rf"{{\large {build_date}}}"
r"\\[2cm]"
r"\vfill"
r"\end{titlepage}"
@@ -679,33 +681,38 @@ def process_macros(duplicated_src_dir_path):
duplicated_src_dir_path = "src_copy/src"
- # Step 1: make a copy of the src directory in the current directory
+ # make a copy of the src directory in the current directory
copy_src()
- # Step 2: run mkdocs macros embedded in markdown files
+ # run mkdocs macros embedded in markdown files
process_macros(duplicated_src_dir_path)
- # Step 3: copy BIDS_logo to images directory of the src_copy directory
+ # remove mkdocs admonition
+ remove_admonitions(
+ input_folder=duplicated_src_dir_path, output_folder=duplicated_src_dir_path
+ )
+
+ # copy BIDS_logo to images directory of the src_copy directory
copy_bids_logo()
- # Step 4: copy images from subdirectories of src_copy directory
+ # copy images from subdirectories of src_copy directory
copy_images(duplicated_src_dir_path)
subprocess.call("mv src_copy/src/images/images/* src_copy/src/images/", shell=True)
- # Step 5: extract the latest version number, date and title
+ # extract the latest version number, date and title
extract_header_string()
add_header()
edit_titlepage()
- # Step 6: modify changelog to be a level 1 heading to facilitate section
+ # modify changelog to be a level 1 heading to facilitate section
# separation
modify_changelog()
- # Step 7: remove all internal links
+ # remove all internal links
assert_no_multiline_links(duplicated_src_dir_path)
remove_internal_links_inline(duplicated_src_dir_path)
remove_internal_links_reference(duplicated_src_dir_path)
- # Step 8: correct number of dashes and fences alignment for rendering tables in PDF
+ # correct number of dashes and fences alignment for rendering tables in PDF
correct_tables(duplicated_src_dir_path)
diff --git a/pdf_build_src/remove_admonitions.py b/pdf_build_src/remove_admonitions.py
new file mode 100644
index 0000000000..1942f8b5b0
--- /dev/null
+++ b/pdf_build_src/remove_admonitions.py
@@ -0,0 +1,64 @@
+"""Script to remove all mkdocs admonition from the markdown files in a directory.
+
+See the pdf_build_src/tests/data/input directory to see what admonitions look like.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+INDENT = " "
+
+ADMONITION_DELIMITERS = ["!!!", "???", "???+"]
+
+
+def remove_admonitions(
+ input_folder: str | Path, output_folder: str | Path, indent: str = None
+):
+
+ if indent is None:
+ indent = INDENT
+
+ md_files = Path(input_folder).glob("**/*.md")
+
+ for file in md_files:
+
+ with open(file, "r", encoding="utf8") as f:
+ content = f.readlines()
+
+ output_file = Path(output_folder) / file.relative_to(input_folder)
+ output_file.parent.mkdir(parents=True, exist_ok=True)
+ print(f"processing: {file}\n to: {output_file}")
+
+ with open(output_file, "w", encoding="utf8") as f:
+
+ is_admonition = False
+ counter = 0
+ for line in content:
+
+ if any(line.startswith(x) for x in ADMONITION_DELIMITERS):
+ is_admonition = True
+ counter = 0
+ continue
+
+ # skip first line after admonition
+ if is_admonition and counter == 0:
+ counter += 1
+ continue
+
+ if line != "\n" and not line.startswith(indent):
+ is_admonition = False
+
+ if is_admonition:
+ line = line.lstrip(indent)
+
+ f.write(line)
+
+
+if __name__ == "__main__":
+ """If run as a script this will just run the main function on test data."""
+ input_folder = Path(__file__).parent / "tests" / "data" / "input"
+ output_folder = Path(__file__).parent / "tests" / "data" / "output"
+ shutil.rmtree(output_folder)
+ remove_admonitions(input_folder, output_folder)
diff --git a/pdf_build_src/test_remove_admonitions.py b/pdf_build_src/test_remove_admonitions.py
new file mode 100644
index 0000000000..e995e42246
--- /dev/null
+++ b/pdf_build_src/test_remove_admonitions.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from remove_admonitions import remove_admonitions
+
+
+def test_remove_admonitions(tmp_path):
+ input_folder = Path(__file__).parent / "tests" / "data" / "input"
+ expected_folder = Path(__file__).parent / "tests" / "data" / "expected"
+
+ remove_admonitions(input_folder, tmp_path)
+
+ generated_files = list(tmp_path.glob("**/*.md"))
+
+ for file in generated_files:
+
+ expected = expected_folder / file.relative_to(tmp_path)
+
+ with open(expected, "r", encoding="utf8") as f:
+ expected_content = f.readlines()
+
+ with open(file, "r", encoding="utf8") as f:
+ generated_content = f.readlines()
+
+ for expected_line, generated_line in zip(expected_content, generated_content):
+ assert generated_line == expected_line
diff --git a/pdf_build_src/tests/data/expected/README.md b/pdf_build_src/tests/data/expected/README.md
new file mode 100644
index 0000000000..37fadc295d
--- /dev/null
+++ b/pdf_build_src/tests/data/expected/README.md
@@ -0,0 +1,32 @@
+# Test inputs
+
+This input directory contains data to use for testing the pdf build code of the BIDS specification.
+
+For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Nulla et euismod nulla.
+Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+nec semper lorem quam in massa.
+
+The `expected` directory should contain the documents
+as they should look like after processing.
+
+[Mkdocs admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#inline-blocks-inline-end)
+come in different type. In aaddtion of the classical admonitions show above you have also:
+
+Collapsible admonitions start with 3 questions marks (`???`).
+
+Collapsible admonitions that will be shown as expanded
+start with 3 questions marks and a plus sign (`???+`).
+
+
+
+Let's see
+
+- [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+- foo bar [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+
+More of the admonition
+
+And here we resume normal thing.
diff --git a/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md
new file mode 100644
index 0000000000..d70af64a49
--- /dev/null
+++ b/pdf_build_src/tests/data/expected/modality-specific-files/magnetic-resonance-imaging-data.md
@@ -0,0 +1,8 @@
+# Magnetic Resonance Imaging
+
+## Common metadata fields
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Nulla et euismod nulla.
+Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+nec semper lorem quam in massa.
diff --git a/pdf_build_src/tests/data/input/README.md b/pdf_build_src/tests/data/input/README.md
new file mode 100644
index 0000000000..cab4817282
--- /dev/null
+++ b/pdf_build_src/tests/data/input/README.md
@@ -0,0 +1,40 @@
+# Test inputs
+
+This input directory contains data to use for testing the pdf build code of the BIDS specification.
+
+For example the following admonition should be removed by `pdf_build_src/remove_admonitions.py`.
+
+!!! note
+
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Nulla et euismod nulla.
+ Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+ nec semper lorem quam in massa.
+
+The `expected` directory should contain the documents
+as they should look like after processing.
+
+[Mkdocs admonitions](https://squidfunk.github.io/mkdocs-material/reference/admonitions/#inline-blocks-inline-end)
+come in different type. In aaddtion of the classical admonitions show above you have also:
+
+??? note "Collapsible admonitions"
+
+ Collapsible admonitions start with 3 questions marks (`???`).
+
+???+ note "Expanded collapsible admonitions"
+
+ Collapsible admonitions that will be shown as expanded
+ start with 3 questions marks and a plus sign (`???+`).
+
+
+
+!!! example "non ordered list should be handle propeler"
+
+ Let's see
+
+ - [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+ - foo bar [`UK biobank`](https://github.com/bids-standard/bids-examples/tree/master/genetics_ukbb)
+
+ More of the admonition
+
+And here we resume normal thing.
diff --git a/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md
new file mode 100644
index 0000000000..bfc1b8509a
--- /dev/null
+++ b/pdf_build_src/tests/data/input/modality-specific-files/magnetic-resonance-imaging-data.md
@@ -0,0 +1,10 @@
+# Magnetic Resonance Imaging
+
+## Common metadata fields
+
+!!! warning "foo bar"
+
+ Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+ Nulla et euismod nulla.
+ Curabitur feugiat, tortor non consequat finibus, justo purus auctor massa,
+ nec semper lorem quam in massa.
diff --git a/readthedocs.yml b/readthedocs.yml
index cc3f461fc7..5611c59505 100644
--- a/readthedocs.yml
+++ b/readthedocs.yml
@@ -2,11 +2,14 @@ version: 2
build:
os: ubuntu-22.04
+ apt_packages:
+ - jq
tools:
python: "3.11"
jobs:
pre_build:
- bst -v export --output src/schema.json
+ - tools/no-bad-schema-paths.sh src/schema.json # README.md might need fixing
mkdocs:
configuration: mkdocs.yml
diff --git a/src/appendices/entity-table.md b/src/appendices/entity-table.md
index 4dff2dcf93..4ee4857d00 100644
--- a/src/appendices/entity-table.md
+++ b/src/appendices/entity-table.md
@@ -1,3 +1,8 @@
+---
+hide:
+- toc
+---
+
# Entity table
This section compiles the entities (key-value pairs within filenames)
diff --git a/src/appendices/schema.md b/src/appendices/schema.md
index 1616a6d5c9..fc0855881d 100644
--- a/src/appendices/schema.md
+++ b/src/appendices/schema.md
@@ -9,7 +9,7 @@ to the BIDS standard.
The BIDS schema is available in two machine readable formats:
-- as a set of [YAML](https://en.wikipedia.org/wiki/YAML) files in the [BIDS specification repository](https://github.com/bids-standard/bids-specification/src/schema)
+- as a set of [YAML](https://en.wikipedia.org/wiki/YAML) files in the [BIDS specification repository](https://github.com/bids-standard/bids-specification/tree/master/src/schema)
- as a [single dereferenced json file](https://bids-specification.readthedocs.io/en/stable/schema.json)
A didactic walkthrough of the schema can be found in the [BEP Guide](https://bids-extensions.readthedocs.io/en/latest/schema/),
diff --git a/src/common-principles.md b/src/common-principles.md
index d3ff223dcb..ac05174e59 100644
--- a/src/common-principles.md
+++ b/src/common-principles.md
@@ -238,10 +238,14 @@ distinguish partial results from the raw data and share the latter.
See [Storage of derived datasets](#storage-of-derived-datasets) for more on
organizing derivatives.
-Similar rules apply to source data, which is defined as data before
-harmonization, reconstruction, and/or file format conversion (for example, E-Prime event logs or DICOM files).
-Storing actual source files with the data is preferred over links to
-external source repositories to maximize long term preservation,
+Similar rules apply to source data, which is defined as data
+before harmonization, reconstruction, and/or file format conversion
+(for example, E-Prime event logs or DICOM files).
+Retaining the source data is especially valuable
+in a case when conversion fails to preserve crucial metadata
+unique to specific acquisition setup.
+Storing actual source files with the data is preferred over links
+to external source repositories to maximize long term preservation,
which would suffer if an external repository would not be available anymore.
This specification currently does not go into the details of
recommending a particular naming scheme for including different types of
@@ -426,35 +430,54 @@ NIfTI header.
### Tabular files
-Tabular data MUST be saved as tab delimited values (`.tsv`) files, that is, CSV
-files where commas are replaced by tabs. Tabs MUST be true tab characters and
-MUST NOT be a series of space characters. Each TSV file MUST start with a header
-line listing the names of all columns (with the exception of
-[physiological and other continuous recordings](modality-specific-files/physiological-and-other-continuous-recordings.md)).
+Tabular data MUST be saved as plain-text, tab-delimited values (TSV) files
+(with [extension `.tsv`](glossary.md#tsv-extensions)),
+that is, [CSV files](https://en.wikipedia.org/wiki/Comma-separated_values) where commas are replaced by tab characters.
+Tabs MUST be true tab characters and MUST NOT be a series of space characters.
+Tabular data such as continuous physiology recordings typically containing
+large numbers of rows MAY be saved as
+[compressed tabular files (with extension `.tsv.gz`)](#compressed-tabular-files),
+which are introduced below.
+Plain-text TSV and compressed TSV are not interchangeable, that is, each section
+of the specification prescribes which one MUST be used for the data type at
+hand.
+Each TSV file MUST start with a header line listing the names of all columns
+with two exceptions:
+
+1. [compressed tabular files](#compressed-tabular-files),
+ for which column names are defined in a sidecar metadata
+ [JSON object](https://www.json.org/json-en.html) described below; and
+1. [motion recording data](modality-specific-files/motion.md),
+ which use plain-text TSV and columns are defined as described
+ in its corresponding section of the specifications.
+
It is RECOMMENDED that the column names in the header of the TSV file are
written in [`snake_case`](https://en.wikipedia.org/wiki/Snake_case) with the
first letter in lower case (for example, `variable_name`, not `Variable_name`).
-As for all other data in the TSV files, column names MUST be separated with tabs.
+Column names defined in the header MUST be separated with tabs as for the data contents.
Furthermore, column names MUST NOT be blank (that is, an empty string) and MUST NOT
be duplicated within a single TSV file.
-String values containing tabs MUST be escaped using double
-quotes. Missing and non-applicable values MUST be coded as `n/a`. Numerical
-values MUST employ the dot (`.`) as decimal separator and MAY be specified
+String values containing tabs MUST be escaped using double quotes.
+Missing and non-applicable values MUST be coded as `n/a`.
+Numerical values MUST employ the dot (`.`) as decimal separator and MAY be specified
in scientific notation, using `e` or `E` to separate the significand from the
-exponent. TSV files MUST be in UTF-8 encoding.
+exponent.
+TSV files MUST be in UTF-8 encoding.
Example:
```Text
-onset duration response_time correct stop_trial go_trial
-200 200 0 n/a n/a n/a
+onset duration response_time trial_type trial_extra
+200 20.0 15.8 word 中国人
+240 5.0 17.34e-1 visual n/a
```
-**Note**: The TSV examples in this document (like the one above this note)
-are occasionally formatted using space characters instead of tabs to improve
-human readability.
-Directly copying and then pasting these examples from the specification
-for use in new BIDS datasets can lead to errors and is discouraged.
+!!! warning "Attention"
+
+ The TSV examples in this document (like the one above this note) are occasionally
+ formatted using space characters instead of tabs to improve human readability.
+ Directly copying and then pasting these examples from the specification
+ for use in new BIDS datasets can lead to errors and is discouraged.
Tabular files MAY be optionally accompanied by a simple data dictionary
in the form of a JSON [object](https://www.json.org/json-en.html)
@@ -531,12 +554,38 @@ like in the example below.
"F": {
"Description": "Female",
"TermURL": "https://www.ncbi.nlm.nih.gov/mesh/68005260"
- },
+ }
}
}
}
```
+### Compressed tabular files
+
+Large tabular information, such as physiological recordings, MUST be stored with
+[compressed tab-delineated (TSV.GZ) files](glossary.md#tsv_gz-extensions) when
+so established by the specifications.
+Rules for formatting plain-text tabular files apply to TSVGZ files with three exceptions:
+
+1. The contents of TSVGZ files MUST be compressed with
+ [gzip](https://datatracker.ietf.org/doc/html/rfc1952).
+1. Compressed tabular files MUST NOT contain a header in the first row
+ indicating the column names.
+1. TSVGZ files MUST have an associated JSON file that defines the columns in the tabular file.
+
+!!! warning "Attention"
+
+ In contrast to plain-text TSV files,
+ compressed tabular files files MUST NOT include a header line.
+ Column names MUST be provided in the JSON file with the
+ [`Columns`](glossary.md#columns-metadata) field.
+ Each column MAY additionally be described with a column description,
+ as described in [Tabular files](#tabular-files).
+
+ TSVGZ are header-less to improve compatibility with existing software
+ (for example, FSL, or PNM), and to facilitate the support for other file formats
+ in the future.
+
### Key-value files (dictionaries)
JavaScript Object Notation (JSON) files MUST be used for storing key-value
diff --git a/src/derivatives/common-data-types.md b/src/derivatives/common-data-types.md
index 55ad6e3bf1..5a2e725e91 100644
--- a/src/derivatives/common-data-types.md
+++ b/src/derivatives/common-data-types.md
@@ -256,24 +256,33 @@ static volume, a `RepetitionTime` property would no longer be relevant).
## descriptions.tsv
+Template:
+
+```Text
+[sub-