Split optional dependencies in pyproject.toml (#309)

* split optional deps

* comment out un-implemented readers

* remove _reader suffix

* add kerchunk as writer to cover fastparquet dependency

* embed the pyproject.toml in the installation docs

* notes on test and docs deps

* wording

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* release note

* update kerchunk version

* separate remote options

* remove sneaky git thing

* Update pyproject.toml

Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com>

* update kerchunk verion

* remove commented-out readers

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Max Jones <14077947+maxrjones@users.noreply.github.com>

docs/installation.md

@@ -12,16 +12,43 @@ and on conda-forge:
 conda install -c conda-forge virtualizarr
 ```
 
+## Optional dependencies
+
+VirtualiZarr has many optional dependencies, split into those for reading various file formats, and those for writing virtual references out to different formats.
+
+Optional dependencies can be installed in groups via pip. For example to read HDF files and write virtual references to icechunk you could install all necessary dependencies via:
+
+```shell
+pip install "virtualizarr[hdf, icechunk]"
+```
+
+The full list of optional dependencies can be seen in the `pyproject.toml` file:
+
+```{literalinclude} ../pyproject.toml
+:start-at: "[project.optional-dependencies]"
+:end-before: "test ="
+```
+
+The compound groups allow you to install multiple sets of dependencies at once, e.g. install every file reader via
+
+```shell
+pip install "virtualizarr[all_readers]"
+```
+
+The basic `pip install virtualizarr` will only install the minimal required dependencies, and so may not be particularly useful on its own.
 
 ## Install Test Dependencies
 
+For local development you will want to install the test dependencies so that you can run all the tests in the test suite:
+
 ```shell
 pip install '-e .[test]'
 ```
 
-
 ## Install Docs Dependencies
 
+To build the documentation locally you will need further dependencies:
+
 ```shell
 pip install '-e .[docs]'
 ```

docs/releases.rst

@@ -9,6 +9,8 @@ v1.2.1 (unreleased)
 New Features
 ~~~~~~~~~~~~
 
+- Optional dependencies can now be installed in groups via pip. See the installation docs.
+  (:pull:`309`) By `Tom Nicholas <https://github.com/TomNicholas>`_.
 - Added a ``.nbytes`` accessor method which displays the bytes needed to hold the virtual references in memory.
   (:issue:`167`, :pull:`227`) By `Tom Nicholas <https://github.com/TomNicholas>`_.
 - Sync with Icechunk v0.1.0a8  (:pull:`368`) By `Matthew Iannucci <https://github.com/mpiannucci>`. This also adds support

pyproject.toml

@@ -22,41 +22,70 @@ dynamic = ["version"]
 dependencies = [
     "xarray>=2025.1.1",
     "numpy>=2.0.0",
-    "packaging",
     "universal-pathlib",
     "numcodecs",
     "ujson",
+    "packaging",
 ]
 
 [project.optional-dependencies]
-hdf_reader = [
+# for creating virtual datasets from objects on remote storage
+remote = [
     "fsspec",
+    "requests",
+    "aiohttp",
+    "s3fs",
+]
+
+# non-kerchunk-based readers
+hdf = [
+    "virtualizarr[remote]",
     "h5py",
     "hdf5plugin",
     "imagecodecs",
     "imagecodecs-numcodecs==2024.6.1",
-    "numcodecs"
+]
+
+# kerchunk-based readers
+hdf5 = [
+    "virtualizarr[remote]",
+    "kerchunk>=0.2.8",
+    "h5py",
 ]
 netcdf3 = [
+    "virtualizarr[remote]",
     "kerchunk>=0.2.8",
     "scipy",
 ]
 fits = [
+    "virtualizarr[remote]",
     "kerchunk>=0.2.8",
     "astropy",
 ]
+all_readers = [
+    "virtualizarr[hdf]",
+    "virtualizarr[hdf5]",
+    "virtualizarr[netcdf3]",
+    "virtualizarr[fits]",
+]
+
+# writers
 icechunk = [
     "icechunk>=0.1.0a12",
 ]
+kerchunk = ["fastparquet"]
+all_writers = [
+    "virtualizarr[icechunk]",
+    "virtualizarr[kerchunk]",
+]
+all = [
+    "virtualizarr[all_readers]",
+    "virtualizarr[all_writers]",
+]
+
 test = [
     "codecov",
-    "fastparquet",
-    "fsspec",
-    "h5py",
-    "kerchunk>=0.2.8",
     "mypy",
-    "netcdf4",
-    "numcodecs",
     "pandas-stubs",
     "pooch",
     "pre-commit",
@@ -65,8 +94,7 @@ test = [
     "pytest",
     "ruff",
     "s3fs",
-    "scipy",
-    "virtualizarr[hdf_reader]"
+    "virtualizarr[all]"
 ]
 
 [project.urls]