Merge pull request #77 from dandi/gh-76

Replace datalad-fuse with davfs2

code/setup.cfg

@@ -22,8 +22,6 @@ install_requires =
     anyio ~= 4.0
     async_generator ~= 1.10; python_version < '3.10'
     click >= 8.0
-    datalad
-    datalad-fuse >= 0.5.1
     ghreq ~= 0.1
     hdmf
     packaging
@@ -36,6 +34,9 @@ install_requires =
 dandi =
     # Needed for timing `dandi ls` runs
     dandi
+datalad-fuse =
+    datalad
+    datalad-fuse >= 0.5.1
 
 [options.packages.find]
 where = src

code/src/healthstatus/__main__.py

@@ -16,7 +16,7 @@
 from .core import AssetPath, AssetTestResult, DandisetStatus, Outcome, TestSummary, log
 from .mounts import (
     AssetInDandiset,
-    FuseMounter,
+    DavFS2Mounter,
     MountBenchmark,
     MountType,
     iter_mounters,
@@ -63,13 +63,6 @@ def main() -> None:
 
 
 @main.command()
-@click.option(
-    "-d",
-    "--dataset-path",
-    type=click.Path(file_okay=False, path_type=Path),
-    help="Directory containing a clone of dandi/dandisets",
-    required=True,
-)
 @click.option(
     "-J",
     "--dandiset-jobs",
@@ -100,7 +93,6 @@ def main() -> None:
 )
 @click.argument("dandisets", nargs=-1)
 def check(
-    dataset_path: Path,
     mount_point: Path,
     dandiset_jobs: int,
     dandisets: tuple[str, ...],
@@ -117,7 +109,7 @@ def check(
         dandiset_jobs=dandiset_jobs,
         versions=pkg_versions,
     )
-    mnt = FuseMounter(dataset_path=dataset_path, mount_path=mount_point, update=True)
+    mnt = DavFS2Mounter(mount_path=mount_point)
     with mnt.mount():
         if mode == "all":
             anyio.run(hs.run_all)

code/src/healthstatus/mounts.py

@@ -10,12 +10,14 @@
 from signal import SIGINT
 import subprocess
 from time import sleep
-from typing import Any
+from typing import TYPE_CHECKING
 import click
-from datalad.api import Dataset
 from ghreq import Client
 from .core import AssetPath, log
 
+if TYPE_CHECKING:
+    from datalad.api import Dataset
+
 DANDIDAV_URL = "https://webdav.dandiarchive.org"
 
 
@@ -77,6 +79,8 @@ def type(self) -> MountType:
 
     @contextmanager
     def mount(self) -> Iterator[None]:
+        from datalad.api import Dataset
+
         if not self.dataset_path.exists():
             log.info("Cloning dandi/dandisets to %s ...", self.dataset_path)
             self.dataset_path.mkdir(parents=True, exist_ok=True)
@@ -91,7 +95,12 @@ def mount(self) -> Iterator[None]:
             )
             get_dandisets(Dataset(self.dataset_path))
         elif self.update:
-            update_dandisets(self.dataset_path)
+            log.info("Updating Dandisets dataset ...")
+            ds = Dataset(self.dataset_path)
+            ds.update(
+                follow="parentds", how="ff-only", recursive=True, recursion_limit=1
+            )
+            get_dandisets(ds)
         self.mount_path.mkdir(parents=True, exist_ok=True)
         with (self.logdir / "fuse.log").open("wb") as fp:
             log.debug("Starting `datalad fusefs` process ...")
@@ -218,14 +227,7 @@ def iter_mounters(
         yield DavFS2Mounter(mount_path=mount_path)
 
 
-def update_dandisets(dataset_path: Path) -> None:
-    log.info("Updating Dandisets dataset ...")
-    ds = Dataset(dataset_path)
-    ds.update(follow="parentds", how="ff-only", recursive=True, recursion_limit=1)
-    get_dandisets(ds)
-
-
-def get_dandisets(ds: Any) -> None:
+def get_dandisets(ds: Dataset) -> None:
     # Fetch just the public repositories from the dandisets org, and then get
     # or update just those subdatasets rather than trying to get all
     # subdatasets and failing on private/embargoed ones

code/tox.ini

@@ -23,6 +23,7 @@ commands =
 [testenv:typing]
 deps =
     mypy
+    datalad
     # trio-typing contains the stubs for async_generator
     trio-typing
     types-pyyaml

tools/run.sh

@@ -2,7 +2,6 @@
 set -ex
 
 PYTHON="$HOME"/miniconda3/bin/python
-DANDISETS_PATH=/mnt/backup/dandi/dandisets-healthstatus/dandisets
 MOUNT_PATH=/mnt/backup/dandi/dandisets-healthstatus/dandisets-fuse
 
 cd "$(dirname "$0")"/..
@@ -18,7 +17,7 @@ cd "$(dirname "$0")"/..
 pip install ./code
 #pip install 'git+https://github.com/fsspec/filesystem_spec'
 #pip install 'git+https://github.com/jwodder/filesystem_spec@rlock-cache'
-dandisets-healthstatus check -d "$DANDISETS_PATH" -m "$MOUNT_PATH" -J 10 "$@"
+dandisets-healthstatus check -m "$MOUNT_PATH" -J 10 "$@"
 dandisets-healthstatus report
 
 # TODO: Uncomment this block when setting up the cronjob: