diff --git a/code/setup.cfg b/code/setup.cfg index fd9549ed5..7b2bf08bc 100644 --- a/code/setup.cfg +++ b/code/setup.cfg @@ -22,8 +22,6 @@ install_requires = anyio ~= 4.0 async_generator ~= 1.10; python_version < '3.10' click >= 8.0 - datalad - datalad-fuse >= 0.5.1 ghreq ~= 0.1 hdmf packaging @@ -36,6 +34,9 @@ install_requires = dandi = # Needed for timing `dandi ls` runs dandi +datalad-fuse = + datalad + datalad-fuse >= 0.5.1 [options.packages.find] where = src diff --git a/code/src/healthstatus/__main__.py b/code/src/healthstatus/__main__.py index 7f2bd548a..5d76dc261 100644 --- a/code/src/healthstatus/__main__.py +++ b/code/src/healthstatus/__main__.py @@ -16,7 +16,7 @@ from .core import AssetPath, AssetTestResult, DandisetStatus, Outcome, TestSummary, log from .mounts import ( AssetInDandiset, - FuseMounter, + DavFS2Mounter, MountBenchmark, MountType, iter_mounters, @@ -63,13 +63,6 @@ def main() -> None: @main.command() -@click.option( - "-d", - "--dataset-path", - type=click.Path(file_okay=False, path_type=Path), - help="Directory containing a clone of dandi/dandisets", - required=True, -) @click.option( "-J", "--dandiset-jobs", @@ -100,7 +93,6 @@ def main() -> None: ) @click.argument("dandisets", nargs=-1) def check( - dataset_path: Path, mount_point: Path, dandiset_jobs: int, dandisets: tuple[str, ...], @@ -117,7 +109,7 @@ def check( dandiset_jobs=dandiset_jobs, versions=pkg_versions, ) - mnt = FuseMounter(dataset_path=dataset_path, mount_path=mount_point, update=True) + mnt = DavFS2Mounter(mount_path=mount_point) with mnt.mount(): if mode == "all": anyio.run(hs.run_all) diff --git a/code/src/healthstatus/mounts.py b/code/src/healthstatus/mounts.py index 6151212e9..7a5e51182 100644 --- a/code/src/healthstatus/mounts.py +++ b/code/src/healthstatus/mounts.py @@ -10,12 +10,14 @@ from signal import SIGINT import subprocess from time import sleep -from typing import Any +from typing import TYPE_CHECKING import click -from datalad.api import Dataset from ghreq import Client from .core import AssetPath, log +if TYPE_CHECKING: + from datalad.api import Dataset + DANDIDAV_URL = "https://webdav.dandiarchive.org" @@ -77,6 +79,8 @@ def type(self) -> MountType: @contextmanager def mount(self) -> Iterator[None]: + from datalad.api import Dataset + if not self.dataset_path.exists(): log.info("Cloning dandi/dandisets to %s ...", self.dataset_path) self.dataset_path.mkdir(parents=True, exist_ok=True) @@ -91,7 +95,12 @@ def mount(self) -> Iterator[None]: ) get_dandisets(Dataset(self.dataset_path)) elif self.update: - update_dandisets(self.dataset_path) + log.info("Updating Dandisets dataset ...") + ds = Dataset(self.dataset_path) + ds.update( + follow="parentds", how="ff-only", recursive=True, recursion_limit=1 + ) + get_dandisets(ds) self.mount_path.mkdir(parents=True, exist_ok=True) with (self.logdir / "fuse.log").open("wb") as fp: log.debug("Starting `datalad fusefs` process ...") @@ -218,14 +227,7 @@ def iter_mounters( yield DavFS2Mounter(mount_path=mount_path) -def update_dandisets(dataset_path: Path) -> None: - log.info("Updating Dandisets dataset ...") - ds = Dataset(dataset_path) - ds.update(follow="parentds", how="ff-only", recursive=True, recursion_limit=1) - get_dandisets(ds) - - -def get_dandisets(ds: Any) -> None: +def get_dandisets(ds: Dataset) -> None: # Fetch just the public repositories from the dandisets org, and then get # or update just those subdatasets rather than trying to get all # subdatasets and failing on private/embargoed ones diff --git a/code/tox.ini b/code/tox.ini index ce720a09b..0b24f18d0 100644 --- a/code/tox.ini +++ b/code/tox.ini @@ -23,6 +23,7 @@ commands = [testenv:typing] deps = mypy + datalad # trio-typing contains the stubs for async_generator trio-typing types-pyyaml diff --git a/tools/run.sh b/tools/run.sh index 603e69e4f..585de7d56 100755 --- a/tools/run.sh +++ b/tools/run.sh @@ -2,7 +2,6 @@ set -ex PYTHON="$HOME"/miniconda3/bin/python -DANDISETS_PATH=/mnt/backup/dandi/dandisets-healthstatus/dandisets MOUNT_PATH=/mnt/backup/dandi/dandisets-healthstatus/dandisets-fuse cd "$(dirname "$0")"/.. @@ -18,7 +17,7 @@ cd "$(dirname "$0")"/.. pip install ./code #pip install 'git+https://github.com/fsspec/filesystem_spec' #pip install 'git+https://github.com/jwodder/filesystem_spec@rlock-cache' -dandisets-healthstatus check -d "$DANDISETS_PATH" -m "$MOUNT_PATH" -J 10 "$@" +dandisets-healthstatus check -m "$MOUNT_PATH" -J 10 "$@" dandisets-healthstatus report # TODO: Uncomment this block when setting up the cronjob: