From 9f5a11021f071a082b126057967d1f713259effe Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 22 May 2024 20:45:38 +0530 Subject: [PATCH 01/24] Add CI job to test out-of-tree Pyodide builds --- .github/workflows/emscripten.yml | 79 ++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/emscripten.yml diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 0000000000..230ce56c33 --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,79 @@ +# Attributed to NumPy https://github.com/numpy/numpy/pull/25894 +# https://github.com/numpy/numpy/blob/d2d2c25fa81b47810f5cbd85ea6485eb3a3ffec3/.github/workflows/emscripten.yml +# + +name: Pyodide wheel + +on: + # TODO: refine after this is ready to merge + [push, pull_request, workflow_dispatch] + +env: + FORCE_COLOR: 3 + PYODIDE_VERSION: 0.25.1 + # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. + # The appropriate versions can be found in the Pyodide repodata.json + # "info" field, or in Makefile.envs: + # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 + PYTHON_VERSION: 3.11.3 + EMSCRIPTEN_VERSION: 3.1.46 + NODE_VERSION: 18 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + build_wasm_emscripten: + name: Build and test Zarr for Pyodide + runs-on: ubuntu-22.04 + # To enable this workflow on a fork, comment out: + # FIXME: uncomment after this is ready to merge + # if: github.repository == 'zarr-developers/zarr-python' + steps: + - name: Checkout Zarr repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up Emscripten toolchain + uses: mymindstorm/setup-emsdk@v14 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Install pyodide-build + run: python -m pip install "pyodide-build==${{ env.PYODIDE_VERSION }}" + + - name: Build Zarr for Pyodide + run: | + pyodide build + + - name: Run Zarr tests for Pyodide + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + python -m pip install dist/*.whl + python -m pip install pytest pytest-cov + python -m pytest -v --cov=zarr --cov-config=pyproject.toml zarr + + - name: Upload Pyodide wheel artifact for debugging + # FIXME: Remove after this is ready to merge + uses: actions/upload-artifact@v4 + with: + name: zarr-pyodide-wheel + path: dist/*.whl + + From 29282fc9455df6543e6fd601fb1ffddd22a39722 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 24 May 2024 00:47:02 +0530 Subject: [PATCH 02/24] Add `[msgpack]` dependency for `numcodecs` --- pyproject.toml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 947bec9369..e995d0b971 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,11 @@ requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.24', - 'fasteners', - 'numcodecs>=0.10.0', - 'crc32c', - 'zstandard', + 'fasteners; sys_platform != "emscripten"', + # 'numcodecs[msgpack]>=0.10.0; sys_platform != "emscripten"', # does not currently work + 'numcodecs[msgpack]>=0.10.0', # this works + 'crc32c', # not available in Pyodide, have to see if we can make it optional for Pyodide + 'zstandard', # not available in Pyodide, have to see if we can make it optional for Pyodide 'typing_extensions', 'donfig' ] From d465742a193e145340e3afe91a7e0bd5b3893419 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 27 May 2024 23:26:24 +0530 Subject: [PATCH 03/24] Bump to Pyodide 0.26.0, update comments --- .github/workflows/emscripten.yml | 9 +++------ pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 230ce56c33..22437d05c7 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -1,6 +1,5 @@ # Attributed to NumPy https://github.com/numpy/numpy/pull/25894 # https://github.com/numpy/numpy/blob/d2d2c25fa81b47810f5cbd85ea6485eb3a3ffec3/.github/workflows/emscripten.yml -# name: Pyodide wheel @@ -10,13 +9,13 @@ on: env: FORCE_COLOR: 3 - PYODIDE_VERSION: 0.25.1 + PYODIDE_VERSION: 0.26.0 # PYTHON_VERSION and EMSCRIPTEN_VERSION are determined by PYODIDE_VERSION. # The appropriate versions can be found in the Pyodide repodata.json # "info" field, or in Makefile.envs: # https://github.com/pyodide/pyodide/blob/main/Makefile.envs#L2 - PYTHON_VERSION: 3.11.3 - EMSCRIPTEN_VERSION: 3.1.46 + PYTHON_VERSION: 3.12.1 + EMSCRIPTEN_VERSION: 3.1.58 NODE_VERSION: 18 concurrency: @@ -75,5 +74,3 @@ jobs: with: name: zarr-pyodide-wheel path: dist/*.whl - - diff --git a/pyproject.toml b/pyproject.toml index e995d0b971..e7e89c5670 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,8 +27,8 @@ dependencies = [ 'fasteners; sys_platform != "emscripten"', # 'numcodecs[msgpack]>=0.10.0; sys_platform != "emscripten"', # does not currently work 'numcodecs[msgpack]>=0.10.0', # this works - 'crc32c', # not available in Pyodide, have to see if we can make it optional for Pyodide - 'zstandard', # not available in Pyodide, have to see if we can make it optional for Pyodide + 'crc32c', + 'zstandard', 'typing_extensions', 'donfig' ] From cdf0bb205cd962ab518b220ab9fc2f9962ceeae6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:06:22 +0530 Subject: [PATCH 04/24] Try to run tests without async --- .github/workflows/emscripten.yml | 2 +- pyproject.toml | 2 +- tests/v3/_shared.py | 16 ++++++++++++++++ tests/v3/test_buffer.py | 5 +++-- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/v3/_shared.py diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 22437d05c7..88fc6ff7e6 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -66,7 +66,7 @@ jobs: source .venv-pyodide/bin/activate python -m pip install dist/*.whl python -m pip install pytest pytest-cov - python -m pytest -v --cov=zarr --cov-config=pyproject.toml zarr + python -m pytest -v --cov=zarr --cov-config=pyproject.toml - name: Upload Pyodide wheel artifact for debugging # FIXME: Remove after this is ready to merge diff --git a/pyproject.toml b/pyproject.toml index e7e89c5670..da57fdbea6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,7 +249,7 @@ minversion = "7" testpaths = ["tests"] log_cli_level = "INFO" xfail_strict = true -asyncio_mode = "auto" +# asyncio_mode = "auto" doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS", diff --git a/tests/v3/_shared.py b/tests/v3/_shared.py new file mode 100644 index 0000000000..d853a02675 --- /dev/null +++ b/tests/v3/_shared.py @@ -0,0 +1,16 @@ +# A common file that can be used to add constants, functions, +# convenience classes, etc. that are shared across multiple tests + +import platform +import sys + +import pytest + +IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] + + +def asyncio_tests_wrapper(func): + if IS_WASM: + return func + else: + return pytest.mark.asyncio(func) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 2f58d116fe..07ef96bf6e 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -5,11 +5,12 @@ import numpy as np import numpy.typing as npt -import pytest from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer +from ._shared import asyncio_tests_wrapper + if TYPE_CHECKING: from typing_extensions import Self @@ -45,7 +46,7 @@ def test_nd_array_like(xp): assert isinstance(ary, NDArrayLike) -@pytest.mark.asyncio +@asyncio_tests_wrapper async def test_async_array_factory(store_path): expect = np.zeros((9, 9), dtype="uint16", order="F") a = await AsyncArray.create( From dfe032115bdac8112ef60f9187bfd3c653da4a6e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:51:53 +0530 Subject: [PATCH 05/24] Move shared file to rootdir, outside v2 and v3 --- tests/{v3 => }/_shared.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{v3 => }/_shared.py (100%) diff --git a/tests/v3/_shared.py b/tests/_shared.py similarity index 100% rename from tests/v3/_shared.py rename to tests/_shared.py From b100ec9d0f7e1473349f5b1e157ff166efe01e0c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 01:52:27 +0530 Subject: [PATCH 06/24] Move `fasteners` import inside ThreadSynchronizer --- src/zarr/v2/sync.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zarr/v2/sync.py b/src/zarr/v2/sync.py index 49684a51ee..e8dff71e32 100644 --- a/src/zarr/v2/sync.py +++ b/src/zarr/v2/sync.py @@ -2,8 +2,6 @@ from collections import defaultdict from threading import Lock -import fasteners - class ThreadSynchronizer: """Provides synchronization using thread locks.""" @@ -42,6 +40,7 @@ def __init__(self, path): def __getitem__(self, item): path = os.path.join(self.path, item) + import fasteners lock = fasteners.InterProcessLock(path) return lock From b0dddca9d6d302e100f22106634cc58fe11f7621 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:02:53 +0530 Subject: [PATCH 07/24] Make the tests directory importable, fix `_shared` --- tests/__init__.py | 0 tests/_shared.py | 1 + tests/v3/test_buffer.py | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/_shared.py b/tests/_shared.py index d853a02675..969936db01 100644 --- a/tests/_shared.py +++ b/tests/_shared.py @@ -1,5 +1,6 @@ # A common file that can be used to add constants, functions, # convenience classes, etc. that are shared across multiple tests +# similar to tests/v2/util.py, but can be used for both v2 and v3 import platform import sys diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 07ef96bf6e..2e9dcde2a5 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -9,7 +9,7 @@ from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer -from ._shared import asyncio_tests_wrapper +from .._shared import asyncio_tests_wrapper if TYPE_CHECKING: from typing_extensions import Self From d227728f23ee33dd9264490281f67f6123434f2f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:03:10 +0530 Subject: [PATCH 08/24] Import list of greetings from `numcodecs` --- tests/v2/test_core.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index f053725b95..75222893bb 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -27,7 +27,14 @@ Zlib, ) from numcodecs.compat import ensure_bytes, ensure_ndarray -from numcodecs.tests.common import greetings + +try: + from numcodecs.tests.common import greetings +except ModuleNotFoundError: + greetings = ['¡Hola mundo!', 'Hej Världen!', 'Servus Woid!', 'Hei maailma!', + 'Xin chào thế giới', 'Njatjeta Botë!', 'Γεια σου κόσμε!', + 'こんにちは世界', '世界，你好！', 'Helló, világ!', 'Zdravo svete!', + 'เฮลโลเวิลด์'] from numpy.testing import assert_array_almost_equal, assert_array_equal import zarr.v2 From fdb2bef749d9ad3be7d0396264ce5d31a6924c02 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:10:21 +0530 Subject: [PATCH 09/24] Skip some tests that use threading --- tests/v2/test_sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index ea6fd0523d..8ff6ab3819 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -8,6 +8,7 @@ import numpy as np from numpy.testing import assert_array_equal +import pytest from zarr.v2.attrs import Attributes from zarr.v2.core import Array @@ -20,7 +21,10 @@ from .test_core import TestArray from .test_hierarchy import TestGroup +from tests._shared import IS_WASM + +@pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") class TestAttributesWithThreadSynchronizer(TestAttributes): def init_attributes(self, store, read_only=False, cache=True): key = ".zattrs" @@ -30,6 +34,7 @@ def init_attributes(self, store, read_only=False, cache=True): ) +@pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") class TestAttributesProcessSynchronizer(TestAttributes): def init_attributes(self, store, read_only=False, cache=True): key = ".zattrs" From 621077a1f7e7ea057b62870209cca625574d469f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:11:04 +0530 Subject: [PATCH 10/24] Skip some tests that use `fcntl` --- tests/v2/test_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 8ff6ab3819..9b9ddd3618 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -153,6 +153,7 @@ def test_hexdigest(self): assert "05b0663ffe1785f38d3a459dec17e57a18f254af" == z.hexdigest() +@pytest.mark.skipif(IS_WASM, reason="fcntl not available in WASM") class TestArrayWithProcessSynchronizer(TestArray, MixinArraySyncTests): def create_array(self, read_only=False, **kwargs): path = tempfile.mkdtemp() @@ -291,6 +292,7 @@ def test_synchronizer_property(self): assert isinstance(g.synchronizer, ThreadSynchronizer) +@pytest.mark.skipif(IS_WASM, reason="fcntl not available in WASM") class TestGroupWithProcessSynchronizer(TestGroup, MixinGroupSyncTests): def create_store(self): path = tempfile.mkdtemp() From 7ae9a975bf42e13a327732ef52b8275ea314a7d0 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:14:47 +0530 Subject: [PATCH 11/24] Skip tests that require `dbm` --- tests/v2/test_core.py | 4 ++++ tests/v2/test_hierarchy.py | 4 ++++ tests/v2/test_storage.py | 3 +++ 3 files changed, 11 insertions(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 75222893bb..976dbccfc9 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -65,6 +65,8 @@ from zarr.v2.util import buffer_size from .util import abs_container, skip_test_env_var, have_fsspec, mktemp +from tests._shared import IS_WASM + # noinspection PyMethodMayBeStatic @@ -1992,6 +1994,7 @@ def create_store(self): return store +@pytest.mark.skipif(IS_WASM, reason="no dbm support in WASM") class TestArrayWithDBMStore(TestArray): def create_store(self): path = mktemp(suffix=".anydbm") @@ -2003,6 +2006,7 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(IS_WASM, reason="no dbm support in WASM") @pytest.mark.skip(reason="can't get bsddb3 to work on CI right now") class TestArrayWithDBMStoreBerkeleyDB(TestArray): def create_store(self): diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 23c5a56edf..31553242f9 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -45,6 +45,8 @@ from zarr.v2.util import InfoReporter from .util import skip_test_env_var, have_fsspec, abs_container, mktemp +from tests._shared import IS_WASM + # noinspection PyStatementEffect @@ -1122,6 +1124,7 @@ def test_move(self): pass +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestGroupWithDBMStore(TestGroup): @staticmethod def create_store(): @@ -1131,6 +1134,7 @@ def create_store(): return store, None +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestGroupWithDBMStoreBerkeleyDB(TestGroup): @staticmethod def create_store(): diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 17b80e6a5c..6e49c09cf2 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -59,6 +59,7 @@ from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps +from tests._shared import IS_WASM @contextmanager def does_not_raise(): @@ -1765,6 +1766,7 @@ def test_store_and_retrieve_ndarray(self): assert np.array_equiv(y, x) +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestDBMStore(StoreTests): def create_store(self, dimension_separator=None): path = mktemp(suffix=".anydbm") @@ -1780,6 +1782,7 @@ def test_context_manager(self): assert 2 == len(store) +@pytest.mark.skipif(IS_WASM, reason="dbm not available in WASM") class TestDBMStoreDumb(TestDBMStore): def create_store(self, **kwargs): path = mktemp(suffix=".dumbdbm") From 22eb6daaa47fbed0ef6772a42d8d523051daa904 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:37:11 +0530 Subject: [PATCH 12/24] Move `IS_WASM` logic to internal `zarr` API --- src/zarr/testing/utils.py | 6 ++++++ tests/__init__.py | 0 tests/_shared.py | 17 ----------------- tests/v2/test_core.py | 2 +- tests/v2/test_hierarchy.py | 2 +- tests/v2/test_storage.py | 2 +- tests/v2/test_sync.py | 2 +- 7 files changed, 10 insertions(+), 21 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/_shared.py diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 04b05d1b1c..2d45d34579 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,5 +1,8 @@ from __future__ import annotations +import platform +import sys + from zarr.buffer import Buffer from zarr.common import BytesLike @@ -16,3 +19,6 @@ def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | N if isinstance(b2, Buffer): b2 = b2.to_bytes() assert b1 == b2 + + +IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/_shared.py b/tests/_shared.py deleted file mode 100644 index 969936db01..0000000000 --- a/tests/_shared.py +++ /dev/null @@ -1,17 +0,0 @@ -# A common file that can be used to add constants, functions, -# convenience classes, etc. that are shared across multiple tests -# similar to tests/v2/util.py, but can be used for both v2 and v3 - -import platform -import sys - -import pytest - -IS_WASM = sys.platform == "emscripten" or platform.machine() in ["wasm32", "wasm64"] - - -def asyncio_tests_wrapper(func): - if IS_WASM: - return func - else: - return pytest.mark.asyncio(func) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 976dbccfc9..9da81cb787 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -65,7 +65,7 @@ from zarr.v2.util import buffer_size from .util import abs_container, skip_test_env_var, have_fsspec, mktemp -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM # noinspection PyMethodMayBeStatic diff --git a/tests/v2/test_hierarchy.py b/tests/v2/test_hierarchy.py index 31553242f9..4936bf1b2f 100644 --- a/tests/v2/test_hierarchy.py +++ b/tests/v2/test_hierarchy.py @@ -45,7 +45,7 @@ from zarr.v2.util import InfoReporter from .util import skip_test_env_var, have_fsspec, abs_container, mktemp -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM # noinspection PyStatementEffect diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 6e49c09cf2..47db11950b 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -59,7 +59,7 @@ from .util import CountingDict, have_fsspec, skip_test_env_var, abs_container, mktemp from zarr.v2.util import ConstantMap, json_dumps -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM @contextmanager def does_not_raise(): diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 9b9ddd3618..5065d066aa 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -21,7 +21,7 @@ from .test_core import TestArray from .test_hierarchy import TestGroup -from tests._shared import IS_WASM +from zarr.testing.utils import IS_WASM @pytest.mark.skipif(IS_WASM, reason="no threading support in WASM") From 683694773e758b3f7b0c919f3ae39bfd1e4dea51 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:37:34 +0530 Subject: [PATCH 13/24] Skip a few tests trying to import `multiprocessing` --- tests/v2/test_sync.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/v2/test_sync.py b/tests/v2/test_sync.py index 5065d066aa..8a86f40d6f 100644 --- a/tests/v2/test_sync.py +++ b/tests/v2/test_sync.py @@ -101,6 +101,7 @@ def test_parallel_append(self): pool.terminate() +@pytest.mark.skipif(IS_WASM, reason="no multiprocessing support in WASM") class TestArrayWithThreadSynchronizer(TestArray, MixinArraySyncTests): def create_array(self, read_only=False, **kwargs): store = KVStore(dict()) @@ -265,6 +266,7 @@ def test_parallel_require_group(self): pool.terminate() +@pytest.mark.skipif(IS_WASM, reason="no multiprocessing support in WASM") class TestGroupWithThreadSynchronizer(TestGroup, MixinGroupSyncTests): def create_group( self, store=None, path=None, read_only=False, chunk_store=None, synchronizer=None From fe3bf274de04489c3f20fff28184bbd60794b6b5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:41:08 +0530 Subject: [PATCH 14/24] Skip tests that use async and threading code This is because they test async code and also use threads. See the following issues: https://github.com/pyodide/pyodide/issues/2221 https://github.com/pyodide/pyodide/issues/237 --- tests/v3/test_codecs.py | 8 +++++++- tests/v3/test_group.py | 4 ++++ tests/v3/test_sync.py | 6 ++++++ tests/v3/test_v2.py | 2 ++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index a595b12494..744699e07a 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -25,7 +25,7 @@ from zarr.config import config from zarr.indexing import morton_order_iter from zarr.store import MemoryStore, StorePath -from zarr.testing.utils import assert_bytes_equal +from zarr.testing.utils import IS_WASM, assert_bytes_equal @dataclass(frozen=True) @@ -65,6 +65,7 @@ def order_from_dim(order: Literal["F", "C"], ndim: int) -> tuple[int, ...]: return tuple(range(ndim)) +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -95,6 +96,7 @@ def test_sharding( assert np.array_equal(sample_data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -128,6 +130,7 @@ def test_sharding_partial( assert np.array_equal(sample_data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_read( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -155,6 +158,7 @@ def test_sharding_partial_read( assert np.all(read_data == 1) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_overwrite( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -191,6 +195,7 @@ def test_sharding_partial_overwrite( assert np.array_equal(data, read_data) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize( "outer_index_location", ["start", "end"], @@ -298,6 +303,7 @@ async def test_order( assert_bytes_equal(await (store / "order/0.0").get(), z._store["0.0"]) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 36b82f413c..4798a0ffdf 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -6,6 +6,7 @@ from zarr.buffer import Buffer from zarr.store.core import make_store_path from zarr.sync import sync +from zarr.testing.utils import IS_WASM if TYPE_CHECKING: from zarr.common import ZarrFormat @@ -18,6 +19,7 @@ from zarr.store import StorePath +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") # todo: put RemoteStore in here @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) def test_group_children(store: MemoryStore | LocalStore) -> None: @@ -55,6 +57,7 @@ def test_group_children(store: MemoryStore | LocalStore) -> None: assert sorted(dict(members_observed)) == sorted(members_expected) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("store", (("local", "memory")), indirect=["store"]) def test_group(store: MemoryStore | LocalStore) -> None: store_path = StorePath(store) @@ -94,6 +97,7 @@ def test_group(store: MemoryStore | LocalStore) -> None: assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) @pytest.mark.parametrize("exists_ok", (True, False)) def test_group_create(store: MemoryStore | LocalStore, exists_ok: bool) -> None: diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 5b953573d8..2dbc8559b6 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -6,6 +6,7 @@ import pytest from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync +from zarr.testing.utils import IS_WASM @pytest.fixture(params=[True, False]) @@ -31,12 +32,14 @@ def test_get_lock() -> None: assert lock is lock2 +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(return_value="foo") assert sync(foo(), loop=sync_loop) == "foo" foo.assert_awaited_once() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(side_effect=ValueError("foo-bar")) with pytest.raises(ValueError, match="foo-bar"): @@ -44,6 +47,7 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo.assert_awaited_once() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_timeout() -> None: duration = 0.002 @@ -54,6 +58,7 @@ async def foo() -> None: sync(foo(), timeout=duration / 2) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: def foo() -> str: return "foo" @@ -97,6 +102,7 @@ def test_sync_raises_if_loop_is_invalid_type() -> None: foo.assert_not_awaited() +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_mixin(sync_loop) -> None: class AsyncFoo: def __init__(self) -> None: diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index 2a38dc8fdc..d884cebc16 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -6,6 +6,7 @@ from zarr.abc.store import Store from zarr.array import Array from zarr.store import MemoryStore, StorePath +from zarr.testing.utils import IS_WASM @pytest.fixture @@ -13,6 +14,7 @@ def store() -> Iterator[Store]: yield StorePath(MemoryStore()) +@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_simple(store: Store): data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) From 08997ec474905903cec7eb24c9de8d1bceb04416 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 02:51:18 +0530 Subject: [PATCH 15/24] Improve `asyncio_tests_wrapper`, fix test imports --- tests/v3/test_buffer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 2e9dcde2a5..7af2915724 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -5,16 +5,21 @@ import numpy as np import numpy.typing as npt +import pytest from zarr.array import AsyncArray from zarr.buffer import ArrayLike, NDArrayLike, NDBuffer - -from .._shared import asyncio_tests_wrapper +from zarr.testing.utils import IS_WASM if TYPE_CHECKING: from typing_extensions import Self +# Helper function to skip async tests on WASM platforms +def asyncio_tests_wrapper(func): + return func if IS_WASM else pytest.mark.asyncio(func) + + class MyNDArrayLike(np.ndarray): """An example of a ndarray-like class""" From 9bfc860c2f84b0667172154cea95400626dd0dda Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:07:15 +0530 Subject: [PATCH 16/24] Skip entire `test_codecs.py` file --- tests/v3/test_codecs.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index 744699e07a..c44451a07b 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -27,6 +27,11 @@ from zarr.store import MemoryStore, StorePath from zarr.testing.utils import IS_WASM, assert_bytes_equal +# Skip entire file if running on WASM platforms, see +# 1. https://github.com/pyodide/pyodide/issues/2221 +# 2. https://github.com/pyodide/pyodide/issues/237 +pytestmark = pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") + @dataclass(frozen=True) class _AsyncArrayProxy: @@ -65,7 +70,6 @@ def order_from_dim(order: Literal["F", "C"], ndim: int) -> tuple[int, ...]: return tuple(range(ndim)) -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -96,7 +100,6 @@ def test_sharding( assert np.array_equal(sample_data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -130,7 +133,6 @@ def test_sharding_partial( assert np.array_equal(sample_data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_read( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -158,7 +160,6 @@ def test_sharding_partial_read( assert np.all(read_data == 1) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("index_location", ["start", "end"]) def test_sharding_partial_overwrite( store: Store, sample_data: np.ndarray, index_location: ShardingCodecIndexLocation @@ -195,7 +196,6 @@ def test_sharding_partial_overwrite( assert np.array_equal(data, read_data) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize( "outer_index_location", ["start", "end"], @@ -303,7 +303,6 @@ async def test_order( assert_bytes_equal(await (store / "order/0.0").get(), z._store["0.0"]) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) @pytest.mark.parametrize("runtime_read_order", ["F", "C"]) From 9bcb350b18ea534d51d6c96ba50d5c6989439431 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:10:06 +0530 Subject: [PATCH 17/24] Skip yet another test that requires threads --- tests/v3/test_codecs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_codecs.py b/tests/v3/test_codecs.py index c44451a07b..6a486b3b76 100644 --- a/tests/v3/test_codecs.py +++ b/tests/v3/test_codecs.py @@ -411,6 +411,7 @@ async def test_transpose( assert await (store / "transpose/0.0").get() == await (store / "transpose_zarr/0.0").get() +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_transpose_invalid( store: Store, ): From 9985abb34fe5e309a43b37af8b8d7a85c8a26029 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:10:29 +0530 Subject: [PATCH 18/24] xfail test where array's fill values are different --- tests/v2/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 9da81cb787..40e678bb97 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -1706,6 +1706,7 @@ def create_store(self): store = N5Store(path) return store + @pytest.mark.xfail(reason="Can't get this to pass under WASM right now") def test_array_0d(self): # test behaviour for array with 0 dimensions From 7ea12ef6acdeafb4ae6c5effd363b8e615df6cc2 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:08:23 +0530 Subject: [PATCH 19/24] xfail test because Emscripten FS --- tests/v2/test_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_storage.py b/tests/v2/test_storage.py index 47db11950b..b5895d7dab 100644 --- a/tests/v2/test_storage.py +++ b/tests/v2/test_storage.py @@ -939,6 +939,7 @@ def create_store(self, normalize_keys=False, dimension_separator=".", **kwargs): ) return store + @pytest.mark.xfail(reason="Emscripten filesystem handles umasks differently") def test_filesystem_path(self): # test behaviour with path that does not exist path = "data/store" From a6565ded85d4acc1e39b44a59f38da08af763baf Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:16:54 +0530 Subject: [PATCH 20/24] Skip last test that tries to run threads --- tests/v3/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 2dbc8559b6..1030b3152e 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -18,6 +18,7 @@ def sync_loop(request) -> asyncio.AbstractEventLoop | None: return None +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_get_loop() -> None: # test that calling _get_loop() twice returns the same loop loop = _get_loop() From 85f621cb8c9e199df9dedc37b5e21e501287d36e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:20:39 +0530 Subject: [PATCH 21/24] Another test that tries to run threads --- tests/v3/test_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 1030b3152e..03acbb044f 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -68,6 +68,7 @@ def foo() -> str: sync(foo(), loop=sync_loop) +@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_closed() -> None: loop = _get_loop() From 1a64255fb996bce57f4dafdbbc030f8268e3f7ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:21:33 +0530 Subject: [PATCH 22/24] xfail another array's differing `fill_values` test --- tests/v2/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index 40e678bb97..72f465df70 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -983,6 +983,7 @@ def test_0len_dim_2d(self): z.store.close() # noinspection PyStatementEffect + @pytest.mark.xfail(reason="Can't get this to pass under WASM right now") def test_array_0d(self): # test behaviour for array with 0 dimensions From c8cb38bb896e841bb34a065e64d168e86d666738 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 03:27:17 +0530 Subject: [PATCH 23/24] Skip entire sync file under WASM, no threading --- tests/v3/test_sync.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 03acbb044f..69cbf815bf 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -8,6 +8,8 @@ from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync from zarr.testing.utils import IS_WASM +pytestmark = pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") + @pytest.fixture(params=[True, False]) def sync_loop(request) -> asyncio.AbstractEventLoop | None: @@ -18,7 +20,6 @@ def sync_loop(request) -> asyncio.AbstractEventLoop | None: return None -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") def test_get_loop() -> None: # test that calling _get_loop() twice returns the same loop loop = _get_loop() @@ -33,14 +34,12 @@ def test_get_lock() -> None: assert lock is lock2 -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(return_value="foo") assert sync(foo(), loop=sync_loop) == "foo" foo.assert_awaited_once() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo = AsyncMock(side_effect=ValueError("foo-bar")) with pytest.raises(ValueError, match="foo-bar"): @@ -48,7 +47,6 @@ def test_sync_raises(sync_loop: asyncio.AbstractEventLoop | None) -> None: foo.assert_awaited_once() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_timeout() -> None: duration = 0.002 @@ -59,7 +57,6 @@ async def foo() -> None: sync(foo(), timeout=duration / 2) -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_raises_if_no_coroutine(sync_loop: asyncio.AbstractEventLoop | None) -> None: def foo() -> str: return "foo" @@ -68,7 +65,6 @@ def foo() -> str: sync(foo(), loop=sync_loop) -@pytest.mark.skipif(IS_WASM, reason="Can't start new threads in WASM") @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") def test_sync_raises_if_loop_is_closed() -> None: loop = _get_loop() @@ -104,7 +100,6 @@ def test_sync_raises_if_loop_is_invalid_type() -> None: foo.assert_not_awaited() -@pytest.mark.skipif(IS_WASM, reason="Can't test async code in WASM") def test_sync_mixin(sync_loop) -> None: class AsyncFoo: def __init__(self) -> None: From eb36d40077b63c0958a20cb84a7be1ec74a3b214 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 28 May 2024 21:46:32 +0530 Subject: [PATCH 24/24] Restore pytest config options, remove when needed --- .github/workflows/emscripten.yml | 2 ++ pyproject.toml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 88fc6ff7e6..c672c8f32d 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -62,6 +62,8 @@ jobs: - name: Run Zarr tests for Pyodide run: | + # Avoid missing asyncio plugin error from pytest, unavailable in Pyodide + if grep -q 'asyncio_mode = "auto"' "pyproject.toml"; then sed '/asyncio_mode = "auto"/d' "pyproject.toml" > temp && mv temp "pyproject.toml"; fi pyodide venv .venv-pyodide source .venv-pyodide/bin/activate python -m pip install dist/*.whl diff --git a/pyproject.toml b/pyproject.toml index da57fdbea6..48eb5b1544 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -249,7 +249,8 @@ minversion = "7" testpaths = ["tests"] log_cli_level = "INFO" xfail_strict = true -# asyncio_mode = "auto" +# Doesn't work under WASM, remove when running Pyodide test suite +asyncio_mode = "auto" doctest_optionflags = [ "NORMALIZE_WHITESPACE", "ELLIPSIS",