diff --git a/docs/usage.md b/docs/usage.md index 3f8ff84c..0ce9a892 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -228,8 +228,16 @@ You can see that the dataset contains a mixture of virtual variables backed by ` Loading variables can be useful in a few scenarios: 1. You need to look at the actual values of a multi-dimensional variable in order to decide what to do next, 2. You want in-memory indexes to use with ``xr.combine_by_coords``, -3. Storing a variable on-disk as a set of references would be inefficient, e.g. because each chunk is very small (saving the values like this is similar to kerchunk's concept of "inlining" data), -4. The variable has complicated encoding, and the simplest way to decode it correctly is to let xarray's standard decoding machinery load it into memory and apply the decoding. +3. Storing a variable on-disk as a set of references would be inefficient, e.g. because it's a very small array (saving the values like this is similar to kerchunk's concept of "inlining" data), +4. The variable has encoding, and the simplest way to decode it correctly is to let xarray's standard decoding machinery load it into memory and apply the decoding, +5. Some of your variables have inconsistent-length chunks, and you want to be able to concatenate them together. For example you might have multiple virtual datasets with coordinates of inconsistent length (e.g., leap years within multi-year daily data). + +### Loading low-dimensional coordinates + +In general, it is recommended to load all of your low-dimensional coordinates. +This will slow down your initial opening of the individual virtual datasets, but by loading your coordinates into memory, they can be inlined in the reference file for fast reads of the virtualized store. +However, doing this for coordinates that are N-dimensional might use a lot of storage duplicating them. +Also, anything duplicated could become out of sync with the referenced original files, especially if not using a transactional storage engine like `Icechunk`. ### CF-encoded time variables diff --git a/virtualizarr/readers/dmrpp.py b/virtualizarr/readers/dmrpp.py index 8af835ba..74e097ad 100644 --- a/virtualizarr/readers/dmrpp.py +++ b/virtualizarr/readers/dmrpp.py @@ -27,7 +27,7 @@ def open_virtual_dataset( virtual_backend_kwargs: Optional[dict] = None, reader_options: Optional[dict] = None, ) -> Dataset: - loadable_variables, drop_variables = check_for_collisions( + drop_variables, loadable_variables = check_for_collisions( drop_variables=drop_variables, loadable_variables=loadable_variables, ) diff --git a/virtualizarr/readers/kerchunk.py b/virtualizarr/readers/kerchunk.py index 6f688d3c..d032bb57 100644 --- a/virtualizarr/readers/kerchunk.py +++ b/virtualizarr/readers/kerchunk.py @@ -38,7 +38,7 @@ def open_virtual_dataset( if group: raise NotImplementedError() - loadable_variables, drop_variables = check_for_collisions( + drop_variables, loadable_variables = check_for_collisions( drop_variables=drop_variables, loadable_variables=loadable_variables, ) diff --git a/virtualizarr/tests/test_backend.py b/virtualizarr/tests/test_backend.py index 403ae637..4d41bb47 100644 --- a/virtualizarr/tests/test_backend.py +++ b/virtualizarr/tests/test_backend.py @@ -234,13 +234,15 @@ class TestReadFromURL: "netcdf4", "https://github.com/pydata/xarray-data/raw/master/ROMS_example.nc", ), - ( + pytest.param( "hdf4", "https://github.com/corteva/rioxarray/raw/master/test/test_data/input/MOD09GA.A2008296.h14v17.006.2015181011753.hdf", + marks=pytest.mark.skip(reason="often times out"), ), - ( + pytest.param( "hdf5", "https://nisar.asf.earthdatacloud.nasa.gov/NISAR-SAMPLE-DATA/GCOV/ALOS1_Rosamond_20081012/NISAR_L2_PR_GCOV_001_005_A_219_4020_SHNA_A_20081012T060910_20081012T060926_P01101_F_N_J_001.h5", + marks=pytest.mark.skip(reason="often times out"), ), # https://github.com/zarr-developers/VirtualiZarr/issues/159 # ("hdf5", "https://github.com/fsspec/kerchunk/raw/main/kerchunk/tests/NEONDSTowerTemperatureData.hdf5"), @@ -284,7 +286,7 @@ def test_read_from_url(self, hdf_backend, filetype, url): vds = open_virtual_dataset(url, indexes={}) assert isinstance(vds, xr.Dataset) - @pytest.mark.xfail(reason="often times out, as nisar file is 200MB") + @pytest.mark.skip(reason="often times out, as nisar file is 200MB") def test_virtualizarr_vs_local_nisar(self, hdf_backend): import fsspec diff --git a/virtualizarr/tests/test_readers/test_dmrpp.py b/virtualizarr/tests/test_readers/test_dmrpp.py index 72ae3ecc..37e45453 100644 --- a/virtualizarr/tests/test_readers/test_dmrpp.py +++ b/virtualizarr/tests/test_readers/test_dmrpp.py @@ -446,3 +446,15 @@ def test_relative_path_to_dmrpp_file(self, basic_dmrpp_temp_filepath: Path): ".dmrpp" ) assert path == expected_datafile_path_uri + + +@pytest.mark.parametrize("drop_variables", ["mask", ["data", "mask"]]) +def test_drop_variables(basic_dmrpp_temp_filepath: Path, drop_variables): + vds = open_virtual_dataset( + str(basic_dmrpp_temp_filepath), + indexes={}, + filetype="dmrpp", + drop_variables=drop_variables, + ) + + assert all(var not in vds for var in drop_variables) diff --git a/virtualizarr/tests/test_readers/test_kerchunk.py b/virtualizarr/tests/test_readers/test_kerchunk.py index 10f385d6..89c6ba31 100644 --- a/virtualizarr/tests/test_readers/test_kerchunk.py +++ b/virtualizarr/tests/test_readers/test_kerchunk.py @@ -248,3 +248,14 @@ def test_notimplemented_read_inline_refs(tmp_path, netcdf4_inlined_ref): open_virtual_dataset( filepath=ref_filepath.as_posix(), filetype="kerchunk", indexes={} ) + + +@pytest.mark.parametrize("drop_variables", ["a", ["a"]]) +def test_drop_variables(refs_file_factory, drop_variables): + refs_file = refs_file_factory() + + vds = open_virtual_dataset( + refs_file, filetype="kerchunk", drop_variables=drop_variables + ) + + assert all(var not in vds for var in drop_variables)