From accfc0195c35dd6aae11c8311f32821a16fd4b06 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 4 Oct 2024 08:59:27 -0500 Subject: [PATCH] creating a ~1Gb dataset for reproducibility porposes --- sigmet2zarr/create_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sigmet2zarr/create_dataset.py b/sigmet2zarr/create_dataset.py index 2beb736..948140e 100644 --- a/sigmet2zarr/create_dataset.py +++ b/sigmet2zarr/create_dataset.py @@ -40,7 +40,7 @@ def radar_convert(): fs = fsspec.filesystem("s3", anon=True) radar_files = [ f"s3://{i}" for i in sorted(fs.glob(f"{str_bucket}{query}*")) - ] + ][:30] for files in batch(radar_files, n=12): bag = db.from_sequence(files, npartitions=len(files)).map( accessor_wrapper