Virtual Ref config support [EAR-1345] (#126)

.github/workflows/python-ci.yaml

@@ -98,7 +98,7 @@ jobs:
           python3 -m venv .venv
           source .venv/bin/activate
           pip install icechunk['test'] --find-links dist --force-reinstall
-          AWS_ALLOW_HTTP=1 AWS_ENDPOINT_URL=http://localhost:9000 AWS_ACCESS_KEY_ID=minio123 AWS_SECRET_ACCESS_KEY=minio123 pytest
+          pytest
       - name: pytest
         if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }}
         uses: uraimo/run-on-arch-action@v2
@@ -114,7 +114,7 @@ jobs:
           run: |
             set -e
             pip3 install icechunk['test'] --find-links dist --force-reinstall
-            AWS_ALLOW_HTTP=1 AWS_ENDPOINT_URL=http://localhost:9000 AWS_ACCESS_KEY_ID=minio123 AWS_SECRET_ACCESS_KEY=minio123 pytest
+            pytest
 
   # musllinux:
   #   runs-on: ${{ matrix.platform.runner }}

icechunk-python/examples/smoke-test.py

@@ -177,7 +177,7 @@ async def create_zarr_store(*, store: Literal["memory", "local", "s3"]) -> Store
 
 if __name__ == "__main__":
     MEMORY = StorageConfig.memory("new")
-    MINIO = StorageConfig.s3_from_credentials(
+    MINIO = StorageConfig.s3_from_config(
         bucket="testbucket",
         prefix="root-icechunk",
         credentials=S3Credentials(

icechunk-python/notebooks/demo-dummy-data.ipynb

@@ -1155,11 +1155,11 @@
     "array = root_group[\"group2/foo3\"]\n",
     "print(array)\n",
     "\n",
-    "array = array.resize((array.shape[0]*2, *array.shape[1:]))\n",
+    "array = array.resize((array.shape[0] * 2, *array.shape[1:]))\n",
     "print(array)\n",
-    "array[array.shape[0]//2:, ...] = expected[\"group2/foo3\"]\n",
-    "print(array[2:, 0,0,0])\n",
-    "expected[\"group2/foo3\"] = np.concatenate([expected[\"group2/foo3\"]]*2, axis=0)\n",
+    "array[array.shape[0] // 2 :, ...] = expected[\"group2/foo3\"]\n",
+    "print(array[2:, 0, 0, 0])\n",
+    "expected[\"group2/foo3\"] = np.concatenate([expected[\"group2/foo3\"]] * 2, axis=0)\n",
     "\n",
     "await store.commit(\"appended to group2/foo3\")"
    ]

icechunk-python/notebooks/demo-s3.ipynb

@@ -19,7 +19,7 @@
    "source": [
     "import zarr\n",
     "\n",
-    "from icechunk import IcechunkStore, Storage"
+    "from icechunk import IcechunkStore, StorageConfig"
    ]
   },
   {
@@ -39,7 +39,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "s3_storage = Storage.s3_from_env(bucket=\"icechunk-test\", prefix=\"oscar-demo-repository\")"
+    "s3_storage = StorageConfig.s3_from_env(bucket=\"icechunk-test\", prefix=\"oscar-demo-repository\")"
    ]
   },
   {
@@ -1085,7 +1085,7 @@
     "    group.create_array(\n",
     "        name=var,\n",
     "        shape=oscar[var].shape,\n",
-    "        chunk_shape = (1, 1, 481, 1201),\n",
+    "        chunk_shape=(1, 1, 481, 1201),\n",
     "        fill_value=-1234567,\n",
     "        dtype=oscar[var].dtype,\n",
     "        data=oscar[var],\n",
@@ -1147,10 +1147,10 @@
    "source": [
     "import zarr\n",
     "\n",
-    "from icechunk import IcechunkStore, Storage\n",
+    "from icechunk import IcechunkStore, StorageConfig\n",
     "\n",
     "# TODO: catalog will handle this\n",
-    "s3_storage = Storage.s3_from_env(bucket=\"icechunk-test\", prefix=\"oscar-demo-repository\")"
+    "s3_storage = StorageConfig.s3_from_env(bucket=\"icechunk-test\", prefix=\"oscar-demo-repository\")"
    ]
   },
   {
@@ -1339,9 +1339,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:icechunk]",
+   "display_name": ".venv",
    "language": "python",
-   "name": "conda-env-icechunk-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1353,7 +1353,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.6"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

icechunk-python/notebooks/memorystore.ipynb

@@ -34,7 +34,9 @@
     }
    ],
    "source": [
-    "store = await icechunk.IcechunkStore.create(storage=icechunk.StorageConfig.memory(\"\"), mode=\"w\")\n",
+    "store = await icechunk.IcechunkStore.create(\n",
+    "    storage=icechunk.StorageConfig.memory(\"\"), mode=\"w\"\n",
+    ")\n",
     "store"
    ]
   },
@@ -83,7 +85,9 @@
     }
    ],
    "source": [
-    "air_temp = group.create_array(\"air_temp\", shape=(1000, 1000), chunk_shape=(100, 100), dtype=\"i4\")\n",
+    "air_temp = group.create_array(\n",
+    "    \"air_temp\", shape=(1000, 1000), chunk_shape=(100, 100), dtype=\"i4\"\n",
+    ")\n",
     "air_temp"
    ]
   },