From 9f51c85d6bc994b20eeb0460a95a8ab00a791a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Galkin?= Date: Thu, 20 Feb 2025 00:17:37 -0300 Subject: [PATCH] Switch to conditional PUT for branch updates (#755) We no longer do the sequence of write-only branch versions. --- docs/docs/icechunk-python/configuration.md | 4 - .../python/icechunk/_icechunk_python.pyi | 25 -- icechunk-python/src/config.rs | 11 +- icechunk-python/src/store.rs | 3 +- ...66KVWRNQGEEWC8QN0 => 09HEW2P03CSMHFAZY7DG} | Bin ...VFJQXWR84R1F1Q58G => 52H0E4NSPN8SVRK9EVGG} | Bin .../test-repo/chunks/AN2V69NWNGNCW91839Q0 | Bin 19 -> 0 bytes ...MGAGQ1FC1GKKHZTK0 => DWQ75SDC624XF9H326RG} | Bin .../test-repo/chunks/HTKY8SN65KBX42E4V9FG | Bin 19 -> 0 bytes .../test-repo/chunks/MDYP3ZEV630YPNCEENC0 | Bin 19 -> 0 bytes ...3J7FEC9CCEB6GEKVG => RW938N1KP2R4BHMW62QG} | Bin .../test-repo/chunks/Y9DTSVWNZV9HKT2R17T0 | Bin 19 -> 0 bytes .../tests/data/test-repo/config.yaml | 9 +- .../test-repo/manifests/0GQQ44D2837GGMHY81CG | Bin 237 -> 0 bytes .../test-repo/manifests/3C9WRKTE3PNDSNYBKD60 | Bin 165 -> 0 bytes .../test-repo/manifests/73Q2CY1JSN768PFJS2M0 | Bin 168 -> 0 bytes .../test-repo/manifests/8WT6R2E6WVC9GJ7BS6GG | Bin 277 -> 0 bytes .../test-repo/manifests/C38XX4Z2517M93GQ5MA0 | Bin 174 -> 0 bytes .../test-repo/manifests/CMYVHDWMSTG9R25780YG | Bin 0 -> 240 bytes .../test-repo/manifests/G3W2W8V6ZG09J6C21WE0 | Bin 0 -> 175 bytes .../test-repo/manifests/G94WC9CN23R53A63CRXG | Bin 278 -> 0 bytes .../test-repo/manifests/MWE7J4Y1V04W0DCXB8Z0 | Bin 174 -> 0 bytes .../test-repo/manifests/Q04J7QW5RQ8D17TPA10G | Bin 0 -> 278 bytes .../test-repo/manifests/SHTEAP8C784YMZSJKBM0 | Bin 0 -> 166 bytes .../test-repo/manifests/T9PRDPYDRCEHC2GAVR8G | Bin 241 -> 0 bytes .../test-repo/refs/branch.main/ZZZZZZZW.json | 1 - .../test-repo/refs/branch.main/ZZZZZZZX.json | 1 - .../test-repo/refs/branch.main/ZZZZZZZY.json | 1 - .../test-repo/refs/branch.main/ZZZZZZZZ.json | 1 - .../data/test-repo/refs/branch.main/ref.json | 1 + .../refs/branch.my-branch/ZZZZZZZX.json | 1 - .../refs/branch.my-branch/ZZZZZZZY.json | 1 - .../refs/branch.my-branch/ZZZZZZZZ.json | 1 - .../test-repo/refs/branch.my-branch/ref.json | 1 + .../data/test-repo/refs/tag.deleted/ref.json | 2 +- .../refs/tag.it also works!/ref.json | 2 +- .../test-repo/refs/tag.it works!/ref.json | 2 +- .../test-repo/snapshots/394QWZDXAY74HP6Q8P3G | Bin 867 -> 0 bytes .../test-repo/snapshots/3EKE17N8YF5ZK5NRMZJ0 | Bin 801 -> 0 bytes .../test-repo/snapshots/4QF8JA0YPDN51MHSSYVG | Bin 0 -> 863 bytes .../test-repo/snapshots/6Q9GDTXKF17BGQVSQZFG | Bin 177 -> 0 bytes .../test-repo/snapshots/7XAF0Q905SH4938DN9CG | Bin 0 -> 860 bytes .../test-repo/snapshots/949AXZ49X764TMDC6D4G | Bin 787 -> 0 bytes .../test-repo/snapshots/A2RD2Y65PR6D3B6BR1K0 | Bin 587 -> 0 bytes .../test-repo/snapshots/GC4YVH5SKBPEZCENYQE0 | Bin 0 -> 792 bytes .../test-repo/snapshots/GNFK0SSWD5B8CVA53XEG | Bin 867 -> 0 bytes .../test-repo/snapshots/HNG82GMS51ECXFXFCYJG | Bin 871 -> 0 bytes .../test-repo/snapshots/K1BMYVG1HNVTNV1FSBH0 | Bin 577 -> 0 bytes .../test-repo/snapshots/NXH3M0HJ7EEJ0699DPP0 | Bin 0 -> 861 bytes .../test-repo/snapshots/P874YS3J196959RDHX7G | Bin 0 -> 172 bytes .../test-repo/snapshots/R7F1RJHPZ428N4AK19K0 | Bin 178 -> 0 bytes .../test-repo/snapshots/RPA0WQCNM2N9HBBRHJQ0 | Bin 513 -> 0 bytes .../test-repo/snapshots/SNF98D1SK7NWD5KQJM20 | Bin 586 -> 0 bytes .../test-repo/snapshots/TNE0TX645A2G7VTXFA1G | Bin 1053 -> 0 bytes .../test-repo/snapshots/XDZ162T1TYBEJMK99NPG | Bin 0 -> 1038 bytes .../transactions/394QWZDXAY74HP6Q8P3G | Bin 147 -> 0 bytes .../transactions/3EKE17N8YF5ZK5NRMZJ0 | Bin 157 -> 0 bytes .../transactions/4QF8JA0YPDN51MHSSYVG | Bin 0 -> 146 bytes .../transactions/7XAF0Q905SH4938DN9CG | Bin 0 -> 237 bytes .../transactions/949AXZ49X764TMDC6D4G | Bin 172 -> 0 bytes .../transactions/A2RD2Y65PR6D3B6BR1K0 | Bin 148 -> 0 bytes .../transactions/GC4YVH5SKBPEZCENYQE0 | Bin 0 -> 157 bytes .../transactions/GNFK0SSWD5B8CVA53XEG | Bin 235 -> 0 bytes .../transactions/HNG82GMS51ECXFXFCYJG | Bin 146 -> 0 bytes .../transactions/K1BMYVG1HNVTNV1FSBH0 | Bin 235 -> 0 bytes .../transactions/NXH3M0HJ7EEJ0699DPP0 | Bin 0 -> 148 bytes .../transactions/RPA0WQCNM2N9HBBRHJQ0 | Bin 167 -> 0 bytes .../transactions/SNF98D1SK7NWD5KQJM20 | Bin 148 -> 0 bytes .../transactions/TNE0TX645A2G7VTXFA1G | Bin 173 -> 0 bytes .../transactions/XDZ162T1TYBEJMK99NPG | Bin 0 -> 169 bytes icechunk-python/tests/test_config.py | 6 +- icechunk/examples/multithreaded_store.rs | 1 - icechunk/src/config.rs | 11 - icechunk/src/format/manifest.rs | 4 +- icechunk/src/ops/gc.rs | 2 +- icechunk/src/refs.rs | 316 ++++-------------- icechunk/src/repository.rs | 118 +++---- icechunk/src/session.rs | 28 +- icechunk/src/storage/logging.rs | 18 +- icechunk/src/storage/mod.rs | 48 ++- icechunk/src/storage/object_store.rs | 149 ++++----- icechunk/src/storage/s3.rs | 78 ++--- icechunk/src/virtual_chunks.rs | 4 +- icechunk/tests/test_gc.rs | 2 - icechunk/tests/test_storage.rs | 190 +++++------ icechunk/tests/test_virtual_refs.rs | 6 +- 86 files changed, 385 insertions(+), 663 deletions(-) rename icechunk-python/tests/data/test-repo/chunks/{0DX66KVWRNQGEEWC8QN0 => 09HEW2P03CSMHFAZY7DG} (100%) rename icechunk-python/tests/data/test-repo/chunks/{20BVFJQXWR84R1F1Q58G => 52H0E4NSPN8SVRK9EVGG} (100%) delete mode 100644 icechunk-python/tests/data/test-repo/chunks/AN2V69NWNGNCW91839Q0 rename icechunk-python/tests/data/test-repo/chunks/{5PCMGAGQ1FC1GKKHZTK0 => DWQ75SDC624XF9H326RG} (100%) delete mode 100644 icechunk-python/tests/data/test-repo/chunks/HTKY8SN65KBX42E4V9FG delete mode 100644 icechunk-python/tests/data/test-repo/chunks/MDYP3ZEV630YPNCEENC0 rename icechunk-python/tests/data/test-repo/chunks/{9HZ3J7FEC9CCEB6GEKVG => RW938N1KP2R4BHMW62QG} (100%) delete mode 100644 icechunk-python/tests/data/test-repo/chunks/Y9DTSVWNZV9HKT2R17T0 delete mode 100644 icechunk-python/tests/data/test-repo/manifests/0GQQ44D2837GGMHY81CG delete mode 100644 icechunk-python/tests/data/test-repo/manifests/3C9WRKTE3PNDSNYBKD60 delete mode 100644 icechunk-python/tests/data/test-repo/manifests/73Q2CY1JSN768PFJS2M0 delete mode 100644 icechunk-python/tests/data/test-repo/manifests/8WT6R2E6WVC9GJ7BS6GG delete mode 100644 icechunk-python/tests/data/test-repo/manifests/C38XX4Z2517M93GQ5MA0 create mode 100644 icechunk-python/tests/data/test-repo/manifests/CMYVHDWMSTG9R25780YG create mode 100644 icechunk-python/tests/data/test-repo/manifests/G3W2W8V6ZG09J6C21WE0 delete mode 100644 icechunk-python/tests/data/test-repo/manifests/G94WC9CN23R53A63CRXG delete mode 100644 icechunk-python/tests/data/test-repo/manifests/MWE7J4Y1V04W0DCXB8Z0 create mode 100644 icechunk-python/tests/data/test-repo/manifests/Q04J7QW5RQ8D17TPA10G create mode 100644 icechunk-python/tests/data/test-repo/manifests/SHTEAP8C784YMZSJKBM0 delete mode 100644 icechunk-python/tests/data/test-repo/manifests/T9PRDPYDRCEHC2GAVR8G delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZW.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZX.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZY.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZZ.json create mode 100644 icechunk-python/tests/data/test-repo/refs/branch.main/ref.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZX.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZY.json delete mode 100644 icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZZ.json create mode 100644 icechunk-python/tests/data/test-repo/refs/branch.my-branch/ref.json delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/394QWZDXAY74HP6Q8P3G delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/3EKE17N8YF5ZK5NRMZJ0 create mode 100644 icechunk-python/tests/data/test-repo/snapshots/4QF8JA0YPDN51MHSSYVG delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/6Q9GDTXKF17BGQVSQZFG create mode 100644 icechunk-python/tests/data/test-repo/snapshots/7XAF0Q905SH4938DN9CG delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/949AXZ49X764TMDC6D4G delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/A2RD2Y65PR6D3B6BR1K0 create mode 100644 icechunk-python/tests/data/test-repo/snapshots/GC4YVH5SKBPEZCENYQE0 delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/GNFK0SSWD5B8CVA53XEG delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/HNG82GMS51ECXFXFCYJG delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/K1BMYVG1HNVTNV1FSBH0 create mode 100644 icechunk-python/tests/data/test-repo/snapshots/NXH3M0HJ7EEJ0699DPP0 create mode 100644 icechunk-python/tests/data/test-repo/snapshots/P874YS3J196959RDHX7G delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/R7F1RJHPZ428N4AK19K0 delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/RPA0WQCNM2N9HBBRHJQ0 delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/SNF98D1SK7NWD5KQJM20 delete mode 100644 icechunk-python/tests/data/test-repo/snapshots/TNE0TX645A2G7VTXFA1G create mode 100644 icechunk-python/tests/data/test-repo/snapshots/XDZ162T1TYBEJMK99NPG delete mode 100644 icechunk-python/tests/data/test-repo/transactions/394QWZDXAY74HP6Q8P3G delete mode 100644 icechunk-python/tests/data/test-repo/transactions/3EKE17N8YF5ZK5NRMZJ0 create mode 100644 icechunk-python/tests/data/test-repo/transactions/4QF8JA0YPDN51MHSSYVG create mode 100644 icechunk-python/tests/data/test-repo/transactions/7XAF0Q905SH4938DN9CG delete mode 100644 icechunk-python/tests/data/test-repo/transactions/949AXZ49X764TMDC6D4G delete mode 100644 icechunk-python/tests/data/test-repo/transactions/A2RD2Y65PR6D3B6BR1K0 create mode 100644 icechunk-python/tests/data/test-repo/transactions/GC4YVH5SKBPEZCENYQE0 delete mode 100644 icechunk-python/tests/data/test-repo/transactions/GNFK0SSWD5B8CVA53XEG delete mode 100644 icechunk-python/tests/data/test-repo/transactions/HNG82GMS51ECXFXFCYJG delete mode 100644 icechunk-python/tests/data/test-repo/transactions/K1BMYVG1HNVTNV1FSBH0 create mode 100644 icechunk-python/tests/data/test-repo/transactions/NXH3M0HJ7EEJ0699DPP0 delete mode 100644 icechunk-python/tests/data/test-repo/transactions/RPA0WQCNM2N9HBBRHJQ0 delete mode 100644 icechunk-python/tests/data/test-repo/transactions/SNF98D1SK7NWD5KQJM20 delete mode 100644 icechunk-python/tests/data/test-repo/transactions/TNE0TX645A2G7VTXFA1G create mode 100644 icechunk-python/tests/data/test-repo/transactions/XDZ162T1TYBEJMK99NPG diff --git a/docs/docs/icechunk-python/configuration.md b/docs/docs/icechunk-python/configuration.md index 0be341df..d037aa8f 100644 --- a/docs/docs/icechunk-python/configuration.md +++ b/docs/docs/icechunk-python/configuration.md @@ -22,10 +22,6 @@ It allows you to configure the following parameters: The threshold for when to inline a chunk into a manifest instead of storing it as a separate object in the storage backend. -### [`unsafe_overwrite_refs`](./reference.md#icechunk.RepositoryConfig.unsafe_overwrite_refs) - -Whether to allow overwriting references in the repository. - ### [`get_partial_values_concurrency`](./reference.md#icechunk.RepositoryConfig.get_partial_values_concurrency) The number of concurrent requests to make when getting partial values from storage. diff --git a/icechunk-python/python/icechunk/_icechunk_python.pyi b/icechunk-python/python/icechunk/_icechunk_python.pyi index 0db86fc5..75a8f4f6 100644 --- a/icechunk-python/python/icechunk/_icechunk_python.pyi +++ b/icechunk-python/python/icechunk/_icechunk_python.pyi @@ -570,7 +570,6 @@ class RepositoryConfig: def __init__( self, inline_chunk_threshold_bytes: int | None = None, - unsafe_overwrite_refs: bool | None = None, get_partial_values_concurrency: int | None = None, compression: CompressionConfig | None = None, caching: CachingConfig | None = None, @@ -585,8 +584,6 @@ class RepositoryConfig: ---------- inline_chunk_threshold_bytes: int | None The maximum size of a chunk that will be stored inline in the repository. - unsafe_overwrite_refs: bool | None - Whether to allow overwriting references in the repository. get_partial_values_concurrency: int | None The number of concurrent requests to make when getting partial values from storage. compression: CompressionConfig | None @@ -618,28 +615,6 @@ class RepositoryConfig: """ ... @property - def unsafe_overwrite_refs(self) -> bool | None: - """ - Whether to allow overwriting references in the repository. - - Returns - ------- - bool | None - Whether to allow overwriting references in the repository. - """ - ... - @unsafe_overwrite_refs.setter - def unsafe_overwrite_refs(self, value: bool | None) -> None: - """ - Set whether to allow overwriting references in the repository. - - Parameters - ---------- - value: bool | None - Whether to allow overwriting references in the repository. - """ - ... - @property def get_partial_values_concurrency(self) -> int | None: """ The number of concurrent requests to make when getting partial values from storage. diff --git a/icechunk-python/src/config.rs b/icechunk-python/src/config.rs index 05133db8..016ff0df 100644 --- a/icechunk-python/src/config.rs +++ b/icechunk-python/src/config.rs @@ -969,8 +969,6 @@ pub struct PyRepositoryConfig { #[pyo3(get, set)] pub inline_chunk_threshold_bytes: Option, #[pyo3(get, set)] - pub unsafe_overwrite_refs: Option, - #[pyo3(get, set)] pub get_partial_values_concurrency: Option, #[pyo3(get, set)] pub compression: Option>, @@ -996,7 +994,6 @@ impl From<&PyRepositoryConfig> for RepositoryConfig { fn from(value: &PyRepositoryConfig) -> Self { Python::with_gil(|py| Self { inline_chunk_threshold_bytes: value.inline_chunk_threshold_bytes, - unsafe_overwrite_refs: value.unsafe_overwrite_refs, get_partial_values_concurrency: value.get_partial_values_concurrency, compression: value.compression.as_ref().map(|c| (&*c.borrow(py)).into()), caching: value.caching.as_ref().map(|c| (&*c.borrow(py)).into()), @@ -1014,7 +1011,6 @@ impl From for PyRepositoryConfig { #[allow(clippy::expect_used)] Python::with_gil(|py| Self { inline_chunk_threshold_bytes: value.inline_chunk_threshold_bytes, - unsafe_overwrite_refs: value.unsafe_overwrite_refs, get_partial_values_concurrency: value.get_partial_values_concurrency, compression: value.compression.map(|c| { Py::new(py, Into::::into(c)) @@ -1049,11 +1045,10 @@ impl PyRepositoryConfig { } #[new] - #[pyo3(signature = (inline_chunk_threshold_bytes = None, unsafe_overwrite_refs = None, get_partial_values_concurrency = None, compression = None, caching = None, storage = None, virtual_chunk_containers = None, manifest = None))] + #[pyo3(signature = (inline_chunk_threshold_bytes = None, get_partial_values_concurrency = None, compression = None, caching = None, storage = None, virtual_chunk_containers = None, manifest = None))] #[allow(clippy::too_many_arguments)] pub fn new( inline_chunk_threshold_bytes: Option, - unsafe_overwrite_refs: Option, get_partial_values_concurrency: Option, compression: Option>, caching: Option>, @@ -1063,7 +1058,6 @@ impl PyRepositoryConfig { ) -> Self { Self { inline_chunk_threshold_bytes, - unsafe_overwrite_refs, get_partial_values_concurrency, compression, caching, @@ -1129,9 +1123,8 @@ impl PyRepositoryConfig { })); // TODO: virtual chunk containers format!( - r#"RepositoryConfig(inline_chunk_threshold_bytes={inl}, unsafe_overwrite_refs={uns}, get_partial_values_concurrency={partial}, compression={comp}, caching={caching}, storage={storage}, manifest={manifest})"#, + r#"RepositoryConfig(inline_chunk_threshold_bytes={inl}, get_partial_values_concurrency={partial}, compression={comp}, caching={caching}, storage={storage}, manifest={manifest})"#, inl = format_option_to_string(self.inline_chunk_threshold_bytes), - uns = format_option(self.unsafe_overwrite_refs.map(format_bool)), partial = format_option_to_string(self.get_partial_values_concurrency), comp = comp, caching = caching, diff --git a/icechunk-python/src/store.rs b/icechunk-python/src/store.rs index 93833f8b..de50a63d 100644 --- a/icechunk-python/src/store.rs +++ b/icechunk-python/src/store.rs @@ -8,6 +8,7 @@ use icechunk::{ manifest::{Checksum, SecondsSinceEpoch, VirtualChunkLocation, VirtualChunkRef}, ChunkLength, ChunkOffset, }, + storage::ETag, store::{StoreError, StoreErrorKind}, Store, }; @@ -37,7 +38,7 @@ enum ChecksumArgument { impl From for Checksum { fn from(value: ChecksumArgument) -> Self { match value { - ChecksumArgument::String(etag) => Checksum::ETag(etag), + ChecksumArgument::String(etag) => Checksum::ETag(ETag(etag)), ChecksumArgument::Datetime(date_time) => { Checksum::LastModified(SecondsSinceEpoch(date_time.timestamp() as u32)) } diff --git a/icechunk-python/tests/data/test-repo/chunks/0DX66KVWRNQGEEWC8QN0 b/icechunk-python/tests/data/test-repo/chunks/09HEW2P03CSMHFAZY7DG similarity index 100% rename from icechunk-python/tests/data/test-repo/chunks/0DX66KVWRNQGEEWC8QN0 rename to icechunk-python/tests/data/test-repo/chunks/09HEW2P03CSMHFAZY7DG diff --git a/icechunk-python/tests/data/test-repo/chunks/20BVFJQXWR84R1F1Q58G b/icechunk-python/tests/data/test-repo/chunks/52H0E4NSPN8SVRK9EVGG similarity index 100% rename from icechunk-python/tests/data/test-repo/chunks/20BVFJQXWR84R1F1Q58G rename to icechunk-python/tests/data/test-repo/chunks/52H0E4NSPN8SVRK9EVGG diff --git a/icechunk-python/tests/data/test-repo/chunks/AN2V69NWNGNCW91839Q0 b/icechunk-python/tests/data/test-repo/chunks/AN2V69NWNGNCW91839Q0 deleted file mode 100644 index 8666a286982f8d95ce7ab61a59741fcac452468f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmdPcs{dCZC6s|dfq_B8iIL&&lMVnkjRn^L diff --git a/icechunk-python/tests/data/test-repo/chunks/5PCMGAGQ1FC1GKKHZTK0 b/icechunk-python/tests/data/test-repo/chunks/DWQ75SDC624XF9H326RG similarity index 100% rename from icechunk-python/tests/data/test-repo/chunks/5PCMGAGQ1FC1GKKHZTK0 rename to icechunk-python/tests/data/test-repo/chunks/DWQ75SDC624XF9H326RG diff --git a/icechunk-python/tests/data/test-repo/chunks/HTKY8SN65KBX42E4V9FG b/icechunk-python/tests/data/test-repo/chunks/HTKY8SN65KBX42E4V9FG deleted file mode 100644 index 8666a286982f8d95ce7ab61a59741fcac452468f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmdPcs{dCZC6s|dfq_B8iIL&&lMVnkjRn^L diff --git a/icechunk-python/tests/data/test-repo/chunks/MDYP3ZEV630YPNCEENC0 b/icechunk-python/tests/data/test-repo/chunks/MDYP3ZEV630YPNCEENC0 deleted file mode 100644 index 8666a286982f8d95ce7ab61a59741fcac452468f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmdPcs{dCZC6s|dfq_B8iIL&&lMVnkjRn^L diff --git a/icechunk-python/tests/data/test-repo/chunks/9HZ3J7FEC9CCEB6GEKVG b/icechunk-python/tests/data/test-repo/chunks/RW938N1KP2R4BHMW62QG similarity index 100% rename from icechunk-python/tests/data/test-repo/chunks/9HZ3J7FEC9CCEB6GEKVG rename to icechunk-python/tests/data/test-repo/chunks/RW938N1KP2R4BHMW62QG diff --git a/icechunk-python/tests/data/test-repo/chunks/Y9DTSVWNZV9HKT2R17T0 b/icechunk-python/tests/data/test-repo/chunks/Y9DTSVWNZV9HKT2R17T0 deleted file mode 100644 index 8666a286982f8d95ce7ab61a59741fcac452468f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19 acmdPcs{dCZC6s|dfq_B8iIL&&lMVnkjRn^L diff --git a/icechunk-python/tests/data/test-repo/config.yaml b/icechunk-python/tests/data/test-repo/config.yaml index 65e4dd1e..80e47459 100644 --- a/icechunk-python/tests/data/test-repo/config.yaml +++ b/icechunk-python/tests/data/test-repo/config.yaml @@ -1,5 +1,4 @@ inline_chunk_threshold_bytes: 12 -unsafe_overwrite_refs: null get_partial_values_concurrency: null compression: null caching: null @@ -13,6 +12,10 @@ virtual_chunk_containers: endpoint_url: https://fly.storage.tigris.dev anonymous: false allow_http: false + az: + name: az + url_prefix: az + store: !azure {} file: name: file url_prefix: file @@ -21,10 +24,6 @@ virtual_chunk_containers: name: gcs url_prefix: gcs store: !gcs {} - az: - name: az - url_prefix: az - store: !azure {} s3: name: s3 url_prefix: s3:// diff --git a/icechunk-python/tests/data/test-repo/manifests/0GQQ44D2837GGMHY81CG b/icechunk-python/tests/data/test-repo/manifests/0GQQ44D2837GGMHY81CG deleted file mode 100644 index 5dff37e3a650db3fcde7ba9659016e1ce2383ea4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 237 zcmeZtcKtAad6%(=7Ai@07k-dZ7D;1*da9k$Hzy~uxgky#C$>CK0WlJ+LE zuPm3)P}ox95hUcJpi?*ZRamlSkECeh^ck%a-FiJZ&Zsu9E|%f%@=iPO%B4l3E;Taj zQ9)n-o>n_^1xM{gnZK8L=Znv4=X(A~P_V{i%kKQ>(*2C5CRB*>GsrtLIILN_%juX* ebD{XG#|GR1QM0~s%{<6!Xsl|$b?dd@6gdFZx>)-F diff --git a/icechunk-python/tests/data/test-repo/manifests/3C9WRKTE3PNDSNYBKD60 b/icechunk-python/tests/data/test-repo/manifests/3C9WRKTE3PNDSNYBKD60 deleted file mode 100644 index a7da5f1cff2276e5701a1168fb0e3470d6f70d30..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 165 zcmeZtcKtAad6%}6EDl)`m>o$pM(A2tQ50bRj|H~)C>;ofNjJ#(pdN`t?GKmzp1wkf5kQ`rWG&Vs%81$8s^0Con9RzHD@1&*CVh(&yfF#=%UB Ji=(!(0|1RxK=c3r diff --git a/icechunk-python/tests/data/test-repo/manifests/73Q2CY1JSN768PFJS2M0 b/icechunk-python/tests/data/test-repo/manifests/73Q2CY1JSN768PFJS2M0 deleted file mode 100644 index 7b1848c26bf262bb5d8e1cd6f6240f2681e7575f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 168 zcmeZtcKtAad6%@xkaqc9Fq3b-aJ^gF+irCk&B@`fnmAyWupsw7Dp+SKKHIO M4rW?h9JQ4l0LLvssQ>@~ diff --git a/icechunk-python/tests/data/test-repo/manifests/8WT6R2E6WVC9GJ7BS6GG b/icechunk-python/tests/data/test-repo/manifests/8WT6R2E6WVC9GJ7BS6GG deleted file mode 100644 index 1d3cb4ab9b7a5bbecb6f982935debdb55edc903e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 277 zcmeZtcKtAad6%_R-uF2K>|DSU@d*9r( zx6>!)O0`dHoV8Zz`OS$=JeC{7*jhiV*E@5X_kT=PmmNDhkCt$^a1g^IgO8W%rK;=> z1}{?*n!7|MM=Q+6B;cQV@d34GN=;=9EIK~Lf4|)O-T0y2{`qvN&S^c?llU?%RsWvX zxAcnH(93t)31$*TjUorv!@ceUK diff --git a/icechunk-python/tests/data/test-repo/manifests/C38XX4Z2517M93GQ5MA0 b/icechunk-python/tests/data/test-repo/manifests/C38XX4Z2517M93GQ5MA0 deleted file mode 100644 index 98ce9238ee6f5cec1f12e8206e80ce2c0209b4ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 174 zcmeZtcKtAad6%fQ=U-|6wqf;`DZy5o9#YR^E diff --git a/icechunk-python/tests/data/test-repo/manifests/CMYVHDWMSTG9R25780YG b/icechunk-python/tests/data/test-repo/manifests/CMYVHDWMSTG9R25780YG new file mode 100644 index 0000000000000000000000000000000000000000..1cd56eb31be31a39dfc02c37e7978158b3994d0d GIT binary patch literal 240 zcmeZtcKtAad6%_%a zYzjZJwzx~}j5Wvd#_4VQDjNKkapg6Bt-Z;-Ki2H`y|M#S^4^R5H$*%Hv?{!APptP?w#qk zw+X+M)Zcl!@Wv0DDFOwn-i6lg64sosC$QtpajxzakKSKByYKjh1sN4${0#EW3=V77 j?s7UN(_AP%>#+fMK-8?STr&^y8XBt_aNT+>I7JQse0*Oh literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/manifests/G3W2W8V6ZG09J6C21WE0 b/icechunk-python/tests/data/test-repo/manifests/G3W2W8V6ZG09J6C21WE0 new file mode 100644 index 0000000000000000000000000000000000000000..a107d98a1b316869bc531de01b4e33463862332e GIT binary patch literal 175 zcmeZtcKtAad6%&oYR zI_oU{$7^awx)PJ*s_VDn=Kd9F^?M{-nrP!UXZyA48%7e{Pr|quqz^OrwJ%FuP`RMe T-Sp;6@s-a$KRPAj_?8g>>lQ@+ literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/manifests/G94WC9CN23R53A63CRXG b/icechunk-python/tests/data/test-repo/manifests/G94WC9CN23R53A63CRXG deleted file mode 100644 index 7727c935562623328af6fdf9b9e64e7be7f61dd0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 278 zcmeZtcKtAad6%U)_0O=>+zbyF7LFW}UM79pxRtj0N>#$wf^Cl@y%c4v{vYIRb^r7EW$M=&*F%rF9`s}cT;XZh|7+#a z=i)wlIa0J()D7juC%AGv?Kk2-5Fy+;sVCd8;>L_0tE=-l4zK^|SidDg$mzB6(qt>W z<@UcGmnt#^37+C-;ym`z`}E`Dm-2tKzMU6ju#gJcrnFOS{f}E`Px0=!Zdg&W(2{R6 Y!_uXZ>nBLw5ikyBTD)Mdoa0MI0A=)ec>n+a diff --git a/icechunk-python/tests/data/test-repo/manifests/MWE7J4Y1V04W0DCXB8Z0 b/icechunk-python/tests/data/test-repo/manifests/MWE7J4Y1V04W0DCXB8Z0 deleted file mode 100644 index fdfbb82dff590d2c8cc1fa697f9787b316db4893..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 174 zcmeZtcKtAad6%wAE1RlRxrd93LHaO*U;DDu1(gdb T-A!-K6kqx5^P^KTj&B(ONj^!| diff --git a/icechunk-python/tests/data/test-repo/manifests/Q04J7QW5RQ8D17TPA10G b/icechunk-python/tests/data/test-repo/manifests/Q04J7QW5RQ8D17TPA10G new file mode 100644 index 0000000000000000000000000000000000000000..d3ea1cd37be7acb19cf516d8a4aae497e8a7a848 GIT binary patch literal 278 zcmeZtcKtAad6%4TS39-+0BoJ+zdf%YgN0PkHl>|v>wnxjdy03*b;F91g_eAq Z8I~@MTt7kbj(~A6)8Ykt$q zJXvzf>eKhu?Q@c7byd-oVhQ;;r!?HjZKj7*IU^TCc>=?7>B~kJ_AHK4Dt+!_=3u6@tWBIHc4|?}VYjnf(>xBZbzVN{yKLLt+jZj3QJaGQ&uQau(R6irH1UOk z<%~o@7~GQ?E*lpW3r!UM{Bm-F`eD_^*4Z04H7hJ) zUhm>_E$N>9Cs&na&RQ3?BfpZaE#KcbbCnjy$}JP@^y2)E?u&e-t>MOdo{679-kHH+ l&Dvc~$7Grd#b-S>;0}nI^_6SpL0&^+RRgYDuLY;b0RZ*PV1fVu diff --git a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZW.json b/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZW.json deleted file mode 100644 index c7dc109c..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZW.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"HNG82GMS51ECXFXFCYJG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZX.json b/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZX.json deleted file mode 100644 index 0fc7cd40..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZX.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"GNFK0SSWD5B8CVA53XEG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZY.json b/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZY.json deleted file mode 100644 index cf192bde..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZY.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"3EKE17N8YF5ZK5NRMZJ0"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZZ.json b/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZZ.json deleted file mode 100644 index 346eeb78..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.main/ZZZZZZZZ.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"R7F1RJHPZ428N4AK19K0"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.main/ref.json b/icechunk-python/tests/data/test-repo/refs/branch.main/ref.json new file mode 100644 index 00000000..8fa8aba9 --- /dev/null +++ b/icechunk-python/tests/data/test-repo/refs/branch.main/ref.json @@ -0,0 +1 @@ +{"snapshot":"NXH3M0HJ7EEJ0699DPP0"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZX.json b/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZX.json deleted file mode 100644 index a52be478..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZX.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"TNE0TX645A2G7VTXFA1G"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZY.json b/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZY.json deleted file mode 100644 index d63e54c4..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZY.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"394QWZDXAY74HP6Q8P3G"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZZ.json b/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZZ.json deleted file mode 100644 index c7dc109c..00000000 --- a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ZZZZZZZZ.json +++ /dev/null @@ -1 +0,0 @@ -{"snapshot":"HNG82GMS51ECXFXFCYJG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ref.json b/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ref.json new file mode 100644 index 00000000..a4b2ffa4 --- /dev/null +++ b/icechunk-python/tests/data/test-repo/refs/branch.my-branch/ref.json @@ -0,0 +1 @@ +{"snapshot":"XDZ162T1TYBEJMK99NPG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json b/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json index d63e54c4..b84c0bfc 100644 --- a/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json +++ b/icechunk-python/tests/data/test-repo/refs/tag.deleted/ref.json @@ -1 +1 @@ -{"snapshot":"394QWZDXAY74HP6Q8P3G"} \ No newline at end of file +{"snapshot":"4QF8JA0YPDN51MHSSYVG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/tag.it also works!/ref.json b/icechunk-python/tests/data/test-repo/refs/tag.it also works!/ref.json index a52be478..a4b2ffa4 100644 --- a/icechunk-python/tests/data/test-repo/refs/tag.it also works!/ref.json +++ b/icechunk-python/tests/data/test-repo/refs/tag.it also works!/ref.json @@ -1 +1 @@ -{"snapshot":"TNE0TX645A2G7VTXFA1G"} \ No newline at end of file +{"snapshot":"XDZ162T1TYBEJMK99NPG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/refs/tag.it works!/ref.json b/icechunk-python/tests/data/test-repo/refs/tag.it works!/ref.json index d63e54c4..b84c0bfc 100644 --- a/icechunk-python/tests/data/test-repo/refs/tag.it works!/ref.json +++ b/icechunk-python/tests/data/test-repo/refs/tag.it works!/ref.json @@ -1 +1 @@ -{"snapshot":"394QWZDXAY74HP6Q8P3G"} \ No newline at end of file +{"snapshot":"4QF8JA0YPDN51MHSSYVG"} \ No newline at end of file diff --git a/icechunk-python/tests/data/test-repo/snapshots/394QWZDXAY74HP6Q8P3G b/icechunk-python/tests/data/test-repo/snapshots/394QWZDXAY74HP6Q8P3G deleted file mode 100644 index 3b09dece9fc5649e0afd7ffa0a2e5c6413940d1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 867 zcmV-p1DyOxLq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zof!ajtB*zC7$uDX1P%ftB4Id+HOK|tF}Rut`X9IAg#4c;@%t; z7vN0?z$b=q&?jXe?&vra$_z4BnrJNreE@v`g8-WbcPc5?n897^0nV;9v%70~*bB;3 zJaur_8u1|lYNRnGyE?sCl9_4KG7j4$X-Dex>{>lIyQAIRjnitRo=Gch90qr3oSsQi zxEGYDxWN&7!AzqjlFsIxFi2AaaArm(LXF%k#f(NlKJ-%qL&FmlpN7z=vg5xa%^6W0 z%qZn?)CBV)k&Pi!I%GCQWJENGLi2rAVhkE|fFJa`{MGka+3`IV4wWd%mAWm`N|L5X zx!OUQaa)AFV2vosl}MCEo)`!+683_UN9*8dFDNl>CsG<$V%oNGJFU}BwUYt=zX*ij ze;faATv_Fm*~5siKSja`k^xeVY2NumB0;T4~PZJ>nri2TB)k5G0{h|Nm ztt$ds&?edhU8qqdLdcN`Mn=Mnl3@;D0udQ2vjY-J|u;NOX-eE*YKIja7jfoToAK(!!CoFu0e8h0f-S)2-UxJl&ppF@3}y& z%j52sos=)?Uxm4tac3-9PEbd-A!k68`c@ZCtLUGKey9(D0+H%aT>->{)HjQ8g6&1f z>qq-&<^HmNdo(Z;T`W{_J=GpGSqE4a_(g6u7ti|~)bt)^1^z2U9Yzj>Y=IvH_>#qT tq`c35nM7#-^X({<m zjTZox=!Hh`vL+ou2E?rML|vq()lmq zE(tUmK06B@h-$3BK_W$mofkYxfXEq9;D6{RNALsZlP-G6KWBO9|4}E+!G8`U#(GAW zIJawK;=EVRW~?cc8;#9u<6XU2Ep8nPV;#K-VX`?T*P~{vYjCLbc`*zl8!5It=Czs=@^DLX2uysnx?|3e2kfd3%>P5xWo=Yz&SbGg)*svT!3 ztT&4$Si5UGZXH!&tS?N}j>~F=F(i}}1&no!k+w^Wb=2ev)(Salk}_l~tvp>DulFMB zy)fDdImSBWS(Fw7|NmIMt9kWWUR_lx{ul6n^wIvSFyx34UdDEAtXy0HPEkQYSD- zFoKz2C}!d&qRhe;8*N9tlU!?Hs;cow2Pd80eecIXJZo=G@d_I%hoJP!T|!K-Bbd diff --git a/icechunk-python/tests/data/test-repo/snapshots/4QF8JA0YPDN51MHSSYVG b/icechunk-python/tests/data/test-repo/snapshots/4QF8JA0YPDN51MHSSYVG new file mode 100644 index 0000000000000000000000000000000000000000..14bba3bfc2a037f609999db60e1d57a438b4dcd2 GIT binary patch literal 863 zcmV-l1EBm#Lq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zeHj2&>WxVNu_hfr5dj=N@I*OH#JQbhzOqO#U3$b{SNjoMaAX|GzTFb(wb{096KO{i z<)SGKT8HXApcu7~iF9aluJ8u8ly5?d*jt+de*kmh9Jkw2U-1T{w1YHovIEFY5VE`aei4EHKD{u%lB7{SQMPvwtY~L3;j>5&uUA zS;0{Sm?7fAPn!9EVgiN$r9mV}iR9`;iWe|2W-y>2kivn+f&Zna+`#)PH@)PeyS((j zTwKh5Qx#WN#x#SgdUKu>+sSGfVIY2Sl%-{~w1(O);~BL%R~uVKZ`RJEu^ZDYTValN z-%HC#K|5806uhhYsL>=)dVdDGY;n{u_7xcS5<>j^_~6tiAfnyU_d_!D{@N&~gaI;w3Sjo7o}w{0+`daQ?;-YDiI@75 zwpVfc*t-;1-`M61_EVtYJVKP}78Xvc>z^ur=net3D;3VVv-pD2H$8J!{ns`706LMP zl)XDFWL?}=;ZCY;=vW6{0381u7A~asiK%lv%mQ>hvFJuFp-)g=0p$u=`wZW(`%6ML p0J?U=8iX7*(s|6 zCW-c)jOMA=562u--k{}QBO~OUa`40H)%N%9$%q(fH6}HD_+a9oH+NfCklT^Dy+I5P VX`jSc7(Q~9&v_xeOF_GX2>{IPM3?{o diff --git a/icechunk-python/tests/data/test-repo/snapshots/7XAF0Q905SH4938DN9CG b/icechunk-python/tests/data/test-repo/snapshots/7XAF0Q905SH4938DN9CG new file mode 100644 index 0000000000000000000000000000000000000000..d5dc1dfb067c959d4b63b25d2a32dfbab686ffc9 GIT binary patch literal 860 zcmV-i1Ec&&Lq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zWf=g*>W)e9&?X&V{xIv5dqiLIBWeuT871(ni@}dgHy$5!i0b(hk^KToq;Igzl1=1p zk;NtG%)B7`j7Qa^aYxT%7v}z>{xyz_K8CiXfdF*?egO5hRxeO=b@C;HRm$>e=IP1q zO-0qcrMkPf2+c_vn=UN7H=AFi^NV`^Yk!BPVnWOfgApJ|q5tEEqry81{2x0%NY4M! zKTH`?Bs(7-{F;LQCopdKl(0d=1Bw+jLNGwk0k8o9%hW2M1_%C=p7H_jtGx7*kM45P z|8jLQ|4RXlE3U4LX%<=a<~%93^J`ARMEvULN^{au8fv|aYpBh+TG^ZgXZ1iTyD`l& z7VfAQzBDJPXeZ6e+WN_pY);-V&pTrjXDiN+dM&VeVF%KcjJ{^w+f3Ds>h1=^n&%Z; z^#j9BcHe2m7QM1LNnsO|HqA*dGcuK8X~{FR-wF!F;urlSm@yLzu;9f>Mh>3;$17_m z44+UjF}$tQf0O?nqY}vllnxL21ekDG7{?%-haS4e|Iq)^e{CFtc>dX@x)gT94}ZHMW-O@}f-TL-JYhHdR`<@D6%&g~bCT9nEK$u#+Ev-ScQ)-R*fal6;{QiL|3CBe z+S>a45BYx@b^l`&IQYC62wTKJ5q;z*2f4{huD$e;{~+TyO87t9tQEoLRP3g;4*V-{7EZ8mDZQtc+Un5=VDgDXH+l#Mg3k(gRY>*?zRd0rNh|<4 m+bQiH+RtT?x_7rXYB%N?hr{AW3_&+!s^So@sz-qecJeCmEuugG literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/snapshots/949AXZ49X764TMDC6D4G b/icechunk-python/tests/data/test-repo/snapshots/949AXZ49X764TMDC6D4G deleted file mode 100644 index 391bc09fe71ec7954d598ca79876b7bf35971a58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 787 zcmV+u1MK`sLq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m z9Txx=>5NFglGYr$aiI)~&St-x;H?ca{iJ3O8 zuFCT7^}+Rung!@+>Ne2Y#&~&G+s19Y-60FK=i9@*2N|RKUSQf28-Y(s#Mh*YV zZ{7>Gaulxc1=*rq%G(+5r5Dm(kx8g!lZeB#*S2bJYSpPiYdW>WrY%xXaX9>Ah8H+g zOFUNB>nWb+P-dpp{dz-GApx|iU&l#~1IUwNS5x^z>7NDa^>jWK7jxTj6w+$iHAgI; zYT8vLo=;ML&cBWatu8MSDiS7ak)Ygr%@XF`TWk8WK+FgbgrY(Wt8&A$Fp+!1gfIhg zRB*MEKp4x5e7H9Zt@^^D2#*g!ocIMK4cwbF-k4bd=|)6>ByIjve(K(2OYH;yzgd?2 z7ytj*DKOaXg20wVQ{c~sroJEhll(X(+2*umFEjaFT=3}egM)xK5-Z>2*|%lerp3&d z-hA7?Q$nP`MTi5aB6gvO)F52U$PJ18W`iLXMjoUsssL7w>8IVM{k5B)Hf{6T0Yj*f zG^CJIrX-j^Do{|;!U4bnnF*-q0TSTAakN4yM+Fm2?8ME)%ZmXpubcU>gIy#i(ELZ< zxP*}H|DYAt(B=r=wVXn3+x#Km z-4FoAUTaA3usCaOC)51%aPi_Pnv+48Z9I@;0AjI$*NKbm3gYp|_PwS_ITk-Gn#0lsE^<`;VPk2-;UD?yy+1Z`vrX$_&D#nV80s1=IoilpQuM_}vu2u>X3Z&< zE2piAL2Vja-Dos0m7XeF42gh@2?M7ENmwDkU$h@p^}xlk{=wrJ{tT-|IObcQsaP+$SsD<4w!%R zf@z8@C@KL4Sj(%0IwrneVra+m48;&FOQ0p0+Ky1NV_!+Yn)gSday zT6G@H7ACAwm~yKa$e@8ok@V+OUh5;Fh* diff --git a/icechunk-python/tests/data/test-repo/snapshots/GC4YVH5SKBPEZCENYQE0 b/icechunk-python/tests/data/test-repo/snapshots/GC4YVH5SKBPEZCENYQE0 new file mode 100644 index 0000000000000000000000000000000000000000..53a1a52d0313b1401b7a4c01a6f32fb061e6cc2a GIT binary patch literal 792 zcmV+z1LyonLq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zMHc`jr-4NfuO=P9E9B$Fw`O;g38EfRBs2@m04jl8j3B13_#(1zw!5v(HETDCEx{oP zbsObXfUJv%$=~-WyE|?sJ%1u@v3_DxcK~VtZ2*mOXKdBY-hEp$Um`5b*NB<Z9!*9`8I3OYe&$z}2g%L-}#OJ?f#s7~L z3j;F5EV$~nvN2PFMNJ14C@y#iks`o^{~iyyf%8QU@{VWxQFP*mEKoh3fe2VM}k{v z#*JzdW-54EF<&bdW)U;nhGDF?SWDSyawaXmWrG1a zp^_@FLL!voBIo~YCgnrfkSN8`S_=IKy>kHnJO4laf4c6)&A+->DnwOEvh&rO!P2Yr z+J+mQCXDrcs7gs$t?z|oq~wILhA+{!h_QxRRJ~dsLoH5zY@>~)YvJWuV!ii4D<6pi zuyR-9>b0!8DHQzA@jsNI{ZC1VsX;+NN8k{!zVVNH4tmEw2l>W-PuF!2{;w`6xye|+ z`?hx>D%;?FTa?sJs$Ii>ntw={Ina@DU{k^Bx)fXf|F`~!{MSDMeznR|>CvdfDiT1XWr|^ziN;KCA`T7eqCJmp^2ztbrJi_XqRD!-8+PdfJ&vYkxChUyV zUkXQ5iM9)qC##NSej}VPg1{o_ro8(h=7$e6Ip(E@Q1)_Sj&(VzDvVS-&{7V}JUF!o zl}qY9_p0q4{RPnu;y6bw!cA~xz}Wz7zry$Jc9R4K$k^^%Bx3z+h-A1s+dvyD{Nf=S W;wQ0B-p6Qs^TUdM<_V{SA-oEbTY0$v literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/snapshots/GNFK0SSWD5B8CVA53XEG b/icechunk-python/tests/data/test-repo/snapshots/GNFK0SSWD5B8CVA53XEG deleted file mode 100644 index 93decc5f58bed160705f4ec7d083e1d5aca16fdd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 867 zcmV-p1DyOxLq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zof!a@>W)g_vNj!}LTW&r0RTM8?lPdDAcxNJGLvz*9t;$~0zh9lYS6a+)a)N?t(R?k z%Zb7O4b-?`(EKoAUus;Jy_)5ZRD?i}o7sGu0(<~=0Dl0k%+XA}*{!Lhx;2;XZtX&E zv$m*eoZTAEFFNsyV*j&0B#Xh&BcTIOj)BnsvIKfP1m+)!{Ujm&k3O=}5)n&4x5*;MRaZ~F&0BBk_44?tH)nfiZ!@IU3+wD2Yj)X;UG>D5-ex5)z2WY) z-!#qMCKfZrv{lD^N%~a}46P@2VdY$P1?$#escuwvcPQFSF$vl)6ua4t=_SF6XK%Bz zXgG0to1VoWw+cyBH(zpSXWt2ha`B61K=?3W$A+t3@Bl&a|08!VWsN3Oixe?v`b9=0 z1p+uUh>&tar^ZiBSqAMq^w35AhyIuTXJr|b;)iJ|Tv=HB#*pI7RghvakF$DW*wWj~ zl!e8Q!rajuxs`6MfX=8B?|3BhEGxhpj`~5HZ zZyI&~TN!l3DCyxdv(rNKlMgt_Nk@71(og=Alx0ctAEw<)hP_RisYi2VabC^TvsWBL zvAc=?v9r+m!4afK#tHx)98OjGWjq1ukvfpOs-8CDl=VN zSXwd_Z(pckB|?ai2}VSMnUV~1Z~_q-BeMe%fPjjc;z@{vL;{gOD5IPbGYT0FU?{0w ztL1Q5o44-?Hax1F<*|vJDG%RN+~F85n>XBLkkhpiDChwRJ6QsilSds^h0|H6hA7t>Jq?XOW$PTMBR%| zm-PFtTJ}Z!_ExYMy3|m0PO8e#gB?%+xYlVHF422#SPwnZGgKC0(MFntp8&W9%x$Fh tdA`&BmqgnD|F**vXXRgs$gn%_i(#Aa{Rf?iZ!im8Gq0GOc)U7Cj;m_VrtSa$ diff --git a/icechunk-python/tests/data/test-repo/snapshots/HNG82GMS51ECXFXFCYJG b/icechunk-python/tests/data/test-repo/snapshots/HNG82GMS51ECXFXFCYJG deleted file mode 100644 index bd82739d03887139f7d8951db0b8c096b03e55d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 871 zcmV-t1DO0tLq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zy%_)|tB*eg1eyR{3w zN$Q#_EW0(DUnJre?flRFj!Xdto(Ua*Y7B(_k0TK59WDRI&JP0O|L7klE}b03(+VE0az7toY^O5Lb0aKE*Ktqhpf6`4}@&oUmUOLG~XF2KrS*s4h z{|++s<~%93Q`MVniI~;VQ*YAJ8EU`mS<&pTZdXDdz>^**qAVFyy~MOm?KEtcv=b$5ed&GU+1Re@nAyYIB(N3ZNn z($)keO>fex_v2WgxauTJ1_LN5d4^VXLZMLnqMryk6l&OfrQ!pQ5&w@lYiA3aP%bgN zpXv7)krE8xz(7Jtj2jp}G20_#gUT`k#$sP>DaLr7msVP=&#HGe^I9!;B^M zLNKK_nP=;UD(>D1JDs4Y^d>2+Vu|Wa(yqSdy|Zao!JdsPuB(h`76bo33JLiCx1D0X zb*3}L{{j3rjk^D?3^ZJX+-UKFb3*iy6FA68CwcbLNB)D1;|TLVrdcb3y~&xUH#2Q< z9?jFMHpF1qJ;eW=o4E0jk%Q+&4+7CBzbX^pBM&QVJiz&xks`#ZIQXBDME;BZoBY}{ zk!#Xry2zE4U#MXvLWr3OMnuAll3@;T0udP_vjY-@0+B!{qnr{m3K_}hD_z^?gu*~3sBmUS~5AxFfBoCW#JHPvtb&UPa;33hLD9R9}qJ@Q1l>MsBAKHUJair2)7jvL6%T1~| z;r1fsCD6Wy7JSj)-U*gMmlmp4PgMjxuLJY~*OKPuQh!g1>$ztnfl442{YW$6B!FlE xCmN}Joo}?CCDAm%ep^Wjx&F5|^2VL@V%TbY?}KjQgD^9%@hC1Q9Mpstf=C diff --git a/icechunk-python/tests/data/test-repo/snapshots/K1BMYVG1HNVTNV1FSBH0 b/icechunk-python/tests/data/test-repo/snapshots/K1BMYVG1HNVTNV1FSBH0 deleted file mode 100644 index bfcd527f6b50ebac99cd5bc30782bc066aca903d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 577 zcmV-H0>1r8Lq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zjSv9V9&1S8vN&r}fkX%j%JW!2+7H1f1%`T{EXedH_gtCE6ILX1(n&$D%%Im?*dwES zF(O3)a1M&lG*OyEfkZxhL|ovV>bKjL*gFMP0AB!I05*S%5i$U%P`to6!Et{tked(s zuJqmPuX`vgE)LS~yPyxLAM$rSepAoQh7vR$5g;lAnEBlJK>_h-_`sPFDA4}Wrwv?S z@RUuHuBR^jw@9(&2Ls|L+MNzR%3tpV!sCW>Ic6Bk+6ou!Y>6W&?$oT%Qzn&-C5qZo z#|-Vt9Cc{U5sj&8GTE0x&6bpgJj2qZ#^t0?)25*u+RyTb{vW)AU~n6up<+PHWV`oP zF6G`k3JcIMri8JHL^9exq5G=<8zNB&0TwT6cyPriaVr1!cYXBtkA4zA&`FL1I!)2*q(70b`TC?^LJ6zE2?xh3 zMR#W`t0K9gzj63kv4ui11py}@aDt%m zZ5aRt>y1S4c9X_H*&ZGosRsk+q|q7pXYPzW2FuBA=Hs$(h-|AYk%n~9woRmMtvh+x z$A1Mzj8CAUTDjhcY4@U8;KVUHn@s_J0D1s_0NJD|)6C|{VDP1G6V_BytTBVTR)m;c z%Vl@hGO-twt0>dKT}#D#cDx734|)KgAt*A;2+<=)sPlgx6cz)p@B)5(3XcD?fYA$= z14L?y2YL7xl-N-q#)i{qW1fh>z(WOLWo=Yw@mwb#f9MB0c#p7X1b?1z=x1cG4!bYu&``mUeeHZq`V>)I@C@ z26t)Hyp*JHFDO-UgDUod0T3p_Ko^%52{m#WiltO4@*X7|oWQIYd`LoS5(VeKGih2> zb)?eD<7kTXtNZ~zq602gXq;NQoqMX%CH zvsHF~f}ixk{?!T@NM0^*MwqO4Ht>Q|RvE#dMH~22zVAEn-&mUFZF|93V-~AanKD^p zHZ`u>IBXq1kcD)>q0pHzLJ17d2UGJD`lA5^i5o|b6nU|v!GIt?NQJ=P`91%Gw+0c| z3SFpiAwr0e2}VSM8A=OtfC3R2qp|}Mpn!@P!byk-kpv=vP)0c=W)w0U5dHt%Rnw&= zCIm4O4MiQbV}WI=!Xy=ExCO!HU3VGb{(8&vMm~ nX!oj<>wjz`q206(f!zm>b?i#C5N6D4M*7T&f7SCu1-C;LX_%7v literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/snapshots/P874YS3J196959RDHX7G b/icechunk-python/tests/data/test-repo/snapshots/P874YS3J196959RDHX7G new file mode 100644 index 0000000000000000000000000000000000000000..5af21bb17de2ee0f9cfd2d888559780e4072aba3 GIT binary patch literal 172 zcmeZtcKtAad6%!#6Ya4@{d*NiJ-6R8iF2@pMA)@-@xQOiV`?B#Jpb49%>Hq)$ literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/snapshots/R7F1RJHPZ428N4AK19K0 b/icechunk-python/tests/data/test-repo/snapshots/R7F1RJHPZ428N4AK19K0 deleted file mode 100644 index 493bb36234807ae0c37188edabd877d288654fe1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 178 zcmeZtcKtAad6%MjMjwhGnx0--)FtE zUSEuOKV-Y7O(uuG^c?jXZK|xVWGQ?3w=*0dNOYQZqW6sxXpZFK)OiPmuIJp z3Yf&F?_@L=P0%l9dA~_5yhbLx$?53BY2o#I_NEGmv@kzsRLbP#p6++sRWtd~IX6uP W2H{F>7KV>p<#S$0?^4k2U;+R;xk4uZ diff --git a/icechunk-python/tests/data/test-repo/snapshots/RPA0WQCNM2N9HBBRHJQ0 b/icechunk-python/tests/data/test-repo/snapshots/RPA0WQCNM2N9HBBRHJQ0 deleted file mode 100644 index 574db73faaa944dbb300439c2f6eec11f28ed4c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 513 zcmV+c0{;C;Lq+hPr;0;JRZdH3V=XW)F)lM8ARr(hARr(hARr(hARr(C0RbqrFZ}>m zjSc{In_xu%u{LY+)_t0Wg37f&0d2Z0)o>Wb@z3h}9yirqJ}^m%ev9#St!PGAwmkmg&`MaUxbU zDOtekg(olUMZ}7<{JgmSlK;@XH{|q}A$9tcq&>0{#+*ANy@>?(H{^pQKa61_qJ@U`K|%#~hE|7R_;d z%WE(CCk#M2Ai2yKak2}8qNa`u`kW0cNM=5t9RT3?SmmCWb|CIQYOUeCkF@Z$aHduO z*~1yR3|~7nwuZ|_4>0cG#GXymH3I{G<}{4kOm@>B1Bg=(w)Hy)VP5k4Gv71wF$m z)er!7-fBqTvN&rZ9iWLFVl>9G_%Uf5*ywSD!0@Tz&?Eb**N3$_l@?^L%piDQHk0nR zH4sZIaUNDH>CoSu;&b z6Xz7m)oF2NP@BT2Y_l17lAfvB42g;+i><3UNzbw-s=4`9{wtKo`Pev7z`_L?0tVcB zYiDxrB?S#q@x^52j7GDYKZEyI39J`43_vBD13amisYCg}!Rw^KpY-eeIY@pI0Fl^y zXnx52qZ>?9bU{%IIKWm`PEEYDC3(Gmq(l4)%UC^;KbqP5O-2LFHUME1uNK~= zwMtdU9pPk;0}&%7Mm zZ65&EbFN0fT1{&}RNyQ!0>h^BiBvO&9mz?6GNkfOvH(l7l%GZ-pZU9c^kO4&ds#K$ zfSh4Q&|{{akB&CVP?iAX6AubEh9@K>1(g7s0Hy#P<>|Rq1Zgzs&g!imowKDgG-pd^ zCpPDtEf15U_NW$-bk3HDR~32H)6aUEo7IpVG9;T79I3x7zr6TYPXqL;2@gQ_HLtHR z^DWs6(D67kew%9`9Np+a<-z9&eUA4zAU+4^f2E)h1as;Q7w42fX@c*S}Fwcj2D_JhI7AtMrkv$x+*VNp&3SwA~lZI?0dxiY{sI zjp!Tit(F8UZF1ChBaJ${GdT)cBnx_%RI=c`A4n%h!}@ulrE^l!-XtQjv!2+|pc-@o zE7A-EKN^M6$-IX!dp?VqG3t+@FN2YGA9(I$S574EXP2Nh;lt8Xo1CP z42{Pr%cw9VIuMB9#N(tX%Exb@ETdRPs_013;6gzcF-l%#DaHkYw{UW_-s;tBZ8Oi_ z>V2R{hGI94zsEw(U}z$0K_LT%D$JB0Z5VzqA6XC2(M=p{u%8a#*01^-{H(GRv*llE z4UN_|iZn8;xp5{<^oW#GT*|5w@ZcDvhn$!SN? zHmyx0{{la3)%{05BFmG_mn9pRh^WT-zyWqWVAij3{#Ys0f{B$gKcED0pekwYBZ9>U zysCkOiv=dAD@JTUS%shFy7^N&KZ|v93koQP1}!QsI%1=Y(?2-_!D=I6vDK>S2$&9D zP!gUH47&W3r5UDwbt?70%<%Y${oL^LQRN7As39Rj3X&NTQqU%aHUn@15gDqY9ulB% znwY{Vh6o}eB7sOC5+{K~lN1!BU4k&L!Ws$NCRXdiOC|};oNgZ>_=q{sN zB+DI^w9C?lL}sbCfR>K_ZHeAna=@WoOdexn^EE{bIgZW=A%Am|Z|slGa^~On4&Xg@ zr95ApAc>g3m z^&S95@~TF_T1{&LoGR7IK1fwX9AwmY*X-=8nhR?gDkwte+6j8gXa4RUzu1V}UeuZ) z+RO7JUvnw3&4w}R5(&Msp){*U5rT}g6qEp!0H6S#NRuUjBoaq|fNl-H8C|-vdTU4L zbnzI?>Eh|}nR8ATh`CX#RFOtHr>n%TjQk4fZ+*-Q0tT6nAT2;>QT=7fMaIwi7z@uL zdH_K$1ACdNcLn_br>BYW&n(kGg;0%+=-~+v{W(CAI8c&dgbxHeETSx^UGO$sfzd>Q zfSVIpew%kz@SJDZHM0)d=97>(3O_6jeuCykEt5C;%#B*kTdJa1r{z3w);W5khjd4K z$sdzX$z&UPV3i)s=1 zRT8rA^JX+g=SIcwR-6Jvm%`vwY3V;5_6+nA|kxl&;y5!>dT1(l%*FaPBc8i==cDq#qq z*mA?^MoP$Epe&_cG^Ua?AZ#beAwkGTBt)2IUBN%e$_4WL4jpvdwrmCk(A}sjGwhi^~0u~ zhJ1!D=n1WK{3@nJBz34UAwtNR84^;^CWYp30udRiq8<`}V4@u26hj0N5s^S75Q&pO zqDcx0Qm_L6ti@y>;D0aH!`nv@m?*McD^!A+I9K;(7V{E`W^kmzYchAFRpuhIuJ-RD zeHZj=5*OQq*RUIryaDBIs54Twp=$SX6XeTFTCX{C8E^r%by)T;%U2SarN#nUI`Fq8 z8gI!4hZZq;X941CC3nlwKPL`(mYaVgE_9YN|NeER?r|%X`Qigf!~_m*gDCJZxW>`Q z$Zn^eZu;OU@<9Lg2*Dcj1_5inV{x&^Y6m^)-3xUtVDkYYTI(^0e}P^LkvNYd#Fas2 z334-pnO_403zqS-T&v$JWy=e2x21Sjj^hMyUT#o2{e?OR)q~u`$77biO=(0!uYGf& IRj@`?0l*~UfdBvi literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/transactions/394QWZDXAY74HP6Q8P3G b/icechunk-python/tests/data/test-repo/transactions/394QWZDXAY74HP6Q8P3G deleted file mode 100644 index 0b4e36ae78131bce64313a5d0f482c8b76b910bf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 147 zcmeZtcKtAad6%tNCAM~{K>`X6k`e6NTTi#2(YW0NR{ma9B=O+ew1{et*y!zElG$FeDpG=iw(T)TG p;nt^&olNNqo6lc6X{N7OY$969hx!o6m6yvK)G%?#i`ai#2M=G1U;mrAhOJUwq&>oZ#!^GVA=u z$bh&>hcUj diff --git a/icechunk-python/tests/data/test-repo/transactions/4QF8JA0YPDN51MHSSYVG b/icechunk-python/tests/data/test-repo/transactions/4QF8JA0YPDN51MHSSYVG new file mode 100644 index 0000000000000000000000000000000000000000..b92c29da513042b0998a756d738d6230b6e8ba1c GIT binary patch literal 146 zcmeZtcKtAad6%3IMXL^a ztO*QtcdPsG&qOzQuhOlS--#9ZmJ>wwFAw*fpIX?Vp(b=7^m*+nwPHTos_KPBh6zG^ oUJD$Jn%A#Qn9{oLy!Q?{UbaU%tPIwkR#P4>-qgK$N4cyB0I(!9zW@LL literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/transactions/7XAF0Q905SH4938DN9CG b/icechunk-python/tests/data/test-repo/transactions/7XAF0Q905SH4938DN9CG new file mode 100644 index 0000000000000000000000000000000000000000..c3df8b96cd7316e17a6ca423c3d56558c8b68e0b GIT binary patch literal 237 zcmeZtcKtAad6%yPd6ZCg3#Pf^$*x4Nrq zy=5M_J0;e}C@C=ITfN_Wv*M%p?rTv@hotq6Iwn6jc!Xi;)@vWFkL#*FzqmZuzJr7F zL&OS=8ERn)B794Q=6Ef>@O-V4q=XpHbS5zd=Oqsp-aV6W;7!2C19y+eR3F~!*DD;s itnIL8sSNYS-8WOGzYklPa9D8DyEL>z?mU{O8{g<7v2rAypi}>|H`#n-rn6?zv_l2OL3*!(&-MxYsEL7zMI9#EWEXQ=_dk literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/data/test-repo/transactions/GNFK0SSWD5B8CVA53XEG b/icechunk-python/tests/data/test-repo/transactions/GNFK0SSWD5B8CVA53XEG deleted file mode 100644 index 29b3ccfcb87cae95bc4ed88da024bc9821fe7dc7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 235 zcmeZtcKtAad6%y0aX8-v;2aLWHNQPP{ztf)gP?SYTaZ7OhwmoNB7tXMGu%@N=&x+{w z9S6=&5=i{vu~J~p%{_i@$w#d>WUp0}sCnoh1egYEM)4M-{GgMplNE3D*|#$WM)(b9C?P1ppqhH|GEV diff --git a/icechunk-python/tests/data/test-repo/transactions/K1BMYVG1HNVTNV1FSBH0 b/icechunk-python/tests/data/test-repo/transactions/K1BMYVG1HNVTNV1FSBH0 deleted file mode 100644 index 4490f2cb31409419ad10e82185bf4766dab9299d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 235 zcmeZtcKtAad6%}S^BvotmOaa^{`>lib2Ec@rqd;p{R#?!+x7bxq=5hIl)2`|) znittTRX<|ga8`UM;CPv_Hgtb@`cm_9zFMo&W}yj8 zB7EFEt{X2*_^4j`AcU*dYW-7VMaG2%nj#F6*+;}bHP%eO74nep216wKw42gDWp(%K d2kl$4cd{Up1aA+UTe?(P_P?D@VLZ|5ekjsI9+BlS8d4BDV-^TJ#qiF)Rg4J>h|6}w*K%%pE?d!hG0vcgFD<6 LgC^XUXpjT|e>Om@ diff --git a/icechunk-python/tests/data/test-repo/transactions/SNF98D1SK7NWD5KQJM20 b/icechunk-python/tests/data/test-repo/transactions/SNF98D1SK7NWD5KQJM20 deleted file mode 100644 index 271c8d02de150762851ae5032e5301a040213812..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 148 zcmeZtcKtAad6%MWutX=LXq|Bo=;Z1Ao8u|o$}Pcn>zQ70hKT?F^Lz-@tLb$ zFjnNXeNsHQp?J=st(A-2zSSGI2dHt(GCJeLI+N-D+^MWPBic-|`jdTIkEt@Gep2RN zp3`v9=KAlsj|@3GO@yxA-_~gV@5}o7LyuOt1kAp~^g(Z>e4M$@{NoQ>?shCJ)$FNf NV`8xA|9!@h5daEHL_`1p literal 0 HcmV?d00001 diff --git a/icechunk-python/tests/test_config.py b/icechunk-python/tests/test_config.py index 6fb9fccf..df509752 100644 --- a/icechunk-python/tests/test_config.py +++ b/icechunk-python/tests/test_config.py @@ -127,7 +127,7 @@ def test_virtual_chunk_containers() -> None: container = icechunk.VirtualChunkContainer("custom", "s3://", store_config) config.set_virtual_chunk_container(container) assert re.match( - r"RepositoryConfig\(inline_chunk_threshold_bytes=None, unsafe_overwrite_refs=None, get_partial_values_concurrency=None, compression=None, caching=None, storage=None, manifest=.*\)", + r"RepositoryConfig\(inline_chunk_threshold_bytes=None, get_partial_values_concurrency=None, compression=None, caching=None, storage=None, manifest=.*\)", repr(config), ) assert config.virtual_chunk_containers @@ -158,11 +158,9 @@ def test_can_change_deep_config_values() -> None: ) config = icechunk.RepositoryConfig( inline_chunk_threshold_bytes=11, - unsafe_overwrite_refs=False, compression=icechunk.CompressionConfig(level=0), ) config.inline_chunk_threshold_bytes = 5 - config.unsafe_overwrite_refs = True config.get_partial_values_concurrency = 42 config.compression = icechunk.CompressionConfig(level=8) config.compression.level = 2 @@ -180,7 +178,7 @@ def test_can_change_deep_config_values() -> None: ) assert re.match( - r"RepositoryConfig\(inline_chunk_threshold_bytes=5, unsafe_overwrite_refs=True, get_partial_values_concurrency=42, compression=CompressionConfig\(algorithm=None, level=2\), caching=CachingConfig\(num_snapshot_nodes=None, num_chunk_refs=8, num_transaction_changes=None, num_bytes_attributes=None, num_bytes_chunks=None\), storage=StorageSettings\(concurrency=StorageConcurrencySettings\(max_concurrent_requests_for_object=5, ideal_concurrent_request_size=1000000\)\), manifest=.*\)", + r"RepositoryConfig\(inline_chunk_threshold_bytes=5, get_partial_values_concurrency=42, compression=CompressionConfig\(algorithm=None, level=2\), caching=CachingConfig\(num_snapshot_nodes=None, num_chunk_refs=8, num_transaction_changes=None, num_bytes_attributes=None, num_bytes_chunks=None\), storage=StorageSettings\(concurrency=StorageConcurrencySettings\(max_concurrent_requests_for_object=5, ideal_concurrent_request_size=1000000\)\), manifest=.*\)", repr(config), ) repo = icechunk.Repository.open( diff --git a/icechunk/examples/multithreaded_store.rs b/icechunk/examples/multithreaded_store.rs index d64a7c03..1e77e9e8 100644 --- a/icechunk/examples/multithreaded_store.rs +++ b/icechunk/examples/multithreaded_store.rs @@ -13,7 +13,6 @@ async fn main() -> Result<(), Box> { let storage = new_in_memory_storage().await?; let config = RepositoryConfig { inline_chunk_threshold_bytes: Some(128), - unsafe_overwrite_refs: Some(true), ..Default::default() }; let repo = Repository::create(Some(config), storage, HashMap::new()).await?; diff --git a/icechunk/src/config.rs b/icechunk/src/config.rs index bbb9b391..a026d24b 100644 --- a/icechunk/src/config.rs +++ b/icechunk/src/config.rs @@ -205,11 +205,6 @@ impl ManifestConfig { pub struct RepositoryConfig { /// Chunks smaller than this will be stored inline in the manifst pub inline_chunk_threshold_bytes: Option, - /// Unsafely overwrite refs on write. This is not recommended, users should only use it at their - /// own risk in object stores for which we don't support write-object-if-not-exists. There is - /// the possibility of race conditions if this variable is set to true and there are concurrent - /// commit attempts. - pub unsafe_overwrite_refs: Option, /// Concurrency used by the get_partial_values operation to fetch different keys in parallel pub get_partial_values_concurrency: Option, @@ -236,9 +231,6 @@ impl RepositoryConfig { pub fn inline_chunk_threshold_bytes(&self) -> u16 { self.inline_chunk_threshold_bytes.unwrap_or(512) } - pub fn unsafe_overwrite_refs(&self) -> bool { - self.unsafe_overwrite_refs.unwrap_or(false) - } pub fn get_partial_values_concurrency(&self) -> u16 { self.get_partial_values_concurrency.unwrap_or(10) } @@ -268,9 +260,6 @@ impl RepositoryConfig { inline_chunk_threshold_bytes: other .inline_chunk_threshold_bytes .or(self.inline_chunk_threshold_bytes), - unsafe_overwrite_refs: other - .unsafe_overwrite_refs - .or(self.unsafe_overwrite_refs), get_partial_values_concurrency: other .get_partial_values_concurrency .or(self.get_partial_values_concurrency), diff --git a/icechunk/src/format/manifest.rs b/icechunk/src/format/manifest.rs index 8e92cd34..bcf665f1 100644 --- a/icechunk/src/format/manifest.rs +++ b/icechunk/src/format/manifest.rs @@ -366,7 +366,7 @@ fn ref_to_payload( fn checksum(payload: &gen::ChunkRef<'_>) -> Option { if let Some(etag) = payload.checksum_etag() { - Some(Checksum::ETag(etag.to_string())) + Some(Checksum::ETag(ETag(etag.to_string()))) } else if payload.checksum_last_modified() > 0 { Some(Checksum::LastModified(SecondsSinceEpoch(payload.checksum_last_modified()))) } else { @@ -398,7 +398,7 @@ fn mk_chunk_ref<'bldr>( Some(cs) => match cs { Checksum::LastModified(_) => None, Checksum::ETag(etag) => { - Some(builder.create_string(etag.as_str())) + Some(builder.create_string(etag.0.as_str())) } }, None => None, diff --git a/icechunk/src/ops/gc.rs b/icechunk/src/ops/gc.rs index 5af763ee..73bbc422 100644 --- a/icechunk/src/ops/gc.rs +++ b/icechunk/src/ops/gc.rs @@ -527,7 +527,7 @@ pub async fn expire( ExpireRefResult::RefIsExpired => match &r { Ref::Tag(name) => { if expired_tags == ExpiredRefAction::Delete { - delete_tag(storage, storage_settings, name.as_str(), false) + delete_tag(storage, storage_settings, name.as_str()) .await .map_err(GCError::Ref)?; result.deleted_refs.insert(r); diff --git a/icechunk/src/refs.rs b/icechunk/src/refs.rs index f6902d35..e1309374 100644 --- a/icechunk/src/refs.rs +++ b/icechunk/src/refs.rs @@ -8,7 +8,7 @@ use async_recursion::async_recursion; use bytes::Bytes; use futures::{ stream::{FuturesOrdered, FuturesUnordered}, - FutureExt, Stream, StreamExt, TryStreamExt, + FutureExt, StreamExt, }; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -19,22 +19,10 @@ use tracing::instrument; use crate::{ error::ICError, format::SnapshotId, - storage::{self, GetRefResult, StorageErrorKind, WriteRefResult}, + storage::{self, GetRefResult, StorageErrorKind, VersionInfo, WriteRefResult}, Storage, StorageError, }; -fn crock_encode_int(n: u64) -> String { - // skip the first 3 bytes (zeroes) - base32::encode(base32::Alphabet::Crockford, &n.to_be_bytes()[3..=7]) -} - -fn crock_decode_int(data: &str) -> Option { - // re insert the first 3 bytes removed during encoding - let mut bytes = vec![0, 0, 0]; - bytes.extend(base32::decode(base32::Alphabet::Crockford, data)?); - Some(u64::from_be_bytes(bytes.as_slice().try_into().ok()?)) -} - #[derive(Debug, Error)] pub enum RefErrorKind { #[error(transparent)] @@ -49,9 +37,6 @@ pub enum RefErrorKind { #[error("invalid ref name `{0}`")] InvalidRefName(String), - #[error("invalid branch version `{0}`")] - InvalidBranchVersion(String), - #[error("tag already exists, tags are immutable: `{0}`")] TagAlreadyExists(String), @@ -128,35 +113,6 @@ impl Ref { } } -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct BranchVersion(pub u64); - -impl BranchVersion { - const MAX_VERSION_NUMBER: u64 = 1099511627775; - - fn decode(version: &str) -> RefResult { - let n = crock_decode_int(version) - .ok_or(RefErrorKind::InvalidBranchVersion(version.to_string()))?; - Ok(BranchVersion(BranchVersion::MAX_VERSION_NUMBER - n)) - } - - fn encode(&self) -> String { - crock_encode_int(BranchVersion::MAX_VERSION_NUMBER - self.0) - } - - fn to_path(&self, branch_name: &str) -> RefResult { - branch_key(branch_name, self.encode().as_str()) - } - - fn initial() -> Self { - Self(0) - } - - fn inc(&self) -> Self { - Self(self.0 + 1) - } -} - #[serde_as] #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub struct RefData { @@ -164,7 +120,7 @@ pub struct RefData { pub snapshot: SnapshotId, } -const TAG_KEY_NAME: &str = "ref.json"; +const REF_KEY_NAME: &str = "ref.json"; const TAG_DELETE_MARKER_KEY_NAME: &str = "ref.json.deleted"; fn tag_key(tag_name: &str) -> RefResult { @@ -172,7 +128,7 @@ fn tag_key(tag_name: &str) -> RefResult { return Err(RefErrorKind::InvalidRefName(tag_name.to_string()).into()); } - Ok(format!("tag.{}/{}", tag_name, TAG_KEY_NAME)) + Ok(format!("tag.{}/{}", tag_name, REF_KEY_NAME)) } fn tag_delete_marker_key(tag_name: &str) -> RefResult { @@ -183,24 +139,19 @@ fn tag_delete_marker_key(tag_name: &str) -> RefResult { Ok(format!("tag.{}/{}", tag_name, TAG_DELETE_MARKER_KEY_NAME)) } -fn branch_root(branch_name: &str) -> RefResult { +fn branch_key(branch_name: &str) -> RefResult { if branch_name.contains('/') { return Err(RefErrorKind::InvalidRefName(branch_name.to_string()).into()); } - Ok(format!("branch.{}", branch_name)) + Ok(format!("branch.{}/{}", branch_name, REF_KEY_NAME)) } -fn branch_key(branch_name: &str, version_id: &str) -> RefResult { - branch_root(branch_name).map(|root| format!("{}/{}.json", root, version_id)) -} - -#[instrument(skip(storage, storage_settings, overwrite_refs))] +#[instrument(skip(storage, storage_settings))] pub async fn create_tag( storage: &(dyn Storage + Send + Sync), storage_settings: &storage::Settings, name: &str, snapshot: SnapshotId, - overwrite_refs: bool, ) -> RefResult<()> { let key = tag_key(name)?; let data = RefData { snapshot }; @@ -209,8 +160,8 @@ pub async fn create_tag( .write_ref( storage_settings, key.as_str(), - overwrite_refs, Bytes::copy_from_slice(&content), + &VersionInfo::for_creation(), ) .await { @@ -223,51 +174,47 @@ pub async fn create_tag( } #[async_recursion] -#[instrument(skip(storage, storage_settings, overwrite_refs))] +#[instrument(skip(storage, storage_settings))] pub async fn update_branch( storage: &(dyn Storage + Send + Sync), storage_settings: &storage::Settings, name: &str, new_snapshot: SnapshotId, current_snapshot: Option<&SnapshotId>, - overwrite_refs: bool, -) -> RefResult { - let last_version = last_branch_version(storage, storage_settings, name).await; - let last_ref_data = match last_version { - Ok(version) => fetch_branch(storage, storage_settings, name, &version) - .await - .map(|d| Some((version, d))), - Err(RefError { kind: RefErrorKind::RefNotFound(_), .. }) => Ok(None), - Err(err) => Err(err), - }?; - let last_snapshot = last_ref_data.as_ref().map(|d| &d.1.snapshot); - if last_snapshot != current_snapshot { +) -> RefResult<()> { + let (ref_data, version) = match fetch_branch(storage, storage_settings, name).await { + Ok((ref_data, version)) => (Some(ref_data), version), + Err(RefError { kind: RefErrorKind::RefNotFound(..), .. }) => { + (None, VersionInfo::for_creation()) + } + Err(err) => { + return Err(err); + } + }; + + if ref_data.as_ref().map(|rd| &rd.snapshot) != current_snapshot { return Err(RefErrorKind::Conflict { expected_parent: current_snapshot.cloned(), - actual_parent: last_snapshot.cloned(), + actual_parent: ref_data.map(|rd| rd.snapshot), } .into()); } - let new_version = match last_ref_data { - Some((version, _)) => version.inc(), - None => BranchVersion::initial(), - }; - let key = new_version.to_path(name)?; + let key = branch_key(name)?; let data = RefData { snapshot: new_snapshot }; let content = serde_json::to_vec(&data)?; match storage .write_ref( storage_settings, key.as_str(), - overwrite_refs, Bytes::copy_from_slice(&content), + &version, ) .await { - Ok(WriteRefResult::Written) => Ok(new_version), + Ok(WriteRefResult::Written) => Ok(()), Ok(WriteRefResult::WontOverwrite) => { - // If the branch version already exists, an update happened since we checked + // If the already exists, an update happened since we checked // we can just try again and the conflict will be reported update_branch( storage, @@ -275,7 +222,6 @@ pub async fn update_branch( name, data.snapshot, current_snapshot, - overwrite_refs, ) .await } @@ -345,56 +291,25 @@ pub async fn list_branches( Ok(branches) } -async fn branch_history<'a>( - storage: &'a (dyn Storage + Send + Sync), - storage_settings: &storage::Settings, - branch: &str, -) -> RefResult> + 'a> { - let key = branch_root(branch)?; - let all = storage.ref_versions(storage_settings, key.as_str()).await?; - Ok(all.map_err(|e| e.into()).and_then(move |version_id| async move { - let version = version_id - .strip_suffix(".json") - .ok_or(RefErrorKind::InvalidRefName(version_id.clone()))?; - BranchVersion::decode(version) - })) -} - -async fn last_branch_version( - storage: &(dyn Storage + Send + Sync), - storage_settings: &storage::Settings, - branch: &str, -) -> RefResult { - // TODO! optimize - let mut all = Box::pin(branch_history(storage, storage_settings, branch).await?); - all.try_next().await?.ok_or(RefErrorKind::RefNotFound(branch.to_string()).into()) -} - #[instrument(skip(storage, storage_settings))] pub async fn delete_branch( storage: &(dyn Storage + Send + Sync), storage_settings: &storage::Settings, branch: &str, ) -> RefResult<()> { - let key = branch_root(branch)?; - let key_ref = key.as_str(); - let refs = storage - .ref_versions(storage_settings, key_ref) - .await? - .filter_map(|v| async move { - v.ok().map(|v| format!("{}/{}", key_ref, v).as_str().to_string()) - }) - .boxed(); - storage.delete_refs(storage_settings, refs).await?; + // we make sure the branch exists + _ = fetch_branch_tip(storage, storage_settings, branch).await?; + + let key = branch_key(branch)?; + storage.delete_refs(storage_settings, futures::stream::iter([key]).boxed()).await?; Ok(()) } -#[instrument(skip(storage, storage_settings, overwrite_refs))] +#[instrument(skip(storage, storage_settings))] pub async fn delete_tag( storage: &(dyn Storage + Send + Sync), storage_settings: &storage::Settings, tag: &str, - overwrite_refs: bool, ) -> RefResult<()> { // we make sure the tag exists _ = fetch_tag(storage, storage_settings, tag).await?; @@ -405,8 +320,8 @@ pub async fn delete_tag( .write_ref( storage_settings, key.as_str(), - overwrite_refs, Bytes::from_static(&[]), + &VersionInfo::for_creation(), ) .await { @@ -429,7 +344,7 @@ pub async fn fetch_tag( let fut1: Pin>>> = async move { match storage.get_ref(storage_settings, ref_path.as_str()).await { - Ok(GetRefResult::Found { bytes }) => Ok(bytes), + Ok(GetRefResult::Found { bytes, .. }) => Ok(bytes), Ok(GetRefResult::NotFound) => { Err(RefErrorKind::RefNotFound(name.to_string()).into()) } @@ -472,11 +387,13 @@ async fn fetch_branch( storage: &(dyn Storage + Send + Sync), storage_settings: &storage::Settings, name: &str, - version: &BranchVersion, -) -> RefResult { - let path = version.to_path(name)?; - match storage.get_ref(storage_settings, path.as_str()).await { - Ok(GetRefResult::Found { bytes }) => Ok(serde_json::from_slice(bytes.as_ref())?), +) -> RefResult<(RefData, VersionInfo)> { + let ref_key = branch_key(name)?; + match storage.get_ref(storage_settings, ref_key.as_str()).await { + Ok(GetRefResult::Found { bytes, version }) => { + let data = serde_json::from_slice(bytes.as_ref())?; + Ok((data, version)) + } Ok(GetRefResult::NotFound) => { Err(RefErrorKind::RefNotFound(name.to_string()).into()) } @@ -490,30 +407,13 @@ pub async fn fetch_branch_tip( storage_settings: &storage::Settings, name: &str, ) -> RefResult { - let version = last_branch_version(storage, storage_settings, name).await?; - fetch_branch(storage, storage_settings, name, &version).await -} - -#[instrument(skip(storage, storage_settings))] -pub async fn fetch_ref( - storage: &(dyn Storage + Send + Sync), - storage_settings: &storage::Settings, - ref_name: &str, -) -> RefResult<(Ref, RefData)> { - match fetch_tag(storage, storage_settings, ref_name).await { - Ok(from_ref) => Ok((Ref::Tag(ref_name.to_string()), from_ref)), - Err(RefError { kind: RefErrorKind::RefNotFound(_), .. }) => { - let data = fetch_branch_tip(storage, storage_settings, ref_name).await?; - Ok((Ref::Branch(ref_name.to_string()), data)) - } - Err(err) => Err(err), - } + Ok(fetch_branch(storage, storage_settings, name).await?.0) } #[cfg(test)] #[allow(clippy::panic, clippy::unwrap_used, clippy::expect_used)] mod tests { - use std::{iter::once, sync::Arc}; + use std::sync::Arc; use futures::Future; use pretty_assertions::assert_eq; @@ -523,31 +423,6 @@ mod tests { use super::*; - #[tokio::test] - async fn test_branch_version_encoding() -> Result<(), Box> { - let targets = (0..10u64).chain(once(BranchVersion::MAX_VERSION_NUMBER)); - let encodings = [ - "ZZZZZZZZ", "ZZZZZZZY", "ZZZZZZZX", "ZZZZZZZW", "ZZZZZZZV", - // no U - "ZZZZZZZT", "ZZZZZZZS", "ZZZZZZZR", "ZZZZZZZQ", "ZZZZZZZP", - ]; - - for n in targets { - let encoded = BranchVersion(n).encode(); - - if n < 100 { - assert_eq!(encoded, encodings[n as usize]); - } - if n == BranchVersion::MAX_VERSION_NUMBER { - assert_eq!(encoded, "00000000"); - } - - let round = BranchVersion::decode(encoded.as_str())?; - assert_eq!(round, BranchVersion(n)); - } - Ok(()) - } - /// Execute the passed block with all test implementations of Storage. /// /// Currently this function executes against the in-memory and local filesystem object_store @@ -560,6 +435,7 @@ mod tests { mut f: F, ) -> ((Arc, R), (Arc, R, TempDir)) { let mem_storage = new_in_memory_storage().await.unwrap(); + println!("Using mem storage"); let res1 = f(Arc::clone(&mem_storage) as Arc).await; let dir = tempdir().expect("cannot create temp dir"); @@ -567,6 +443,7 @@ mod tests { .await .expect("Cannot create local Storage"); + println!("Using local file system storage"); let res2 = f(Arc::clone(&local_storage) as Arc).await; ((mem_storage, res1), (local_storage, res2, dir)) } @@ -579,28 +456,18 @@ mod tests { let s2 = SnapshotId::random(); let res = fetch_tag(storage.as_ref(), &storage_settings, "tag1").await; - assert!(matches!(res, Err(RefError{kind: RefErrorKind::RefNotFound(name),..}) if name == *"tag1")); + assert!(matches!(res, Err(RefError{kind: RefErrorKind::RefNotFound(name),..}) if name == "tag1")); assert_eq!(list_refs(storage.as_ref(), &storage_settings).await?, BTreeSet::new()); - create_tag(storage.as_ref(), &storage_settings, "tag1", s1.clone(), false).await?; - create_tag(storage.as_ref(), &storage_settings, "tag2", s2.clone(), false).await?; + create_tag(storage.as_ref(), &storage_settings, "tag1", s1.clone()).await?; + create_tag(storage.as_ref(), &storage_settings, "tag2", s2.clone()).await?; let res = fetch_tag(storage.as_ref(), &storage_settings, "tag1").await?; assert_eq!(res.snapshot, s1); - assert_eq!( - fetch_tag(storage.as_ref(), &storage_settings, "tag1").await?, - fetch_ref(storage.as_ref(), &storage_settings, "tag1").await?.1 - ); - let res = fetch_tag(storage.as_ref(), &storage_settings, "tag2").await?; assert_eq!(res.snapshot, s2); - assert_eq!( - fetch_tag(storage.as_ref(), &storage_settings, "tag2").await?, - fetch_ref(storage.as_ref(), &storage_settings, "tag2").await?.1 - ); - assert_eq!( list_refs(storage.as_ref(), &storage_settings).await?, BTreeSet::from([Ref::Tag("tag1".to_string()), Ref::Tag("tag2".to_string())]) @@ -608,8 +475,8 @@ mod tests { // attempts to recreate a tag fail assert!(matches!( - create_tag(storage.as_ref(), &storage_settings, "tag1", s1.clone(), false).await, - Err(RefError{kind: RefErrorKind::TagAlreadyExists(name), ..}) if name == *"tag1" + create_tag(storage.as_ref(), &storage_settings, "tag1", s1.clone()).await, + Err(RefError{kind: RefErrorKind::TagAlreadyExists(name), ..}) if name == "tag1" )); assert_eq!( list_refs(storage.as_ref(), &storage_settings).await?, @@ -618,7 +485,7 @@ mod tests { // attempting to create a branch that doesn't exist, with a fake parent let res = - update_branch(storage.as_ref(), &storage_settings, "branch0", s1.clone(), Some(&s2), false) + update_branch(storage.as_ref(), &storage_settings, "branch0", s1.clone(), Some(&s2)) .await; assert!(res.is_err()); assert_eq!( @@ -627,34 +494,21 @@ mod tests { ); // create a branch successfully - update_branch(storage.as_ref(), &storage_settings, "branch1", s1.clone(), None, false).await?; + update_branch(storage.as_ref(), &storage_settings, "branch1", s1.clone(), None).await?; + assert_eq!( - branch_history(storage.as_ref(), &storage_settings, "branch1") - .await? - .try_collect::>() - .await?, - vec![BranchVersion(0)] - ); - assert_eq!( - last_branch_version(storage.as_ref(), &storage_settings, "branch1").await?, - BranchVersion(0) - ); - assert_eq!( - fetch_branch(storage.as_ref(), &storage_settings, "branch1", &BranchVersion(0)).await?, + fetch_branch_tip(storage.as_ref(), &storage_settings, "branch1").await?, RefData { snapshot: s1.clone() } ); - assert_eq!( - fetch_branch(storage.as_ref(), &storage_settings, "branch1", &BranchVersion(0)).await?, - fetch_ref(storage.as_ref(), &storage_settings, "branch1").await?.1 - ); + assert_eq!( list_refs(storage.as_ref(), &storage_settings).await?, BTreeSet::from([ Ref::Branch("branch1".to_string()), - Ref::Tag("tag1".to_string()), - Ref::Tag("tag2".to_string()) + Ref::Tag("tag1".to_string()), + Ref::Tag("tag2".to_string()) ]) ); @@ -664,36 +518,18 @@ mod tests { "branch1", s2.clone(), Some(&s1.clone()), - false, ) .await?; assert_eq!( - branch_history(storage.as_ref(), &storage_settings, "branch1") - .await? - .try_collect::>() - .await?, - vec![BranchVersion(1), BranchVersion(0)] - ); - assert_eq!( - last_branch_version(storage.as_ref(), &storage_settings, "branch1").await?, - BranchVersion(1) - ); - - assert_eq!( - fetch_branch(storage.as_ref(), &storage_settings, "branch1", &BranchVersion(1)).await?, + fetch_branch_tip(storage.as_ref(), &storage_settings, "branch1").await?, RefData { snapshot: s2.clone() } ); - assert_eq!( - fetch_branch(storage.as_ref(), &storage_settings, "branch1", &BranchVersion(1)).await?, - fetch_ref(storage.as_ref(), &storage_settings, "branch1").await?.1 - ); - let sid = SnapshotId::random(); // update a branch with the wrong parent let res = - update_branch(storage.as_ref(), &storage_settings, "branch1", sid.clone(), Some(&s1), false) + update_branch(storage.as_ref(), &storage_settings, "branch1", sid.clone(), Some(&s1)) .await; assert!(matches!(res, Err(RefError{kind: RefErrorKind::Conflict { expected_parent, actual_parent }, ..}) @@ -701,35 +537,19 @@ mod tests { )); // update the branch again but now with the right parent - update_branch(storage.as_ref(), &storage_settings, "branch1", sid.clone(), Some(&s2), false) + update_branch(storage.as_ref(), &storage_settings, "branch1", sid.clone(), Some(&s2)) .await?; assert_eq!( - branch_history(storage.as_ref(), &storage_settings, "branch1") - .await? - .try_collect::>() - .await?, - vec![BranchVersion(2), BranchVersion(1), BranchVersion(0)] - ); - assert_eq!( - last_branch_version(storage.as_ref(), &storage_settings, "branch1").await?, - BranchVersion(2) + fetch_branch_tip(storage.as_ref(), &storage_settings, "branch1").await?, + RefData { snapshot: sid.clone() } ); - assert_eq!( - fetch_branch(storage.as_ref(), &storage_settings, "branch1", &BranchVersion(2)).await?, - fetch_ref(storage.as_ref(), &storage_settings, "branch1").await?.1 - ); - - assert_eq!( - fetch_ref(storage.as_ref(), &storage_settings, "branch1").await?, - (Ref::Branch("branch1".to_string()), RefData { snapshot: sid.clone() }) - ); // delete a branch delete_branch(storage.as_ref(), &storage_settings, "branch1").await?; assert!(matches!( - fetch_ref(storage.as_ref(), &storage_settings, "branch1").await, + fetch_branch_tip(storage.as_ref(), &storage_settings, "branch1").await, Err(RefError{kind: RefErrorKind::RefNotFound(name),..}) if name == "branch1" )); @@ -750,13 +570,13 @@ mod tests { let storage_settings = storage.default_settings(); let s1 = SnapshotId::random(); let s2 = SnapshotId::random(); - create_tag(storage.as_ref(), &storage_settings, "tag1", s1, false).await?; + create_tag(storage.as_ref(), &storage_settings, "tag1", s1).await?; // we can delete tags - delete_tag(storage.as_ref(), &storage_settings, "tag1", false).await?; + delete_tag(storage.as_ref(), &storage_settings, "tag1").await?; // cannot delete twice - assert!(delete_tag(storage.as_ref(), &storage_settings, "tag1", false) + assert!(delete_tag(storage.as_ref(), &storage_settings, "tag1") .await .is_err()); @@ -765,7 +585,6 @@ mod tests { storage.as_ref(), &storage_settings, "doesnt_exist", - false ) .await .is_err()); @@ -776,14 +595,13 @@ mod tests { &storage_settings, "tag1", s2.clone(), - false ) .await, Err(RefError{kind: RefErrorKind::TagAlreadyExists(name),..}) if name == "tag1"); assert!(list_tags(storage.as_ref(), &storage_settings).await?.is_empty()); // can create different tag - create_tag(storage.as_ref(), &storage_settings, "tag2", s2, false).await?; + create_tag(storage.as_ref(), &storage_settings, "tag2", s2).await?; // listing doesn't include deleted tags assert_eq!( diff --git a/icechunk/src/repository.rs b/icechunk/src/repository.rs index 86b35e9c..44ab628e 100644 --- a/icechunk/src/repository.rs +++ b/icechunk/src/repository.rs @@ -6,6 +6,7 @@ use std::{ }; use bytes::Bytes; +use err_into::ErrorInto as _; use futures::{ stream::{FuturesOrdered, FuturesUnordered}, Stream, StreamExt, TryStreamExt, @@ -28,11 +29,10 @@ use crate::{ }, refs::{ create_tag, delete_branch, delete_tag, fetch_branch_tip, fetch_tag, - list_branches, list_tags, update_branch, BranchVersion, Ref, RefError, - RefErrorKind, + list_branches, list_tags, update_branch, Ref, RefError, RefErrorKind, }, session::{Session, SessionErrorKind, SessionResult}, - storage::{self, ETag, FetchConfigResult, StorageErrorKind, UpdateConfigResult}, + storage::{self, FetchConfigResult, StorageErrorKind, UpdateConfigResult}, virtual_chunks::{ContainerName, VirtualChunkResolver}, Storage, StorageError, }; @@ -125,7 +125,7 @@ pub type RepositoryResult = Result; pub struct Repository { config: RepositoryConfig, storage_settings: storage::Settings, - config_etag: Option, + config_version: storage::VersionInfo, storage: Arc, asset_manager: Arc, virtual_resolver: Arc, @@ -147,7 +147,6 @@ impl Repository { let config = config.map(|c| RepositoryConfig::default().merge(c)).unwrap_or_default(); let compression = config.compression().level(); - let overwrite_refs = config.unsafe_overwrite_refs(); let storage_c = Arc::clone(&storage); let storage_settings = config.storage().cloned().unwrap_or_else(|| storage.default_settings()); @@ -174,7 +173,6 @@ impl Repository { Ref::DEFAULT_BRANCH, new_snapshot.id().clone(), None, - overwrite_refs, ) .await?; Ok::<(), RepositoryError>(()) @@ -187,23 +185,26 @@ impl Repository { let handle2 = tokio::spawn( async move { if has_overriden_config { - let etag = - Repository::store_config(storage_c.as_ref(), &config_c, None) - .await?; - Ok::<_, RepositoryError>(Some(etag)) + let version = Repository::store_config( + storage_c.as_ref(), + &config_c, + &storage::VersionInfo::for_creation(), + ) + .await?; + Ok::<_, RepositoryError>(version) } else { - Ok(None) + Ok(storage::VersionInfo::for_creation()) } } .in_current_span(), ); handle1.await??; - let config_etag = handle2.await??; + let config_version = handle2.await??; debug_assert!(Self::exists(storage.as_ref()).await.unwrap_or(false)); - Self::new(config, config_etag, storage, virtual_chunk_credentials) + Self::new(config, config_version, storage, virtual_chunk_credentials) } #[instrument(skip_all)] @@ -233,17 +234,22 @@ impl Repository { #[allow(clippy::expect_used)] handle2.await.expect("Error checking if repo exists")?; #[allow(clippy::expect_used)] - if let Some((default_config, config_etag)) = + if let Some((default_config, config_version)) = handle1.await.expect("Error fetching repo config")? { // Merge the given config with the defaults let config = config.map(|c| default_config.merge(c)).unwrap_or(default_config); - Self::new(config, Some(config_etag), storage, virtual_chunk_credentials) + Self::new(config, config_version, storage, virtual_chunk_credentials) } else { let config = config.unwrap_or_default(); - Self::new(config, None, storage, virtual_chunk_credentials) + Self::new( + config, + storage::VersionInfo::for_creation(), + storage, + virtual_chunk_credentials, + ) } } @@ -261,7 +267,7 @@ impl Repository { fn new( config: RepositoryConfig, - config_etag: Option, + config_version: storage::VersionInfo, storage: Arc, virtual_chunk_credentials: HashMap, ) -> RepositoryResult { @@ -282,7 +288,7 @@ impl Repository { )); Ok(Self { config, - config_etag, + config_version, storage, storage_settings, virtual_resolver, @@ -316,7 +322,7 @@ impl Repository { Self::new( config, - self.config_etag.clone(), + self.config_version.clone(), Arc::clone(&self.storage), virtual_chunk_credentials .unwrap_or_else(|| self.virtual_chunk_credentials.clone()), @@ -326,22 +332,22 @@ impl Repository { #[instrument(skip_all)] pub async fn fetch_config( storage: &(dyn Storage + Send + Sync), - ) -> RepositoryResult> { + ) -> RepositoryResult> { match storage.fetch_config(&storage.default_settings()).await? { - FetchConfigResult::Found { bytes, etag } => { + FetchConfigResult::Found { bytes, version } => { let config = serde_yaml_ng::from_slice(&bytes)?; - Ok(Some((config, etag))) + Ok(Some((config, version))) } FetchConfigResult::NotFound => Ok(None), } } #[instrument(skip_all)] - pub async fn save_config(&self) -> RepositoryResult { + pub async fn save_config(&self) -> RepositoryResult { Repository::store_config( self.storage().as_ref(), self.config(), - self.config_etag.as_ref(), + &self.config_version, ) .await } @@ -350,18 +356,14 @@ impl Repository { pub(crate) async fn store_config( storage: &(dyn Storage + Send + Sync), config: &RepositoryConfig, - config_etag: Option<&ETag>, - ) -> RepositoryResult { + previous_version: &storage::VersionInfo, + ) -> RepositoryResult { let bytes = Bytes::from(serde_yaml_ng::to_string(config)?); match storage - .update_config( - &storage.default_settings(), - bytes, - config_etag.map(|e| e.as_str()), - ) + .update_config(&storage.default_settings(), bytes, previous_version) .await? { - UpdateConfigResult::Updated { new_etag } => Ok(new_etag), + UpdateConfigResult::Updated { new_version } => Ok(new_version), UpdateConfigResult::NotOnLatestVersion => { Err(RepositoryErrorKind::ConfigWasUpdated.into()) } @@ -426,30 +428,23 @@ impl Repository { &self, branch_name: &str, snapshot_id: &SnapshotId, - ) -> RepositoryResult { + ) -> RepositoryResult<()> { // TODO: The parent snapshot should exist? - let version = match update_branch( + update_branch( self.storage.as_ref(), &self.storage_settings, branch_name, snapshot_id.clone(), None, - self.config().unsafe_overwrite_refs(), ) .await - { - Ok(branch_version) => Ok::<_, RepositoryError>(branch_version), - Err(RefError { + .map_err(|e| match e { + RefError { kind: RefErrorKind::Conflict { expected_parent, actual_parent }, .. - }) => { - Err(RepositoryErrorKind::Conflict { expected_parent, actual_parent } - .into()) - } - Err(err) => Err(err.into()), - }?; - - Ok(version) + } => RepositoryErrorKind::Conflict { expected_parent, actual_parent }.into(), + err => err.into(), + }) } /// List all branches in the repository. @@ -477,7 +472,7 @@ impl Repository { &self, branch: &str, snapshot_id: &SnapshotId, - ) -> RepositoryResult { + ) -> RepositoryResult<()> { raise_if_invalid_snapshot_id( self.storage.as_ref(), &self.storage_settings, @@ -485,17 +480,15 @@ impl Repository { ) .await?; let branch_tip = self.lookup_branch(branch).await?; - let version = update_branch( + update_branch( self.storage.as_ref(), &self.storage_settings, branch, snapshot_id.clone(), Some(&branch_tip), - self.config().unsafe_overwrite_refs(), ) - .await?; - - Ok(version) + .await + .err_into() } /// Delete a branch from the repository. @@ -516,13 +509,7 @@ impl Repository { /// chunks or snapshots associated with the tag. #[instrument(skip(self))] pub async fn delete_tag(&self, tag: &str) -> RepositoryResult<()> { - Ok(delete_tag( - self.storage.as_ref(), - &self.storage_settings, - tag, - self.config().unsafe_overwrite_refs(), - ) - .await?) + Ok(delete_tag(self.storage.as_ref(), &self.storage_settings, tag).await?) } /// Create a new tag in the repository at the given snapshot id @@ -537,7 +524,6 @@ impl Repository { &self.storage_settings, tag_name, snapshot_id.clone(), - self.config().unsafe_overwrite_refs(), ) .await?; Ok(()) @@ -851,8 +837,8 @@ mod tests { // it inits with the default config assert_eq!(repo.config(), &RepositoryConfig::default()); // updating the persistent config create a new file with default values - let etag = repo.save_config().await?; - assert_ne!(etag, ""); + let version = repo.save_config().await?; + assert_ne!(version, storage::VersionInfo::for_creation()); assert_eq!( Repository::fetch_config(storage.as_ref()).await?.unwrap().0, RepositoryConfig::default() @@ -872,8 +858,8 @@ mod tests { assert_eq!(repo.config().inline_chunk_threshold_bytes(), 42); // update the persistent config - let etag = repo.save_config().await?; - assert_ne!(etag, ""); + let version = repo.save_config().await?; + assert_ne!(version, storage::VersionInfo::for_creation()); assert_eq!( Repository::fetch_config(storage.as_ref()) .await? @@ -915,8 +901,8 @@ mod tests { assert_eq!(repo.config().caching().num_chunk_refs(), 100); // and we can save the merge - let etag = repo.save_config().await?; - assert_ne!(etag, ""); + let version = repo.save_config().await?; + assert_ne!(version, storage::VersionInfo::for_creation()); assert_eq!( &Repository::fetch_config(storage.as_ref()).await?.unwrap().0, repo.config() diff --git a/icechunk/src/session.rs b/icechunk/src/session.rs index 88cba53a..93988195 100644 --- a/icechunk/src/session.rs +++ b/icechunk/src/session.rs @@ -812,7 +812,6 @@ impl Session { let id = match current { Err(RefError { kind: RefErrorKind::RefNotFound(_), .. }) => { do_commit( - &self.config, self.storage.as_ref(), Arc::clone(&self.asset_manager), self.storage_settings.as_ref(), @@ -835,7 +834,6 @@ impl Session { .into()) } else { do_commit( - &self.config, self.storage.as_ref(), Arc::clone(&self.asset_manager), self.storage_settings.as_ref(), @@ -1624,7 +1622,6 @@ async fn flush( #[allow(clippy::too_many_arguments)] async fn do_commit( - config: &RepositoryConfig, storage: &(dyn Storage + Send + Sync), asset_manager: Arc, storage_settings: &storage::Settings, @@ -1647,7 +1644,6 @@ async fn do_commit( branch_name, new_snapshot.clone(), Some(&parent_snapshot), - config.unsafe_overwrite_refs(), ) .await { @@ -1747,7 +1743,7 @@ mod tests { detector::ConflictDetector, }, format::manifest::ManifestExtents, - refs::{fetch_ref, Ref}, + refs::{fetch_tag, Ref}, repository::VersionInfo, storage::new_in_memory_storage, strategies::{ @@ -2021,11 +2017,14 @@ mod tests { "main", snapshot.id().clone(), None, - true, ) .await?; - Repository::store_config(storage.as_ref(), &RepositoryConfig::default(), None) - .await?; + Repository::store_config( + storage.as_ref(), + &RepositoryConfig::default(), + &storage::VersionInfo::for_creation(), + ) + .await?; let repo = Repository::open(None, storage, HashMap::new()).await?; let mut ds = repo.writable_session("main").await?; @@ -2864,14 +2863,12 @@ mod tests { let new_snapshot_id = ds.commit("first commit", None).await?; assert_eq!( new_snapshot_id, - fetch_ref(storage.as_ref(), &storage_settings, "main").await?.1.snapshot + fetch_branch_tip(storage.as_ref(), &storage_settings, "main").await?.snapshot ); assert_eq!(&new_snapshot_id, ds.snapshot_id()); repo.create_tag("v1", &new_snapshot_id).await?; - let (ref_name, ref_data) = - fetch_ref(storage.as_ref(), &storage_settings, "v1").await?; - assert_eq!(ref_name, Ref::Tag("v1".to_string())); + let ref_data = fetch_tag(storage.as_ref(), &storage_settings, "v1").await?; assert_eq!(new_snapshot_id, ref_data.snapshot); assert!(matches!( @@ -2906,9 +2903,9 @@ mod tests { ) .await?; let new_snapshot_id = ds.commit("second commit", None).await?; - let (ref_name, ref_data) = - fetch_ref(storage.as_ref(), &storage_settings, Ref::DEFAULT_BRANCH).await?; - assert_eq!(ref_name, Ref::Branch("main".to_string())); + let ref_data = + fetch_branch_tip(storage.as_ref(), &storage_settings, Ref::DEFAULT_BRANCH) + .await?; assert_eq!(new_snapshot_id, ref_data.snapshot); let parents = repo @@ -3416,7 +3413,6 @@ mod tests { "main", conflicting_snap.clone(), Some(¤t_snap), - false, ) .await?; Ok(()) diff --git a/icechunk/src/storage/logging.rs b/icechunk/src/storage/logging.rs index 983629d0..89aae1f1 100644 --- a/icechunk/src/storage/logging.rs +++ b/icechunk/src/storage/logging.rs @@ -12,7 +12,7 @@ use tokio::io::AsyncRead; use super::{ FetchConfigResult, GetRefResult, ListInfo, Reader, Settings, Storage, StorageError, - StorageResult, UpdateConfigResult, WriteRefResult, + StorageResult, UpdateConfigResult, VersionInfo, WriteRefResult, }; use crate::{ format::{ChunkId, ChunkOffset, ManifestId, SnapshotId}, @@ -56,9 +56,9 @@ impl Storage for LoggingStorage { &self, settings: &Settings, config: Bytes, - etag: Option<&str>, + previous_version: &VersionInfo, ) -> StorageResult { - self.backend.update_config(settings, config, etag).await + self.backend.update_config(settings, config, previous_version).await } async fn fetch_snapshot( @@ -178,18 +178,10 @@ impl Storage for LoggingStorage { &self, settings: &Settings, ref_key: &str, - overwrite_refs: bool, bytes: Bytes, + previous_version: &VersionInfo, ) -> StorageResult { - self.backend.write_ref(settings, ref_key, overwrite_refs, bytes).await - } - - async fn ref_versions( - &self, - settings: &Settings, - ref_name: &str, - ) -> StorageResult>> { - self.backend.ref_versions(settings, ref_name).await + self.backend.write_ref(settings, ref_key, bytes, previous_version).await } async fn list_objects<'a>( diff --git a/icechunk/src/storage/mod.rs b/icechunk/src/storage/mod.rs index 1c4b3125..09cffbd7 100644 --- a/icechunk/src/storage/mod.rs +++ b/icechunk/src/storage/mod.rs @@ -104,7 +104,38 @@ const REF_PREFIX: &str = "refs"; const TRANSACTION_PREFIX: &str = "transactions/"; const CONFIG_PATH: &str = "config.yaml"; -pub type ETag = String; +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Hash, PartialOrd, Ord)] +pub struct ETag(pub String); +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Default)] +pub struct Generation(pub String); + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] +pub struct VersionInfo { + pub etag: Option, + pub generation: Option, +} + +impl VersionInfo { + pub fn for_creation() -> Self { + Self { etag: None, generation: None } + } + + pub fn from_etag_only(etag: String) -> Self { + Self { etag: Some(ETag(etag)), generation: None } + } + + pub fn is_create(&self) -> bool { + self.etag.is_none() && self.generation.is_none() + } + + pub fn etag(&self) -> Option<&String> { + self.etag.as_ref().map(|e| &e.0) + } + + pub fn generation(&self) -> Option<&String> { + self.generation.as_ref().map(|e| &e.0) + } +} #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, Default)] pub struct ConcurrencySettings { @@ -197,19 +228,19 @@ impl Reader { #[derive(Debug, Clone, PartialEq, Eq)] pub enum FetchConfigResult { - Found { bytes: Bytes, etag: ETag }, + Found { bytes: Bytes, version: VersionInfo }, NotFound, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum UpdateConfigResult { - Updated { new_etag: ETag }, + Updated { new_version: VersionInfo }, NotOnLatestVersion, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum GetRefResult { - Found { bytes: Bytes }, + Found { bytes: Bytes, version: VersionInfo }, NotFound, } @@ -235,7 +266,7 @@ pub trait Storage: fmt::Debug + private::Sealed + Sync + Send { &self, settings: &Settings, config: Bytes, - etag: Option<&str>, + previous_version: &VersionInfo, ) -> StorageResult; async fn fetch_snapshot( &self, @@ -304,17 +335,12 @@ pub trait Storage: fmt::Debug + private::Sealed + Sync + Send { ref_key: &str, ) -> StorageResult; async fn ref_names(&self, settings: &Settings) -> StorageResult>; - async fn ref_versions( - &self, - settings: &Settings, - ref_name: &str, - ) -> StorageResult>>; async fn write_ref( &self, settings: &Settings, ref_key: &str, - overwrite_refs: bool, bytes: Bytes, + previous_version: &VersionInfo, ) -> StorageResult; async fn list_objects<'a>( diff --git a/icechunk/src/storage/object_store.rs b/icechunk/src/storage/object_store.rs index e3192bdc..38ee9f41 100644 --- a/icechunk/src/storage/object_store.rs +++ b/icechunk/src/storage/object_store.rs @@ -46,10 +46,10 @@ use tokio_util::compat::FuturesAsyncReadCompatExt; use tracing::instrument; use super::{ - ConcurrencySettings, FetchConfigResult, GetRefResult, ListInfo, Reader, Settings, - Storage, StorageError, StorageErrorKind, StorageResult, UpdateConfigResult, - WriteRefResult, CHUNK_PREFIX, CONFIG_PATH, MANIFEST_PREFIX, REF_PREFIX, - SNAPSHOT_PREFIX, TRANSACTION_PREFIX, + ConcurrencySettings, ETag, FetchConfigResult, Generation, GetRefResult, ListInfo, + Reader, Settings, Storage, StorageError, StorageErrorKind, StorageResult, + UpdateConfigResult, VersionInfo, WriteRefResult, CHUNK_PREFIX, CONFIG_PATH, + MANIFEST_PREFIX, REF_PREFIX, SNAPSHOT_PREFIX, TRANSACTION_PREFIX, }; #[derive(Debug, Serialize, Deserialize)] @@ -163,6 +163,11 @@ impl ObjectStorage { self.backend.supports_metadata() } + /// We need this because object_store's local file implementation doesn't support it + pub fn supports_conditional_put_updates(&self) -> bool { + self.backend.supports_conditional_put_updates() + } + /// Return all keys in the store /// /// Intended for testing and debugging purposes only. @@ -219,27 +224,6 @@ impl ObjectStorage { ObjectPath::from(format!("{}/{}/{}", self.backend.prefix(), REF_PREFIX, ref_key)) } - async fn do_ref_versions(&self, ref_name: &str) -> BoxStream> { - let prefix = self.ref_key(ref_name); - self.get_client() - .await - .list(Some(prefix.clone()).as_ref()) - .map_err(|e| e.into()) - .and_then(move |meta| { - ready( - self.drop_prefix(&prefix, &meta.location) - .map(|path| path.to_string()) - .ok_or( - StorageErrorKind::Other( - "Bug in ref prefix logic".to_string(), - ) - .into(), - ), - ) - }) - .boxed() - } - async fn delete_batch( &self, prefix: &str, @@ -279,6 +263,27 @@ impl ObjectStorage { Attributes::new() } } + + fn get_ref_name(&self, prefix: &ObjectPath, meta: &ObjectMeta) -> Option { + let relative_key = self.drop_prefix(prefix, &meta.location)?; + let parent = relative_key.parts().next()?; + Some(parent.as_ref().to_string()) + } + + fn get_put_mode(&self, previous_version: &VersionInfo) -> PutMode { + let degrade_to_overwrite = + !previous_version.is_create() && !self.supports_conditional_put_updates(); + if degrade_to_overwrite { + PutMode::Overwrite + } else if previous_version.is_create() { + PutMode::Create + } else { + PutMode::Update(UpdateVersion { + e_tag: previous_version.etag().cloned(), + version: previous_version.generation().cloned(), + }) + } + } } impl private::Sealed for ObjectStorage {} @@ -300,12 +305,14 @@ impl Storage for ObjectStorage { let response = self.get_client().await.get(&path).await; match response { - Ok(result) => match result.meta.e_tag.clone() { - Some(etag) => { - Ok(FetchConfigResult::Found { bytes: result.bytes().await?, etag }) - } - None => Ok(FetchConfigResult::NotFound), - }, + Ok(result) => { + let version = VersionInfo { + etag: result.meta.e_tag.as_ref().cloned().map(ETag), + generation: result.meta.version.as_ref().cloned().map(Generation), + }; + + Ok(FetchConfigResult::Found { bytes: result.bytes().await?, version }) + } Err(object_store::Error::NotFound { .. }) => Ok(FetchConfigResult::NotFound), Err(err) => Err(err.into()), } @@ -315,7 +322,7 @@ impl Storage for ObjectStorage { &self, _settings: &Settings, config: Bytes, - etag: Option<&str>, + previous_version: &VersionInfo, ) -> StorageResult { let path = self.get_config_path(); let attributes = if self.supports_metadata() { @@ -327,23 +334,17 @@ impl Storage for ObjectStorage { Attributes::new() }; - let mode = if let Some(etag) = etag { - PutMode::Update(UpdateVersion { - e_tag: Some(etag.to_string()), - version: None, - }) - } else { - PutMode::Create - }; + let mode = self.get_put_mode(previous_version); let options = PutOptions { mode, attributes, ..PutOptions::default() }; let res = self.get_client().await.put_opts(&path, config.into(), options).await; match res { Ok(res) => { - let new_etag = res.e_tag.ok_or(StorageErrorKind::Other( - "Config object should have an etag".to_string(), - ))?; - Ok(UpdateConfigResult::Updated { new_etag }) + let new_version = VersionInfo { + etag: res.e_tag.map(ETag), + generation: res.version.map(Generation), + }; + Ok(UpdateConfigResult::Updated { new_version }) } Err(object_store::Error::Precondition { .. }) => { Ok(UpdateConfigResult::NotOnLatestVersion) @@ -479,7 +480,14 @@ impl Storage for ObjectStorage { ) -> StorageResult { let key = self.ref_key(ref_key); match self.get_client().await.get(&key).await { - Ok(res) => Ok(GetRefResult::Found { bytes: res.bytes().await? }), + Ok(res) => { + let etag = res.meta.e_tag.clone().map(ETag); + let generation = res.meta.version.clone().map(Generation); + Ok(GetRefResult::Found { + bytes: res.bytes().await?, + version: VersionInfo { etag, generation }, + }) + } Err(object_store::Error::NotFound { .. }) => Ok(GetRefResult::NotFound), Err(err) => Err(err.into()), } @@ -487,42 +495,15 @@ impl Storage for ObjectStorage { #[instrument(skip(self, _settings))] async fn ref_names(&self, _settings: &Settings) -> StorageResult> { - // FIXME: i don't think object_store's implementation of list_with_delimiter is any good - // we need to test if it even works beyond 1k refs let prefix = self.ref_key(""); Ok(self .get_client() .await - .list_with_delimiter(Some(prefix.clone()).as_ref()) - .await? - .common_prefixes - .iter() - .filter_map(|path| { - self.drop_prefix(&prefix, path).map(|path| path.to_string()) - }) - .collect()) - } - - #[instrument(skip(self, _settings))] - async fn ref_versions( - &self, - _settings: &Settings, - ref_name: &str, - ) -> StorageResult>> { - let res = self.do_ref_versions(ref_name).await; - if self.artificially_sort_refs_in_mem() { - #[allow(clippy::expect_used)] - // This branch is used for local tests, not in production. We don't expect the size of - // these streams to be large, so we can collect in memory and fail early if there is an - // error - let mut all = - res.try_collect::>().await.expect("Error fetching ref versions"); - all.sort(); - Ok(futures::stream::iter(all.into_iter().map(Ok)).boxed()) - } else { - Ok(res) - } + .list(Some(prefix.clone()).as_ref()) + .try_filter_map(|meta| ready(Ok(self.get_ref_name(&prefix, &meta)))) + .try_collect() + .await?) } #[instrument(skip(self, _settings, bytes))] @@ -530,11 +511,11 @@ impl Storage for ObjectStorage { &self, _settings: &Settings, ref_key: &str, - overwrite_refs: bool, bytes: Bytes, + previous_version: &VersionInfo, ) -> StorageResult { let key = self.ref_key(ref_key); - let mode = if overwrite_refs { PutMode::Overwrite } else { PutMode::Create }; + let mode = self.get_put_mode(previous_version); let opts = PutOptions { mode, ..PutOptions::default() }; match self @@ -544,7 +525,8 @@ impl Storage for ObjectStorage { .await { Ok(_) => Ok(WriteRefResult::Written), - Err(object_store::Error::AlreadyExists { .. }) => { + Err(object_store::Error::Precondition { .. }) + | Err(object_store::Error::AlreadyExists { .. }) => { Ok(WriteRefResult::WontOverwrite) } Err(err) => Err(err.into()), @@ -668,6 +650,11 @@ pub trait ObjectStoreBackend: Debug + Sync + Send { true } + /// We need this because object_store's local file implementation doesn't support it + fn supports_conditional_put_updates(&self) -> bool { + true + } + fn default_settings(&self) -> Settings { Settings::default() } @@ -734,6 +721,10 @@ impl ObjectStoreBackend for LocalFileSystemObjectStoreBackend { false } + fn supports_conditional_put_updates(&self) -> bool { + false + } + fn default_settings(&self) -> Settings { Settings { concurrency: Some(ConcurrencySettings { diff --git a/icechunk/src/storage/s3.rs b/icechunk/src/storage/s3.rs index e67fd7a3..be105a21 100644 --- a/icechunk/src/storage/s3.rs +++ b/icechunk/src/storage/s3.rs @@ -13,7 +13,6 @@ use crate::{ format::{ChunkId, ChunkOffset, FileTypeTag, ManifestId, ObjectId, SnapshotId}, private, Storage, StorageError, }; -use async_stream::try_stream; use async_trait::async_trait; use aws_config::{ meta::region::RegionProviderChain, retry::ProvideErrorKind, AppName, BehaviorVersion, @@ -41,8 +40,8 @@ use tracing::instrument; use super::{ FetchConfigResult, GetRefResult, ListInfo, Reader, Settings, StorageErrorKind, - StorageResult, UpdateConfigResult, WriteRefResult, CHUNK_PREFIX, CONFIG_PATH, - MANIFEST_PREFIX, REF_PREFIX, SNAPSHOT_PREFIX, TRANSACTION_PREFIX, + StorageResult, UpdateConfigResult, VersionInfo, WriteRefResult, CHUNK_PREFIX, + CONFIG_PATH, MANIFEST_PREFIX, REF_PREFIX, SNAPSHOT_PREFIX, TRANSACTION_PREFIX, }; #[derive(Debug, Serialize, Deserialize)] @@ -248,6 +247,14 @@ impl S3Storage { Ok(res.deleted().len()) } + + fn get_ref_name<'a>(&self, key: Option<&'a str>) -> Option<&'a str> { + let key = key?; + let prefix = self.ref_key("").ok()?; + let relative_key = key.strip_prefix(&prefix)?; + let ref_name = relative_key.split('/').next()?; + Some(ref_name) + } } pub fn range_to_header(range: &Range) -> String { @@ -278,7 +285,7 @@ impl Storage for S3Storage { Ok(output) => match output.e_tag { Some(etag) => Ok(FetchConfigResult::Found { bytes: output.body.collect().await?.into_bytes(), - etag, + version: VersionInfo::from_etag_only(etag), }), None => Ok(FetchConfigResult::NotFound), }, @@ -294,7 +301,7 @@ impl Storage for S3Storage { &self, _settings: &Settings, config: Bytes, - etag: Option<&str>, + previous_version: &VersionInfo, ) -> StorageResult { let key = self.get_config_path()?; let mut req = self @@ -306,7 +313,7 @@ impl Storage for S3Storage { .content_type("application/yaml") .body(config.into()); - if let Some(etag) = etag { + if let Some(etag) = previous_version.etag() { req = req.if_match(etag) } else { req = req.if_none_match("*") @@ -322,7 +329,8 @@ impl Storage for S3Storage { "Config object should have an etag".to_string(), ))? .to_string(); - Ok(UpdateConfigResult::Updated { new_etag }) + let new_version = VersionInfo::from_etag_only(new_etag); + Ok(UpdateConfigResult::Updated { new_version }) } // minio returns this Err(SdkError::ServiceError(err)) => { @@ -471,7 +479,11 @@ impl Storage for S3Storage { match res { Ok(res) => { let bytes = res.body.collect().await?.into_bytes(); - Ok(GetRefResult::Found { bytes }) + if let Some(version) = res.e_tag.map(VersionInfo::from_etag_only) { + Ok(GetRefResult::Found { bytes, version }) + } else { + Ok(GetRefResult::NotFound) + } } Err(err) if err @@ -494,21 +506,16 @@ impl Storage for S3Storage { .list_objects_v2() .bucket(self.bucket.clone()) .prefix(prefix.clone()) - .delimiter("/") .into_paginator() .send(); let mut res = Vec::new(); while let Some(page) = paginator.try_next().await? { - for common_prefix in page.common_prefixes() { - if let Some(key) = common_prefix - .prefix() - .as_ref() - .and_then(|key| key.strip_prefix(prefix.as_str())) - .and_then(|key| key.strip_suffix('/')) - { - res.push(key.to_string()); + for obj in page.contents.unwrap_or_else(Vec::new) { + let name = self.get_ref_name(obj.key()); + if let Some(name) = name { + res.push(name.to_string()); } } } @@ -516,42 +523,13 @@ impl Storage for S3Storage { Ok(res) } - #[instrument(skip(self, _settings))] - async fn ref_versions( - &self, - _settings: &Settings, - ref_name: &str, - ) -> StorageResult>> { - let prefix = self.ref_key(ref_name)?; - let mut paginator = self - .get_client() - .await - .list_objects_v2() - .bucket(self.bucket.clone()) - .prefix(prefix.clone()) - .into_paginator() - .send(); - - let prefix = prefix + "/"; - let stream = try_stream! { - while let Some(page) = paginator.try_next().await? { - for object in page.contents() { - if let Some(key) = object.key.as_ref().and_then(|key| key.strip_prefix(prefix.as_str())) { - yield key.to_string() - } - } - } - }; - Ok(stream.boxed()) - } - #[instrument(skip(self, _settings, bytes))] async fn write_ref( &self, _settings: &Settings, ref_key: &str, - overwrite_refs: bool, bytes: Bytes, + previous_version: &VersionInfo, ) -> StorageResult { let key = self.ref_key(ref_key)?; let mut builder = self @@ -561,8 +539,10 @@ impl Storage for S3Storage { .bucket(self.bucket.clone()) .key(key.clone()); - if !overwrite_refs { - builder = builder.if_none_match("*") + if let Some(etag) = previous_version.etag() { + builder = builder.if_match(etag); + } else { + builder = builder.if_none_match("*"); } let res = builder.body(bytes.into()).send().await; diff --git a/icechunk/src/virtual_chunks.rs b/icechunk/src/virtual_chunks.rs index ae8bc70a..465ea42b 100644 --- a/icechunk/src/virtual_chunks.rs +++ b/icechunk/src/virtual_chunks.rs @@ -325,7 +325,7 @@ impl S3Fetcher { ) } Some(Checksum::ETag(etag)) => { - b = b.if_match(etag); + b = b.if_match(&etag.0); } None => {} }; @@ -459,7 +459,7 @@ impl ChunkFetcher for LocalFSFetcher { .expect("Bad last modified field in virtual chunk reference"); options.if_unmodified_since = Some(d); } - Some(Checksum::ETag(etag)) => options.if_match = Some(etag.clone()), + Some(Checksum::ETag(etag)) => options.if_match = Some(etag.0.clone()), None => {} } diff --git a/icechunk/tests/test_gc.rs b/icechunk/tests/test_gc.rs index f64d08fc..59183e55 100644 --- a/icechunk/tests/test_gc.rs +++ b/icechunk/tests/test_gc.rs @@ -58,7 +58,6 @@ pub async fn test_gc() -> Result<(), Box> { let repo = Repository::create( Some(RepositoryConfig { inline_chunk_threshold_bytes: Some(0), - unsafe_overwrite_refs: Some(true), ..Default::default() }), Arc::clone(&storage), @@ -127,7 +126,6 @@ pub async fn test_gc() -> Result<(), Box> { "main", first_snap_id, Some(&second_snap_id), - false, ) .await?; diff --git a/icechunk/tests/test_storage.rs b/icechunk/tests/test_storage.rs index 802a7bad..43765fa7 100644 --- a/icechunk/tests/test_storage.rs +++ b/icechunk/tests/test_storage.rs @@ -8,18 +8,20 @@ use bytes::Bytes; use icechunk::{ config::{S3Credentials, S3Options, S3StaticCredentials}, format::{ChunkId, ManifestId, SnapshotId}, + new_local_filesystem_storage, refs::{ create_tag, fetch_branch_tip, fetch_tag, list_refs, update_branch, Ref, RefError, RefErrorKind, }, storage::{ new_in_memory_storage, new_s3_storage, FetchConfigResult, StorageResult, - UpdateConfigResult, + UpdateConfigResult, VersionInfo, }, ObjectStorage, Storage, }; use object_store::azure::AzureConfigKey; use pretty_assertions::{assert_eq, assert_ne}; +use tempfile::tempdir; use tokio::io::AsyncReadExt; #[allow(clippy::expect_used)] @@ -89,9 +91,10 @@ async fn mk_azure_blob_storage( Ok(storage) } +#[allow(clippy::expect_used)] async fn with_storage(f: F) -> Result<(), Box> where - F: Fn(Arc) -> Fut, + F: Fn(&'static str, Arc) -> Fut, Fut: Future>>, { let prefix = format!("{:?}", ChunkId::random()); @@ -100,16 +103,27 @@ where let s2 = new_in_memory_storage().await.unwrap(); let s3 = mk_s3_object_store_storage(format!("{prefix}2").as_str()).await?; let s4 = mk_azure_blob_storage(prefix.as_str()).await?; - f(s1).await?; - f(s2).await?; - f(s3).await?; - f(s4).await?; + let dir = tempdir().expect("cannot create temp dir"); + let s5 = new_local_filesystem_storage(dir.path()) + .await + .expect("Cannot create local Storage"); + + println!("Using in memory storage"); + f("in_memory", s2).await?; + println!("Using local filesystem storage"); + f("local_filesystem", s5).await?; + println!("Using s3 native storage"); + f("s3_native", s1).await?; + println!("Using s3 object_store storage"); + f("s3_object_store", s3).await?; + println!("Using azure_blob storage"); + f("azure_blob", s4).await?; Ok(()) } #[tokio::test] pub async fn test_snapshot_write_read() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); let bytes: [u8; 1024] = core::array::from_fn(|_| rand::random()); @@ -133,7 +147,7 @@ pub async fn test_snapshot_write_read() -> Result<(), Box #[tokio::test] pub async fn test_manifest_write_read() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = ManifestId::random(); let bytes: [u8; 1024] = core::array::from_fn(|_| rand::random()); @@ -165,7 +179,7 @@ pub async fn test_manifest_write_read() -> Result<(), Box #[tokio::test] pub async fn test_chunk_write_read() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = ChunkId::random(); let bytes = Bytes::from_static(b"hello"); @@ -181,11 +195,10 @@ pub async fn test_chunk_write_read() -> Result<(), Box> { #[tokio::test] pub async fn test_tag_write_get() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); - create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone(), false) - .await?; + create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone()).await?; let back = fetch_tag(storage.as_ref(), &storage_settings, "mytag").await?; assert_eq!(id, back.snapshot); Ok(()) @@ -196,10 +209,10 @@ pub async fn test_tag_write_get() -> Result<(), Box> { #[tokio::test] pub async fn test_fetch_non_existing_tag() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); - create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone(), false) + create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone()) .await?; let back = @@ -213,14 +226,14 @@ pub async fn test_fetch_non_existing_tag() -> Result<(), Box Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); - create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone(), false) + create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone()) .await?; let res = - create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone(), false) + create_tag(storage.as_ref(), &storage_settings, "mytag", id.clone()) .await; assert!(matches!(res, Err(RefError{kind: RefErrorKind::TagAlreadyExists(r), ..}) if r == "mytag")); Ok(()) @@ -231,20 +244,18 @@ pub async fn test_create_existing_tag() -> Result<(), Box #[tokio::test] pub async fn test_branch_initialization() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); - let res = update_branch( + update_branch( storage.as_ref(), &storage_settings, "some-branch", id.clone(), None, - false, ) .await?; - assert_eq!(res.0, 0); let res = fetch_branch_tip(storage.as_ref(), &storage_settings, "some-branch").await?; @@ -258,7 +269,7 @@ pub async fn test_branch_initialization() -> Result<(), Box Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id = SnapshotId::random(); update_branch( @@ -267,7 +278,6 @@ pub async fn test_fetch_non_existing_branch() -> Result<(), Box Result<(), Box Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id1 = SnapshotId::random(); let id2 = SnapshotId::random(); let id3 = SnapshotId::random(); - let res = update_branch( + update_branch( storage.as_ref(), &storage_settings, "some-branch", id1.clone(), None, - false, ) .await?; - assert_eq!(res.0, 0); - let res = update_branch( + update_branch( storage.as_ref(), &storage_settings, "some-branch", id2.clone(), Some(&id1), - false, ) .await?; - assert_eq!(res.0, 1); - let res = update_branch( + update_branch( storage.as_ref(), &storage_settings, "some-branch", id3.clone(), Some(&id2), - false, ) .await?; - assert_eq!(res.0, 2); let res = fetch_branch_tip(storage.as_ref(), &storage_settings, "some-branch").await?; @@ -336,56 +340,27 @@ pub async fn test_branch_update() -> Result<(), Box> { #[tokio::test] pub async fn test_ref_names() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let id1 = SnapshotId::random(); let id2 = SnapshotId::random(); - update_branch( - storage.as_ref(), - &storage_settings, - "main", - id1.clone(), - None, - false, - ) - .await?; + update_branch(storage.as_ref(), &storage_settings, "main", id1.clone(), None) + .await?; update_branch( storage.as_ref(), &storage_settings, "main", id2.clone(), Some(&id1), - false, ) .await?; - update_branch( - storage.as_ref(), - &storage_settings, - "foo", - id1.clone(), - None, - false, - ) - .await?; - update_branch( - storage.as_ref(), - &storage_settings, - "bar", - id1.clone(), - None, - false, - ) - .await?; - create_tag(storage.as_ref(), &storage_settings, "my-tag", id1.clone(), false) + update_branch(storage.as_ref(), &storage_settings, "foo", id1.clone(), None) + .await?; + update_branch(storage.as_ref(), &storage_settings, "bar", id1.clone(), None) + .await?; + create_tag(storage.as_ref(), &storage_settings, "my-tag", id1.clone()).await?; + create_tag(storage.as_ref(), &storage_settings, "my-other-tag", id1.clone()) .await?; - create_tag( - storage.as_ref(), - &storage_settings, - "my-other-tag", - id1.clone(), - false, - ) - .await?; let res: HashSet<_> = HashSet::from_iter(list_refs(storage.as_ref(), &storage_settings).await?); @@ -408,17 +383,17 @@ pub async fn test_ref_names() -> Result<(), Box> { #[tokio::test] #[allow(clippy::panic)] pub async fn test_write_config_on_empty() -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); let config = Bytes::copy_from_slice(b"hello"); - let etag = match storage.update_config(&storage_settings, config.clone(), None).await? { - UpdateConfigResult::Updated { new_etag } => new_etag, + let version = match storage.update_config(&storage_settings, config.clone(), &VersionInfo::for_creation()).await? { + UpdateConfigResult::Updated { new_version } => new_version, UpdateConfigResult::NotOnLatestVersion => panic!(), }; - assert_ne!(etag, ""); + assert_ne!(version, VersionInfo::for_creation()); let res = storage.fetch_config(&storage_settings, ).await?; assert!( - matches!(res, FetchConfigResult::Found{bytes, etag: actual_etag} if actual_etag == etag && bytes == config ) + matches!(res, FetchConfigResult::Found{bytes, version: actual_version} if actual_version == version && bytes == config ) ); Ok(()) }).await?; @@ -428,21 +403,21 @@ pub async fn test_write_config_on_empty() -> Result<(), Box Result<(), Box> { - with_storage(|storage| async move { + with_storage(|_, storage| async move { let storage_settings = storage.default_settings(); - let first_etag = match storage.update_config(&storage_settings, Bytes::copy_from_slice(b"hello"), None).await? { - UpdateConfigResult::Updated { new_etag } => new_etag, + let first_version = match storage.update_config(&storage_settings, Bytes::copy_from_slice(b"hello"), &VersionInfo::for_creation()).await? { + UpdateConfigResult::Updated { new_version } => new_version, _ => panic!(), }; let config = Bytes::copy_from_slice(b"bye"); - let second_etag = match storage.update_config(&storage_settings, config.clone(), Some(first_etag.as_str())).await? { - UpdateConfigResult::Updated { new_etag } => new_etag, + let second_version = match storage.update_config(&storage_settings, config.clone(), &first_version).await? { + UpdateConfigResult::Updated { new_version } => new_version, _ => panic!(), }; - assert_ne!(second_etag, first_etag); + assert_ne!(second_version, first_version); let res = storage.fetch_config(&storage_settings, ).await?; assert!( - matches!(res, FetchConfigResult::Found{bytes, etag: actual_etag} if actual_etag == second_etag && bytes == config ) + matches!(res, FetchConfigResult::Found{bytes, version: actual_version} if actual_version == second_version && bytes == config ) ); Ok(()) }).await?; @@ -450,48 +425,63 @@ pub async fn test_write_config_on_existing() -> Result<(), Box Result<(), Box> { // FIXME: this test fails in MiniIO but seems to work on S3 #[allow(clippy::unwrap_used)] let storage = new_in_memory_storage().await.unwrap(); let storage_settings = storage.default_settings(); - let etag = storage + let version = storage .update_config( &storage_settings, Bytes::copy_from_slice(b"hello"), - Some("00000000000000000000000000000000"), + &VersionInfo::from_etag_only("00000000000000000000000000000000".to_string()), ) .await; - assert!(matches!(etag, Ok(UpdateConfigResult::NotOnLatestVersion))); + assert!(matches!(version, Ok(UpdateConfigResult::NotOnLatestVersion))); Ok(()) } #[tokio::test] #[allow(clippy::panic)] -pub async fn test_write_config_fails_on_bad_etag_when_existing( +pub async fn test_write_config_fails_on_bad_version_when_existing( ) -> Result<(), Box> { - with_storage(|storage| async move { + with_storage(|storage_type, storage| async move { let storage_settings = storage.default_settings(); let config = Bytes::copy_from_slice(b"hello"); - let etag = match storage.update_config(&storage_settings, config.clone(), None).await? { - UpdateConfigResult::Updated { new_etag } => new_etag, + let version = match storage.update_config(&storage_settings, config.clone(), &VersionInfo::for_creation()).await? { + UpdateConfigResult::Updated { new_version } => new_version, _ => panic!(), }; - let res = storage + let update_res = storage .update_config(&storage_settings, Bytes::copy_from_slice(b"bye"), - Some("00000000000000000000000000000000"), + &VersionInfo::from_etag_only("00000000000000000000000000000000".to_string()), ) .await?; - assert!(matches!(res, UpdateConfigResult::NotOnLatestVersion)); - - let res = storage.fetch_config(&storage_settings, ).await?; - assert!( - matches!(res, FetchConfigResult::Found{bytes, etag: actual_etag} if actual_etag == etag && bytes == config ) - ); - Ok(()) + if storage_type == "local_filesystem" { + // FIXME: local file system doesn't have conditional updates yet + assert!(matches!(update_res, UpdateConfigResult::Updated{..})); + + } else { + assert!(matches!(update_res, UpdateConfigResult::NotOnLatestVersion)); + } + + let fetch_res = storage.fetch_config(&storage_settings, ).await?; + if storage_type == "local_filesystem" { + // FIXME: local file system doesn't have conditional updates yet + assert!( + matches!(fetch_res, FetchConfigResult::Found{bytes, version: actual_version} + if actual_version != version && bytes == Bytes::copy_from_slice(b"bye")) + ); + } else { + assert!( + matches!(fetch_res, FetchConfigResult::Found{bytes, version: actual_version} + if actual_version == version && bytes == config ) + ); + } + Ok(()) }).await?; Ok(()) } diff --git a/icechunk/tests/test_virtual_refs.rs b/icechunk/tests/test_virtual_refs.rs index 03cfd665..9ab5153f 100644 --- a/icechunk/tests/test_virtual_refs.rs +++ b/icechunk/tests/test_virtual_refs.rs @@ -15,7 +15,7 @@ mod tests { repository::VersionInfo, session::{get_chunk, SessionErrorKind}, storage::{ - self, new_s3_storage, s3::mk_client, ConcurrencySettings, ObjectStorage, + self, new_s3_storage, s3::mk_client, ConcurrencySettings, ETag, ObjectStorage, }, store::{StoreError, StoreErrorKind}, virtual_chunks::VirtualChunkContainer, @@ -711,7 +711,7 @@ mod tests { ))?, offset: 1, length: 5, - checksum: Some(Checksum::ETag(String::from("invalid etag"))), + checksum: Some(Checksum::ETag(ETag(String::from("invalid etag")))), }; store.set_virtual_ref("array/c/0/0/2", ref1, false).await?; @@ -733,7 +733,7 @@ mod tests { )?, offset: 22306, length: 288, - checksum: Some(Checksum::ETag(String::from("invalid etag"))), + checksum: Some(Checksum::ETag(ETag(String::from("invalid etag")))), }; store.set_virtual_ref("array/c/1/1/1", public_ref, false).await?;