From fa2017755948e5e7723b6390e9b930ad47df01da Mon Sep 17 00:00:00 2001 From: Nikos Papailiou Date: Tue, 1 Oct 2024 11:43:44 +0300 Subject: [PATCH 1/3] Disable consolidate_update_fragments for TileDB Cloud URIs --- apis/python/src/tiledb/vector_search/index.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apis/python/src/tiledb/vector_search/index.py b/apis/python/src/tiledb/vector_search/index.py index e99765b0b..d731657a1 100644 --- a/apis/python/src/tiledb/vector_search/index.py +++ b/apis/python/src/tiledb/vector_search/index.py @@ -800,6 +800,10 @@ def _set_has_updates(self, has_updates: bool = True): self.group = tiledb.Group(self.uri, "r", ctx=tiledb.Ctx(self.config)) def _consolidate_update_fragments(self): + # Disable update fragment consolidation for TileDB Cloud URIs + # as this is not supported. + if self.uri.startswith("tiledb://"): + return with tiledb.scope_ctx(ctx_or_config=self.config): fragments_info = tiledb.array_fragments(self.updates_array_uri) count_fragments = 0 From 6b10b548095fec4190394859f32a1c802d9e5cdb Mon Sep 17 00:00:00 2001 From: Nikos Papailiou Date: Wed, 2 Oct 2024 10:49:17 +0300 Subject: [PATCH 2/3] Add warning --- apis/python/src/tiledb/vector_search/index.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apis/python/src/tiledb/vector_search/index.py b/apis/python/src/tiledb/vector_search/index.py index d731657a1..c36982726 100644 --- a/apis/python/src/tiledb/vector_search/index.py +++ b/apis/python/src/tiledb/vector_search/index.py @@ -2,6 +2,7 @@ import json import os import time +import warnings from abc import ABCMeta from abc import abstractmethod from typing import Any, Mapping, Optional @@ -803,6 +804,13 @@ def _consolidate_update_fragments(self): # Disable update fragment consolidation for TileDB Cloud URIs # as this is not supported. if self.uri.startswith("tiledb://"): + warnings.warn( + "Update fragment consolidation is not supported for `tiledb://` URIs." + "Executing multiple updates without consolidating the update fragments can" + "result in poor search performance. Please make sure that you periodically" + "execute `_consolidate_update_fragments` using the storage filesystem URI.", + stacklevel=2, + ) return with tiledb.scope_ctx(ctx_or_config=self.config): fragments_info = tiledb.array_fragments(self.updates_array_uri) From e1cf44a5f8a9859175cef5671dda2c699c100662 Mon Sep 17 00:00:00 2001 From: Nikos Papailiou Date: Wed, 2 Oct 2024 10:50:18 +0300 Subject: [PATCH 3/3] Fix --- apis/python/src/tiledb/vector_search/index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apis/python/src/tiledb/vector_search/index.py b/apis/python/src/tiledb/vector_search/index.py index c36982726..a4495fb1b 100644 --- a/apis/python/src/tiledb/vector_search/index.py +++ b/apis/python/src/tiledb/vector_search/index.py @@ -805,9 +805,9 @@ def _consolidate_update_fragments(self): # as this is not supported. if self.uri.startswith("tiledb://"): warnings.warn( - "Update fragment consolidation is not supported for `tiledb://` URIs." - "Executing multiple updates without consolidating the update fragments can" - "result in poor search performance. Please make sure that you periodically" + "Update fragment consolidation is not supported for `tiledb://` URIs. " + "Executing multiple updates without consolidating the update fragments can " + "result in poor search performance. Please make sure that you periodically " "execute `_consolidate_update_fragments` using the storage filesystem URI.", stacklevel=2, )