issue #57: update to support ai-cfia/llamaindex-db#13

ai-cfia · Apr 29, 2024 · aaf81fe · aaf81fe
1 parent a9b1f43
commit aaf81fe
Show file tree

Hide file tree

Showing 7 changed files with 11 additions and 31 deletions.
diff --git a/.env.template b/.env.template
@@ -79,7 +79,7 @@ FINESSE_BACKEND_AZURE_SEARCH_PARAMS={"highlight_fields": "content", "highlight_p
 FINESSE_BACKEND_LLAMAINDEX_EMBED_MODEL_PARAMS='{"api_key": "<openai_api_key>", "api_version": "2023-07-01-preview", "azure_endpoint": "<azure_endpoint>", "deployment_name": "ada", "model": "text-embedding-ada-002"}'
 
 # Parameters for the vector store configuration. Contact your cloud admin for secrets.
-FINESSE_BACKEND_LLAMAINDEX_VECTOR_STORE_PARAMS='{"database": "llamaindex_db_legacy", "embed_dim": 1536, "host": "<postgres_host>", "password": "<postgres_password>", "port": "5432", "user": "<postgres_user>"}'
+FINESSE_BACKEND_LLAMAINDEX_VECTOR_STORE_PARAMS='{"database": "llamaindex_db_legacy", "schema_name": "v_0_0_1", "embed_dim": 1536, "host": "<postgres_host>", "password": "<postgres_password>", "port": "5432", "user": "<postgres_user>"}'
 
 # Paths for transforming search results from the vector store.
 FINESSE_BACKEND_LLAMAINDEX_TRANS_PATHS='{"chunk_id": "node/metadata/chunk_id", "content": "node/text", "id": "node/metadata/id", "last_updated": "node/metadata/last_updated", "llamaindex_id": "node/id_", "llamaindex_score": "score", "score": "node/metadata/score", "subtitle": "node/metadata/subtitle", "title": "node/metadata/title", "tokens_count": "node/metadata/tokens_count", "url": "node/metadata/url"}'
diff --git a/app/blueprints/search.py b/app/blueprints/search.py
@@ -65,10 +65,8 @@ def get_non_empty_query():
 @search_blueprint.route("/azure", methods=["POST"])
 def search_azure():
     config: Config = current_app.config
-    skip = request.args.get(
-        "skip", default=config["DEFAULT_AZURE_SEARCH_SKIP"], type=int
-    )
-    top = request.args.get("top", default=config["DEFAULT_AZURE_SEARCH_TOP"], type=int)
+    skip = request.args.get("skip", config["DEFAULT_AZURE_SEARCH_SKIP"], int)
+    top = request.args.get("top", config["DEFAULT_AZURE_SEARCH_TOP"], int)
     query = get_non_empty_query()
     search_params = {**config["AZURE_SEARCH_PARAMS"], "skip": skip, "top": top}
     client = config["AZURE_SEARCH_CLIENT"]
@@ -96,15 +94,9 @@ def search_ailab_db():
 @search_blueprint.route("/llamaindex", methods=["POST"])
 def search_ailab_llamaindex():
     config: Config = current_app.config
-    top = request.args.get(
-        "top", default=config["DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP"], type=int
-    )
+    top = request.args.get("top", config["DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP"], int)
     query = get_non_empty_query()
     index = config["AILAB_LLAMAINDEX_SEARCH_INDEX"]
-    search_params = {
-        **config["AILAB_LLAMAINDEX_SEARCH_PARAMS"],
-        "similarity_top_k": top,
-    }
     trans_paths = config["AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS"]
-    results = llamaindex_search(query, index, search_params, trans_paths)
+    results = llamaindex_search(query, index, top, trans_paths)
     return jsonify(results)
diff --git a/app/config.py b/app/config.py
@@ -20,7 +20,6 @@ class Config(TypedDict):
     AZURE_SEARCH_PARAMS: dict
     AZURE_SEARCH_TRANSFORM_MAP: dict
     AILAB_LLAMAINDEX_SEARCH_INDEX: VectorStoreIndex
-    AILAB_LLAMAINDEX_SEARCH_PARAMS: dict
     AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS: dict
     DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP: int
     FINESSE_DATA_URL: str
@@ -56,10 +55,8 @@ def create_config() -> Config:
     vector_store_params = json.loads(
         os.getenv("FINESSE_BACKEND_LLAMAINDEX_VECTOR_STORE_PARAMS", "{}")
     )
-    ailab_llamaindex_index = create_index_object(embed_model_params, vector_store_params)
-    ailab_llamaindex_search_params = (
-        json.loads(os.getenv("FINESSE_BACKEND_AZURE_SEARCH_PARAMS", "{}"))
-        or constants.DEFAULT_AILAB_LLAMAINDEX_SEARCH_PARAMS
+    ailab_llamaindex_index = create_index_object(
+        embed_model_params, vector_store_params
     )
     ailab_llamaindex_trans_paths = (
         json.loads(os.getenv("FINESSE_BACKEND_LLAMAINDEX_TRANS_PATHS", "{}"))
@@ -73,7 +70,6 @@ def create_config() -> Config:
         "AZURE_SEARCH_PARAMS": azure_search_params,
         "AZURE_SEARCH_TRANSFORM_MAP": azure_search_transform_map,
         "AILAB_LLAMAINDEX_SEARCH_INDEX": ailab_llamaindex_index,
-        "AILAB_LLAMAINDEX_SEARCH_PARAMS": ailab_llamaindex_search_params,
         "AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS": ailab_llamaindex_trans_paths,
         "DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP": constants.DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP,
         "FINESSE_DATA_URL": os.getenv("FINESSE_BACKEND_STATIC_FILE_URL"),

diff --git a/app/constants.py b/app/constants.py
@@ -65,6 +65,3 @@
     "llamaindex_score": "score",
     "content": "node/text",
 }
-
-# Default Ailab LlamaIndex parameters
-DEFAULT_AILAB_LLAMAINDEX_SEARCH_PARAMS = {}
diff --git a/requirements-production.txt b/requirements-production.txt
@@ -1,4 +1,4 @@
-git+https://github.com/ai-cfia/llamaindex-db@8-rename-ailab-llama-search-to-ailab-llamaindex-search#subdirectory=ailab-llamaindex-search
+git+https://github.com/ai-cfia/llamaindex-db@12-duplicated-url-results-with-ai-lab-llamaindex-search-in-llamaindex-db#subdirectory=ailab-llamaindex-search
 azure-search-documents==11.4.0
 flask==3.0.2
 flask-cors==4.0.0

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,4 @@
-# todo: back to main
-git+https://github.com/ai-cfia/llamaindex-db@8-rename-ailab-llama-search-to-ailab-llamaindex-search#subdirectory=ailab-llamaindex-search
+git+https://github.com/ai-cfia/llamaindex-db@12-duplicated-url-results-with-ai-lab-llamaindex-search-in-llamaindex-db#subdirectory=ailab-llamaindex-search
 azure-search-documents
 flask
 flask-cors

diff --git a/tests/test_ailab_llamaindex_search.py b/tests/test_ailab_llamaindex_search.py
@@ -17,7 +17,6 @@ def setUp(self):
             "ERROR_UNEXPECTED": constants.DEFAULT_ERROR_UNEXPECTED,
             "ERROR_EMPTY_QUERY": constants.DEFAULT_ERROR_EMPTY_QUERY,
             "AILAB_LLAMAINDEX_SEARCH_INDEX": Mock(),
-            "AILAB_LLAMAINDEX_SEARCH_PARAMS": constants.DEFAULT_AILAB_LLAMAINDEX_SEARCH_PARAMS,
             "AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS": constants.DEFAULT_AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS,
             "DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP": constants.DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP,
         }
@@ -37,7 +36,7 @@ def test_search_llamaindex_success(self):
             mock_search.assert_called_with(
                 "some query",
                 self.config["AILAB_LLAMAINDEX_SEARCH_INDEX"],
-                {"similarity_top_k": 5, **self.config["AILAB_LLAMAINDEX_SEARCH_PARAMS"]},
+                5,
                 self.config["AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS"],
             )
 
@@ -53,10 +52,7 @@ def test_search_llamaindex_success_with_defaults(self):
             mock_search.assert_called_with(
                 "some query",
                 self.config["AILAB_LLAMAINDEX_SEARCH_INDEX"],
-                {
-                    "similarity_top_k": self.config["DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP"],
-                    **self.config["AILAB_LLAMAINDEX_SEARCH_PARAMS"],
-                },
+                self.config["DEFAULT_AILAB_LLAMAINDEX_SEARCH_TOP"],
                 self.config["AILAB_LLAMAINDEX_SEARCH_TRANS_PATHS"],
             )