Merge pull request #279 from weaviate/add-support-for-multi2vec-voyageai-module

Add support for multi2vec-voyageai module

Author: antas-marcin

.cicd/test.sh

@@ -158,13 +158,17 @@ function check_creates_template() {
   check_modules "--set modules.generative-anyscale.enabled=true" "value: generative-anyscale"
   check_modules "--set modules.generative-anyscale.enabled=true --set modules.generative-anyscale.apiKey=apiKey" "value: generative-anyscale"
   check_modules "--set modules.text2vec-voyageai.enabled=true" "value: text2vec-voyageai"
+  check_modules "--set modules.multi2vec-voyageai.enabled=true" "value: multi2vec-voyageai"
   check_modules "--set modules.generative-mistral.enabled=true" "value: generative-mistral"
   check_modules "--set modules.generative-mistral.enabled=true --set modules.generative-mistral.apiKey=apiKey" "value: generative-mistral"
   check_modules "--set modules.multi2vec-palm.enabled=true" "value: multi2vec-palm"
   check_modules "--set modules.multi2vec-palm.enabled=true --set modules.multi2vec-palm.apiKey=apiKey --set modules.generative-palm.enabled=true --set modules.generative-palm.apiKey=apiKey" "value: generative-palm,multi2vec-palm"
   check_modules "--set modules.reranker-voyageai.enabled=true" "value: reranker-voyageai"
   check_modules "--set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: reranker-voyageai"
+  check_modules "--set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey" "value: multi2vec-voyageai"
   check_modules "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: text2vec-voyageai,reranker-voyageai"
+  check_modules "--set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey" "value: multi2vec-voyageai,reranker-voyageai"
+  check_modules "--set modules.text2vec-voyageai.enabled=true --set modules.text2vec-voyageai.apiKey=apiKey --set modules.reranker-voyageai.enabled=true --set modules.reranker-voyageai.apiKey=apiKey --set modules.multi2vec-voyageai.enabled=true --set modules.multi2vec-voyageai.apiKey=apiKey" "value: multi2vec-voyageai,text2vec-voyageai,reranker-voyageai"
   check_modules "--set modules.text2vec-ollama.enabled=true" "value: text2vec-ollama"
   check_modules "--set modules.generative-ollama.enabled=true" "value: generative-ollama"
   check_modules "--set modules.generative-ollama.enabled=true --set modules.text2vec-ollama.enabled=true" "value: generative-ollama,text2vec-ollama"

weaviate/templates/_helpers.tpl

@@ -86,6 +86,9 @@
   {{- if (index .Values "modules" "multi2vec-jinaai" "enabled") -}}
     {{ $modules = append $modules "multi2vec-jinaai" }}
   {{- end -}}
+  {{- if (index .Values "modules" "multi2vec-voyageai" "enabled") -}}
+    {{ $modules = append $modules "multi2vec-voyageai" }}
+  {{- end -}}
   {{- if (index .Values "modules" "text2vec-openai" "enabled") -}}
     {{ $modules = append $modules "text2vec-openai" }}
   {{- end -}}

weaviate/templates/apiKeyVoyageAISecret.yaml

@@ -1,6 +1,7 @@
 {{-  $t2vVoyageAI := and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey") -}}
 {{-  $rerankerVoyageAI := and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey") }}
-{{-  $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) }}
+{{-  $multi2vecVoyageAI := and (index .Values "modules" "multi2vec-voyageai" "enabled") (index .Values "modules" "multi2vec-voyageai" "apiKey") }}
+{{-  $voyageai := or ($t2vVoyageAI) ($rerankerVoyageAI) ($multi2vecVoyageAI) }}
 {{- if $voyageai }}
 
 {{- $apiKeys := list }}
@@ -10,6 +11,9 @@
 {{- if $rerankerVoyageAI }}
   {{- $apiKeys = append $apiKeys (index .Values "modules" "reranker-voyageai" "apiKey") -}}
 {{- end }}
+{{- if $multi2vecVoyageAI }}
+  {{- $apiKeys = append $apiKeys (index .Values "modules" "multi2vec-voyageai" "apiKey") -}}
+{{- end }}
 
 {{- if gt (len ($apiKeys | uniq)) 1 -}}
   {{- fail "VoyageAI modules activated, but their keys differ. Use the same key on all activated VoyageAI modules." }}
@@ -26,6 +30,8 @@ type: Opaque
 data:
   {{- if (index .Values "modules" "reranker-voyageai" "apiKey") }}
   apiKey: {{ index .Values "modules" "reranker-voyageai" "apiKey" | b64enc }}
+  {{- else if (index .Values "modules" "multi2vec-voyageai" "apiKey") }}
+  apiKey: {{ index .Values "modules" "multi2vec-voyageai" "apiKey" | b64enc }}
   {{- else }}
   apiKey: {{ index .Values "modules" "text2vec-voyageai" "apiKey" | b64enc }}
   {{- end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -246,7 +246,7 @@ spec:
                 name: weaviate-jinaai
                 key: apiKey
           {{- end  }}
-          {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) }}
+          {{- if or (and (index .Values "modules" "text2vec-voyageai" "enabled") (index .Values "modules" "text2vec-voyageai" "apiKey")) (and (index .Values "modules" "reranker-voyageai" "enabled") (index .Values "modules" "reranker-voyageai" "apiKey")) (and (index .Values "modules" "multi2vec-voyageai" "enabled") (index .Values "modules" "multi2vec-voyageai" "apiKey")) }}
           - name: VOYAGEAI_APIKEY
             valueFrom:
               secretKeyRef:

weaviate/values.yaml

@@ -1274,6 +1274,19 @@ modules:
     # an environment variable
     apiKey: ''
 
+  # The multi2vec-voyageai module uses VoaygeAI Multi Modal Embeddings API
+  # to dynamically compute vector embeddings.
+  # More information about VoaygeAI Embeddings API can be found here:
+  # https://www.voyageai.com/
+  multi2vec-voyageai:
+
+    # enable if you want to use VoaygeAI module
+    enabled: false
+
+    # Set your VoaygeAI API Key to be passed to Weaviate pod as
+    # an environment variable
+    apiKey: ''
+
   # The qna-transformers module uses neural networks, such as BERT,
   # DistilBERT, to find an answer in text to a given question
   qna-transformers: