Fix tgi inference error

Signed-off-by: Dolpher Du <dolpher.du@intel.com>
opea-project · Nov 8, 2024 · ab88eee · ab88eee
1 parent cf372ec
commit ab88eee
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 0 deletions.
diff --git a/helm-charts/agentqna/gaudi-values.yaml b/helm-charts/agentqna/gaudi-values.yaml
@@ -16,6 +16,7 @@ tgi:
   MAX_TOTAL_TOKENS: "8192"
   CUDA_GRAPHS: ""
   OMPI_MCA_btl_vader_single_copy_mechanism: "none"
+  PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
   ENABLE_HPU_GRAPH: "true"
   LIMIT_HPU_GRAPH: "true"
   USE_FLASH_ATTENTION: "true"

diff --git a/helm-charts/common/agent/gaudi-values.yaml b/helm-charts/common/agent/gaudi-values.yaml
@@ -17,6 +17,7 @@ tgi:
   MAX_TOTAL_TOKENS: "8192"
   CUDA_GRAPHS: ""
   OMPI_MCA_btl_vader_single_copy_mechanism: "none"
+  PT_HPU_ENABLE_LAZY_COLLECTIVES: "true"
   ENABLE_HPU_GRAPH: "true"
   LIMIT_HPU_GRAPH: "true"
   USE_FLASH_ATTENTION: "true"

diff --git a/helm-charts/common/tgi/templates/configmap.yaml b/helm-charts/common/tgi/templates/configmap.yaml
@@ -39,6 +39,9 @@ data:
   {{- if .Values.OMPI_MCA_btl_vader_single_copy_mechanism }}
   OMPI_MCA_btl_vader_single_copy_mechanism: {{ .Values.OMPI_MCA_btl_vader_single_copy_mechanism | quote }}
   {{- end }}
+  {{- if .Values.PT_HPU_ENABLE_LAZY_COLLECTIVES }}
+  PT_HPU_ENABLE_LAZY_COLLECTIVES: {{ .Values.PT_HPU_ENABLE_LAZY_COLLECTIVES | quote }}
+  {{- end }}
   {{- if .Values.ENABLE_HPU_GRAPH }}
   ENABLE_HPU_GRAPH: {{ .Values.ENABLE_HPU_GRAPH | quote }}
   {{- end }}