Enable max token setting for llama.cpp

TNG · Feb 27, 2025 · 44850df · 44850df
1 parent 9815347
commit 44850df
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/LlamaCPP/llama_adapter.py b/LlamaCPP/llama_adapter.py
@@ -148,7 +148,7 @@ def text_conversation_run(
                 )
 
             full_prompt = convert_prompt(prompt)
-            stream = self.llm_interface.create_chat_completion(full_prompt)
+            stream = self.llm_interface.create_chat_completion(full_prompt, params.max_tokens)
             self.stream_function(stream)	
 
         except Exception as ex:

diff --git a/LlamaCPP/llama_cpp_backend.py b/LlamaCPP/llama_cpp_backend.py
@@ -34,9 +34,10 @@ def load_model(self, params: LLMParams, n_gpu_layers: int = -1, context_length:
             if callback is not None:
                 callback("finish")
 
-    def create_chat_completion(self, messages: List[Dict[str, str]]):
+    def create_chat_completion(self, messages: List[Dict[str, str]], max_tokens: int = 1024):
         completion: Iterator[CreateChatCompletionStreamResponse] = self._model.create_chat_completion(
             messages=messages,
+            max_tokens=max_tokens,
             stream=True,
         )
         return completion

diff --git a/WebUI/src/components/SettingsBasic.vue b/WebUI/src/components/SettingsBasic.vue
@@ -105,7 +105,7 @@
           ></button>
         </div>
       </div>
-      <div v-if="textInference.backend !== 'llamaCPP'" class="flex flex-col gap-2">
+      <div class="flex flex-col gap-2">
         <p>Max Tokens</p>
         <slide-bar
           v-model:current="textInference.maxTokens"