From 4cf1e0aef954b0ac5c33e4f023834bd36737784c Mon Sep 17 00:00:00 2001 From: Markus Schuettler Date: Sun, 23 Feb 2025 22:34:20 +0100 Subject: [PATCH] refactor: clean up llm logic in frontend code --- WebUI/src/App.vue | 2 +- WebUI/src/assets/js/store/backendServices.ts | 38 +++- WebUI/src/assets/js/store/globalSetup.ts | 56 ------ WebUI/src/assets/js/store/imageGeneration.ts | 10 +- WebUI/src/assets/js/store/models.ts | 133 +++++++----- WebUI/src/assets/js/store/stableDiffusion.ts | 2 +- WebUI/src/assets/js/store/textInference.ts | 126 ++++++++---- WebUI/src/components/AddLLMDialog.vue | 46 +---- .../src/components/InstallationManagement.vue | 2 + WebUI/src/components/ModelDropDownItem.vue | 8 +- WebUI/src/components/SettingsBasic.vue | 28 +-- WebUI/src/components/SettingsModel.vue | 2 +- WebUI/src/env.d.ts | 4 +- WebUI/src/views/Answer.vue | 189 ++++-------------- WebUI/src/views/Create.vue | 8 +- WebUI/src/views/Enhance.vue | 11 +- 16 files changed, 278 insertions(+), 387 deletions(-) diff --git a/WebUI/src/App.vue b/WebUI/src/App.vue index fc2bb2e7..6abccca3 100644 --- a/WebUI/src/App.vue +++ b/WebUI/src/App.vue @@ -369,7 +369,7 @@ function switchTab(index: number) { } watch(textInference, (newSetting, _oldSetting) => { - if (newSetting.backend === 'LLAMA.CPP') { + if (newSetting.backend === 'llamaCPP') { answer.value!.disableRag() } else { answer.value!.restoreRagState() diff --git a/WebUI/src/assets/js/store/backendServices.ts b/WebUI/src/assets/js/store/backendServices.ts index f922288d..ee68be8e 100644 --- a/WebUI/src/assets/js/store/backendServices.ts +++ b/WebUI/src/assets/js/store/backendServices.ts @@ -1,15 +1,15 @@ import { defineStore } from 'pinia' +const backends = ['ai-backend', 'comfyui-backend', 'llamacpp-backend', 'openvino-backend'] as const +export type BackendServiceName = (typeof backends)[number] + export const useBackendServices = defineStore( 'backendServices', () => { const currentServiceInfo = ref([]) - const serviceListeners: Map = new Map([ - ['ai-backend', new BackendServiceSetupProgressListener('ai-backend')], - ['comfyui-backend', new BackendServiceSetupProgressListener('comfyui-backend')], - ['llamacpp-backend', new BackendServiceSetupProgressListener('llamacpp-backend')], - ['openvino-backend', new BackendServiceSetupProgressListener('openvino-backend')], - ]) + const serviceListeners = new Map( + backends.map((b) => [b, new BackendServiceSetupProgressListener(b)]), + ) window.electronAPI .getServices() @@ -95,12 +95,38 @@ export const useBackendServices = defineStore( return window.electronAPI.sendStopSignal(serviceName) } + const lastUsedBackend = ref(null) + + function updateLastUsedBackend(currentInferenceBackend: BackendServiceName) { + lastUsedBackend.value = currentInferenceBackend + } + + async function resetLastUsedInferenceBackend(currentInferenceBackend: BackendServiceName) { + const lastUsedBackendSnapshot = lastUsedBackend.value + if (lastUsedBackendSnapshot === null || lastUsedBackendSnapshot === currentInferenceBackend) { + return + } + try { + const stopStatus = await stopService(lastUsedBackendSnapshot) + console.info(`unused service ${lastUsedBackendSnapshot} now in state ${stopStatus}`) + const startStatus = await startService(lastUsedBackendSnapshot) + console.info(`service ${lastUsedBackendSnapshot} now in state ${startStatus}`) + } catch (e) { + console.warn( + `Could not reset last used inference backend ${lastUsedBackendSnapshot} due to ${e}`, + ) + } + } + return { info: currentServiceInfo, serviceInfoUpdateReceived: serviceInfoUpdatePresent, allRequiredSetUp, allRequiredRunning, initalStartupRequestComplete, + lastUsedBackend, + updateLastUsedBackend, + resetLastUsedInferenceBackend, startAllSetUpServices, setUpService, startService, diff --git a/WebUI/src/assets/js/store/globalSetup.ts b/WebUI/src/assets/js/store/globalSetup.ts index 1cc525a1..ec2bcef2 100644 --- a/WebUI/src/assets/js/store/globalSetup.ts +++ b/WebUI/src/assets/js/store/globalSetup.ts @@ -1,10 +1,8 @@ import { defineStore } from 'pinia' import * as util from '../util' import { useI18N } from './i18n' -import { useBackendServices } from './backendServices' type GlobalSetupState = 'running' | 'verifyBackend' | 'manageInstallations' | 'loading' | 'failed' -type LastUsedBackend = BackendServiceName | 'None' export const useGlobalSetup = defineStore('globalSetup', () => { const state = reactive({ @@ -14,7 +12,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { }) const defaultBackendBaseUrl = ref('http://127.0.0.1:9999') - const lastUsedBackend = ref('None') const models = ref({ llm: new Array(), @@ -31,9 +28,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { resolution: 0, quality: 0, enableRag: false, - llm_model: 'microsoft/Phi-3-mini-4k-instruct', - ggufLLM_model: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q4_K_S.gguf', - openvinoLLM_model: 'OpenVINO/TinyLlama-1.1B-Chat-v1.0-int4-ov', sd_model: 'Lykon/dreamshaper-8', inpaint_model: 'Lykon/dreamshaper-8-inpainting', negativePrompt: 'bad hands, nsfw', @@ -74,8 +68,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { const errorMessage = ref('') const hdPersistentConfirmation = ref(localStorage.getItem('HdPersistentConfirmation') === 'true') - const backendServices = useBackendServices() - watchEffect(() => { localStorage.setItem('HdPersistentConfirmation', hdPersistentConfirmation.value.toString()) }) @@ -222,27 +214,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { assertSelectExist() } - function updateLastUsedBackend(currentInferenceBackend: BackendServiceName) { - lastUsedBackend.value = currentInferenceBackend - } - - async function resetLastUsedInferenceBackend(currentInferenceBackend: BackendServiceName) { - const lastUsedBackendSnapshot = lastUsedBackend.value - if (lastUsedBackendSnapshot === 'None' || lastUsedBackendSnapshot === currentInferenceBackend) { - return - } - try { - const stopStatus = await backendServices.stopService(lastUsedBackendSnapshot) - console.info(`unused service ${lastUsedBackendSnapshot} now in state ${stopStatus}`) - const startStatus = await backendServices.startService(lastUsedBackendSnapshot) - console.info(`service ${lastUsedBackendSnapshot} now in state ${startStatus}`) - } catch (e) { - console.warn( - `Could not reset last used inference backend ${lastUsedBackendSnapshot} due to ${e}`, - ) - } - } - function assertSelectExist() { let changeUserSetup = false if (models.value.llm.length > 0 && !models.value.llm.includes(modelSettings.llm_model)) { @@ -288,28 +259,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { } } - async function checkModelAlreadyLoaded(params: CheckModelAlreadyLoadedParameters[]) { - const response = await fetch(`${defaultBackendBaseUrl.value}/api/checkModelAlreadyLoaded`, { - method: 'POST', - body: JSON.stringify({ data: params }), - headers: { - 'Content-Type': 'application/json', - }, - }) - const parsedResponse = (await response.json()) as ApiResponse & { - data: CheckModelAlreadyLoadedResult[] - } - return parsedResponse.data - } - - async function checkIfHuggingFaceUrlExists(repo_id: string) { - const response = await fetch( - `${defaultBackendBaseUrl.value}/api/checkHFRepoExists?repo_id=${repo_id}`, - ) - const data = await response.json() - return data.exists - } - return { state, modelSettings, @@ -319,11 +268,8 @@ export const useGlobalSetup = defineStore('globalSetup', () => { apiHost: defaultBackendBaseUrl, graphicsList, loadingState, - lastUsedBackend, errorMessage, hdPersistentConfirmation, - updateLastUsedBackend, - resetLastUsedInferenceBackend, initSetup, applyPathsSettings, applyModelSettings, @@ -331,8 +277,6 @@ export const useGlobalSetup = defineStore('globalSetup', () => { refreshSDModles, refreshInpaintModles, refreshLora, - checkModelAlreadyLoaded: checkModelAlreadyLoaded, - checkIfHuggingFaceUrlExists, applyPresetModelSettings, restorePathsSettings, } diff --git a/WebUI/src/assets/js/store/imageGeneration.ts b/WebUI/src/assets/js/store/imageGeneration.ts index d6ae35e8..797b703f 100644 --- a/WebUI/src/assets/js/store/imageGeneration.ts +++ b/WebUI/src/assets/js/store/imageGeneration.ts @@ -4,8 +4,8 @@ import { useComfyUi } from './comfyUi' import { useStableDiffusion } from './stableDiffusion' import { useI18N } from './i18n' import * as Const from '../const' -import { useGlobalSetup } from './globalSetup' import * as toast from '@/assets/js/toast.ts' +import { useModels } from './models' export type StableDiffusionSettings = { resolution: 'standard' | 'hd' | 'manual' // ~ modelSettings.resolution 0, 1, 3 @@ -386,7 +386,7 @@ export const useImageGeneration = defineStore( const comfyUi = useComfyUi() const stableDiffusion = useStableDiffusion() - const globalSetup = useGlobalSetup() + const models = useModels() const i18nState = useI18N().state const hdWarningDismissed = ref(false) @@ -690,12 +690,12 @@ export const useImageGeneration = defineStore( const checkList: CheckModelAlreadyLoadedParameters[] = workflow.comfyUIRequirements.requiredModels.map(extractDownloadModelParamsFromString) const checkedModels: CheckModelAlreadyLoadedResult[] = - await globalSetup.checkModelAlreadyLoaded(checkList) + await models.checkModelAlreadyLoaded(checkList) const modelsToBeLoaded = checkedModels.filter( (checkModelExistsResult) => !checkModelExistsResult.already_loaded, ) for (const item of modelsToBeLoaded) { - if (!(await globalSetup.checkIfHuggingFaceUrlExists(item.repo_id))) { + if (!(await models.checkIfHuggingFaceUrlExists(item.repo_id))) { toast.error(`declared model ${item.repo_id} does not exist. Aborting Generation.`) return [] } @@ -723,7 +723,7 @@ export const useImageGeneration = defineStore( }) } - const result = await globalSetup.checkModelAlreadyLoaded(checkList) + const result = await models.checkModelAlreadyLoaded(checkList) return result.filter((checkModelExistsResult) => !checkModelExistsResult.already_loaded) } diff --git a/WebUI/src/assets/js/store/models.ts b/WebUI/src/assets/js/store/models.ts index 1e1a8f42..bcae70ec 100644 --- a/WebUI/src/assets/js/store/models.ts +++ b/WebUI/src/assets/js/store/models.ts @@ -1,69 +1,64 @@ import { acceptHMRUpdate, defineStore } from 'pinia' +import { type LlmBackend } from './textInference' +import { useBackendServices } from './backendServices' export type ModelType = - | 'llm' - | 'embedding' - | 'stableDiffusion' - | 'inpaint' - | 'lora' - | 'vae' - | 'undefined' - | 'ggufLLM' - | 'openvinoLLM' +| 'embedding' +| 'stableDiffusion' +| 'inpaint' +| 'lora' +| 'vae' +| 'undefined' +| LlmBackend export type Model = { name: string downloaded: boolean type: ModelType + default: boolean } -const predefinedModels: Model[] = [ - { name: 'Qwen/Qwen2-1.5B-Instruct', type: 'llm', downloaded: false }, - { name: 'microsoft/Phi-3-mini-4k-instruct', type: 'llm', downloaded: false }, - // { name: 'meta-llama/Meta-Llama-3.1-8B-Instruct', type: 'llm', downloaded: false }, - { name: 'mistralai/Mistral-7B-Instruct-v0.3', type: 'llm', downloaded: false }, - // { name: 'google/gemma-7b', type: 'llm', downloaded: false }, - // { name: 'THUDM/chatglm3-6b', type: 'llm', downloaded: false }, +const predefinedModels: Omit[] = [ + { name: 'Qwen/Qwen2-1.5B-Instruct', type: 'ipexLLM', default: false }, + { name: 'microsoft/Phi-3-mini-4k-instruct', type: 'ipexLLM', default: true }, + { name: 'mistralai/Mistral-7B-Instruct-v0.3', type: 'ipexLLM', default: false }, + { name: 'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B', type: 'ipexLLM', default: false }, + { name: 'deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B', type: 'ipexLLM', default: false }, { name: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q4_K_S.gguf', - type: 'ggufLLM', - downloaded: false, + type: 'llamaCPP', + default: true, }, { name: 'bartowski/Llama-3.2-3B-Instruct-GGUF/Llama-3.2-3B-Instruct-Q8_0.gguf', - type: 'ggufLLM', - downloaded: false, + type: 'llamaCPP', + default: false, }, { name: 'bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/Meta-Llama-3.1-8B-Instruct-Q5_K_S.gguf', - type: 'ggufLLM', - downloaded: false, + type: 'llamaCPP', + default: false, }, { name: 'HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/smollm2-1.7b-instruct-q4_k_m.gguf', - type: 'ggufLLM', - downloaded: false, + type: 'llamaCPP', + default: false, }, - { name: 'OpenVINO/Phi-3-medium-4k-instruct-int4-ov', type: 'openvinoLLM', downloaded: false }, - { name: 'OpenVINO/mixtral-8x7b-instruct-v0.1-int4-ov', type: 'openvinoLLM', downloaded: false }, - { name: 'OpenVINO/Mistral-7B-Instruct-v0.2-fp16-ov', type: 'openvinoLLM', downloaded: false }, - { name: 'OpenVINO/TinyLlama-1.1B-Chat-v1.0-int4-ov', type: 'openvinoLLM', downloaded: false }, - { name: 'OpenVINO/Phi-3.5-mini-instruct-fp16-ov', type: 'openvinoLLM', downloaded: false }, + { name: 'OpenVINO/Phi-3-medium-4k-instruct-int4-ov', type: 'openVINO', default: false }, + { name: 'OpenVINO/mixtral-8x7b-instruct-v0.1-int4-ov', type: 'openVINO', default: false }, + { name: 'OpenVINO/Mistral-7B-Instruct-v0.2-fp16-ov', type: 'openVINO', default: false }, + { name: 'OpenVINO/TinyLlama-1.1B-Chat-v1.0-int4-ov', type: 'openVINO', default: true }, + { name: 'OpenVINO/Phi-3.5-mini-instruct-fp16-ov', type: 'openVINO', default: false }, ] -export const userModels: Model[] = [] - export const useModels = defineStore( 'models', () => { const hfToken = ref(undefined) - const models = ref(predefinedModels) - const llms = computed(() => models.value.filter((m) => m.type === 'llm')) + const models = ref([]) + const backendServices = useBackendServices() const downloadList = ref([]) - const ggufLLMs = computed(() => models.value.filter((m) => m.type === 'ggufLLM')) - - const openVINOLLMModels = computed(() => models.value.filter((m) => m.type === 'openvinoLLM')) async function refreshModels() { const sdModels = await window.electronAPI.getDownloadedDiffusionModels() @@ -75,42 +70,78 @@ export const useModels = defineStore( const embeddingModels = await window.electronAPI.getDownloadedEmbeddingModels() const downloadedModels = [ - ...sdModels.map((name) => ({ name, type: 'stableDiffusion', downloaded: true })), - ...llmModels.map((name) => ({ name, type: 'llm', downloaded: true })), - ...ggufModels.map((name) => ({ name, type: 'ggufLLM', downloaded: true })), - ...openVINOLLMModels.map((name) => ({ + ...sdModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'stableDiffusion' })), + ...llmModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'ipexLLM' })), + ...ggufModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'llamaCPP' })), + ...openVINOLLMModels.map<{name: string, type: ModelType}>((name) => ({ name, - type: 'openvinoLLM', - downloaded: true, + type: 'openVINO', })), - ...loraModels.map((name) => ({ name, type: 'lora', downloaded: true })), - ...inpaintModels.map((name) => ({ name, type: 'inpaint', downloaded: true })), - ...embeddingModels.map((name) => ({ name, type: 'embedding', downloaded: true })), + ...loraModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'lora' })), + ...inpaintModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'inpaint' })), + ...embeddingModels.map<{name: string, type: ModelType}>((name) => ({ name, type: 'embedding' })), ] - const notYetDownloaded = (model: Model) => + const notYetDownloaded = (model: { name: string }) => !downloadedModels.map((m) => m.name).includes(model.name) + const notPredefined = (model: { name: string }) => + !predefinedModels.map((m) => m.name).includes(model.name) models.value = [ ...downloadedModels, - ...userModels.filter(notYetDownloaded), ...predefinedModels.filter(notYetDownloaded), - ] + ...models.value.filter(notPredefined).filter(notYetDownloaded), + ].map((m) => ({ ...m, + downloaded: downloadedModels.map((dm) => dm.name).includes(m.name), + default: predefinedModels.find((pm) => pm.name === m.name)?.default ?? false })) } async function download(_models: DownloadModelParam[]) {} + async function addModel(model: Model) { + models.value.push(model) + await refreshModels() + } + + const aipgBackendUrl = () => { + const aiBackendUrl = backendServices.info.find((item) => item.serviceName === 'ai-backend')?.baseUrl + if (!aiBackendUrl) throw new Error('AIPG Backend not running') + return aiBackendUrl + } + + async function checkIfHuggingFaceUrlExists(repo_id: string) { + const response = await fetch( + `${aipgBackendUrl()}/api/checkHFRepoExists?repo_id=${repo_id}`, + ) + const data = await response.json() + return data.exists + } + + async function checkModelAlreadyLoaded(params: CheckModelAlreadyLoadedParameters[]) { + const response = await fetch(`${aipgBackendUrl()}/api/checkModelAlreadyLoaded`, { + method: 'POST', + body: JSON.stringify({ data: params }), + headers: { + 'Content-Type': 'application/json', + }, + }) + const parsedResponse = (await response.json()) as ApiResponse & { + data: CheckModelAlreadyLoadedResult[] + } + return parsedResponse.data + } + refreshModels() return { models, - llms, - ggufLLMs, - openVINOLLMModels, hfToken, hfTokenIsValid: computed(() => hfToken.value?.startsWith('hf_')), downloadList, + addModel, refreshModels, download, + checkIfHuggingFaceUrlExists, + checkModelAlreadyLoaded, } }, { diff --git a/WebUI/src/assets/js/store/stableDiffusion.ts b/WebUI/src/assets/js/store/stableDiffusion.ts index 29e96937..abd6b1b7 100644 --- a/WebUI/src/assets/js/store/stableDiffusion.ts +++ b/WebUI/src/assets/js/store/stableDiffusion.ts @@ -97,7 +97,7 @@ export const useStableDiffusion = defineStore( backend: 'default', }) } - const result = await globalSetup.checkModelAlreadyLoaded(checkList) + const result = await models.checkModelAlreadyLoaded(checkList) const downloadList: CheckModelAlreadyLoadedParameters[] = [] for (const item of result) { if (!item.already_loaded) { diff --git a/WebUI/src/assets/js/store/textInference.ts b/WebUI/src/assets/js/store/textInference.ts index 598c42ad..d6552edc 100644 --- a/WebUI/src/assets/js/store/textInference.ts +++ b/WebUI/src/assets/js/store/textInference.ts @@ -1,61 +1,98 @@ import { acceptHMRUpdate, defineStore } from 'pinia' -import { useGlobalSetup } from './globalSetup' import { z } from 'zod' import { useBackendServices } from './backendServices' +import { useModels } from './models' +import * as Const from '@/assets/js/const' -export const backendTypes = ['IPEX-LLM', 'LLAMA.CPP', 'OpenVINO'] as const -const BackendSchema = z.enum(backendTypes) -export type Backend = z.infer +export const llmBackendTypes = ['ipexLLM', 'llamaCPP', 'openVINO'] as const +const LlmBackendSchema = z.enum(llmBackendTypes) +export type LlmBackend = z.infer -const backendModelKey = { - 'IPEX-LLM': 'llm_model', - 'LLAMA.CPP': 'ggufLLM_model', - OpenVINO: 'openvinoLLM_model', +const backendToService = { + ipexLLM: 'ai-backend', + llamaCPP: 'llamacpp-backend', + openVINO: 'openvino-backend', +} as const + +export type LlmModel = { + name: string + type: LlmBackend + active: boolean + downloaded: boolean } + export const useTextInference = defineStore( 'textInference', () => { - const globalSetup = useGlobalSetup() const backendServices = useBackendServices() - const backend = ref('IPEX-LLM') - const activeModel = ref(null) - const metricsEnabled = ref(false) + const models = useModels() + const backend = ref('ipexLLM') - const llamaBackendUrl = computed(() => { - const url = backendServices.info.find( - (item) => item.serviceName === 'llamacpp-backend', - )?.baseUrl - console.log('llama url', url) - return url + const selectedModels = ref<{ [key in LlmBackend]: string | null }>({ + ipexLLM: null, + llamaCPP: null, + openVINO: null, }) - const openVINOBackendUrl = computed(() => { - const url = backendServices.info.find( - (item) => item.serviceName === 'openvino-backend', - )?.baseUrl - console.log( - 'openvino url', - backendServices.info.find((item) => item.serviceName === 'openvino-backend')?.baseUrl, + const llmModels = computed(() => { + const llmTypeModels = models.models.filter((m) => + ['ipexLLM', 'llamaCPP', 'openVINO'].includes(m.type), ) - return url + const newModels = llmTypeModels.map((m) => { + const selectedModelForType = selectedModels.value[m.type as LlmBackend] + return { + name: m.name, + type: m.type as LlmBackend, + downloaded: m.downloaded, + active: + m.name === selectedModelForType || + (!llmTypeModels.some((m) => m.name === selectedModelForType) && m.default), + } + }) + console.log('llmModels changed', newModels) + return newModels }) - watch([llamaBackendUrl], () => { - console.log('llamaBackendUrl changed', llamaBackendUrl.value) - }) + const selectModel = (backend: LlmBackend, modelName: string) => { + selectedModels.value[backend] = modelName + } - watch( - [openVINOBackendUrl], - () => { - console.log('openVINOBackendUrl changed', openVINOBackendUrl.value) - }, - { immediate: true }, - ) + const backendToAipgBackendName = { + ipexLLM: 'default', + llamaCPP: 'llama_cpp', + openVINO: 'openvino', + } as const - watch([activeModel], () => { - console.log('activeModel changed', activeModel.value) - globalSetup.applyModelSettings({ [backendModelKey[backend.value]]: activeModel.value }) + const backendToAipgModelTypeNumber = { + ipexLLM: Const.MODEL_TYPE_LLM, + llamaCPP: Const.MODEL_TYPE_LLAMA_CPP, + openVINO: Const.MODEL_TYPE_OPENVINO, + } as const + + async function getDownloadParamsForCurrentModelIfRequired() { + if (!activeModel.value) return [] + const checkList = { + repo_id: activeModel.value, + type: backendToAipgModelTypeNumber[backend.value], + backend: backendToAipgBackendName[backend.value], + } + const checkedModels = await models.checkModelAlreadyLoaded([checkList]) + const notYetDownloaded = checkedModels.filter((m) => !m.already_loaded) + return notYetDownloaded + } + + const activeModel = computed(() => { + const newActiveModel = llmModels.value.filter((m) => m.type === backend.value).find((m) => m.active)?.name + console.log('activeModel changed', newActiveModel) + return newActiveModel }) + const metricsEnabled = ref(false) + + const currentBackendUrl = computed( + () => + backendServices.info.find((item) => item.serviceName === backendToService[backend.value]) + ?.baseUrl, + ) function toggleMetrics() { metricsEnabled.value = !metricsEnabled.value @@ -113,22 +150,25 @@ export const useTextInference = defineStore( return { backend, activeModel, - llamaBackendUrl, - openVINOBackendUrl, + selectedModels, + llmModels, + currentBackendUrl, metricsEnabled, - toggleMetrics, fontSizeClass, nameSizeClass, iconSizeClass, isMaxSize, isMinSize, + selectModel, + getDownloadParamsForCurrentModelIfRequired, + toggleMetrics, increaseFontSize, decreaseFontSize, } }, { persist: { - pick: ['backend', 'activeModel'], + pick: ['backend', 'selectedModels'], }, }, ) diff --git a/WebUI/src/components/AddLLMDialog.vue b/WebUI/src/components/AddLLMDialog.vue index 6460e7d2..1dac500e 100644 --- a/WebUI/src/components/AddLLMDialog.vue +++ b/WebUI/src/components/AddLLMDialog.vue @@ -51,7 +51,7 @@ import { Input } from '@/components/ui/input' import { useGlobalSetup } from '@/assets/js/store/globalSetup' import { useI18N } from '@/assets/js/store/i18n' -import { useModels, userModels, ModelType } from '@/assets/js/store/models' +import { useModels } from '@/assets/js/store/models' import { useTextInference } from '@/assets/js/store/textInference' const i18nState = useI18N().state @@ -70,18 +70,18 @@ const emits = defineEmits<{ }>() const exampleModelName = computed(() => - textInference.backend === 'IPEX-LLM' + textInference.backend === 'ipexLLM' ? i18nState.REQUEST_LLM_MODEL_EXAMPLE : i18nState.REQUEST_LLM_SINGLE_EXAMPLE, ) const examplePlaceholder = computed(() => - textInference.backend === 'IPEX-LLM' + textInference.backend === 'ipexLLM' ? i18nState.COM_LLM_HF_PROMPT : i18nState.COM_LLM_HF_PROMPT_GGUF, ) const isValidModelName = (name: string) => { - if (textInference.backend === 'IPEX-LLM') { + if (textInference.backend === 'ipexLLM') { return name.split('/').length === 2 } else { return name.split('/').length >= 3 @@ -93,10 +93,8 @@ function onShow() { } async function addModel() { - const previousModel = globalSetup.modelSettings.llm_model const cancelAndShowWarning = (text: string) => { - globalSetup.modelSettings.llm_model = previousModel addModelErrorMessage.value = text addModelError.value = true } @@ -113,7 +111,7 @@ async function addModel() { return } - const urlExists = await globalSetup.checkIfHuggingFaceUrlExists(modelRequest.value) + const urlExists = await models.checkIfHuggingFaceUrlExists(modelRequest.value) if (!urlExists) { cancelAndShowWarning(i18nState.ERROR_REPO_NOT_EXISTS) return @@ -123,7 +121,7 @@ async function addModel() { const isLlm = await isLLM(modelRequest.value) const downloadNewModel = async () => { - await registerModel() + await models.addModel({ name: modelRequest.value, type: textInference.backend, downloaded: false, default: false }) emits('callCheckModel') closeAdd() } @@ -135,38 +133,6 @@ async function addModel() { } } -async function registerModel() { - let modelType: ModelType - switch (textInference.backend) { - case 'IPEX-LLM': - modelType = 'llm' - break - case 'LLAMA.CPP': - modelType = 'ggufLLM' - break - case 'OpenVINO': - modelType = 'openvinoLLM' - break - default: - modelType = 'llm' - } - - userModels.push({ name: modelRequest.value, type: modelType, downloaded: false }) - await models.refreshModels() - - switch (textInference.backend) { - case 'IPEX-LLM': - globalSetup.modelSettings.llm_model = modelRequest.value - break - case 'LLAMA.CPP': - globalSetup.modelSettings.ggufLLM_model = modelRequest.value - break - case 'OpenVINO': - globalSetup.modelSettings.openvinoLLM_model = modelRequest.value - break - } -} - async function isLLM(repo_id: string) { const response = await fetch(`${globalSetup.apiHost}/api/isLLM?repo_id=${repo_id}`) const data = await response.json() diff --git a/WebUI/src/components/InstallationManagement.vue b/WebUI/src/components/InstallationManagement.vue index f3b536cb..257bb65a 100644 --- a/WebUI/src/components/InstallationManagement.vue +++ b/WebUI/src/components/InstallationManagement.vue @@ -263,6 +263,8 @@ function getInfoURL(serviceName: string) { return 'https://github.com/comfyanonymous/ComfyUI' case 'llamacpp-backend': return 'https://github.com/abetlen/llama-cpp-python' + case 'openvino-backend': + return 'https://github.com/openvinotoolkit/openvino.genai' default: return undefined } diff --git a/WebUI/src/components/ModelDropDownItem.vue b/WebUI/src/components/ModelDropDownItem.vue index 7e21e4f5..80fd3369 100644 --- a/WebUI/src/components/ModelDropDownItem.vue +++ b/WebUI/src/components/ModelDropDownItem.vue @@ -8,7 +8,7 @@