feat(stage-web): load and save live2d model (#41)

* wip: ui * wip: load local live2d zip * feat(stage-web): save model to indexed db * fix: apply suggestion Co-authored-by: Neko <neko@ayaka.moe> --------- Co-authored-by: Neko <neko@ayaka.moe>
moeru-ai · Mar 2, 2025 · 0528239 · 0528239
1 parent b6abd69
commit 0528239
Show file tree

Hide file tree

Showing 12 changed files with 358 additions and 46 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -49,5 +49,8 @@
     "yaml"
   ],
 
-  "typescript.tsdk": "node_modules/typescript/lib"
+  "typescript.tsdk": "node_modules/typescript/lib",
+  "cSpell.words": [
+    "localforage"
+  ]
 }
diff --git a/apps/stage-web/locales/en.yml b/apps/stage-web/locales/en.yml
@@ -54,6 +54,18 @@ settings:
   microphone: Microphone
   model-provider:
     title: Model Providers
+  live2d:
+    title: Live2D Settings
+    change-model:
+      title: Change Model
+      from-url: Load from URL
+      from-url-placeholder: Enter Live2D model URL
+      from-url-confirm: Load
+      from-file: Load from File
+      from-file-select: Select
+    map-motions:
+      title: Map Motions
+      play: Play Motion
   models: Model
   openai-api-key:
     label: OpenAI API Key

diff --git a/apps/stage-web/locales/zh-CN.yml b/apps/stage-web/locales/zh-CN.yml
@@ -41,6 +41,18 @@ settings:
     title: 语言
   model-provider:
     title: 模型提供商
+  live2d:
+    title: Live2D 设置
+    change-model:
+      title: 更换模型
+      from-url: 从 URL 加载
+      from-url-placeholder: 输入 Live2D 模型 URL
+      from-url-confirm: 加载
+      from-file: 从文件加载
+      from-file-select: 选择
+    map-motions:
+      title: 映射动作
+      play: 播放动作
   models: 模型
   openai-api-key:
     label: OpenAI API 密钥

diff --git a/apps/stage-web/package.json b/apps/stage-web/package.json
@@ -61,6 +61,7 @@
     "drizzle-kit": "^0.30.5",
     "drizzle-orm": "^0.40.0",
     "jszip": "^3.10.1",
+    "localforage": "^1.10.0",
     "nprogress": "^0.2.0",
     "ofetch": "^1.4.1",
     "onnxruntime-web": "^1.20.1",

diff --git a/apps/stage-web/src/components/Widgets/Live2DSettings.vue b/apps/stage-web/src/components/Widgets/Live2DSettings.vue
@@ -0,0 +1,102 @@
+<script setup lang="ts">
+import { Collapsable } from '@proj-airi/stage-ui/components'
+import { useSettings } from '@proj-airi/stage-ui/stores'
+import { useFileDialog } from '@vueuse/core'
+import { ref } from 'vue'
+import { useI18n } from 'vue-i18n'
+
+const { t } = useI18n()
+
+const modelFile = useFileDialog({
+  accept: 'application/zip',
+})
+
+const settings = useSettings()
+const modelUrl = ref(settings.live2dModel)
+
+modelFile.onChange((files) => {
+  if (files && files.length > 0) {
+    settings.live2dModel = files[0]
+  }
+})
+</script>
+
+<template>
+  <Collapsable w-full>
+    <template #trigger="slotProps">
+      <button
+        bg="zinc-100 dark:zinc-800"
+        hover="bg-zinc-200 dark:bg-zinc-700"
+        transition="all ease-in-out duration-250"
+        w-full flex items-center gap-1.5 rounded-lg px-4 py-3 outline-none
+        class="[&_.provider-icon]:grayscale-100 [&_.provider-icon]:hover:grayscale-0"
+        @click="slotProps.setVisible(!slotProps.visible)"
+      >
+        <div flex="~ row 1" items-center gap-1.5>
+          <div
+            i-solar:magic-stick-3-bold-duotone class="provider-icon size-6"
+            transition="filter duration-250 ease-in-out"
+          />
+          <div>
+            {{ t('settings.live2d.change-model.title') }}
+          </div>
+        </div>
+        <div transform transition="transform duration-250" :class="{ 'rotate-180': slotProps.visible }">
+          <div i-solar:alt-arrow-down-bold-duotone />
+        </div>
+      </button>
+    </template>
+    <div p-4>
+      <div class="space-y-4">
+        <div class="flex items-center justify-between">
+          <div>
+            <div class="flex items-center gap-1 text-sm font-medium">
+              {{ t('settings.live2d.change-model.from-url') }}
+            </div>
+          </div>
+          <div>
+            <input
+              v-model="modelUrl"
+              :disabled="settings.loadingLive2dModel"
+              type="text"
+              rounded
+              border="zinc-300 dark:zinc-800 solid 1 focus:zinc-400 dark:focus:zinc-600"
+              transition="border duration-250 ease-in-out"
+              px-2 py-1 text-sm outline-none
+              :placeholder="t('settings.live2d.change-model.from-url-placeholder')"
+            >
+            <button
+              :disabled="settings.loadingLive2dModel"
+
+              bg="zinc-100 dark:zinc-800"
+              hover="bg-zinc-200 dark:bg-zinc-700"
+              transition="all ease-in-out duration-250"
+              ml-2 rounded px-2 py-1 text-sm outline-none
+              @click="settings.live2dModel = modelUrl"
+            >
+              {{ t('settings.live2d.change-model.from-url-confirm') }}
+            </button>
+          </div>
+        </div>
+        <div class="flex items-center justify-between">
+          <div>
+            <div class="flex items-center gap-1 text-sm font-medium">
+              {{ t('settings.live2d.change-model.from-file') }}
+            </div>
+          </div>
+          <button
+            :disabled="settings.loadingLive2dModel"
+            rounded
+            bg="zinc-100 dark:zinc-800"
+            hover="bg-zinc-200 dark:bg-zinc-700"
+            transition="all ease-in-out duration-250"
+            px-2 py-1 text-sm outline-none
+            @click="modelFile.open()"
+          >
+            {{ t('settings.live2d.change-model.from-file-select') }}
+          </button>
+        </div>
+      </div>
+    </div>
+  </Collapsable>
+</template>
diff --git a/apps/stage-web/src/components/Widgets/MobileSettings.vue b/apps/stage-web/src/components/Widgets/MobileSettings.vue
@@ -24,6 +24,11 @@ function navigateToProviders() {
   currentView.value = 'providers'
 }
 
+function navigateToLive2D() {
+  slideDirection.value = 'forward'
+  currentView.value = 'live2d'
+}
+
 function navigateBack() {
   slideDirection.value = 'backward'
   currentView.value = 'main'
@@ -122,6 +127,22 @@ function navigateBack() {
                 </div>
               </div>
             </label>
+            <!-- Live2D Setting -->
+            <div
+              grid="~ cols-[150px_1fr]"
+              bg="zinc-100 dark:zinc-800"
+              hover="bg-zinc-200 dark:bg-zinc-700"
+              transition="all ease-in-out duration-250"
+              cursor-pointer items-center gap-1.5 rounded-lg px-4 py-3
+              @click="navigateToLive2D"
+            >
+              <div text="sm">
+                <span>{{ t('settings.live2d.title') }}</span>
+              </div>
+              <div flex="~ row" w-full justify-end>
+                <div i-solar:alt-arrow-right-bold-duotone />
+              </div>
+            </div>
           </div>
         </div>
       </div>
@@ -141,6 +162,21 @@ function navigateBack() {
         </div>
         <ModelProviderSettings />
       </div>
+      <!-- Live2D Settings View -->
+      <div v-else-if="currentView === 'live2d'" key="live2d">
+        <div mb-4 flex items-center gap-2>
+          <button
+            text="zinc-800/80 dark:zinc-200/80"
+            @click="navigateBack"
+          >
+            <div i-solar:alt-arrow-left-bold-duotone />
+          </button>
+          <h2 text="zinc-800/80 dark:zinc-200/80 xl" font-bold>
+            {{ t('settings.live2d.title') }}
+          </h2>
+        </div>
+        <Live2DSettings />
+      </div>
     </Transition>
   </div>
 </template>

diff --git a/packages/stage-ui/src/components/Live2D/Model.vue b/packages/stage-ui/src/components/Live2D/Model.vue
@@ -6,16 +6,18 @@ import type { Ref } from 'vue'
 import { extensions } from '@pixi/extensions'
 import { InteractionManager } from '@pixi/interaction'
 import { Ticker, TickerPlugin } from '@pixi/ticker'
-import { breakpointsTailwind, useBreakpoints, useDark, useDebounceFn } from '@vueuse/core'
+import { breakpointsTailwind, useBreakpoints, useDark, useDebounceFn, watchDebounced } from '@vueuse/core'
+import localforage from 'localforage'
+import { storeToRefs } from 'pinia'
 import { DropShadowFilter } from 'pixi-filters'
-import { Live2DModel, MotionPreloadStrategy, MotionPriority } from 'pixi-live2d-display/cubism4'
+import { Live2DFactory, Live2DModel, MotionPriority } from 'pixi-live2d-display/cubism4'
 import { computed, onMounted, onUnmounted, ref, toRef, watch } from 'vue'
 
 import { useLive2DIdleEyeFocus } from '../../composables/live2d'
+import { useSettings } from '../../stores'
 
 const props = withDefaults(defineProps<{
   app?: Application
-  model: string
   mouthOpenSize?: number
   width: number
   height: number
@@ -58,16 +60,31 @@ function setScale(model: Ref<Live2DModel<InternalModel> | undefined>) {
   model.value.scale.set(scale, scale)
 }
 
-async function initLive2DPixiStage() {
+const { live2dModel, loadingLive2dModel } = storeToRefs(useSettings())
+
+// FIXME: it cannot blink if loading other model
+async function loadModel(source: string | Blob) {
   if (!pixiApp.value)
     return
 
-  // https://guansss.github.io/pixi-live2d-display/#package-importing
-  Live2DModel.registerTicker(Ticker)
-  extensions.add(TickerPlugin)
-  extensions.add(InteractionManager)
+  if (model.value) {
+    pixiApp.value.stage.removeChild(model.value)
+    model.value.destroy()
+    model.value = undefined
+  }
+
+  loadingLive2dModel.value = true
 
-  model.value = await Live2DModel.from(props.model, { motionPreload: MotionPreloadStrategy.ALL })
+  const modelInstance = new Live2DModel()
+
+  if (source instanceof Blob) {
+    await Live2DFactory.setupLive2DModel(modelInstance, [source])
+  }
+  else {
+    await Live2DFactory.setupLive2DModel(modelInstance, source)
+  }
+
+  model.value = modelInstance
   pixiApp.value.stage.addChild(model.value as any)
   initialModelWidth.value = model.value.width
   initialModelHeight.value = model.value.height
@@ -112,6 +129,30 @@ async function initLive2DPixiStage() {
     }
     return true
   }
+
+  // save to indexdb
+  await localforage.setItem('live2dModel', source)
+
+  loadingLive2dModel.value = false
+}
+
+async function initLive2DPixiStage() {
+  if (!pixiApp.value)
+    return
+
+  // https://guansss.github.io/pixi-live2d-display/#package-importing
+  Live2DModel.registerTicker(Ticker)
+  extensions.add(TickerPlugin)
+  extensions.add(InteractionManager)
+
+  // load indexdb model first
+  const live2dModelBlob = await localforage.getItem<Blob>('live2dModel')
+  if (live2dModelBlob) {
+    await loadModel(live2dModelBlob)
+    return
+  }
+
+  await loadModel(live2dModel.value)
 }
 
 async function setMotion(motionName: string) {
@@ -148,8 +189,17 @@ watch(paused, (value) => {
   value ? pixiApp.value?.stop() : pixiApp.value?.start()
 })
 
+watchDebounced(live2dModel, (value) => {
+  if (!value)
+    return
+
+  loadModel(value)
+}, { debounce: 1000 })
+
 onMounted(updateDropShadowFilter)
-onUnmounted(() => model.value && pixiApp.value?.stage.removeChild(model.value))
+onUnmounted(() => {
+  model.value && pixiApp.value?.stage.removeChild(model.value)
+})
 </script>
 
 <template>

diff --git a/packages/stage-ui/src/components/Scenes/Live2D.vue b/packages/stage-ui/src/components/Scenes/Live2D.vue
@@ -16,7 +16,6 @@ import Screen from '../Screen.vue'
 import TransitionVertical from '../TransitionVertical.vue'
 
 withDefaults(defineProps<{
-  model: string
   paused: boolean
   mouthOpenSize?: number
 }>(), {
@@ -30,7 +29,7 @@ const show = ref(false)
 <template>
   <Screen v-slot="{ width, height }" relative>
     <Live2DCanvas v-slot="{ app }" :width="width" :height="height">
-      <Live2DModel :app="app" :model="model" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :motion="motion" :paused="paused" />
+      <Live2DModel :app="app" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :motion="motion" :paused="paused" />
     </Live2DCanvas>
     <div absolute bottom="3" right="3">
       <div flex="~ row" cursor-pointer>

diff --git a/packages/stage-ui/src/components/Widgets/Stage.vue b/packages/stage-ui/src/components/Widgets/Stage.vue
@@ -234,7 +234,6 @@ onMounted(async () => {
         v-if="stageView === '2d'"
         :motion="motion"
         :mouth-open-size="mouthOpenSize"
-        model="./assets/live2d/models/hiyori_pro_zh.zip"
         min-w="50% <lg:full" min-h="100 sm:100" h-full w-full flex-1
         :paused="paused"
       />

diff --git a/packages/stage-ui/src/stores/settings.ts b/packages/stage-ui/src/stores/settings.ts
@@ -23,6 +23,11 @@ export const useSettings = defineStore('settings', () => {
   const elevenlabsVoiceEnglish = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/en', Voice.Myriam)
   const elevenlabsVoiceJapanese = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/ja', Voice.Morioki)
 
+  // TODO: extract to a separate store
+  const live2dModel = ref<File | string>('./assets/live2d/models/hiyori_pro_zh.zip')
+  const live2dPosition = useLocalStorage('settings/live2d/position', { x: 0, y: 0 }) // position is relative to the center of the screen
+  const loadingLive2dModel = ref(false)
+
   watch(isAudioInputOn, (value) => {
     if (value === 'false') {
       selectedAudioDevice.value = undefined
@@ -49,6 +54,9 @@ export const useSettings = defineStore('settings', () => {
     openAiApiBaseURL,
     openAiModel,
     elevenLabsApiKey,
+    live2dModel,
+    live2dPosition,
+    loadingLive2dModel,
     language,
     stageView,
     isAudioInputOn,