From aee70fb1a4a0885933df23de04b4c9f3fcc3229c Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 28 Jan 2025 16:29:44 +0800 Subject: [PATCH] refactor: split llm-agent --- src/manager/action.ts | 203 --------------- src/manager/conversation.ts | 382 ---------------------------- src/plugins/llm-agent/chat.ts | 53 ++++ src/plugins/llm-agent/completion.ts | 27 ++ src/plugins/llm-agent/index.ts | 62 +++++ src/plugins/llm-agent/type.ts | 15 ++ src/plugins/llm-agent/voice.ts | 58 +++++ 7 files changed, 215 insertions(+), 585 deletions(-) delete mode 100644 src/manager/action.ts delete mode 100644 src/manager/conversation.ts create mode 100644 src/plugins/llm-agent/chat.ts create mode 100644 src/plugins/llm-agent/completion.ts create mode 100644 src/plugins/llm-agent/index.ts create mode 100644 src/plugins/llm-agent/type.ts create mode 100644 src/plugins/llm-agent/voice.ts diff --git a/src/manager/action.ts b/src/manager/action.ts deleted file mode 100644 index 7f78c0b..0000000 --- a/src/manager/action.ts +++ /dev/null @@ -1,203 +0,0 @@ -import type { Mineflayer } from '../libs/mineflayer/core' - -import { useLogg } from '@guiiai/logg' -import EventEmitter from 'eventemitter3' - -// Types and interfaces -type ActionFn = (...args: any[]) => void - -interface ActionResult { - success: boolean - message: string | null - timedout: boolean -} - -interface QueuedAction { - label: string - fn: ActionFn - timeout: number - resume: boolean - resolve: (result: ActionResult) => void - reject: (error: Error) => void -} - -export class ActionManager extends EventEmitter { - private state = { - executing: false, - currentActionLabel: '', - currentActionFn: undefined as ActionFn | undefined, - timedout: false, - resume: { - func: undefined as ActionFn | undefined, - name: undefined as string | undefined, - }, - } - - // Action queue to store pending actions - private actionQueue: QueuedAction[] = [] - - private logger = useLogg('ActionManager').useGlobalConfig() - private mineflayer: Mineflayer - - constructor(mineflayer: Mineflayer) { - super() - this.mineflayer = mineflayer - } - - public async resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { - return this.queueAction({ - label: actionLabel, - fn: actionFn, - timeout, - resume: true, - }) - } - - public async runAction( - actionLabel: string, - actionFn: ActionFn, - options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }, - ): Promise { - return this.queueAction({ - label: actionLabel, - fn: actionFn, - timeout: options.timeout, - resume: options.resume, - }) - } - - public async stop(): Promise { - this.mineflayer.emit('interrupt') - // Clear the action queue when stopping - this.actionQueue = [] - } - - public cancelResume(): void { - this.state.resume.func = undefined - this.state.resume.name = undefined - } - - private async queueAction(action: Omit): Promise { - return new Promise((resolve, reject) => { - this.actionQueue.push({ - ...action, - resolve, - reject, - }) - - if (!this.state.executing) { - this.processQueue().catch(reject) - } - }) - } - - private async processQueue(): Promise { - while (this.actionQueue.length > 0) { - const action = this.actionQueue[0] - - try { - const result = action.resume - ? await this.executeResume(action.label, action.fn, action.timeout) - : await this.executeAction(action.label, action.fn, action.timeout) - - this.actionQueue.shift()?.resolve(result) - - if (!result.success) { - this.actionQueue.forEach(pendingAction => - pendingAction.reject(new Error('Queue cleared due to action failure')), - ) - this.actionQueue = [] - return result - } - } - catch (error) { - this.actionQueue.shift()?.reject(error as Error) - this.actionQueue.forEach(pendingAction => - pendingAction.reject(new Error('Queue cleared due to error')), - ) - this.actionQueue = [] - throw error - } - } - - return { success: true, message: 'success', timedout: false } - } - - private async executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { - const isNewResume = actionFn != null - - if (isNewResume) { - if (!actionLabel) { - throw new Error('actionLabel is required for new resume') - } - this.state.resume.func = actionFn - this.state.resume.name = actionLabel - } - - const canExecute = this.state.resume.func != null && isNewResume - - if (!canExecute) { - return { success: false, message: null, timedout: false } - } - - this.state.currentActionLabel = this.state.resume.name || '' - const result = await this.executeAction(this.state.resume.name || '', this.state.resume.func, timeout) - this.state.currentActionLabel = '' - return result - } - - private async executeAction(actionLabel: string, actionFn?: ActionFn, timeout = 10): Promise { - let timeoutHandle: NodeJS.Timeout | undefined - - try { - this.logger.log('executing action...\n') - - if (this.state.executing) { - this.logger.log(`action "${actionLabel}" trying to interrupt current action "${this.state.currentActionLabel}"`) - } - - await this.stop() - - // Set execution state - this.state.executing = true - this.state.currentActionLabel = actionLabel - this.state.currentActionFn = actionFn - - if (timeout > 0) { - timeoutHandle = this.startTimeout(timeout) - } - - await actionFn?.() - - // Reset state after successful execution - this.resetExecutionState(timeoutHandle) - - return { success: true, message: 'success', timedout: false } - } - catch (err) { - this.resetExecutionState(timeoutHandle) - this.cancelResume() - this.logger.withError(err).error('Code execution triggered catch') - await this.stop() - - return { success: false, message: 'failed', timedout: false } - } - } - - private resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { - this.state.executing = false - this.state.currentActionLabel = '' - this.state.currentActionFn = undefined - if (timeoutHandle) - clearTimeout(timeoutHandle) - } - - private startTimeout(timeoutMins = 10): NodeJS.Timeout { - return setTimeout(async () => { - this.logger.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - this.state.timedout = true - this.emit('timeout', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - await this.stop() - }, timeoutMins * 60 * 1000) - } -} diff --git a/src/manager/conversation.ts b/src/manager/conversation.ts deleted file mode 100644 index 891fb7a..0000000 --- a/src/manager/conversation.ts +++ /dev/null @@ -1,382 +0,0 @@ -// import { useLogg } from '@guiiai/logg' - -// let self_prompter_paused = false - -// interface ConversationMessage { -// message: string -// start: boolean -// end: boolean -// } - -// function compileInMessages(inQueue: ConversationMessage[]) { -// let pack: ConversationMessage | undefined -// let fullMessage = '' -// while (inQueue.length > 0) { -// pack = inQueue.shift() -// if (!pack) -// continue - -// fullMessage += pack.message -// } -// if (pack) { -// pack.message = fullMessage -// } - -// return pack -// } - -// type Conversation = ReturnType - -// function useConversations(name: string, agent: Agent) { -// const active = { value: false } -// const ignoreUntilStart = { value: false } -// const blocked = { value: false } -// let inQueue: ConversationMessage[] = [] -// const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } - -// function reset() { -// active.value = false -// ignoreUntilStart.value = false -// inQueue = [] -// } - -// function end() { -// active.value = false -// ignoreUntilStart.value = true -// const fullMessage = compileInMessages(inQueue) -// if (!fullMessage) -// return - -// if (fullMessage.message.trim().length > 0) { -// agent.history.add(name, fullMessage.message) -// } - -// if (agent.lastSender === name) { -// agent.lastSender = undefined -// } -// } - -// function queue(message: ConversationMessage) { -// inQueue.push(message) -// } - -// return { -// reset, -// end, -// queue, -// name, -// inMessageTimer, -// blocked, -// active, -// ignoreUntilStart, -// inQueue, -// } -// } - -// const WAIT_TIME_START = 30000 - -// export type ConversationStore = ReturnType - -// export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { -// const conversations: Record = {} -// const activeConversation: { value: Conversation | undefined } = { value: undefined } -// const awaitingResponse = { value: false } -// const waitTimeLimit = { value: WAIT_TIME_START } -// const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } -// const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } -// const agent = options.agent -// let agentsInGame = options.agentNames || [] -// const log = useLogg('ConversationStore').useGlobalConfig() - -// const conversationStore = { -// getConvo: (name: string) => { -// if (!conversations[name]) -// conversations[name] = useConversations(name, agent) -// return conversations[name] -// }, -// startMonitor: () => { -// clearInterval(connectionMonitor.value) -// let waitTime = 0 -// let lastTime = Date.now() -// connectionMonitor.value = setInterval(() => { -// if (!activeConversation.value) { -// conversationStore.stopMonitor() -// return // will clean itself up -// } - -// const delta = Date.now() - lastTime -// lastTime = Date.now() -// const convo_partner = activeConversation.value.name - -// if (awaitingResponse.value && agent.isIdle()) { -// waitTime += delta -// if (waitTime > waitTimeLimit.value) { -// agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) -// waitTime = 0 -// waitTimeLimit.value *= 2 -// } -// } -// else if (!awaitingResponse.value) { -// waitTimeLimit.value = WAIT_TIME_START -// waitTime = 0 -// } - -// if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { -// connectionTimeout.value = setTimeout(() => { -// if (conversationStore.otherAgentInGame(convo_partner)) { -// conversationStore.clearMonitorTimeouts() -// return -// } -// if (!self_prompter_paused) { -// conversationStore.endConversation(convo_partner) -// agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) -// } -// else { -// conversationStore.endConversation(convo_partner) -// } -// }, 10000) -// } -// }, 1000) -// }, -// stopMonitor: () => { -// clearInterval(connectionMonitor.value) -// connectionMonitor.value = undefined -// conversationStore.clearMonitorTimeouts() -// }, -// clearMonitorTimeouts: () => { -// awaitingResponse.value = false -// clearTimeout(connectionTimeout.value) -// connectionTimeout.value = undefined -// }, -// startConversation: (send_to: string, message: string) => { -// const convo = conversationStore.getConvo(send_to) -// convo.reset() - -// if (agent.self_prompter.on) { -// agent.self_prompter.stop() -// self_prompter_paused = true -// } -// if (convo.active.value) -// return - -// convo.active.value = true -// activeConversation.value = convo -// conversationStore.startMonitor() -// conversationStore.sendToBot(send_to, message, true, false) -// }, -// startConversationFromOtherBot: (name: string) => { -// const convo = conversationStore.getConvo(name) -// convo.active.value = true -// activeConversation.value = convo -// conversationStore.startMonitor() -// }, -// sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { -// if (!conversationStore.isOtherAgent(send_to)) { -// console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) -// return -// } -// const convo = conversationStore.getConvo(send_to) - -// if (options.chatBotMessages && open_chat) -// agent.openChat(`(To ${send_to}) ${message}`) - -// if (convo.ignoreUntilStart.value) -// return -// convo.active.value = true - -// const end = message.includes('!endConversation') -// const json = { -// message, -// start, -// end, -// } - -// awaitingResponse.value = true -// // TODO: -// // sendBotChatToServer(send_to, json) -// log.withField('json', json).log(`Sending message to ${send_to}`) -// }, -// receiveFromBot: async (sender: string, received: ConversationMessage) => { -// const convo = conversationStore.getConvo(sender) - -// if (convo.ignoreUntilStart.value && !received.start) -// return - -// // check if any convo is active besides the sender -// if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { -// conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) -// conversationStore.endConversation(sender) -// return -// } - -// if (received.start) { -// convo.reset() -// conversationStore.startConversationFromOtherBot(sender) -// } - -// conversationStore.clearMonitorTimeouts() -// convo.queue(received) - -// // responding to conversation takes priority over self prompting -// if (agent.self_prompter.on) { -// await agent.self_prompter.stopLoop() -// self_prompter_paused = true -// } - -// _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) -// }, -// responseScheduledFor: (sender: string) => { -// if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) -// return false -// const convo = conversationStore.getConvo(sender) -// return !!convo.inMessageTimer -// }, -// isOtherAgent: (name: string) => { -// return !!options.agentNames?.includes(name) -// }, -// otherAgentInGame: (name: string) => { -// return agentsInGame.includes(name) -// }, -// updateAgents: (agents: Agent[]) => { -// options.agentNames = agents.map(a => a.name) -// agentsInGame = agents.filter(a => a.in_game).map(a => a.name) -// }, -// getInGameAgents: () => { -// return agentsInGame -// }, -// inConversation: (other_agent?: string) => { -// if (other_agent) -// return conversations[other_agent]?.active -// return Object.values(conversations).some(c => c.active) -// }, -// endConversation: (sender: string) => { -// if (conversations[sender]) { -// conversations[sender].end() -// if (activeConversation.value?.name === sender) { -// conversationStore.stopMonitor() -// activeConversation.value = undefined -// if (self_prompter_paused && !conversationStore.inConversation()) { -// _resumeSelfPrompter(agent, conversationStore) -// } -// } -// } -// }, -// endAllConversations: () => { -// for (const sender in conversations) { -// conversationStore.endConversation(sender) -// } -// if (self_prompter_paused) { -// _resumeSelfPrompter(agent, conversationStore) -// } -// }, -// forceEndCurrentConversation: () => { -// if (activeConversation.value) { -// const sender = activeConversation.value.name -// conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) -// conversationStore.endConversation(sender) -// } -// }, -// scheduleSelfPrompter: () => { -// self_prompter_paused = true -// }, -// cancelSelfPrompter: () => { -// self_prompter_paused = false -// }, -// } - -// return conversationStore -// } - -// function containsCommand(message: string) { -// // TODO: mock -// return message -// } - -// /* -// This function controls conversation flow by deciding when the bot responds. -// The logic is as follows: -// - If neither bot is busy, respond quickly with a small delay. -// - If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) -// - If I'm busy but other bot isn't, let LLM decide whether to respond -// - If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses -// - New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk -// */ -// const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions -// const fastDelay = 200 -// const longDelay = 5000 - -// async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { -// if (convo.inMessageTimer) -// clearTimeout(convo.inMessageTimer.value) -// const otherAgentBusy = containsCommand(received.message) - -// const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) - -// if (!agent.isIdle() && otherAgentBusy) { -// // both are busy -// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) -// if (canTalkOver) -// scheduleResponse(fastDelay) -// // otherwise don't respond -// } -// else if (otherAgentBusy) { -// // other bot is busy but I'm not -// scheduleResponse(longDelay) -// } -// else if (!agent.isIdle()) { -// // I'm busy but other bot isn't -// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) -// if (canTalkOver) { -// scheduleResponse(fastDelay) -// } -// else { -// const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) -// useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) -// if (shouldRespond) -// scheduleResponse(fastDelay) -// } -// } -// else { -// // neither are busy -// scheduleResponse(fastDelay) -// } -// } - -// function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { -// const convo = conversationStore.getConvo(name) -// _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) -// } - -// function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { -// if (!received) -// return - -// useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) - -// const convo = conversationStore.getConvo(sender) -// convo.active.value = true - -// let message = _tagMessage(received.message) -// if (received.end) { -// conversationStore.endConversation(sender) -// message = `Conversation with ${sender} ended with message: "${message}"` -// sender = 'system' // bot will respond to system instead of the other bot -// } -// else if (received.start) { -// agent.shut_up = false -// } -// convo.inMessageTimer.value = undefined -// agent.handleMessage(sender, message) -// } - -// function _tagMessage(message: string) { -// return `(FROM OTHER BOT)${message}` -// } - -// async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { -// await new Promise(resolve => setTimeout(resolve, 5000)) -// if (self_prompter_paused && !conversationStore.inConversation()) { -// self_prompter_paused = false -// agent.self_prompter.start() -// } -// } diff --git a/src/plugins/llm-agent/chat.ts b/src/plugins/llm-agent/chat.ts new file mode 100644 index 0000000..e993e1b --- /dev/null +++ b/src/plugins/llm-agent/chat.ts @@ -0,0 +1,53 @@ +import type { useLogg } from '@guiiai/logg' +import type { Neuri, NeuriContext } from 'neuri' +import type { MineflayerWithAgents } from './type' + +import { system, user } from 'neuri/openai' + +import { generateStatusPrompt } from '../../agents/prompt/llm-agent.plugin' +import { toRetriable } from '../../utils/helper' +import { handleLLMCompletion } from './completion' + +export async function handleChatMessage(username: string, message: string, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { + logger.withFields({ username, message }).log('Chat message received') + bot.memory.chatHistory.push(user(`${username}: ${message}`)) + + logger.log('thinking...') + + try { + // Create and execute plan + const plan = await bot.planning.createPlan(message) + logger.withFields({ plan }).log('Plan created') + await bot.planning.executePlan(plan) + logger.log('Plan executed successfully') + + // Generate response + // TODO: use chat agent and conversion manager + const statusPrompt = await generateStatusPrompt(bot) + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('handling response...') + return toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + { onError: err => logger.withError(err).log('error occurred') }, + )(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) + } +} diff --git a/src/plugins/llm-agent/completion.ts b/src/plugins/llm-agent/completion.ts new file mode 100644 index 0000000..d7f7e10 --- /dev/null +++ b/src/plugins/llm-agent/completion.ts @@ -0,0 +1,27 @@ +import type { useLogg } from '@guiiai/logg' +import type { NeuriContext } from 'neuri' +import type { MineflayerWithAgents } from './type' + +import { assistant, type ChatCompletion } from 'neuri/openai' + +import { openaiConfig } from '../../composables/config' + +export async function handleLLMCompletion(context: NeuriContext, bot: MineflayerWithAgents, logger: ReturnType): Promise { + logger.log('rerouting...') + + const completion = await context.reroute('action', context.messages, { + model: openaiConfig.model, + }) as ChatCompletion | { error: { message: string } } & ChatCompletion + + if (!completion || 'error' in completion) { + logger.withFields({ completion }).error('Completion') + logger.withFields({ messages: context.messages }).log('messages') + return completion?.error?.message ?? 'Unknown error' + } + + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('output') + + bot.memory.chatHistory.push(assistant(content)) + return content +} diff --git a/src/plugins/llm-agent/index.ts b/src/plugins/llm-agent/index.ts new file mode 100644 index 0000000..f6918b5 --- /dev/null +++ b/src/plugins/llm-agent/index.ts @@ -0,0 +1,62 @@ +import type { LLMAgentOptions, MineflayerWithAgents } from './type' + +import { useLogg } from '@guiiai/logg' +import { system } from 'neuri/openai' + +import { generateActionAgentPrompt } from '../../agents/prompt/llm-agent.plugin' +import { openaiConfig } from '../../composables/config' +import { createAppContainer } from '../../container' +import { ChatMessageHandler, type MineflayerPlugin } from '../../libs/mineflayer' +import { handleChatMessage } from './chat' +import { handleVoiceInput } from './voice' + +export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { + return { + async created(bot) { + const logger = useLogg('LLMAgent').useGlobalConfig() + + // Create container and get required services + const container = createAppContainer({ + neuri: options.agent, + model: openaiConfig.model, + maxHistoryLength: 50, + idleTimeout: 5 * 60 * 1000, + }) + + const actionAgent = container.resolve('actionAgent') + const planningAgent = container.resolve('planningAgent') + const chatAgent = container.resolve('chatAgent') + + // Initialize agents + await actionAgent.init() + await planningAgent.init() + await chatAgent.init() + + // Type conversion + const botWithAgents = bot as unknown as MineflayerWithAgents + botWithAgents.action = actionAgent + botWithAgents.planning = planningAgent + botWithAgents.chat = chatAgent + + // Initialize system prompt + bot.memory.chatHistory.push(system(generateActionAgentPrompt(bot))) + + // Set message handling + const onChat = new ChatMessageHandler(bot.username).handleChat((username, message) => + handleChatMessage(username, message, botWithAgents, options.agent, logger)) + + options.airiClient.onEvent('input:text:voice', event => + handleVoiceInput(event, botWithAgents, options.agent, logger)) + + bot.bot.on('chat', onChat) + }, + + async beforeCleanup(bot) { + const botWithAgents = bot as unknown as MineflayerWithAgents + await botWithAgents.action?.destroy() + await botWithAgents.planning?.destroy() + await botWithAgents.chat?.destroy() + bot.bot.removeAllListeners('chat') + }, + } +} diff --git a/src/plugins/llm-agent/type.ts b/src/plugins/llm-agent/type.ts new file mode 100644 index 0000000..d5444a1 --- /dev/null +++ b/src/plugins/llm-agent/type.ts @@ -0,0 +1,15 @@ +import type { Client } from '@proj-airi/server-sdk' +import type { Neuri } from 'neuri' +import type { Mineflayer } from '../../libs/mineflayer' +import type { ActionAgent, ChatAgent, PlanningAgent } from '../../libs/mineflayer/base-agent' + +export interface MineflayerWithAgents extends Mineflayer { + planning: PlanningAgent + action: ActionAgent + chat: ChatAgent +} + +export interface LLMAgentOptions { + agent: Neuri + airiClient: Client +} diff --git a/src/plugins/llm-agent/voice.ts b/src/plugins/llm-agent/voice.ts new file mode 100644 index 0000000..ff1ad50 --- /dev/null +++ b/src/plugins/llm-agent/voice.ts @@ -0,0 +1,58 @@ +import type { useLogg } from '@guiiai/logg' +import type { Neuri, NeuriContext } from 'neuri' +import type { MineflayerWithAgents } from './type' + +import { system, user } from 'neuri/openai' + +import { generateStatusPrompt } from '../../agents/prompt/llm-agent.plugin' +import { toRetriable } from '../../utils/helper' +import { handleLLMCompletion } from './completion' + +export async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { + logger + .withFields({ + user: event.data.discord?.guildMember, + message: event.data.transcription, + }) + .log('Chat message received') + + const statusPrompt = await generateStatusPrompt(bot) + bot.memory.chatHistory.push(system(statusPrompt)) + bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) + + try { + // Create and execute plan + const plan = await bot.planning.createPlan(event.data.transcription) + logger.withFields({ plan }).log('Plan created') + await bot.planning.executePlan(plan) + logger.log('Plan executed successfully') + + // Generate response + const retryHandler = toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + ) + + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('thinking...') + return retryHandler(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) + } +}