From c4d8878d4d5e44306a6041d5cc8ad4beeaa06835 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 10 Jan 2025 02:14:12 +0800 Subject: [PATCH 01/22] wip: 4 agents implimemtion --- package.json | 2 +- src/agents/action/index.ts | 78 ++++ src/agents/chat/index.ts | 169 ++++++++ src/agents/chat/llm.ts | 99 +++++ src/agents/chat/types.ts | 25 ++ src/agents/memory/index.ts | 67 ++++ src/agents/planning/factory.ts | 69 ++++ src/agents/planning/index.ts | 457 ++++++++++++++++++++++ src/agents/planning/llm.ts | 119 ++++++ src/composables/action.ts | 53 +-- src/libs/mineflayer/core/agent-factory.ts | 70 ++++ src/libs/mineflayer/core/base-agent.ts | 77 ++++ src/libs/mineflayer/interfaces/agents.ts | 53 +++ src/mineflayer/llm-agent.ts | 196 ++++++---- 14 files changed, 1422 insertions(+), 112 deletions(-) create mode 100644 src/agents/action/index.ts create mode 100644 src/agents/chat/index.ts create mode 100644 src/agents/chat/llm.ts create mode 100644 src/agents/chat/types.ts create mode 100644 src/agents/memory/index.ts create mode 100644 src/agents/planning/factory.ts create mode 100644 src/agents/planning/index.ts create mode 100644 src/agents/planning/llm.ts create mode 100644 src/libs/mineflayer/core/agent-factory.ts create mode 100644 src/libs/mineflayer/core/base-agent.ts create mode 100644 src/libs/mineflayer/interfaces/agents.ts diff --git a/package.json b/package.json index 1998cf5..962a630 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "vitest": "^3.0.0" }, "simple-git-hooks": { - "pre-commit": "pnpm lint-staged && pnpm typecheck" + "pre-commit": "pnpm lint-staged" }, "lint-staged": { "*": "eslint --fix" diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts new file mode 100644 index 0000000..97a8aea --- /dev/null +++ b/src/agents/action/index.ts @@ -0,0 +1,78 @@ +import type { Action } from '../../libs/mineflayer/action' +import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/interfaces/agents' +import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import { actionsList } from '../actions' + +export class ActionAgentImpl extends AbstractAgent implements ActionAgent { + public readonly type = 'action' as const + private actions: Map + + constructor(config: AgentConfig) { + super(config) + this.actions = new Map() + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing action agent') + actionsList.forEach(action => this.actions.set(action.name, action)) + + // Set up event listeners + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + } + + protected async destroyAgent(): Promise { + this.actions.clear() + this.removeAllListeners() + } + + public async performAction(name: string, params: unknown[]): Promise { + if (!this.initialized) { + throw new Error('Action agent not initialized') + } + + const action = this.actions.get(name) + if (!action) { + throw new Error(`Action not found: ${name}`) + } + + try { + this.logger.withFields({ name, params }).log('Performing action') + return await this.actionManager.runAction( + name, + async () => { + const fn = action.perform + return await fn(...params) + }, + { timeout: 60, resume: false }, + ) + } + catch (error) { + this.logger.withFields({ name, params, error }).error('Failed to perform action') + throw error + } + } + + public getAvailableActions(): Action[] { + return Array.from(this.actions.values()) + } + + private async handleAgentMessage(sender: string, message: string): Promise { + // Handle messages from other agents or system + if (sender === 'system') { + // Handle system messages + if (message.includes('interrupt')) { + await this.actionManager.stop() + } + } + else { + // Handle agent messages + const convo = this.conversationStore.getConvo(sender) + if (convo.active.value) { + // Process message and potentially perform actions + this.logger.withFields({ sender, message }).log('Processing agent message') + } + } + } +} diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts new file mode 100644 index 0000000..62e143d --- /dev/null +++ b/src/agents/chat/index.ts @@ -0,0 +1,169 @@ +import type { ChatAgent } from '../../libs/mineflayer/interfaces/agents' +import type { ChatAgentConfig, ChatContext } from './types' +import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import { generateChatResponse } from './llm' + +export class ChatAgentImpl extends AbstractAgent implements ChatAgent { + public readonly type = 'chat' as const + private activeChats: Map + private maxHistoryLength: number + private idleTimeout: number + private llmConfig: ChatAgentConfig['llm'] + + constructor(config: ChatAgentConfig) { + super(config) + this.activeChats = new Map() + this.maxHistoryLength = config.maxHistoryLength ?? 50 + this.idleTimeout = config.idleTimeout ?? 5 * 60 * 1000 // 5 minutes + this.llmConfig = config.llm + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing chat agent') + + // Set up event listeners + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + + // Set up idle timeout checker + setInterval(() => { + this.checkIdleChats() + }, 60 * 1000) // Check every minute + } + + protected async destroyAgent(): Promise { + this.activeChats.clear() + this.removeAllListeners() + } + + public async processMessage(message: string, sender: string): Promise { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withFields({ sender, message }).log('Processing message') + + try { + // Get or create chat context + const context = this.getOrCreateContext(sender) + + // Add message to history + this.addToHistory(context, sender, message) + + // Update last activity time + context.lastUpdate = Date.now() + + // Generate response using LLM + const response = await this.generateResponse(message, context) + + // Add response to history + this.addToHistory(context, this.id, response) + + return response + } + catch (error) { + this.logger.withError(error).error('Failed to process message') + throw error + } + } + + public startConversation(player: string): void { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withField('player', player).log('Starting conversation') + + const context = this.getOrCreateContext(player) + context.startTime = Date.now() + context.lastUpdate = Date.now() + } + + public endConversation(player: string): void { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withField('player', player).log('Ending conversation') + + if (this.activeChats.has(player)) { + const context = this.activeChats.get(player)! + // Archive chat history if needed + this.archiveChat(context) + this.activeChats.delete(player) + } + } + + private getOrCreateContext(player: string): ChatContext { + let context = this.activeChats.get(player) + if (!context) { + context = { + player, + startTime: Date.now(), + lastUpdate: Date.now(), + history: [], + } + this.activeChats.set(player, context) + } + return context + } + + private addToHistory(context: ChatContext, sender: string, message: string): void { + context.history.push({ + sender, + message, + timestamp: Date.now(), + }) + + // Trim history if too long + if (context.history.length > this.maxHistoryLength) { + context.history = context.history.slice(-this.maxHistoryLength) + } + } + + private async generateResponse(message: string, context: ChatContext): Promise { + return await generateChatResponse(message, context.history, { + agent: this.llmConfig.agent, + model: this.llmConfig.model, + maxContextLength: this.maxHistoryLength, + }) + } + + private checkIdleChats(): void { + const now = Date.now() + for (const [player, context] of this.activeChats.entries()) { + if (now - context.lastUpdate > this.idleTimeout) { + this.logger.withField('player', player).log('Ending idle conversation') + this.endConversation(player) + } + } + } + + private async archiveChat(context: ChatContext): Promise { + // Archive chat history to persistent storage if needed + this.logger.withFields({ + player: context.player, + messageCount: context.history.length, + duration: Date.now() - context.startTime, + }).log('Archiving chat history') + } + + private async handleAgentMessage(sender: string, message: string): Promise { + if (sender === 'system') { + if (message.includes('interrupt')) { + // Handle system interrupt + for (const player of this.activeChats.keys()) { + this.endConversation(player) + } + } + } + else { + // Handle messages from other agents + const convo = this.conversationStore.getConvo(sender) + if (convo.active.value) { + await this.processMessage(message, sender) + } + } + } +} diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts new file mode 100644 index 0000000..2875a8b --- /dev/null +++ b/src/agents/chat/llm.ts @@ -0,0 +1,99 @@ +import type { Neuri } from 'neuri' +import type { ChatHistory } from './types' +import { useLogg } from '@guiiai/logg' +import { system, user } from 'neuri/openai' +import { toRetriable } from '../../utils/reliability' + +const logger = useLogg('chat-llm').useGlobalConfig() + +interface LLMChatConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number + maxContextLength?: number +} + +export async function generateChatResponse( + message: string, + history: ChatHistory[], + config: LLMChatConfig, +): Promise { + const systemPrompt = generateSystemPrompt() + const chatHistory = formatChatHistory(history, config.maxContextLength ?? 10) + const userPrompt = message + + const messages = [ + system(systemPrompt), + ...chatHistory, + user(userPrompt), + ] + + const content = await config.agent.handleStateless(messages, async (c) => { + logger.log('Generating response...') + + const handleCompletion = async (c: any): Promise => { + const completion = await c.reroute('chat', c.messages, { + model: config.model ?? 'openai/gpt-4-mini', + }) + + if (!completion || 'error' in completion) { + logger.withFields(c).error('Completion failed') + throw new Error(completion?.error?.message ?? 'Unknown error') + } + + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('Response generated') + return content + } + + const retriableHandler = toRetriable( + config.retryLimit ?? 3, + config.delayInterval ?? 1000, + handleCompletion, + ) + + return await retriableHandler(c) + }) + + if (!content) { + throw new Error('Failed to generate response') + } + + return content +} + +function generateSystemPrompt(): string { + return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. + +Guidelines: +1. Be friendly and helpful +2. Keep responses concise but informative +3. Use game-appropriate language +4. Acknowledge player's emotions and intentions +5. Ask for clarification when needed +6. Remember context from previous messages +7. Be proactive in suggesting helpful actions + +You can: +- Answer questions about the game +- Help with tasks and crafting +- Give directions and suggestions +- Engage in casual conversation +- Coordinate with other bots + +Remember that you're operating in a Minecraft world and should maintain that context in your responses.` +} + +function formatChatHistory( + history: ChatHistory[], + maxLength: number, +): Array<{ role: 'user' | 'assistant', content: string }> { + // Take the most recent messages up to maxLength + const recentHistory = history.slice(-maxLength) + + return recentHistory.map(entry => ({ + role: entry.sender === 'bot' ? 'assistant' : 'user', + content: entry.message, + })) +} diff --git a/src/agents/chat/types.ts b/src/agents/chat/types.ts new file mode 100644 index 0000000..32410be --- /dev/null +++ b/src/agents/chat/types.ts @@ -0,0 +1,25 @@ +import type { Neuri } from 'neuri' + +export interface ChatHistory { + sender: string + message: string + timestamp: number +} + +export interface ChatContext { + player: string + startTime: number + lastUpdate: number + history: ChatHistory[] +} + +export interface ChatAgentConfig { + id: string + type: 'chat' + llm: { + agent: Neuri + model?: string + } + maxHistoryLength?: number + idleTimeout?: number +} diff --git a/src/agents/memory/index.ts b/src/agents/memory/index.ts new file mode 100644 index 0000000..fb125a5 --- /dev/null +++ b/src/agents/memory/index.ts @@ -0,0 +1,67 @@ +import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/interfaces/agents' +import { useLogg } from '@guiiai/logg' + +const logger = useLogg('memory-agent').useGlobalConfig() + +export class MemoryAgentImpl implements MemoryAgent { + public readonly type = 'memory' as const + public readonly id: string + private memory: Map + private initialized: boolean + + constructor(config: AgentConfig) { + this.id = config.id + this.memory = new Map() + this.initialized = false + } + + async init(): Promise { + if (this.initialized) { + return + } + + logger.log('Initializing memory agent') + this.initialized = true + } + + async destroy(): Promise { + this.memory.clear() + this.initialized = false + } + + remember(key: string, value: unknown): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + logger.withFields({ key, value }).log('Storing memory') + this.memory.set(key, value) + } + + recall(key: string): T | undefined { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + const value = this.memory.get(key) as T | undefined + logger.withFields({ key, value }).log('Recalling memory') + return value + } + + forget(key: string): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + logger.withFields({ key }).log('Forgetting memory') + this.memory.delete(key) + } + + getMemorySnapshot(): Record { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return Object.fromEntries(this.memory.entries()) + } +} diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts new file mode 100644 index 0000000..e8cfd49 --- /dev/null +++ b/src/agents/planning/factory.ts @@ -0,0 +1,69 @@ +import type { Neuri } from 'neuri' +import type { PlanningAgentConfig } from '.' +import type { Mineflayer } from '../../libs/mineflayer' +import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' +import { useLogg } from '@guiiai/logg' +import { AgentFactory, AgentRegistry } from '../../libs/mineflayer/core/agent-factory' + +const logger = useLogg('planning-factory').useGlobalConfig() + +interface PlanningPluginOptions { + agent: Neuri + model?: string +} + +export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin { + return { + async created(bot: Mineflayer) { + logger.log('Initializing planning plugin') + + // Create and register agents + const registry = AgentRegistry.getInstance() + + // Create action agent if not exists + if (!registry.has('action-agent')) { + const actionAgent = AgentFactory.createAgent({ + id: 'action-agent', + type: 'action', + }) + registry.register(actionAgent) + await actionAgent.init() + } + + // Create memory agent if not exists + if (!registry.has('memory-agent')) { + const memoryAgent = AgentFactory.createAgent({ + id: 'memory-agent', + type: 'memory', + }) + registry.register(memoryAgent) + await memoryAgent.init() + } + + // Create planning agent + const planningAgent = AgentFactory.createAgent({ + id: 'planning-agent', + type: 'planning', + llm: { + agent: options.agent, + model: options.model, + }, + } as PlanningAgentConfig) + + registry.register(planningAgent) + await planningAgent.init() + + // Add planning agent to bot + bot.planning = planningAgent + }, + + async beforeCleanup() { + logger.log('Destroying planning plugin') + + const registry = AgentRegistry.getInstance() + await registry.destroy() + + // delete bot.planning + }, + } +} diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts new file mode 100644 index 0000000..e47267c --- /dev/null +++ b/src/agents/planning/index.ts @@ -0,0 +1,457 @@ +import type { Neuri } from 'neuri' +import type { Action } from '../../libs/mineflayer/action' +import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/interfaces/agents' +import { AgentRegistry } from '../../libs/mineflayer/core/agent-factory' +import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import { generatePlanWithLLM } from './llm' + +interface PlanContext { + goal: string + currentStep: number + startTime: number + lastUpdate: number + retryCount: number +} + +interface PlanTemplate { + goal: string + conditions: string[] + steps: Array<{ + action: string + params: unknown[] + }> + requiresAction: boolean +} + +export interface PlanningAgentConfig extends AgentConfig { + llm: { + agent: Neuri + model?: string + } +} + +export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { + public readonly type = 'planning' as const + private currentPlan: Plan | null = null + private context: PlanContext | null = null + private actionAgent: ActionAgent | null = null + private memoryAgent: MemoryAgent | null = null + private planTemplates: Map + private llmConfig: PlanningAgentConfig['llm'] + + constructor(config: PlanningAgentConfig) { + super(config) + this.planTemplates = new Map() + this.llmConfig = config.llm + this.initializePlanTemplates() + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing planning agent') + + // Get agent references + const registry = AgentRegistry.getInstance() + this.actionAgent = registry.get('action-agent', 'action') + this.memoryAgent = registry.get('memory-agent', 'memory') + + // Set up event listeners + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + + this.on('interrupt', () => { + this.handleInterrupt() + }) + } + + protected async destroyAgent(): Promise { + this.currentPlan = null + this.context = null + this.actionAgent = null + this.memoryAgent = null + this.planTemplates.clear() + this.removeAllListeners() + } + + public async createPlan(goal: string): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + this.logger.withField('goal', goal).log('Creating plan') + + try { + // Check memory for existing plan + const cachedPlan = await this.loadCachedPlan(goal) + if (cachedPlan) { + this.logger.log('Using cached plan') + return cachedPlan + } + + // Get available actions from action agent + const availableActions = this.actionAgent?.getAvailableActions() ?? [] + + // Check if the goal requires actions + const requirements = this.parseGoalRequirements(goal) + const requiresAction = this.doesGoalRequireAction(requirements) + + // If no actions needed, return empty plan + if (!requiresAction) { + this.logger.log('Goal does not require actions') + return { + goal, + steps: [], + status: 'completed', + requiresAction: false, + } + } + + // Create plan steps based on available actions and goal + const steps = await this.generatePlanSteps(goal, availableActions) + + // Create new plan + const plan: Plan = { + goal, + steps, + status: 'pending', + requiresAction: true, + } + + // Cache the plan + await this.cachePlan(plan) + + this.currentPlan = plan + this.context = { + goal, + currentStep: 0, + startTime: Date.now(), + lastUpdate: Date.now(), + retryCount: 0, + } + + return plan + } + catch (error) { + this.logger.withError(error).error('Failed to create plan') + throw error + } + } + + public async executePlan(plan: Plan): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + if (!plan.requiresAction) { + this.logger.log('Plan does not require actions, skipping execution') + return + } + + if (!this.actionAgent) { + throw new Error('Action agent not available') + } + + this.logger.withField('plan', plan).log('Executing plan') + + try { + plan.status = 'in_progress' + this.currentPlan = plan + + for (let i = 0; i < plan.steps.length; i++) { + if (!this.context) + break + + const step = plan.steps[i] + this.context.currentStep = i + + try { + this.logger.withFields({ step, index: i }).log('Executing plan step') + await this.actionAgent.performAction(step.action, step.params) + this.context.lastUpdate = Date.now() + } + catch (stepError) { + this.logger.withError(stepError).error('Failed to execute plan step') + + // Attempt to adjust plan and retry + if (this.context.retryCount < 3) { + this.context.retryCount++ + const adjustedPlan = await this.adjustPlan(plan, stepError instanceof Error ? stepError.message : 'Unknown error') + await this.executePlan(adjustedPlan) + return + } + + plan.status = 'failed' + throw stepError + } + } + + plan.status = 'completed' + } + catch (error) { + plan.status = 'failed' + throw error + } + finally { + this.context = null + } + } + + public async adjustPlan(plan: Plan, feedback: string): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + this.logger.withFields({ plan, feedback }).log('Adjusting plan') + + try { + // If there's a current context, use it to adjust the plan + if (this.context) { + const currentStep = this.context.currentStep + const availableActions = this.actionAgent?.getAvailableActions() ?? [] + + // Generate new steps from the current point + const newSteps = await this.generatePlanSteps(plan.goal, availableActions, feedback) + + // Create adjusted plan + const adjustedPlan: Plan = { + goal: plan.goal, + steps: [ + ...plan.steps.slice(0, currentStep), + ...newSteps, + ], + status: 'pending', + } + + return adjustedPlan + } + + // If no context, create a new plan + return this.createPlan(plan.goal) + } + catch (error) { + this.logger.withError(error).error('Failed to adjust plan') + throw error + } + } + + private async generatePlanSteps( + goal: string, + availableActions: Action[], + feedback?: string, + ): Promise> { + // First, try to find a matching template + const template = this.findMatchingTemplate(goal) + if (template) { + this.logger.log('Using plan template') + return template.steps + } + + // If no template matches, use LLM to generate plan + this.logger.log('Generating plan using LLM') + return await generatePlanWithLLM(goal, availableActions, { + agent: this.llmConfig.agent, + model: this.llmConfig.model, + }, feedback) + } + + private findMatchingTemplate(goal: string): PlanTemplate | undefined { + for (const [pattern, template] of this.planTemplates.entries()) { + if (goal.toLowerCase().includes(pattern.toLowerCase())) { + return template + } + } + return undefined + } + + private parseGoalRequirements(goal: string): { + needsItems: boolean + items?: string[] + needsMovement: boolean + location?: { x?: number, y?: number, z?: number } + needsInteraction: boolean + target?: string + needsCrafting: boolean + needsCombat: boolean + } { + const requirements = { + needsItems: false, + needsMovement: false, + needsInteraction: false, + needsCrafting: false, + needsCombat: false, + } + + const goalLower = goal.toLowerCase() + + // Check for item-related actions + if (goalLower.includes('collect') || goalLower.includes('get') || goalLower.includes('find')) { + requirements.needsItems = true + requirements.needsMovement = true + } + + // Check for movement-related actions + if (goalLower.includes('go to') || goalLower.includes('move to') || goalLower.includes('follow')) { + requirements.needsMovement = true + } + + // Check for interaction-related actions + if (goalLower.includes('interact') || goalLower.includes('use') || goalLower.includes('open')) { + requirements.needsInteraction = true + } + + // Check for crafting-related actions + if (goalLower.includes('craft') || goalLower.includes('make') || goalLower.includes('build')) { + requirements.needsCrafting = true + requirements.needsItems = true + } + + // Check for combat-related actions + if (goalLower.includes('attack') || goalLower.includes('fight') || goalLower.includes('kill')) { + requirements.needsCombat = true + requirements.needsMovement = true + } + + return requirements + } + + private generateGatheringSteps(items?: string[]): Array<{ action: string, params: unknown[] }> { + const steps: Array<{ action: string, params: unknown[] }> = [] + + if (items) { + for (const item of items) { + steps.push( + { action: 'searchForBlock', params: [item, 64] }, + { action: 'collectBlocks', params: [item, 1] }, + ) + } + } + + return steps + } + + private generateMovementSteps(location?: { x?: number, y?: number, z?: number }): Array<{ action: string, params: unknown[] }> { + if (location?.x !== undefined && location?.y !== undefined && location?.z !== undefined) { + return [{ + action: 'goToCoordinates', + params: [location.x, location.y, location.z, 1], + }] + } + return [] + } + + private generateInteractionSteps(target?: string): Array<{ action: string, params: unknown[] }> { + if (target) { + return [{ + action: 'activate', + params: [target], + }] + } + return [] + } + + private generateRecoverySteps(feedback: string): Array<{ action: string, params: unknown[] }> { + const steps: Array<{ action: string, params: unknown[] }> = [] + + if (feedback.includes('not found')) { + steps.push({ action: 'searchForBlock', params: ['any', 128] }) + } + + if (feedback.includes('inventory full')) { + steps.push({ action: 'discard', params: ['cobblestone', 64] }) + } + + return steps + } + + private async loadCachedPlan(goal: string): Promise { + if (!this.memoryAgent) + return null + + const cachedPlan = this.memoryAgent.recall(`plan:${goal}`) + if (cachedPlan && this.isPlanValid(cachedPlan)) { + return cachedPlan + } + return null + } + + private async cachePlan(plan: Plan): Promise { + if (!this.memoryAgent) + return + + this.memoryAgent.remember(`plan:${plan.goal}`, plan) + } + + private isPlanValid(plan: Plan): boolean { + // Add validation logic here + return true + } + + private initializePlanTemplates(): void { + // Add common plan templates + this.planTemplates.set('collect wood', { + goal: 'collect wood', + conditions: ['needs_axe', 'near_trees'], + steps: [ + { action: 'searchForBlock', params: ['log', 64] }, + { action: 'collectBlocks', params: ['log', 1] }, + ], + requiresAction: true, + }) + + this.planTemplates.set('find shelter', { + goal: 'find shelter', + conditions: ['is_night', 'unsafe'], + steps: [ + { action: 'searchForBlock', params: ['bed', 64] }, + { action: 'goToBed', params: [] }, + ], + requiresAction: true, + }) + + // Add templates for non-action goals + this.planTemplates.set('hello', { + goal: 'hello', + conditions: [], + steps: [], + requiresAction: false, + }) + + this.planTemplates.set('how are you', { + goal: 'how are you', + conditions: [], + steps: [], + requiresAction: false, + }) + } + + private async handleAgentMessage(sender: string, message: string): Promise { + if (sender === 'system') { + if (message.includes('interrupt')) { + this.handleInterrupt() + } + } + else { + const convo = this.conversationStore.getConvo(sender) + if (convo.active.value) { + // Process message and potentially adjust plan + this.logger.withFields({ sender, message }).log('Processing agent message') + } + } + } + + private handleInterrupt(): void { + if (this.currentPlan) { + this.currentPlan.status = 'failed' + this.context = null + } + } + + private doesGoalRequireAction(requirements: ReturnType): boolean { + // Check if any requirement indicates need for action + return requirements.needsItems + || requirements.needsMovement + || requirements.needsInteraction + || requirements.needsCrafting + || requirements.needsCombat + } +} diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts new file mode 100644 index 0000000..a0b133c --- /dev/null +++ b/src/agents/planning/llm.ts @@ -0,0 +1,119 @@ +import type { Neuri } from 'neuri' +import { useLogg } from '@guiiai/logg' +import { system, user } from 'neuri/openai' +import { toRetriable } from '../../utils/reliability' + +const logger = useLogg('planning-llm').useGlobalConfig() + +interface LLMPlanningConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number +} + +export async function generatePlanWithLLM( + goal: string, + availableActions: Array<{ name: string, description: string }>, + config: LLMPlanningConfig, + feedback?: string, +): Promise> { + const systemPrompt = generateSystemPrompt(availableActions) + const userPrompt = generateUserPrompt(goal, feedback) + + const messages = [ + system(systemPrompt), + user(userPrompt), + ] + + const content = await config.agent.handleStateless(messages, async (c) => { + logger.log('Generating plan...') + + const handleCompletion = async (c: any): Promise => { + const completion = await c.reroute('action', c.messages, { + model: config.model ?? 'openai/gpt-4o-mini', + }) + + if (!completion || 'error' in completion) { + logger.withFields(c).error('Completion failed') + throw new Error(completion?.error?.message ?? 'Unknown error') + } + + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('Plan generated') + return content + } + + const retirableHandler = toRetriable( + config.retryLimit ?? 3, + config.delayInterval ?? 1000, + handleCompletion, + ) + + return await retirableHandler(c) + }) + + if (!content) { + throw new Error('Failed to generate plan') + } + + return parsePlanContent(content) +} + +function generateSystemPrompt(availableActions: Array<{ name: string, description: string }>): string { + const actionsList = availableActions + .map(action => `- ${action.name}: ${action.description}`) + .join('\n') + + return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. +Available actions: +${actionsList} + +Respond with a JSON array of steps, where each step has: +- action: The name of the action to perform +- params: Array of parameters for the action + +Example response: +[ + { + "action": "searchForBlock", + "params": ["log", 64] + }, + { + "action": "collectBlocks", + "params": ["log", 1] + } +]` +} + +function generateUserPrompt(goal: string, feedback?: string): string { + let prompt = `Create a plan to: ${goal}` + if (feedback) { + prompt += `\nPrevious attempt feedback: ${feedback}` + } + return prompt +} + +function parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { + try { + // Find JSON array in the content + const match = content.match(/\[[\s\S]*\]/) + if (!match) { + throw new Error('No plan found in response') + } + + const plan = JSON.parse(match[0]) + if (!Array.isArray(plan)) { + throw new TypeError('Invalid plan format') + } + + return plan.map(step => ({ + action: step.action, + params: step.params, + })) + } + catch (error) { + logger.withError(error).error('Failed to parse plan') + throw error + } +} diff --git a/src/composables/action.ts b/src/composables/action.ts index d22b43a..9f91f15 100644 --- a/src/composables/action.ts +++ b/src/composables/action.ts @@ -1,9 +1,9 @@ -import type { Agent } from './agent' +import type { Mineflayer } from '../libs/mineflayer/core' import { useLogg } from '@guiiai/logg' type Fn = (...args: any[]) => void -export function useActionManager(agent: Agent) { +export function useActionManager(mineflayer: Mineflayer) { const executing: { value: boolean } = { value: false } const currentActionLabel: { value: string | undefined } = { value: '' } const currentActionFn: { value: (Fn) | undefined } = { value: undefined } @@ -26,17 +26,18 @@ export function useActionManager(agent: Agent) { } async function stop() { - if (!executing.value) - return - const timeout = setTimeout(() => { - agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.') - }, 10000) - while (executing.value) { - agent.requestInterrupt() - log.log('waiting for code to finish executing...') - await new Promise(resolve => setTimeout(resolve, 300)) - } - clearTimeout(timeout) + // if (!executing.value) + // return + // const timeout = setTimeout(() => { + // mineflayer.cleanKill('Code execution refused stop after 10 seconds. Killing process.') + // }, 10000) + // while (executing.value) { + // mineflayer.requestInterrupt() + // log.log('waiting for code to finish executing...') + // await new Promise(resolve => setTimeout(resolve, 300)) + // } + // clearTimeout(timeout) + mineflayer.emit('interrupt') } function cancelResume() { @@ -53,7 +54,7 @@ export function useActionManager(agent: Agent) { } resume_name.value = actionLabel } - if (resume_func.value != null && (agent.isIdle() || new_resume) && (!agent.self_prompter.on || new_resume)) { + if (resume_func.value != null && (mineflayer.isIdle() || new_resume) && (!mineflayer.self_prompter.on || new_resume)) { currentActionLabel.value = resume_name.value const res = await _executeAction(resume_name.value, resume_func.value, timeout) currentActionLabel.value = '' @@ -70,14 +71,14 @@ export function useActionManager(agent: Agent) { log.log('executing code...\n') // await current action to finish (executing=false), with 10 seconds timeout - // also tell agent.bot to stop various actions + // also tell mineflayer.bot to stop various actions if (executing.value) { log.log(`action "${actionLabel}" trying to interrupt current action "${currentActionLabel.value}"`) } await stop() // clear bot logs and reset interrupt code - agent.clearBotLogs() + mineflayer.clearBotLogs() executing.value = true currentActionLabel.value = actionLabel @@ -99,12 +100,12 @@ export function useActionManager(agent: Agent) { // get bot activity summary const output = _getBotOutputSummary() - const interrupted = agent.bot.interrupt_code - agent.clearBotLogs() + const interrupted = mineflayer.bot.interrupt_code + mineflayer.clearBotLogs() // if not interrupted and not generating, emit idle event - if (!interrupted && !agent.coder.generating) { - agent.bot.emit('idle') + if (!interrupted && !mineflayer.coder.generating) { + mineflayer.bot.emit('idle') } // return action status report @@ -124,17 +125,17 @@ export function useActionManager(agent: Agent) { + `Error: ${err}\n` + `Stack trace:\n${(err as Error).stack}` - const interrupted = agent.bot.interrupt_code - agent.clearBotLogs() - if (!interrupted && !agent.coder.generating) { - agent.bot.emit('idle') + const interrupted = mineflayer.bot.interrupt_code + mineflayer.clearBotLogs() + if (!interrupted && !mineflayer.coder.generating) { + mineflayer.bot.emit('idle') } return { success: false, message, interrupted, timedout: false } } } function _getBotOutputSummary() { - const { bot } = agent + const { bot } = mineflayer if (bot.interrupt_code && !timedout.value) return '' let output = bot.output @@ -154,7 +155,7 @@ export function useActionManager(agent: Agent) { return setTimeout(async () => { log.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) timedout.value = true - agent.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) + mineflayer.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) await stop() // last attempt to stop }, TIMEOUT_MINS * 60 * 1000) } diff --git a/src/libs/mineflayer/core/agent-factory.ts b/src/libs/mineflayer/core/agent-factory.ts new file mode 100644 index 0000000..5fddfba --- /dev/null +++ b/src/libs/mineflayer/core/agent-factory.ts @@ -0,0 +1,70 @@ +import type { ChatAgentConfig } from '../../../agents/chat/types' +import type { AgentConfig, AgentType, BaseAgent } from '../interfaces/agents' +import { ActionAgentImpl } from '../../../agents/action' +import { ChatAgentImpl } from '../../../agents/chat' +import { MemoryAgentImpl } from '../../../agents/memory' +import { type PlanningAgentConfig, PlanningAgentImpl } from '../../../agents/planning' + +export class AgentFactory { + static createAgent(config: AgentConfig): BaseAgent { + switch (config.type) { + case 'action': + return new ActionAgentImpl(config) + case 'memory': + return new MemoryAgentImpl(config) + case 'planning': + return new PlanningAgentImpl(config as PlanningAgentConfig) + case 'chat': + return new ChatAgentImpl(config as ChatAgentConfig) + default: + throw new Error(`Unknown agent type: ${config.type satisfies never}`) + } + } +} + +export class AgentRegistry { + private static instance: AgentRegistry + private agents: Map + + private constructor() { + this.agents = new Map() + } + + static getInstance(): AgentRegistry { + if (!this.instance) { + this.instance = new AgentRegistry() + } + return this.instance + } + + has(id: string): boolean { + return this.agents.has(id) + } + + register(agent: BaseAgent): void { + if (this.agents.has(agent.id)) { + throw new Error(`Agent with id ${agent.id} already exists`) + } + this.agents.set(agent.id, agent) + } + + get(id: string, type: AgentType): T { + const agent = this.agents.get(id) + if (!agent) { + throw new Error(`Agent not found: ${id}`) + } + if (agent.type !== type) { + throw new Error(`Agent ${id} is not of type ${type}`) + } + return agent as T + } + + getAll(): BaseAgent[] { + return Array.from(this.agents.values()) + } + + async destroy(): Promise { + await Promise.all(Array.from(this.agents.values()).map(agent => agent.destroy())) + this.agents.clear() + } +} diff --git a/src/libs/mineflayer/core/base-agent.ts b/src/libs/mineflayer/core/base-agent.ts new file mode 100644 index 0000000..b05a73b --- /dev/null +++ b/src/libs/mineflayer/core/base-agent.ts @@ -0,0 +1,77 @@ +import type { AgentConfig, BaseAgent } from '../interfaces/agents' +import { useLogg } from '@guiiai/logg' +import EventEmitter3 from 'eventemitter3' + +export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { + public readonly id: string + public readonly type: AgentConfig['type'] + public readonly name: string + + protected initialized: boolean + protected logger: ReturnType + // protected actionManager: ReturnType + // protected conversationStore: ReturnType + + constructor(config: AgentConfig) { + super() + this.id = config.id + this.type = config.type + this.name = `${this.type}-${this.id}` + this.initialized = false + this.logger = useLogg(this.name).useGlobalConfig() + + // Initialize managers + // this.actionManager = useActionManager(this) + // this.conversationStore = useConversationStore({ + // agent: this, + // chatBotMessages: true, + // }) + } + + public async init(): Promise { + if (this.initialized) { + return + } + + this.logger.log('Initializing agent') + await this.initializeAgent() + this.initialized = true + } + + public async destroy(): Promise { + if (!this.initialized) { + return + } + + this.logger.log('Destroying agent') + await this.destroyAgent() + this.initialized = false + } + + // Agent interface implementation + // public isIdle(): boolean { + // return !this.actionManager.executing + // } + + public handleMessage(sender: string, message: string): void { + this.logger.withFields({ sender, message }).log('Received message') + this.emit('message', { sender, message }) + } + + public openChat(message: string): void { + this.logger.withField('message', message).log('Opening chat') + this.emit('chat', message) + } + + public clearBotLogs(): void { + // Implement if needed + } + + public requestInterrupt(): void { + this.emit('interrupt') + } + + // Methods to be implemented by specific agents + protected abstract initializeAgent(): Promise + protected abstract destroyAgent(): Promise +} diff --git a/src/libs/mineflayer/interfaces/agents.ts b/src/libs/mineflayer/interfaces/agents.ts new file mode 100644 index 0000000..62140a5 --- /dev/null +++ b/src/libs/mineflayer/interfaces/agents.ts @@ -0,0 +1,53 @@ +import type { Action } from '../action' + +export type AgentType = 'action' | 'memory' | 'planning' | 'chat' + +export interface AgentConfig { + id: string + type: AgentType +} + +export interface BaseAgent { + readonly id: string + readonly type: AgentType + init: () => Promise + destroy: () => Promise +} + +export interface ActionAgent extends BaseAgent { + type: 'action' + performAction: (name: string, params: unknown[]) => Promise + getAvailableActions: () => Action[] +} + +export interface MemoryAgent extends BaseAgent { + type: 'memory' + remember: (key: string, value: unknown) => void + recall: (key: string) => T | undefined + forget: (key: string) => void + getMemorySnapshot: () => Record +} + +export interface Plan { + goal: string + steps: Array<{ + action: string + params: unknown[] + }> + status: 'pending' | 'in_progress' | 'completed' | 'failed' + requiresAction: boolean +} + +export interface PlanningAgent extends BaseAgent { + type: 'planning' + createPlan: (goal: string) => Promise + executePlan: (plan: Plan) => Promise + adjustPlan: (plan: Plan, feedback: string) => Promise +} + +export interface ChatAgent extends BaseAgent { + type: 'chat' + processMessage: (message: string, sender: string) => Promise + startConversation: (player: string) => void + endConversation: (player: string) => void +} diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts index 145e9f8..d1a6c5d 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/mineflayer/llm-agent.ts @@ -1,112 +1,138 @@ import type { Client } from '@proj-airi/server-sdk' import type { Neuri, NeuriContext } from 'neuri' +import type { ChatCompletion } from 'neuri/openai' +import type { PlanningAgent } from '../libs/mineflayer/interfaces/agents' import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' +import { PlanningPlugin } from '../agents/planning/factory' import { formBotChat } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' import { toRetriable } from '../utils/reliability' -export function LLMAgent(options: { agent: Neuri, airiClient: Client }): MineflayerPlugin { - return { - async created(bot) { - const agent = options.agent - - const logger = useLogg('LLMAgent').useGlobalConfig() +interface LLMAgentOptions { + agent: Neuri + airiClient: Client +} - bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) +async function handleLLMCompletion(context: NeuriContext, bot: any, logger: any): Promise { + const completion = await context.reroute('action', context.messages, { + model: 'openai/gpt-4o-mini', + }) as ChatCompletion | { error: { message: string } } & ChatCompletion - // todo: get system message - const onChat = formBotChat(bot.username, async (username, message) => { - logger.withFields({ username, message }).log('Chat message received') - - // long memory - bot.memory.chatHistory.push(user(`${username}: ${message}`)) - - // short memory - const statusPrompt = await genStatusPrompt(bot) - const content = await agent.handleStateless([...bot.memory.chatHistory, system(statusPrompt)], async (c: NeuriContext) => { - logger.log('thinking...') - - const handleCompletion = async (c: NeuriContext): Promise => { - logger.log('rerouting...') - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - if (!completion || 'error' in completion) { - logger.withFields({ completion }).error('Completion') - throw completion?.error || new Error('Unknown error') - } - - const content = await completion?.firstContent() - logger.withFields({ usage: completion.usage, content }).log('output') - - bot.memory.chatHistory.push(assistant(content)) - - return content - } - - const retirableHandler = toRetriable( - 3, // retryLimit - 1000, // delayInterval in ms - handleCompletion, - { onError: err => logger.withError(err).log('error occurred') }, - ) - - logger.log('handling...') - return await retirableHandler(c) - }) - - if (content) { - logger.withFields({ content }).log('responded') - bot.bot.chat(content) - } - }) + if (!completion || 'error' in completion) { + logger.withFields({ completion }).error('Completion') + logger.withFields({ messages: context.messages }).log('messages') + throw new Error(completion?.error?.message ?? 'Unknown error') + } - options.airiClient.onEvent('input:text:voice', async (event) => { - logger.withFields({ user: event.data.discord?.guildMember, message: event.data.transcription }).log('Chat message received') + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('output') - // long memory - bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) + bot.memory.chatHistory.push(assistant(content)) + return content +} - // short memory - const statusPrompt = await genStatusPrompt(bot) - const content = await agent.handleStateless([...bot.memory.chatHistory, system(statusPrompt)], async (c: NeuriContext) => { - logger.log('thinking...') +async function handleChatMessage(username: string, message: string, bot: any, agent: Neuri, logger: any): Promise { + logger.withFields({ username, message }).log('Chat message received') + bot.memory.chatHistory.push(user(`${username}: ${message}`)) + + const statusPrompt = await genStatusPrompt(bot) + const retryHandler = toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + ) + + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('thinking...') + return retryHandler(c) + }, + ) - const handleCompletion = async (c: NeuriContext): Promise => { - logger.log('rerouting...') - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - if (!completion || 'error' in completion) { - logger.withFields({ completion }).error('Completion') - throw completion?.error || new Error('Unknown error') - } + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } +} - const content = await completion?.firstContent() - logger.withFields({ usage: completion.usage, content }).log('output') +async function handleVoiceInput(event: any, bot: any, agent: Neuri, logger: any): Promise { + logger + .withFields({ + user: event.data.discord?.guildMember, + message: event.data.transcription, + }) + .log('Chat message received') + + const statusPrompt = await genStatusPrompt(bot) + bot.memory.chatHistory.push(system(statusPrompt)) + bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) + + try { + const planningAgent = bot.planning as PlanningAgent + const plan = await planningAgent.createPlan(event.data.transcription) + logger.withFields({ plan }).log('Plan created') + + await planningAgent.executePlan(plan) + logger.log('Plan executed successfully') + + const retryHandler = toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + ) + + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('thinking...') + return retryHandler(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) + } +} - bot.memory.chatHistory.push(assistant(content)) +export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { + return { + async created(bot) { + const agent = options.agent + const logger = useLogg('LLMAgent').useGlobalConfig() - return content - } + const planningPlugin = PlanningPlugin({ + agent: options.agent, + model: 'openai/gpt-4o-mini', + }) + await planningPlugin.created!(bot) - const retirableHandler = toRetriable( - 3, // retryLimit - 1000, // delayInterval in ms - handleCompletion, - { onError: err => logger.withError(err).log('error occurred') }, - ) + bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) - logger.log('handling...') - return await retirableHandler(c) - }) + const onChat = formBotChat(bot.username, (username, message) => + handleChatMessage(username, message, bot, agent, logger)) - if (content) { - logger.withFields({ content }).log('responded') - bot.bot.chat(content) - } - }) + options.airiClient.onEvent('input:text:voice', event => + handleVoiceInput(event, bot, agent, logger)) bot.bot.on('chat', onChat) }, + + async beforeCleanup(bot) { + bot.bot.removeAllListeners('chat') + }, } } From db3c7929f70357a4bd6f4f5a5512f8f4efcc0f7f Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 17 Jan 2025 03:34:14 +0800 Subject: [PATCH 02/22] feat: parallel actions --- src/agents/action/index.ts | 112 ++++++++- src/agents/actions.test.ts | 9 +- src/agents/chat/index.ts | 4 +- src/agents/planning/factory.ts | 52 ++-- src/agents/planning/index.ts | 440 ++++++++++++++++++++++++--------- src/agents/planning/llm.ts | 32 +-- src/composables/action.ts | 232 ++++++++++------- src/mineflayer/llm-agent.ts | 98 +++++--- src/prompts/agent.ts | 30 ++- 9 files changed, 689 insertions(+), 320 deletions(-) diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 97a8aea..98aca80 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -1,15 +1,34 @@ +import type { MineflayerWithExtensions } from '../../composables/action' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/interfaces/agents' +import { useActionManager } from '../../composables/action' +import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' import { actionsList } from '../actions' +interface ActionState { + executing: boolean + label: string + startTime: number +} + export class ActionAgentImpl extends AbstractAgent implements ActionAgent { public readonly type = 'action' as const private actions: Map + private actionManager: ReturnType + private mineflayer: MineflayerWithExtensions + private currentActionState: ActionState constructor(config: AgentConfig) { super(config) this.actions = new Map() + this.mineflayer = useBot().bot as MineflayerWithExtensions + this.actionManager = useActionManager(this.mineflayer) + this.currentActionState = { + executing: false, + label: '', + startTime: 0, + } } protected async initializeAgent(): Promise { @@ -23,11 +42,22 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { } protected async destroyAgent(): Promise { + await this.actionManager.stop() + this.actionManager.cancelResume() this.actions.clear() this.removeAllListeners() + this.currentActionState = { + executing: false, + label: '', + startTime: 0, + } } - public async performAction(name: string, params: unknown[]): Promise { + public async performAction( + name: string, + params: unknown[], + options: { timeout?: number, resume?: boolean } = {}, + ): Promise { if (!this.initialized) { throw new Error('Action agent not initialized') } @@ -38,20 +68,66 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { } try { + this.updateActionState(true, name) this.logger.withFields({ name, params }).log('Performing action') - return await this.actionManager.runAction( + + const result = await this.actionManager.runAction( name, async () => { - const fn = action.perform + const fn = action.perform(this.mineflayer) return await fn(...params) }, - { timeout: 60, resume: false }, + { + timeout: options.timeout ?? 60, + resume: options.resume ?? false, + }, ) + + if (!result.success) { + throw new Error(result.message ?? 'Action failed') + } + + return this.formatActionOutput(result) } catch (error) { this.logger.withFields({ name, params, error }).error('Failed to perform action') throw error } + finally { + this.updateActionState(false) + } + } + + public async resumeAction(name: string, params: unknown[]): Promise { + const action = this.actions.get(name) + if (!action) { + throw new Error(`Action not found: ${name}`) + } + + try { + this.updateActionState(true, name) + const result = await this.actionManager.resumeAction( + name, + async () => { + const fn = action.perform(this.mineflayer) + return await fn(...params) + }, + 60, + ) + + if (!result.success) { + throw new Error(result.message ?? 'Action failed') + } + + return this.formatActionOutput(result) + } + catch (error) { + this.logger.withFields({ name, params, error }).error('Failed to resume action') + throw error + } + finally { + this.updateActionState(false) + } } public getAvailableActions(): Action[] { @@ -59,20 +135,32 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { } private async handleAgentMessage(sender: string, message: string): Promise { - // Handle messages from other agents or system if (sender === 'system') { - // Handle system messages if (message.includes('interrupt')) { await this.actionManager.stop() } } else { - // Handle agent messages - const convo = this.conversationStore.getConvo(sender) - if (convo.active.value) { - // Process message and potentially perform actions - this.logger.withFields({ sender, message }).log('Processing agent message') - } + this.logger.withFields({ sender, message }).log('Processing agent message') + } + } + + private updateActionState(executing: boolean, label = ''): void { + this.currentActionState = { + executing, + label, + startTime: executing ? Date.now() : 0, + } + this.emit('actionStateChanged', this.currentActionState) + } + + private formatActionOutput(result: { message: string | null, timedout: boolean, interrupted: boolean }): string { + if (result.timedout) { + return `Action timed out: ${result.message}` + } + if (result.interrupted) { + return 'Action was interrupted' } + return result.message ?? '' } } diff --git a/src/agents/actions.test.ts b/src/agents/actions.test.ts index ac6d974..c537b62 100644 --- a/src/agents/actions.test.ts +++ b/src/agents/actions.test.ts @@ -25,7 +25,6 @@ describe('actions agent', { timeout: 0 }, () => { user('What\'s your status?'), ), async (c) => { const completion = await c.reroute('query', c.messages, { model: 'openai/gpt-4o-mini' }) - console.log(JSON.stringify(completion, null, 2)) return await completion?.firstContent() }) @@ -40,24 +39,18 @@ describe('actions agent', { timeout: 0 }, () => { const { bot } = useBot() const agent = await initAgent(bot) - // console.log(JSON.stringify(agent, null, 2)) - await new Promise((resolve) => { bot.bot.on('spawn', async () => { const text = await agent.handle(messages( system(genActionAgentPrompt(bot)), user('goToPlayer: luoling8192'), ), async (c) => { - console.log(JSON.stringify(c, null, 2)) - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - console.log(JSON.stringify(completion, null, 2)) - return await completion?.firstContent() }) - console.log(JSON.stringify(text, null, 2)) + expect(text).toContain('goToPlayer') await sleep(10000) resolve() diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts index 62e143d..6f41da2 100644 --- a/src/agents/chat/index.ts +++ b/src/agents/chat/index.ts @@ -160,8 +160,8 @@ export class ChatAgentImpl extends AbstractAgent implements ChatAgent { } else { // Handle messages from other agents - const convo = this.conversationStore.getConvo(sender) - if (convo.active.value) { + const context = this.activeChats.get(sender) + if (context) { await this.processMessage(message, sender) } } diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts index e8cfd49..a277d1b 100644 --- a/src/agents/planning/factory.ts +++ b/src/agents/planning/factory.ts @@ -1,46 +1,42 @@ import type { Neuri } from 'neuri' +import type { AgentType } from 'src/libs/mineflayer/interfaces/agents' import type { PlanningAgentConfig } from '.' import type { Mineflayer } from '../../libs/mineflayer' import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { AgentFactory, AgentRegistry } from '../../libs/mineflayer/core/agent-factory' -const logger = useLogg('planning-factory').useGlobalConfig() - interface PlanningPluginOptions { agent: Neuri model?: string } +interface MineflayerWithPlanning extends Mineflayer { + planning: any +} + +const logger = useLogg('planning-factory').useGlobalConfig() + +async function initializeAgent(registry: AgentRegistry, id: string, type: string): Promise { + if (!registry.has(id)) { + const agent = AgentFactory.createAgent({ id, type: type as AgentType }) + registry.register(agent) + await agent.init() + } +} + export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin { return { - async created(bot: Mineflayer) { + async created(mineflayer: Mineflayer) { logger.log('Initializing planning plugin') - // Create and register agents const registry = AgentRegistry.getInstance() - // Create action agent if not exists - if (!registry.has('action-agent')) { - const actionAgent = AgentFactory.createAgent({ - id: 'action-agent', - type: 'action', - }) - registry.register(actionAgent) - await actionAgent.init() - } - - // Create memory agent if not exists - if (!registry.has('memory-agent')) { - const memoryAgent = AgentFactory.createAgent({ - id: 'memory-agent', - type: 'memory', - }) - registry.register(memoryAgent) - await memoryAgent.init() - } + // Initialize required agents + await initializeAgent(registry, 'action-agent', 'action') + await initializeAgent(registry, 'memory-agent', 'memory') - // Create planning agent + // Create and initialize planning agent const planningAgent = AgentFactory.createAgent({ id: 'planning-agent', type: 'planning', @@ -54,16 +50,12 @@ export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin await planningAgent.init() // Add planning agent to bot - bot.planning = planningAgent + ;(mineflayer as MineflayerWithPlanning).planning = planningAgent }, async beforeCleanup() { logger.log('Destroying planning plugin') - - const registry = AgentRegistry.getInstance() - await registry.destroy() - - // delete bot.planning + await AgentRegistry.getInstance().destroy() }, } } diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index e47267c..48ab48a 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -11,6 +11,11 @@ interface PlanContext { startTime: number lastUpdate: number retryCount: number + isGenerating: boolean + pendingSteps: Array<{ + action: string + params: unknown[] + }> } interface PlanTemplate { @@ -127,6 +132,8 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { startTime: Date.now(), lastUpdate: Date.now(), retryCount: 0, + isGenerating: false, + pendingSteps: [], } return plan @@ -157,45 +164,204 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { plan.status = 'in_progress' this.currentPlan = plan - for (let i = 0; i < plan.steps.length; i++) { - if (!this.context) + // Start generating and executing steps in parallel + await this.generateAndExecutePlanSteps(plan) + + plan.status = 'completed' + } + catch (error) { + plan.status = 'failed' + throw error + } + finally { + this.context = null + } + } + + private async generateAndExecutePlanSteps(plan: Plan): Promise { + if (!this.context || !this.actionAgent) { + return + } + + // Initialize step generation + this.context.isGenerating = true + this.context.pendingSteps = [] + + // Get available actions + const availableActions = this.actionAgent.getAvailableActions() + + // Start step generation + const generationPromise = this.generateStepsStream(plan.goal, availableActions) + + // Start step execution + const executionPromise = this.executeStepsStream() + + // Wait for both generation and execution to complete + await Promise.all([generationPromise, executionPromise]) + } + + private async generateStepsStream( + goal: string, + availableActions: Action[], + ): Promise { + if (!this.context) { + return + } + + try { + // Generate steps in chunks + const generator = this.createStepGenerator(goal, availableActions) + for await (const steps of generator) { + if (!this.context.isGenerating) { break + } - const step = plan.steps[i] - this.context.currentStep = i + // Add generated steps to pending queue + this.context.pendingSteps.push(...steps) + this.logger.withField('steps', steps).log('Generated new steps') + } + } + catch (error) { + this.logger.withError(error).error('Failed to generate steps') + throw error + } + finally { + this.context.isGenerating = false + } + } + + private async executeStepsStream(): Promise { + if (!this.context || !this.actionAgent) { + return + } + + try { + while (this.context.isGenerating || this.context.pendingSteps.length > 0) { + // Wait for steps to be available + if (this.context.pendingSteps.length === 0) { + await new Promise(resolve => setTimeout(resolve, 100)) + continue + } + + // Execute next step + const step = this.context.pendingSteps.shift() + if (!step) { + continue + } try { - this.logger.withFields({ step, index: i }).log('Executing plan step') + this.logger.withField('step', step).log('Executing step') await this.actionAgent.performAction(step.action, step.params) this.context.lastUpdate = Date.now() + this.context.currentStep++ } catch (stepError) { - this.logger.withError(stepError).error('Failed to execute plan step') + this.logger.withError(stepError).error('Failed to execute step') // Attempt to adjust plan and retry if (this.context.retryCount < 3) { this.context.retryCount++ - const adjustedPlan = await this.adjustPlan(plan, stepError instanceof Error ? stepError.message : 'Unknown error') + // Stop current generation + this.context.isGenerating = false + this.context.pendingSteps = [] + // Adjust plan and restart + const adjustedPlan = await this.adjustPlan( + this.currentPlan!, + stepError instanceof Error ? stepError.message : 'Unknown error', + ) await this.executePlan(adjustedPlan) return } - plan.status = 'failed' throw stepError } } - - plan.status = 'completed' } catch (error) { - plan.status = 'failed' + this.logger.withError(error).error('Failed to execute steps') throw error } - finally { - this.context = null + } + + private async *createStepGenerator( + goal: string, + availableActions: Action[], + ): AsyncGenerator, void, unknown> { + // First, try to find a matching template + const template = this.findMatchingTemplate(goal) + if (template) { + this.logger.log('Using plan template') + yield template.steps + return + } + + // If no template matches, use LLM to generate plan in chunks + this.logger.log('Generating plan using LLM') + const chunkSize = 3 // Generate 3 steps at a time + let currentChunk = 1 + + while (true) { + const steps = await generatePlanWithLLM( + goal, + availableActions, + { + agent: this.llmConfig.agent, + model: this.llmConfig.model, + }, + `Generate steps ${currentChunk * chunkSize - 2} to ${currentChunk * chunkSize}`, + ) + + if (steps.length === 0) { + break + } + + yield steps + currentChunk++ + + // Check if we've generated enough steps or if the goal is achieved + if (steps.length < chunkSize || await this.isGoalAchieved(goal)) { + break + } } } + private async isGoalAchieved(goal: string): Promise { + if (!this.context || !this.actionAgent) { + return false + } + + const requirements = this.parseGoalRequirements(goal) + + // Check inventory for required items + if (requirements.needsItems && requirements.items) { + const inventorySteps = this.generateGatheringSteps(requirements.items) + if (inventorySteps.length > 0) { + this.context.pendingSteps.push(...inventorySteps) + return false + } + } + + // Check location requirements + if (requirements.needsMovement && requirements.location) { + const movementSteps = this.generateMovementSteps(requirements.location) + if (movementSteps.length > 0) { + this.context.pendingSteps.push(...movementSteps) + return false + } + } + + // Check interaction requirements + if (requirements.needsInteraction && requirements.target) { + const interactionSteps = this.generateInteractionSteps(requirements.target) + if (interactionSteps.length > 0) { + this.context.pendingSteps.push(...interactionSteps) + return false + } + } + + return true + } + public async adjustPlan(plan: Plan, feedback: string): Promise { if (!this.initialized) { throw new Error('Planning agent not initialized') @@ -209,6 +375,9 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { const currentStep = this.context.currentStep const availableActions = this.actionAgent?.getAvailableActions() ?? [] + // Generate recovery steps based on feedback + const recoverySteps = this.generateRecoverySteps(feedback) + // Generate new steps from the current point const newSteps = await this.generatePlanSteps(plan.goal, availableActions, feedback) @@ -217,9 +386,11 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { goal: plan.goal, steps: [ ...plan.steps.slice(0, currentStep), + ...recoverySteps, ...newSteps, ], status: 'pending', + requiresAction: true, } return adjustedPlan @@ -234,103 +405,21 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - private async generatePlanSteps( - goal: string, - availableActions: Action[], - feedback?: string, - ): Promise> { - // First, try to find a matching template - const template = this.findMatchingTemplate(goal) - if (template) { - this.logger.log('Using plan template') - return template.steps - } - - // If no template matches, use LLM to generate plan - this.logger.log('Generating plan using LLM') - return await generatePlanWithLLM(goal, availableActions, { - agent: this.llmConfig.agent, - model: this.llmConfig.model, - }, feedback) - } - - private findMatchingTemplate(goal: string): PlanTemplate | undefined { - for (const [pattern, template] of this.planTemplates.entries()) { - if (goal.toLowerCase().includes(pattern.toLowerCase())) { - return template - } - } - return undefined - } - - private parseGoalRequirements(goal: string): { - needsItems: boolean - items?: string[] - needsMovement: boolean - location?: { x?: number, y?: number, z?: number } - needsInteraction: boolean - target?: string - needsCrafting: boolean - needsCombat: boolean - } { - const requirements = { - needsItems: false, - needsMovement: false, - needsInteraction: false, - needsCrafting: false, - needsCombat: false, - } - - const goalLower = goal.toLowerCase() - - // Check for item-related actions - if (goalLower.includes('collect') || goalLower.includes('get') || goalLower.includes('find')) { - requirements.needsItems = true - requirements.needsMovement = true - } - - // Check for movement-related actions - if (goalLower.includes('go to') || goalLower.includes('move to') || goalLower.includes('follow')) { - requirements.needsMovement = true - } - - // Check for interaction-related actions - if (goalLower.includes('interact') || goalLower.includes('use') || goalLower.includes('open')) { - requirements.needsInteraction = true - } - - // Check for crafting-related actions - if (goalLower.includes('craft') || goalLower.includes('make') || goalLower.includes('build')) { - requirements.needsCrafting = true - requirements.needsItems = true - } - - // Check for combat-related actions - if (goalLower.includes('attack') || goalLower.includes('fight') || goalLower.includes('kill')) { - requirements.needsCombat = true - requirements.needsMovement = true - } - - return requirements - } - - private generateGatheringSteps(items?: string[]): Array<{ action: string, params: unknown[] }> { + private generateGatheringSteps(items: string[]): Array<{ action: string, params: unknown[] }> { const steps: Array<{ action: string, params: unknown[] }> = [] - if (items) { - for (const item of items) { - steps.push( - { action: 'searchForBlock', params: [item, 64] }, - { action: 'collectBlocks', params: [item, 1] }, - ) - } + for (const item of items) { + steps.push( + { action: 'searchForBlock', params: [item, 64] }, + { action: 'collectBlocks', params: [item, 1] }, + ) } return steps } - private generateMovementSteps(location?: { x?: number, y?: number, z?: number }): Array<{ action: string, params: unknown[] }> { - if (location?.x !== undefined && location?.y !== undefined && location?.z !== undefined) { + private generateMovementSteps(location: { x?: number, y?: number, z?: number }): Array<{ action: string, params: unknown[] }> { + if (location.x !== undefined && location.y !== undefined && location.z !== undefined) { return [{ action: 'goToCoordinates', params: [location.x, location.y, location.z, 1], @@ -339,14 +428,11 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return [] } - private generateInteractionSteps(target?: string): Array<{ action: string, params: unknown[] }> { - if (target) { - return [{ - action: 'activate', - params: [target], - }] - } - return [] + private generateInteractionSteps(target: string): Array<{ action: string, params: unknown[] }> { + return [{ + action: 'activate', + params: [target], + }] } private generateRecoverySteps(feedback: string): Array<{ action: string, params: unknown[] }> { @@ -360,6 +446,21 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { steps.push({ action: 'discard', params: ['cobblestone', 64] }) } + if (feedback.includes('blocked') || feedback.includes('cannot reach')) { + steps.push({ action: 'moveAway', params: [5] }) + } + + if (feedback.includes('too far')) { + steps.push({ action: 'moveAway', params: [-3] }) // Move closer + } + + if (feedback.includes('need tool')) { + steps.push( + { action: 'craftRecipe', params: ['wooden_pickaxe', 1] }, + { action: 'equip', params: ['wooden_pickaxe'] }, + ) + } + return steps } @@ -381,7 +482,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { this.memoryAgent.remember(`plan:${plan.goal}`, plan) } - private isPlanValid(plan: Plan): boolean { + private isPlanValid(_plan: Plan): boolean { // Add validation logic here return true } @@ -431,10 +532,12 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } else { - const convo = this.conversationStore.getConvo(sender) - if (convo.active.value) { - // Process message and potentially adjust plan - this.logger.withFields({ sender, message }).log('Processing agent message') + // Process message and potentially adjust plan + this.logger.withFields({ sender, message }).log('Processing agent message') + + // If there's a current plan, try to adjust it based on the message + if (this.currentPlan) { + await this.adjustPlan(this.currentPlan, message) } } } @@ -454,4 +557,109 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { || requirements.needsCrafting || requirements.needsCombat } + + private async generatePlanSteps( + goal: string, + availableActions: Action[], + feedback?: string, + ): Promise> { + // First, try to find a matching template + const template = this.findMatchingTemplate(goal) + if (template) { + this.logger.log('Using plan template') + return template.steps + } + + // If no template matches, use LLM to generate plan + this.logger.log('Generating plan using LLM') + return await generatePlanWithLLM(goal, availableActions, { + agent: this.llmConfig.agent, + model: this.llmConfig.model, + }, feedback) + } + + private findMatchingTemplate(goal: string): PlanTemplate | undefined { + for (const [pattern, template] of this.planTemplates.entries()) { + if (goal.toLowerCase().includes(pattern.toLowerCase())) { + return template + } + } + return undefined + } + + private parseGoalRequirements(goal: string): { + needsItems: boolean + items?: string[] + needsMovement: boolean + location?: { x?: number, y?: number, z?: number } + needsInteraction: boolean + target?: string + needsCrafting: boolean + needsCombat: boolean + } { + const requirements = { + needsItems: false, + items: [] as string[], + needsMovement: false, + location: undefined as { x?: number, y?: number, z?: number } | undefined, + needsInteraction: false, + target: undefined as string | undefined, + needsCrafting: false, + needsCombat: false, + } + + const goalLower = goal.toLowerCase() + + // Extract items from goal + const itemMatches = goalLower.match(/(collect|get|find|craft|make|build|use|equip) (\w+)/g) + if (itemMatches) { + requirements.needsItems = true + requirements.items = itemMatches.map(match => match.split(' ')[1]) + } + + // Extract location from goal + const locationMatches = goalLower.match(/(go to|move to|at) (\d+)[, ]+(\d+)[, ]+(\d+)/g) + if (locationMatches) { + requirements.needsMovement = true + const [x, y, z] = locationMatches[0].split(/[, ]+/).slice(-3).map(Number) + requirements.location = { x, y, z } + } + + // Extract target from goal + const targetMatches = goalLower.match(/(interact with|use|open|activate) (\w+)/g) + if (targetMatches) { + requirements.needsInteraction = true + requirements.target = targetMatches[0].split(' ').pop() + } + + // Check for item-related actions + if (goalLower.includes('collect') || goalLower.includes('get') || goalLower.includes('find')) { + requirements.needsItems = true + requirements.needsMovement = true + } + + // Check for movement-related actions + if (goalLower.includes('go to') || goalLower.includes('move to') || goalLower.includes('follow')) { + requirements.needsMovement = true + } + + // Check for interaction-related actions + if (goalLower.includes('interact') || goalLower.includes('use') || goalLower.includes('open')) { + requirements.needsInteraction = true + } + + // Check for crafting-related actions + if (goalLower.includes('craft') || goalLower.includes('make') || goalLower.includes('build')) { + requirements.needsCrafting = true + requirements.needsItems = true + } + + // Check for combat-related actions + if (goalLower.includes('attack') || goalLower.includes('fight') || goalLower.includes('kill')) { + requirements.needsCombat = true + requirements.needsMovement = true + } + + return requirements + } } diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index a0b133c..c1187b0 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -1,6 +1,8 @@ import type { Neuri } from 'neuri' +import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' +import { genPlanningAgentPrompt } from '../../prompts/agent' import { toRetriable } from '../../utils/reliability' const logger = useLogg('planning-llm').useGlobalConfig() @@ -14,11 +16,11 @@ interface LLMPlanningConfig { export async function generatePlanWithLLM( goal: string, - availableActions: Array<{ name: string, description: string }>, + availableActions: Action[], config: LLMPlanningConfig, feedback?: string, ): Promise> { - const systemPrompt = generateSystemPrompt(availableActions) + const systemPrompt = genPlanningAgentPrompt(availableActions) const userPrompt = generateUserPrompt(goal, feedback) const messages = [ @@ -60,32 +62,6 @@ export async function generatePlanWithLLM( return parsePlanContent(content) } -function generateSystemPrompt(availableActions: Array<{ name: string, description: string }>): string { - const actionsList = availableActions - .map(action => `- ${action.name}: ${action.description}`) - .join('\n') - - return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. -Available actions: -${actionsList} - -Respond with a JSON array of steps, where each step has: -- action: The name of the action to perform -- params: Array of parameters for the action - -Example response: -[ - { - "action": "searchForBlock", - "params": ["log", 64] - }, - { - "action": "collectBlocks", - "params": ["log", 1] - } -]` -} - function generateUserPrompt(goal: string, feedback?: string): string { let prompt = `Create a plan to: ${goal}` if (feedback) { diff --git a/src/composables/action.ts b/src/composables/action.ts index 9f91f15..5b38658 100644 --- a/src/composables/action.ts +++ b/src/composables/action.ts @@ -1,163 +1,207 @@ +import type { Bot } from 'mineflayer' import type { Mineflayer } from '../libs/mineflayer/core' import { useLogg } from '@guiiai/logg' -type Fn = (...args: any[]) => void +// Types and interfaces +type ActionFn = (...args: any[]) => void + +interface ActionResult { + success: boolean + message: string | null + interrupted: boolean + timedout: boolean +} + +interface BotWithExtensions extends Bot { + isIdle: () => boolean + interrupt_code: boolean + output: string +} + +export interface MineflayerWithExtensions extends Mineflayer { + bot: BotWithExtensions + self_prompter: { + on: boolean + } + coder: { + generating: boolean + } + clearBotLogs: () => void + history: { + add: (source: string, message: string) => void + } +} + +interface ActionState { + executing: { value: boolean } + currentActionLabel: { value: string | undefined } + currentActionFn: { value: ActionFn | undefined } + timedout: { value: boolean } + resume: { + func: { value: ActionFn | undefined } + name: { value: string | undefined } + } +} + +export function useActionManager(mineflayer: MineflayerWithExtensions) { + // Initialize state + const state: ActionState = { + executing: { value: false }, + currentActionLabel: { value: '' }, + currentActionFn: { value: undefined }, + timedout: { value: false }, + resume: { + func: { value: undefined }, + name: { value: undefined }, + }, + } -export function useActionManager(mineflayer: Mineflayer) { - const executing: { value: boolean } = { value: false } - const currentActionLabel: { value: string | undefined } = { value: '' } - const currentActionFn: { value: (Fn) | undefined } = { value: undefined } - const timedout: { value: boolean } = { value: false } - const resume_func: { value: (Fn) | undefined } = { value: undefined } - const resume_name: { value: string | undefined } = { value: undefined } const log = useLogg('ActionManager').useGlobalConfig() - async function resumeAction(actionLabel: string, actionFn: Fn, timeout: number) { + // Public API + async function resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { return _executeResume(actionLabel, actionFn, timeout) } - async function runAction(actionLabel: string, actionFn: Fn, options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }) { - if (options.resume) { - return _executeResume(actionLabel, actionFn, options.timeout) - } - else { - return _executeAction(actionLabel, actionFn, options.timeout) - } + async function runAction( + actionLabel: string, + actionFn: ActionFn, + options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }, + ): Promise { + return options.resume + ? _executeResume(actionLabel, actionFn, options.timeout) + : _executeAction(actionLabel, actionFn, options.timeout) } - async function stop() { - // if (!executing.value) - // return - // const timeout = setTimeout(() => { - // mineflayer.cleanKill('Code execution refused stop after 10 seconds. Killing process.') - // }, 10000) - // while (executing.value) { - // mineflayer.requestInterrupt() - // log.log('waiting for code to finish executing...') - // await new Promise(resolve => setTimeout(resolve, 300)) - // } - // clearTimeout(timeout) + async function stop(): Promise { mineflayer.emit('interrupt') } - function cancelResume() { - resume_func.value = undefined - resume_name.value = undefined + function cancelResume(): void { + state.resume.func.value = undefined + state.resume.name.value = undefined } - async function _executeResume(actionLabel?: string, actionFn?: Fn, timeout = 10) { - const new_resume = actionFn != null - if (new_resume) { // start new resume - resume_func.value = actionFn - if (actionLabel == null) { + // Private helpers + async function _executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { + const isNewResume = actionFn != null + + if (isNewResume) { + if (!actionLabel) { throw new Error('actionLabel is required for new resume') } - resume_name.value = actionLabel + state.resume.func.value = actionFn + state.resume.name.value = actionLabel } - if (resume_func.value != null && (mineflayer.isIdle() || new_resume) && (!mineflayer.self_prompter.on || new_resume)) { - currentActionLabel.value = resume_name.value - const res = await _executeAction(resume_name.value, resume_func.value, timeout) - currentActionLabel.value = '' - return res - } - else { + + const canExecute = state.resume.func.value != null + && (mineflayer.bot.isIdle() || isNewResume) + && (!mineflayer.self_prompter.on || isNewResume) + + if (!canExecute) { return { success: false, message: null, interrupted: false, timedout: false } } + + state.currentActionLabel.value = state.resume.name.value + const result = await _executeAction(state.resume.name.value, state.resume.func.value, timeout) + state.currentActionLabel.value = '' + return result } - async function _executeAction(actionLabel?: string, actionFn?: Fn, timeout = 10) { - let TIMEOUT + async function _executeAction(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { + let timeoutHandle: NodeJS.Timeout | undefined + try { log.log('executing code...\n') - // await current action to finish (executing=false), with 10 seconds timeout - // also tell mineflayer.bot to stop various actions - if (executing.value) { - log.log(`action "${actionLabel}" trying to interrupt current action "${currentActionLabel.value}"`) + if (state.executing.value) { + log.log(`action "${actionLabel}" trying to interrupt current action "${state.currentActionLabel.value}"`) } - await stop() - // clear bot logs and reset interrupt code + await stop() mineflayer.clearBotLogs() - executing.value = true - currentActionLabel.value = actionLabel - currentActionFn.value = actionFn + // Set execution state + state.executing.value = true + state.currentActionLabel.value = actionLabel + state.currentActionFn.value = actionFn - // timeout in minutes if (timeout > 0) { - TIMEOUT = _startTimeout(timeout) + timeoutHandle = _startTimeout(timeout) } - // start the action await actionFn?.() - // mark action as finished + cleanup - executing.value = false - currentActionLabel.value = '' - currentActionFn.value = undefined - clearTimeout(TIMEOUT) + // Reset state after successful execution + _resetExecutionState(timeoutHandle) - // get bot activity summary const output = _getBotOutputSummary() const interrupted = mineflayer.bot.interrupt_code mineflayer.clearBotLogs() - // if not interrupted and not generating, emit idle event if (!interrupted && !mineflayer.coder.generating) { - mineflayer.bot.emit('idle') + mineflayer.bot.emit('idle' as any) } - // return action status report - return { success: true, message: output, interrupted, timedout } + return { success: true, message: output, interrupted, timedout: false } } catch (err) { - executing.value = false - currentActionLabel.value = '' - currentActionFn.value = undefined - clearTimeout(TIMEOUT) + _resetExecutionState(timeoutHandle) cancelResume() log.withError(err).error('Code execution triggered catch') await stop() - const message = `${_getBotOutputSummary() - }!!Code threw exception!!\n` - + `Error: ${err}\n` - + `Stack trace:\n${(err as Error).stack}` - + const message = _formatErrorMessage(err as Error) const interrupted = mineflayer.bot.interrupt_code mineflayer.clearBotLogs() + if (!interrupted && !mineflayer.coder.generating) { - mineflayer.bot.emit('idle') + mineflayer.bot.emit('idle' as any) } + return { success: false, message, interrupted, timedout: false } } } - function _getBotOutputSummary() { + function _resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { + state.executing.value = false + state.currentActionLabel.value = '' + state.currentActionFn.value = undefined + if (timeoutHandle) + clearTimeout(timeoutHandle) + } + + function _formatErrorMessage(error: Error): string { + return `${_getBotOutputSummary()}!!Code threw exception!!\nError: ${error}\nStack trace:\n${error.stack}` + } + + function _getBotOutputSummary(): string { const { bot } = mineflayer - if (bot.interrupt_code && !timedout.value) + if (bot.interrupt_code && !state.timedout.value) { return '' - let output = bot.output - const MAX_OUT = 500 - if (output.length > MAX_OUT) { - output = `Code output is very long (${output.length} chars) and has been shortened.\n - First outputs:\n${output.substring(0, MAX_OUT / 2)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - MAX_OUT / 2)}` - } - else { - output = `Code output:\n${output}` } + const MAX_OUT = 500 + const output = bot.output.length > MAX_OUT + ? _truncateOutput(bot.output, MAX_OUT) + : `Code output:\n${bot.output}` + return output } - function _startTimeout(TIMEOUT_MINS = 10) { + function _truncateOutput(output: string, maxLength: number): string { + const halfLength = maxLength / 2 + return `Code output is very long (${output.length} chars) and has been shortened.\n + First outputs:\n${output.substring(0, halfLength)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - halfLength)}` + } + + function _startTimeout(timeoutMins = 10): NodeJS.Timeout { return setTimeout(async () => { - log.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) - timedout.value = true - mineflayer.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) - await stop() // last attempt to stop - }, TIMEOUT_MINS * 60 * 1000) + log.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + state.timedout.value = true + mineflayer.history.add('system', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + await stop() + }, timeoutMins * 60 * 1000) } return { diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts index d1a6c5d..f865e1d 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/mineflayer/llm-agent.ts @@ -1,22 +1,31 @@ import type { Client } from '@proj-airi/server-sdk' import type { Neuri, NeuriContext } from 'neuri' import type { ChatCompletion } from 'neuri/openai' -import type { PlanningAgent } from '../libs/mineflayer/interfaces/agents' +import type { Mineflayer } from '../libs/mineflayer' +import type { ActionAgent, PlanningAgent } from '../libs/mineflayer/interfaces/agents' import type { MineflayerPlugin } from '../libs/mineflayer/plugin' + import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' - import { PlanningPlugin } from '../agents/planning/factory' +import { AgentRegistry } from '../libs/mineflayer/core/agent-factory' import { formBotChat } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' import { toRetriable } from '../utils/reliability' +interface MineflayerWithAgents extends Mineflayer { + planning: PlanningAgent + action: ActionAgent +} + interface LLMAgentOptions { agent: Neuri airiClient: Client } -async function handleLLMCompletion(context: NeuriContext, bot: any, logger: any): Promise { +async function handleLLMCompletion(context: NeuriContext, bot: MineflayerWithAgents, logger: ReturnType): Promise { + logger.log('rerouting...') + const completion = await context.reroute('action', context.messages, { model: 'openai/gpt-4o-mini', }) as ChatCompletion | { error: { message: string } } & ChatCompletion @@ -24,7 +33,7 @@ async function handleLLMCompletion(context: NeuriContext, bot: any, logger: any) if (!completion || 'error' in completion) { logger.withFields({ completion }).error('Completion') logger.withFields({ messages: context.messages }).log('messages') - throw new Error(completion?.error?.message ?? 'Unknown error') + return completion?.error?.message ?? 'Unknown error' } const content = await completion.firstContent() @@ -34,32 +43,52 @@ async function handleLLMCompletion(context: NeuriContext, bot: any, logger: any) return content } -async function handleChatMessage(username: string, message: string, bot: any, agent: Neuri, logger: any): Promise { +async function handleChatMessage(username: string, message: string, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { logger.withFields({ username, message }).log('Chat message received') bot.memory.chatHistory.push(user(`${username}: ${message}`)) - const statusPrompt = await genStatusPrompt(bot) - const retryHandler = toRetriable( - 3, - 1000, - ctx => handleLLMCompletion(ctx, bot, logger), - ) - - const content = await agent.handleStateless( - [...bot.memory.chatHistory, system(statusPrompt)], - async (c: NeuriContext) => { - logger.log('thinking...') - return retryHandler(c) - }, - ) + logger.log('thinking...') + + try { + // 创建并执行计划 + const plan = await bot.planning.createPlan(message) + logger.withFields({ plan }).log('Plan created') + await bot.planning.executePlan(plan) + logger.log('Plan executed successfully') + + // 生成回复 + const statusPrompt = await genStatusPrompt(bot) + const retryHandler = toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + { onError: err => logger.withError(err).log('error occurred') }, + ) - if (content) { - logger.withFields({ content }).log('responded') - bot.bot.chat(content) + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('handling...') + return retryHandler(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) } } -async function handleVoiceInput(event: any, bot: any, agent: Neuri, logger: any): Promise { +async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { logger .withFields({ user: event.data.discord?.guildMember, @@ -72,13 +101,13 @@ async function handleVoiceInput(event: any, bot: any, agent: Neuri, logger: any) bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) try { - const planningAgent = bot.planning as PlanningAgent - const plan = await planningAgent.createPlan(event.data.transcription) + // 创建并执行计划 + const plan = await bot.planning.createPlan(event.data.transcription) logger.withFields({ plan }).log('Plan created') - - await planningAgent.executePlan(plan) + await bot.planning.executePlan(plan) logger.log('Plan executed successfully') + // 生成回复 const retryHandler = toRetriable( 3, 1000, @@ -114,19 +143,30 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { const agent = options.agent const logger = useLogg('LLMAgent').useGlobalConfig() + // 初始化 Planning Agent const planningPlugin = PlanningPlugin({ agent: options.agent, model: 'openai/gpt-4o-mini', }) await planningPlugin.created!(bot) + // 获取 Action Agent + const registry = AgentRegistry.getInstance() + const actionAgent = registry.get('action-agent', 'action') + + // 类型转换 + const botWithAgents = bot as unknown as MineflayerWithAgents + botWithAgents.action = actionAgent + + // 初始化系统提示 bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) + // 设置消息处理 const onChat = formBotChat(bot.username, (username, message) => - handleChatMessage(username, message, bot, agent, logger)) + handleChatMessage(username, message, botWithAgents, agent, logger)) options.airiClient.onEvent('input:text:voice', event => - handleVoiceInput(event, bot, agent, logger)) + handleVoiceInput(event, botWithAgents, agent, logger)) bot.bot.on('chat', onChat) }, diff --git a/src/prompts/agent.ts b/src/prompts/agent.ts index c27144b..6e52276 100644 --- a/src/prompts/agent.ts +++ b/src/prompts/agent.ts @@ -1,4 +1,4 @@ -import type { Mineflayer } from '../libs/mineflayer' +import type { Action, Mineflayer } from '../libs/mineflayer' import { listInventory } from '../skills/actions/inventory' export function genSystemBasicPrompt(botName: string): string { @@ -60,3 +60,31 @@ ${mineflayer.status.toOneLiner()} return prompt } + +export function genPlanningAgentPrompt(availableActions: Action[]): string { + const actionsList = availableActions + .map(action => `- ${action.name}: ${action.description}`) + .join('\n') + + return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. +Available actions: +${actionsList} + +Respond with a Valid JSON array of steps, where each step has: +- action: The name of the action to perform +- params: Array of parameters for the action + +DO NOT contains any \`\`\` or explation, otherwise agent will be interrupted. + +Example response: +[ + { + "action": "searchForBlock", + "params": ["log", 64] + }, + { + "action": "collectBlocks", + "params": ["log", 1] + } + ]` +} From 00d2b063f6ed58bdd17155be64d247ba8590b275 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 17 Jan 2025 18:58:28 +0800 Subject: [PATCH 03/22] fix: action manager --- src/agents/action/index.ts | 28 ++-- src/composables/action.ts | 257 +++++++++++++++++-------------------- 2 files changed, 140 insertions(+), 145 deletions(-) diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 98aca80..5968de2 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -1,7 +1,7 @@ -import type { MineflayerWithExtensions } from '../../composables/action' +import type { Mineflayer } from '../../libs/mineflayer' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/interfaces/agents' -import { useActionManager } from '../../composables/action' +import { ActionManager } from '../../composables/action' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' import { actionsList } from '../actions' @@ -12,18 +12,22 @@ interface ActionState { startTime: number } +/** + * ActionAgentImpl implements the ActionAgent interface to handle action execution + * Manages action lifecycle, state tracking and error handling + */ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { public readonly type = 'action' as const private actions: Map - private actionManager: ReturnType - private mineflayer: MineflayerWithExtensions + private actionManager: ActionManager + private mineflayer: Mineflayer private currentActionState: ActionState constructor(config: AgentConfig) { super(config) this.actions = new Map() - this.mineflayer = useBot().bot as MineflayerWithExtensions - this.actionManager = useActionManager(this.mineflayer) + this.mineflayer = useBot().bot + this.actionManager = new ActionManager(this.mineflayer) this.currentActionState = { executing: false, label: '', @@ -87,7 +91,11 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { throw new Error(result.message ?? 'Action failed') } - return this.formatActionOutput(result) + return this.formatActionOutput({ + message: result.message, + timedout: result.timedout, + interrupted: false, + }) } catch (error) { this.logger.withFields({ name, params, error }).error('Failed to perform action') @@ -119,7 +127,11 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { throw new Error(result.message ?? 'Action failed') } - return this.formatActionOutput(result) + return this.formatActionOutput({ + message: result.message, + timedout: result.timedout, + interrupted: false, + }) } catch (error) { this.logger.withFields({ name, params, error }).error('Failed to resume action') diff --git a/src/composables/action.ts b/src/composables/action.ts index 5b38658..96eeaed 100644 --- a/src/composables/action.ts +++ b/src/composables/action.ts @@ -1,6 +1,6 @@ -import type { Bot } from 'mineflayer' import type { Mineflayer } from '../libs/mineflayer/core' import { useLogg } from '@guiiai/logg' +import EventEmitter from 'eventemitter3' // Types and interfaces type ActionFn = (...args: any[]) => void @@ -8,206 +8,189 @@ type ActionFn = (...args: any[]) => void interface ActionResult { success: boolean message: string | null - interrupted: boolean timedout: boolean } -interface BotWithExtensions extends Bot { - isIdle: () => boolean - interrupt_code: boolean - output: string +interface QueuedAction { + label: string + fn: ActionFn + timeout: number + resume: boolean } -export interface MineflayerWithExtensions extends Mineflayer { - bot: BotWithExtensions - self_prompter: { - on: boolean - } - coder: { - generating: boolean - } - clearBotLogs: () => void - history: { - add: (source: string, message: string) => void - } -} - -interface ActionState { - executing: { value: boolean } - currentActionLabel: { value: string | undefined } - currentActionFn: { value: ActionFn | undefined } - timedout: { value: boolean } - resume: { - func: { value: ActionFn | undefined } - name: { value: string | undefined } - } -} - -export function useActionManager(mineflayer: MineflayerWithExtensions) { - // Initialize state - const state: ActionState = { - executing: { value: false }, - currentActionLabel: { value: '' }, - currentActionFn: { value: undefined }, - timedout: { value: false }, +export class ActionManager extends EventEmitter { + private state = { + executing: false, + currentActionLabel: '', + currentActionFn: undefined as ActionFn | undefined, + timedout: false, resume: { - func: { value: undefined }, - name: { value: undefined }, + func: undefined as ActionFn | undefined, + name: undefined as string | undefined, }, } - const log = useLogg('ActionManager').useGlobalConfig() + // Action queue to store pending actions + private actionQueue: QueuedAction[] = [] + + private logger = useLogg('ActionManager').useGlobalConfig() + private mineflayer: Mineflayer - // Public API - async function resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { - return _executeResume(actionLabel, actionFn, timeout) + constructor(mineflayer: Mineflayer) { + super() + this.mineflayer = mineflayer } - async function runAction( + public async resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { + return this.queueAction({ + label: actionLabel, + fn: actionFn, + timeout, + resume: true, + }) + } + + public async runAction( actionLabel: string, actionFn: ActionFn, options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }, ): Promise { - return options.resume - ? _executeResume(actionLabel, actionFn, options.timeout) - : _executeAction(actionLabel, actionFn, options.timeout) + return this.queueAction({ + label: actionLabel, + fn: actionFn, + timeout: options.timeout, + resume: options.resume, + }) } - async function stop(): Promise { - mineflayer.emit('interrupt') + public async stop(): Promise { + this.mineflayer.emit('interrupt') + // Clear the action queue when stopping + this.actionQueue = [] } - function cancelResume(): void { - state.resume.func.value = undefined - state.resume.name.value = undefined + public cancelResume(): void { + this.state.resume.func = undefined + this.state.resume.name = undefined + } + + private async queueAction(action: QueuedAction): Promise { + // Add action to queue + this.actionQueue.push(action) + + // If not executing, start processing queue + if (!this.state.executing) { + return this.processQueue() + } + + // Return a promise that will resolve when the action is executed + return new Promise((resolve) => { + const checkQueue = setInterval(() => { + const index = this.actionQueue.findIndex(a => a === action) + if (index === -1) { + clearInterval(checkQueue) + resolve({ success: true, message: 'success', timedout: false }) + } + }, 100) + }) + } + + private async processQueue(): Promise { + while (this.actionQueue.length > 0) { + const action = this.actionQueue[0] + + const result = action.resume + ? await this.executeResume(action.label, action.fn, action.timeout) + : await this.executeAction(action.label, action.fn, action.timeout) + + // Remove completed action from queue + this.actionQueue.shift() + + if (!result.success) { + // Clear queue on failure + this.actionQueue = [] + return result + } + } + + return { success: true, message: 'success', timedout: false } } - // Private helpers - async function _executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { + private async executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { const isNewResume = actionFn != null if (isNewResume) { if (!actionLabel) { throw new Error('actionLabel is required for new resume') } - state.resume.func.value = actionFn - state.resume.name.value = actionLabel + this.state.resume.func = actionFn + this.state.resume.name = actionLabel } - const canExecute = state.resume.func.value != null - && (mineflayer.bot.isIdle() || isNewResume) - && (!mineflayer.self_prompter.on || isNewResume) + const canExecute = this.state.resume.func != null && isNewResume if (!canExecute) { - return { success: false, message: null, interrupted: false, timedout: false } + return { success: false, message: null, timedout: false } } - state.currentActionLabel.value = state.resume.name.value - const result = await _executeAction(state.resume.name.value, state.resume.func.value, timeout) - state.currentActionLabel.value = '' + this.state.currentActionLabel = this.state.resume.name || '' + const result = await this.executeAction(this.state.resume.name || '', this.state.resume.func, timeout) + this.state.currentActionLabel = '' return result } - async function _executeAction(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { + private async executeAction(actionLabel: string, actionFn?: ActionFn, timeout = 10): Promise { let timeoutHandle: NodeJS.Timeout | undefined try { - log.log('executing code...\n') + this.logger.log('executing action...\n') - if (state.executing.value) { - log.log(`action "${actionLabel}" trying to interrupt current action "${state.currentActionLabel.value}"`) + if (this.state.executing) { + this.logger.log(`action "${actionLabel}" trying to interrupt current action "${this.state.currentActionLabel}"`) } - await stop() - mineflayer.clearBotLogs() + await this.stop() // Set execution state - state.executing.value = true - state.currentActionLabel.value = actionLabel - state.currentActionFn.value = actionFn + this.state.executing = true + this.state.currentActionLabel = actionLabel + this.state.currentActionFn = actionFn if (timeout > 0) { - timeoutHandle = _startTimeout(timeout) + timeoutHandle = this.startTimeout(timeout) } await actionFn?.() // Reset state after successful execution - _resetExecutionState(timeoutHandle) - - const output = _getBotOutputSummary() - const interrupted = mineflayer.bot.interrupt_code - mineflayer.clearBotLogs() + this.resetExecutionState(timeoutHandle) - if (!interrupted && !mineflayer.coder.generating) { - mineflayer.bot.emit('idle' as any) - } - - return { success: true, message: output, interrupted, timedout: false } + return { success: true, message: 'success', timedout: false } } catch (err) { - _resetExecutionState(timeoutHandle) - cancelResume() - log.withError(err).error('Code execution triggered catch') - await stop() - - const message = _formatErrorMessage(err as Error) - const interrupted = mineflayer.bot.interrupt_code - mineflayer.clearBotLogs() + this.resetExecutionState(timeoutHandle) + this.cancelResume() + this.logger.withError(err).error('Code execution triggered catch') + await this.stop() - if (!interrupted && !mineflayer.coder.generating) { - mineflayer.bot.emit('idle' as any) - } - - return { success: false, message, interrupted, timedout: false } + return { success: false, message: 'failed', timedout: false } } } - function _resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { - state.executing.value = false - state.currentActionLabel.value = '' - state.currentActionFn.value = undefined + private resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { + this.state.executing = false + this.state.currentActionLabel = '' + this.state.currentActionFn = undefined if (timeoutHandle) clearTimeout(timeoutHandle) } - function _formatErrorMessage(error: Error): string { - return `${_getBotOutputSummary()}!!Code threw exception!!\nError: ${error}\nStack trace:\n${error.stack}` - } - - function _getBotOutputSummary(): string { - const { bot } = mineflayer - if (bot.interrupt_code && !state.timedout.value) { - return '' - } - - const MAX_OUT = 500 - const output = bot.output.length > MAX_OUT - ? _truncateOutput(bot.output, MAX_OUT) - : `Code output:\n${bot.output}` - - return output - } - - function _truncateOutput(output: string, maxLength: number): string { - const halfLength = maxLength / 2 - return `Code output is very long (${output.length} chars) and has been shortened.\n - First outputs:\n${output.substring(0, halfLength)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - halfLength)}` - } - - function _startTimeout(timeoutMins = 10): NodeJS.Timeout { + private startTimeout(timeoutMins = 10): NodeJS.Timeout { return setTimeout(async () => { - log.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - state.timedout.value = true - mineflayer.history.add('system', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - await stop() + this.logger.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + this.state.timedout = true + this.emit('timeout', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + await this.stop() }, timeoutMins * 60 * 1000) } - - return { - runAction, - resumeAction, - stop, - cancelResume, - } } From ce805d25f8eb5300841ea67ddbbb7cd30a147671 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 18 Jan 2025 07:11:47 +0800 Subject: [PATCH 04/22] refactor: remove agent-factory --- src/agents/chat/index.ts | 6 +- src/agents/planning/factory.ts | 43 ++++++-------- src/agents/planning/index.ts | 14 +++-- src/libs/mineflayer/core/agent-factory.ts | 70 ----------------------- src/mineflayer/llm-agent.ts | 14 +++-- 5 files changed, 38 insertions(+), 109 deletions(-) delete mode 100644 src/libs/mineflayer/core/agent-factory.ts diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts index 6f41da2..c4bbd13 100644 --- a/src/agents/chat/index.ts +++ b/src/agents/chat/index.ts @@ -21,15 +21,15 @@ export class ChatAgentImpl extends AbstractAgent implements ChatAgent { protected async initializeAgent(): Promise { this.logger.log('Initializing chat agent') - // Set up event listeners + // 设置事件监听 this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) - // Set up idle timeout checker + // 设置空闲超时检查 setInterval(() => { this.checkIdleChats() - }, 60 * 1000) // Check every minute + }, 60 * 1000) } protected async destroyAgent(): Promise { diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts index a277d1b..459b8c4 100644 --- a/src/agents/planning/factory.ts +++ b/src/agents/planning/factory.ts @@ -1,10 +1,9 @@ import type { Neuri } from 'neuri' -import type { AgentType } from 'src/libs/mineflayer/interfaces/agents' -import type { PlanningAgentConfig } from '.' import type { Mineflayer } from '../../libs/mineflayer' import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' -import { AgentFactory, AgentRegistry } from '../../libs/mineflayer/core/agent-factory' +import { PlanningAgentImpl } from '.' +import { ActionAgentImpl } from '../action' interface PlanningPluginOptions { agent: Neuri @@ -17,45 +16,37 @@ interface MineflayerWithPlanning extends Mineflayer { const logger = useLogg('planning-factory').useGlobalConfig() -async function initializeAgent(registry: AgentRegistry, id: string, type: string): Promise { - if (!registry.has(id)) { - const agent = AgentFactory.createAgent({ id, type: type as AgentType }) - registry.register(agent) - await agent.init() - } -} - export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin { return { async created(mineflayer: Mineflayer) { logger.log('Initializing planning plugin') - const registry = AgentRegistry.getInstance() + // 直接创建 action agent + const actionAgent = new ActionAgentImpl({ + id: 'action', + type: 'action', + }) + await actionAgent.init() - // Initialize required agents - await initializeAgent(registry, 'action-agent', 'action') - await initializeAgent(registry, 'memory-agent', 'memory') - - // Create and initialize planning agent - const planningAgent = AgentFactory.createAgent({ - id: 'planning-agent', + // 创建并初始化 planning agent + const planningAgent = new PlanningAgentImpl({ + id: 'planning', type: 'planning', llm: { agent: options.agent, model: options.model, }, - } as PlanningAgentConfig) - - registry.register(planningAgent) + }) await planningAgent.init() - // Add planning agent to bot + // 添加到 bot ;(mineflayer as MineflayerWithPlanning).planning = planningAgent }, - async beforeCleanup() { - logger.log('Destroying planning plugin') - await AgentRegistry.getInstance().destroy() + async beforeCleanup(bot) { + logger.log('Cleaning up planning plugin') + const botWithPlanning = bot as MineflayerWithPlanning + await botWithPlanning.planning?.destroy() }, } } diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 48ab48a..29eaa24 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -1,8 +1,8 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/interfaces/agents' -import { AgentRegistry } from '../../libs/mineflayer/core/agent-factory' import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import { ActionAgentImpl } from '../action' import { generatePlanWithLLM } from './llm' interface PlanContext { @@ -54,12 +54,14 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { protected async initializeAgent(): Promise { this.logger.log('Initializing planning agent') - // Get agent references - const registry = AgentRegistry.getInstance() - this.actionAgent = registry.get('action-agent', 'action') - this.memoryAgent = registry.get('memory-agent', 'memory') + // 直接创建 action agent + this.actionAgent = new ActionAgentImpl({ + id: 'action', + type: 'action', + }) + await this.actionAgent.init() - // Set up event listeners + // 设置事件监听 this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) diff --git a/src/libs/mineflayer/core/agent-factory.ts b/src/libs/mineflayer/core/agent-factory.ts deleted file mode 100644 index 5fddfba..0000000 --- a/src/libs/mineflayer/core/agent-factory.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { ChatAgentConfig } from '../../../agents/chat/types' -import type { AgentConfig, AgentType, BaseAgent } from '../interfaces/agents' -import { ActionAgentImpl } from '../../../agents/action' -import { ChatAgentImpl } from '../../../agents/chat' -import { MemoryAgentImpl } from '../../../agents/memory' -import { type PlanningAgentConfig, PlanningAgentImpl } from '../../../agents/planning' - -export class AgentFactory { - static createAgent(config: AgentConfig): BaseAgent { - switch (config.type) { - case 'action': - return new ActionAgentImpl(config) - case 'memory': - return new MemoryAgentImpl(config) - case 'planning': - return new PlanningAgentImpl(config as PlanningAgentConfig) - case 'chat': - return new ChatAgentImpl(config as ChatAgentConfig) - default: - throw new Error(`Unknown agent type: ${config.type satisfies never}`) - } - } -} - -export class AgentRegistry { - private static instance: AgentRegistry - private agents: Map - - private constructor() { - this.agents = new Map() - } - - static getInstance(): AgentRegistry { - if (!this.instance) { - this.instance = new AgentRegistry() - } - return this.instance - } - - has(id: string): boolean { - return this.agents.has(id) - } - - register(agent: BaseAgent): void { - if (this.agents.has(agent.id)) { - throw new Error(`Agent with id ${agent.id} already exists`) - } - this.agents.set(agent.id, agent) - } - - get(id: string, type: AgentType): T { - const agent = this.agents.get(id) - if (!agent) { - throw new Error(`Agent not found: ${id}`) - } - if (agent.type !== type) { - throw new Error(`Agent ${id} is not of type ${type}`) - } - return agent as T - } - - getAll(): BaseAgent[] { - return Array.from(this.agents.values()) - } - - async destroy(): Promise { - await Promise.all(Array.from(this.agents.values()).map(agent => agent.destroy())) - this.agents.clear() - } -} diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts index f865e1d..6b1ae7f 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/mineflayer/llm-agent.ts @@ -7,8 +7,8 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' +import { ActionAgentImpl } from '../agents/action' import { PlanningPlugin } from '../agents/planning/factory' -import { AgentRegistry } from '../libs/mineflayer/core/agent-factory' import { formBotChat } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' import { toRetriable } from '../utils/reliability' @@ -150,9 +150,12 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { }) await planningPlugin.created!(bot) - // 获取 Action Agent - const registry = AgentRegistry.getInstance() - const actionAgent = registry.get('action-agent', 'action') + // 创建 Action Agent + const actionAgent = new ActionAgentImpl({ + id: 'action', + type: 'action', + }) + await actionAgent.init() // 类型转换 const botWithAgents = bot as unknown as MineflayerWithAgents @@ -172,6 +175,9 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { }, async beforeCleanup(bot) { + const botWithAgents = bot as unknown as MineflayerWithAgents + await botWithAgents.action?.destroy() + await botWithAgents.planning?.destroy() bot.bot.removeAllListeners('chat') }, } From a0386625617a8d1f4e390fb6cf1ec8c4b1f49af6 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 18 Jan 2025 07:34:30 +0800 Subject: [PATCH 05/22] feat: DI --- cspell.config.yaml | 1 + package.json | 1 + pnpm-lock.yaml | 43 +++++++++++++++++++++ src/agents/planning/factory.ts | 25 +++++-------- src/container.ts | 68 ++++++++++++++++++++++++++++++++++ src/mineflayer/llm-agent.ts | 37 ++++++++++-------- 6 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 src/container.ts diff --git a/cspell.config.yaml b/cspell.config.yaml index 853e9c7..8cca917 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -6,6 +6,7 @@ words: - aichat - airi - antfu + - awilix - bumpp - collectblock - convo diff --git a/package.json b/package.json index 962a630..b51fb42 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@guiiai/logg": "^1.0.7", "@proj-airi/server-sdk": "^0.1.4", "@typeschema/zod": "^0.14.0", + "awilix": "^12.0.4", "dotenv": "^16.4.7", "es-toolkit": "^1.31.0", "eventemitter3": "^5.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89e8e3c..9dc5368 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@typeschema/zod': specifier: ^0.14.0 version: 0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.1(zod@3.24.1))(zod@3.24.1) + awilix: + specifier: ^12.0.4 + version: 12.0.4 dotenv: specifier: ^16.4.7 version: 16.4.7 @@ -1064,6 +1067,10 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + awilix@12.0.4: + resolution: {integrity: sha512-P6bd20vqMiUyjgBAVl+4WixM/MR9O9zsTzd9vS5lTd1eLpFEn6Re4+GeeYzDDE8U1DXL8cO/nTOHofKDEJUfAQ==} + engines: {node: '>=16.3.0'} + axios@0.21.4: resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} @@ -1133,6 +1140,9 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} + camel-case@4.1.2: + resolution: {integrity: sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==} + caniuse-lite@1.0.30001690: resolution: {integrity: sha512-5ExiE3qQN6oF8Clf8ifIDcMRCRE/dMGcETG/XGMD8/XiXm6HXQgQTh1yZYLXXpSOsEUlJm1Xr7kGULZTuGtP/w==} @@ -2062,6 +2072,9 @@ packages: loupe@3.1.2: resolution: {integrity: sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==} + lower-case@2.0.2: + resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} + macaddress@0.5.3: resolution: {integrity: sha512-vGBKTA+jwM4KgjGZ+S/8/Mkj9rWzePyGY6jManXPGhiWu63RYwW8dKPyk5koP+8qNVhPhHgFa1y/MJ4wrjsNrg==} @@ -2346,6 +2359,9 @@ packages: neuri@0.0.19: resolution: {integrity: sha512-Fr1sFbFlyyg0+xWd6UT90wmPcFqxQUhDh9qs5YFc+T8SbRul/LbGy0Cg52y/6VTVbp1cu4yb8AIfpJCmvxtzrA==} + no-case@3.0.4: + resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2467,6 +2483,9 @@ packages: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} + pascal-case@3.1.2: + resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4042,6 +4061,11 @@ snapshots: asynckit@0.4.0: {} + awilix@12.0.4: + dependencies: + camel-case: 4.1.2 + fast-glob: 3.3.2 + axios@0.21.4(debug@4.4.0): dependencies: follow-redirects: 1.15.9(debug@4.4.0) @@ -4120,6 +4144,11 @@ snapshots: callsites@3.1.0: {} + camel-case@4.1.2: + dependencies: + pascal-case: 3.1.2 + tslib: 2.8.1 + caniuse-lite@1.0.30001690: {} ccount@2.0.1: {} @@ -5172,6 +5201,10 @@ snapshots: loupe@3.1.2: {} + lower-case@2.0.2: + dependencies: + tslib: 2.8.1 + macaddress@0.5.3: {} magic-string@0.30.17: @@ -5729,6 +5762,11 @@ snapshots: - encoding - zod + no-case@3.0.4: + dependencies: + lower-case: 2.0.2 + tslib: 2.8.1 + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -5847,6 +5885,11 @@ snapshots: parseurl@1.3.3: {} + pascal-case@3.1.2: + dependencies: + no-case: 3.0.4 + tslib: 2.8.1 + path-exists@4.0.0: {} path-key@3.1.1: {} diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts index 459b8c4..f652626 100644 --- a/src/agents/planning/factory.ts +++ b/src/agents/planning/factory.ts @@ -2,8 +2,7 @@ import type { Neuri } from 'neuri' import type { Mineflayer } from '../../libs/mineflayer' import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' -import { PlanningAgentImpl } from '.' -import { ActionAgentImpl } from '../action' +import { createAppContainer } from '../../container' interface PlanningPluginOptions { agent: Neuri @@ -21,22 +20,16 @@ export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin async created(mineflayer: Mineflayer) { logger.log('Initializing planning plugin') - // 直接创建 action agent - const actionAgent = new ActionAgentImpl({ - id: 'action', - type: 'action', + // 创建容器并获取所需的服务 + const container = createAppContainer({ + neuri: options.agent, + model: options.model, }) - await actionAgent.init() + const actionAgent = container.resolve('actionAgent') + const planningAgent = container.resolve('planningAgent') - // 创建并初始化 planning agent - const planningAgent = new PlanningAgentImpl({ - id: 'planning', - type: 'planning', - llm: { - agent: options.agent, - model: options.model, - }, - }) + // 初始化 agents + await actionAgent.init() await planningAgent.init() // 添加到 bot diff --git a/src/container.ts b/src/container.ts new file mode 100644 index 0000000..18911f9 --- /dev/null +++ b/src/container.ts @@ -0,0 +1,68 @@ +import type { Neuri } from 'neuri' +import { useLogg } from '@guiiai/logg' +import { asClass, asFunction, createContainer, InjectionMode } from 'awilix' +import { ActionAgentImpl } from './agents/action' +import { ChatAgentImpl } from './agents/chat' +import { PlanningAgentImpl } from './agents/planning' + +export interface ContainerServices { + logger: ReturnType + actionAgent: ActionAgentImpl + planningAgent: PlanningAgentImpl + chatAgent: ChatAgentImpl + neuri: Neuri +} + +export function createAppContainer(options: { + neuri: Neuri + model?: string + maxHistoryLength?: number + idleTimeout?: number +}) { + const container = createContainer({ + injectionMode: InjectionMode.PROXY, + }) + + // Register services + container.register({ + // Create independent logger for each agent + logger: asFunction(() => useLogg('app').useGlobalConfig()).singleton(), + + // Register neuri client + neuri: asFunction(() => options.neuri).singleton(), + + // Register agents + actionAgent: asClass(ActionAgentImpl) + .singleton() + .inject(() => ({ + id: 'action', + type: 'action' as const, + })), + + planningAgent: asClass(PlanningAgentImpl) + .singleton() + .inject(() => ({ + id: 'planning', + type: 'planning' as const, + llm: { + agent: options.neuri, + model: options.model, + }, + })), + + chatAgent: asClass(ChatAgentImpl) + .singleton() + .inject(() => ({ + id: 'chat', + type: 'chat' as const, + llm: { + agent: options.neuri, + model: options.model, + }, + maxHistoryLength: options.maxHistoryLength, + idleTimeout: options.idleTimeout, + })), + }) + + return container +} diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts index 6b1ae7f..25f4fc5 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/mineflayer/llm-agent.ts @@ -2,13 +2,12 @@ import type { Client } from '@proj-airi/server-sdk' import type { Neuri, NeuriContext } from 'neuri' import type { ChatCompletion } from 'neuri/openai' import type { Mineflayer } from '../libs/mineflayer' -import type { ActionAgent, PlanningAgent } from '../libs/mineflayer/interfaces/agents' +import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/interfaces/agents' import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' -import { ActionAgentImpl } from '../agents/action' -import { PlanningPlugin } from '../agents/planning/factory' +import { createAppContainer } from '../container' import { formBotChat } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' import { toRetriable } from '../utils/reliability' @@ -16,6 +15,7 @@ import { toRetriable } from '../utils/reliability' interface MineflayerWithAgents extends Mineflayer { planning: PlanningAgent action: ActionAgent + chat: ChatAgent } interface LLMAgentOptions { @@ -140,36 +140,40 @@ async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Ne export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { return { async created(bot) { - const agent = options.agent const logger = useLogg('LLMAgent').useGlobalConfig() - // 初始化 Planning Agent - const planningPlugin = PlanningPlugin({ - agent: options.agent, - model: 'openai/gpt-4o-mini', + // 创建容器并获取所需的服务 + const container = createAppContainer({ + neuri: options.agent, + model: 'openai/gpt-4-mini', + maxHistoryLength: 50, + idleTimeout: 5 * 60 * 1000, }) - await planningPlugin.created!(bot) - // 创建 Action Agent - const actionAgent = new ActionAgentImpl({ - id: 'action', - type: 'action', - }) + const actionAgent = container.resolve('actionAgent') + const planningAgent = container.resolve('planningAgent') + const chatAgent = container.resolve('chatAgent') + + // 初始化 agents await actionAgent.init() + await planningAgent.init() + await chatAgent.init() // 类型转换 const botWithAgents = bot as unknown as MineflayerWithAgents botWithAgents.action = actionAgent + botWithAgents.planning = planningAgent + botWithAgents.chat = chatAgent // 初始化系统提示 bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) // 设置消息处理 const onChat = formBotChat(bot.username, (username, message) => - handleChatMessage(username, message, botWithAgents, agent, logger)) + handleChatMessage(username, message, botWithAgents, options.agent, logger)) options.airiClient.onEvent('input:text:voice', event => - handleVoiceInput(event, botWithAgents, agent, logger)) + handleVoiceInput(event, botWithAgents, options.agent, logger)) bot.bot.on('chat', onChat) }, @@ -178,6 +182,7 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { const botWithAgents = bot as unknown as MineflayerWithAgents await botWithAgents.action?.destroy() await botWithAgents.planning?.destroy() + await botWithAgents.chat?.destroy() bot.bot.removeAllListeners('chat') }, } From 483719ffe272962afa56e4b842fc7b285a5369a3 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 18 Jan 2025 18:28:49 +0800 Subject: [PATCH 06/22] refactor: agents --- src/agents/{ => action}/actions.test.ts | 12 +- src/agents/{ => action}/actions.ts | 12 +- src/agents/action/index.ts | 2 +- src/agents/{ => action}/openai.test.ts | 0 src/agents/{ => action}/openai.ts | 4 +- src/agents/memory/index.ts | 39 +++ src/composables/agent.ts | 36 -- src/composables/bot.ts | 25 +- src/composables/config.ts | 58 +++- src/composables/conversation.ts | 383 --------------------- src/composables/deprecated/conversation.ts | 382 ++++++++++++++++++++ src/libs/mineflayer/core.ts | 4 +- src/libs/mineflayer/core/base-agent.ts | 6 +- src/libs/mineflayer/index.ts | 1 - src/libs/mineflayer/interfaces.ts | 3 - src/libs/mineflayer/message.ts | 57 +-- src/libs/mineflayer/status.ts | 2 +- src/libs/mineflayer/types.ts | 4 + src/main.ts | 2 +- src/mineflayer/echo.ts | 4 +- src/mineflayer/index.ts | 2 - src/mineflayer/llm-agent.ts | 6 +- 22 files changed, 547 insertions(+), 497 deletions(-) rename src/agents/{ => action}/actions.test.ts (87%) rename src/agents/{ => action}/actions.ts (98%) rename src/agents/{ => action}/openai.test.ts (100%) rename src/agents/{ => action}/openai.ts (92%) delete mode 100644 src/composables/agent.ts delete mode 100644 src/composables/conversation.ts create mode 100644 src/composables/deprecated/conversation.ts delete mode 100644 src/libs/mineflayer/interfaces.ts delete mode 100644 src/mineflayer/index.ts diff --git a/src/agents/actions.test.ts b/src/agents/action/actions.test.ts similarity index 87% rename from src/agents/actions.test.ts rename to src/agents/action/actions.test.ts index c537b62..5e255f3 100644 --- a/src/agents/actions.test.ts +++ b/src/agents/action/actions.test.ts @@ -1,11 +1,11 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' -import { initBot, useBot } from '../composables/bot' -import { botConfig, initEnv } from '../composables/config' -import { genActionAgentPrompt, genQueryAgentPrompt } from '../prompts/agent' -import { sleep } from '../utils/helper' -import { initLogger } from '../utils/logger' -import { initAgent } from './openai' +import { initBot, useBot } from '../../composables/bot' +import { botConfig, initEnv } from '../../composables/config' +import { genActionAgentPrompt, genQueryAgentPrompt } from '../../prompts/agent' +import { sleep } from '../../utils/helper' +import { initLogger } from '../../utils/logger' +import { initAgent } from '../openai' describe('actions agent', { timeout: 0 }, () => { beforeAll(() => { diff --git a/src/agents/actions.ts b/src/agents/action/actions.ts similarity index 98% rename from src/agents/actions.ts rename to src/agents/action/actions.ts index 709943e..3d9e2fb 100644 --- a/src/agents/actions.ts +++ b/src/agents/action/actions.ts @@ -1,11 +1,11 @@ -import type { Action } from '../libs/mineflayer' +import type { Action } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { z } from 'zod' -import * as world from '../composables/world' -import * as skills from '../skills' -import { collectBlock } from '../skills/actions/collect-block' -import { discard, equip, putInChest, takeFromChest, viewChest } from '../skills/actions/inventory' -import { activateNearestBlock, placeBlock } from '../skills/actions/world-interactions' +import * as world from '../../composables/world' +import * as skills from '../../skills' +import { collectBlock } from '../../skills/actions/collect-block' +import { discard, equip, putInChest, takeFromChest, viewChest } from '../../skills/actions/inventory' +import { activateNearestBlock, placeBlock } from '../../skills/actions/world-interactions' // Utils const pad = (str: string): string => `\n${str}\n` diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 5968de2..3868c6f 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -4,7 +4,7 @@ import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/interfaces/ import { ActionManager } from '../../composables/action' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' -import { actionsList } from '../actions' +import { actionsList } from './actions' interface ActionState { executing: boolean diff --git a/src/agents/openai.test.ts b/src/agents/action/openai.test.ts similarity index 100% rename from src/agents/openai.test.ts rename to src/agents/action/openai.test.ts diff --git a/src/agents/openai.ts b/src/agents/action/openai.ts similarity index 92% rename from src/agents/openai.ts rename to src/agents/action/openai.ts index 9258485..f8fa327 100644 --- a/src/agents/openai.ts +++ b/src/agents/action/openai.ts @@ -1,8 +1,8 @@ import type { Agent, Neuri } from 'neuri' -import type { Mineflayer } from '../libs/mineflayer' +import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { agent, neuri } from 'neuri' -import { openaiConfig } from '../composables/config' +import { openaiConfig } from '../../composables/config' import { actionsList } from './actions' let neuriAgent: Neuri | undefined diff --git a/src/agents/memory/index.ts b/src/agents/memory/index.ts index fb125a5..ce53ead 100644 --- a/src/agents/memory/index.ts +++ b/src/agents/memory/index.ts @@ -1,5 +1,8 @@ +import type { Message } from 'neuri/openai' +import type { Action } from 'src/libs/mineflayer' import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/interfaces/agents' import { useLogg } from '@guiiai/logg' +import { Memory } from '../../libs/mineflayer/memory' const logger = useLogg('memory-agent').useGlobalConfig() @@ -8,11 +11,13 @@ export class MemoryAgentImpl implements MemoryAgent { public readonly id: string private memory: Map private initialized: boolean + private memoryInstance: Memory constructor(config: AgentConfig) { this.id = config.id this.memory = new Map() this.initialized = false + this.memoryInstance = new Memory() } async init(): Promise { @@ -64,4 +69,38 @@ export class MemoryAgentImpl implements MemoryAgent { return Object.fromEntries(this.memory.entries()) } + + addChatMessage(message: Message): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + this.memoryInstance.chatHistory.push(message) + logger.withFields({ message }).log('Adding chat message to memory') + } + + addAction(action: Action): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + this.memoryInstance.actions.push(action) + logger.withFields({ action }).log('Adding action to memory') + } + + getChatHistory(): Message[] { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return this.memoryInstance.chatHistory + } + + getActions(): Action[] { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return this.memoryInstance.actions + } } diff --git a/src/composables/agent.ts b/src/composables/agent.ts deleted file mode 100644 index bee29c5..0000000 --- a/src/composables/agent.ts +++ /dev/null @@ -1,36 +0,0 @@ -export interface Agent { - name: string - history: { - add: (name: string, message: string) => void - } - lastSender?: string - isIdle: () => boolean - handleMessage: (sender: string, message: string) => void - openChat: (message: string) => void - self_prompter: { - on: boolean - stop: () => Promise - stopLoop: () => Promise - start: () => Promise - promptShouldRespondToBot: (message: string) => Promise - } - actions: { - currentActionLabel: string - } - prompter: { - promptShouldRespondToBot: (message: string) => Promise - } - shut_up: boolean - in_game: boolean - cleanKill: (message: string) => void - clearBotLogs: () => void - bot: { - interrupt_code: boolean - output: string - emit: (event: string) => void - } - coder: { - generating: boolean - } - requestInterrupt: () => void -} diff --git a/src/composables/bot.ts b/src/composables/bot.ts index 2638736..5a513aa 100644 --- a/src/composables/bot.ts +++ b/src/composables/bot.ts @@ -1,18 +1,31 @@ import { Mineflayer, type MineflayerOptions } from '../libs/mineflayer' -let mineflayer: Mineflayer +// Singleton instance of the Mineflayer bot +let botInstance: Mineflayer | null = null +/** + * Initialize a new Mineflayer bot instance. + * Follows singleton pattern to ensure only one bot exists at a time. + */ export async function initBot(options: MineflayerOptions): Promise<{ bot: Mineflayer }> { - mineflayer = await Mineflayer.asyncBuild(options) - return { bot: mineflayer } + if (botInstance) { + throw new Error('Bot already initialized') + } + + botInstance = await Mineflayer.asyncBuild(options) + return { bot: botInstance } } -export function useBot() { - if (!mineflayer) { +/** + * Get the current bot instance. + * Throws if bot is not initialized. + */ +export function useBot(): { bot: Mineflayer } { + if (!botInstance) { throw new Error('Bot not initialized') } return { - bot: mineflayer, + bot: botInstance, } } diff --git a/src/composables/config.ts b/src/composables/config.ts index 2cbfd06..03fbe52 100644 --- a/src/composables/config.ts +++ b/src/composables/config.ts @@ -4,35 +4,57 @@ import { useLogg } from '@guiiai/logg' const logger = useLogg('config').useGlobalConfig() +// Configuration interfaces interface OpenAIConfig { apiKey: string baseUrl: string } -export const botConfig: BotOptions = { - username: '', - host: '', - port: 0, - password: '', - version: '1.20', +interface EnvConfig { + openai: OpenAIConfig + bot: BotOptions } -export const openaiConfig: OpenAIConfig = { - apiKey: '', - baseUrl: '', +// Default configurations +const defaultConfig: EnvConfig = { + openai: { + apiKey: '', + baseUrl: '', + }, + bot: { + username: '', + host: '', + port: 0, + password: '', + version: '1.20', + }, } -export function initEnv() { - logger.log('Initializing environment variables') +// Exported configurations +export const botConfig: BotOptions = { ...defaultConfig.bot } +export const openaiConfig: OpenAIConfig = { ...defaultConfig.openai } - openaiConfig.apiKey = env.OPENAI_API_KEY || '' - openaiConfig.baseUrl = env.OPENAI_API_BASEURL || '' +// Load environment variables into config +export function initEnv(): void { + logger.log('Initializing environment variables') - botConfig.username = env.BOT_USERNAME || '' - botConfig.host = env.BOT_HOSTNAME || '' - botConfig.port = Number.parseInt(env.BOT_PORT || '49415') - botConfig.password = env.BOT_PASSWORD || '' - botConfig.version = env.BOT_VERSION || '1.20' + const config: EnvConfig = { + openai: { + apiKey: env.OPENAI_API_KEY || defaultConfig.openai.apiKey, + baseUrl: env.OPENAI_API_BASEURL || defaultConfig.openai.baseUrl, + }, + bot: { + username: env.BOT_USERNAME || defaultConfig.bot.username, + host: env.BOT_HOSTNAME || defaultConfig.bot.host, + port: Number.parseInt(env.BOT_PORT || '49415'), + password: env.BOT_PASSWORD || defaultConfig.bot.password, + version: env.BOT_VERSION || defaultConfig.bot.version, + }, + } + + // Update exported configs + Object.assign(openaiConfig, config.openai) + Object.assign(botConfig, config.bot) logger.withFields({ openaiConfig }).log('Environment variables initialized') } diff --git a/src/composables/conversation.ts b/src/composables/conversation.ts deleted file mode 100644 index 5d61f65..0000000 --- a/src/composables/conversation.ts +++ /dev/null @@ -1,383 +0,0 @@ -import type { Agent } from './agent' -import { useLogg } from '@guiiai/logg' - -let self_prompter_paused = false - -interface ConversationMessage { - message: string - start: boolean - end: boolean -} - -function compileInMessages(inQueue: ConversationMessage[]) { - let pack: ConversationMessage | undefined - let fullMessage = '' - while (inQueue.length > 0) { - pack = inQueue.shift() - if (!pack) - continue - - fullMessage += pack.message - } - if (pack) { - pack.message = fullMessage - } - - return pack -} - -type Conversation = ReturnType - -function useConversations(name: string, agent: Agent) { - const active = { value: false } - const ignoreUntilStart = { value: false } - const blocked = { value: false } - let inQueue: ConversationMessage[] = [] - const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } - - function reset() { - active.value = false - ignoreUntilStart.value = false - inQueue = [] - } - - function end() { - active.value = false - ignoreUntilStart.value = true - const fullMessage = compileInMessages(inQueue) - if (!fullMessage) - return - - if (fullMessage.message.trim().length > 0) { - agent.history.add(name, fullMessage.message) - } - - if (agent.lastSender === name) { - agent.lastSender = undefined - } - } - - function queue(message: ConversationMessage) { - inQueue.push(message) - } - - return { - reset, - end, - queue, - name, - inMessageTimer, - blocked, - active, - ignoreUntilStart, - inQueue, - } -} - -const WAIT_TIME_START = 30000 - -export type ConversationStore = ReturnType - -export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { - const conversations: Record = {} - const activeConversation: { value: Conversation | undefined } = { value: undefined } - const awaitingResponse = { value: false } - const waitTimeLimit = { value: WAIT_TIME_START } - const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } - const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } - const agent = options.agent - let agentsInGame = options.agentNames || [] - const log = useLogg('ConversationStore').useGlobalConfig() - - const conversationStore = { - getConvo: (name: string) => { - if (!conversations[name]) - conversations[name] = useConversations(name, agent) - return conversations[name] - }, - startMonitor: () => { - clearInterval(connectionMonitor.value) - let waitTime = 0 - let lastTime = Date.now() - connectionMonitor.value = setInterval(() => { - if (!activeConversation.value) { - conversationStore.stopMonitor() - return // will clean itself up - } - - const delta = Date.now() - lastTime - lastTime = Date.now() - const convo_partner = activeConversation.value.name - - if (awaitingResponse.value && agent.isIdle()) { - waitTime += delta - if (waitTime > waitTimeLimit.value) { - agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) - waitTime = 0 - waitTimeLimit.value *= 2 - } - } - else if (!awaitingResponse.value) { - waitTimeLimit.value = WAIT_TIME_START - waitTime = 0 - } - - if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { - connectionTimeout.value = setTimeout(() => { - if (conversationStore.otherAgentInGame(convo_partner)) { - conversationStore.clearMonitorTimeouts() - return - } - if (!self_prompter_paused) { - conversationStore.endConversation(convo_partner) - agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) - } - else { - conversationStore.endConversation(convo_partner) - } - }, 10000) - } - }, 1000) - }, - stopMonitor: () => { - clearInterval(connectionMonitor.value) - connectionMonitor.value = undefined - conversationStore.clearMonitorTimeouts() - }, - clearMonitorTimeouts: () => { - awaitingResponse.value = false - clearTimeout(connectionTimeout.value) - connectionTimeout.value = undefined - }, - startConversation: (send_to: string, message: string) => { - const convo = conversationStore.getConvo(send_to) - convo.reset() - - if (agent.self_prompter.on) { - agent.self_prompter.stop() - self_prompter_paused = true - } - if (convo.active.value) - return - - convo.active.value = true - activeConversation.value = convo - conversationStore.startMonitor() - conversationStore.sendToBot(send_to, message, true, false) - }, - startConversationFromOtherBot: (name: string) => { - const convo = conversationStore.getConvo(name) - convo.active.value = true - activeConversation.value = convo - conversationStore.startMonitor() - }, - sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { - if (!conversationStore.isOtherAgent(send_to)) { - console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) - return - } - const convo = conversationStore.getConvo(send_to) - - if (options.chatBotMessages && open_chat) - agent.openChat(`(To ${send_to}) ${message}`) - - if (convo.ignoreUntilStart.value) - return - convo.active.value = true - - const end = message.includes('!endConversation') - const json = { - message, - start, - end, - } - - awaitingResponse.value = true - // TODO: - // sendBotChatToServer(send_to, json) - log.withField('json', json).log(`Sending message to ${send_to}`) - }, - receiveFromBot: async (sender: string, received: ConversationMessage) => { - const convo = conversationStore.getConvo(sender) - - if (convo.ignoreUntilStart.value && !received.start) - return - - // check if any convo is active besides the sender - if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { - conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) - conversationStore.endConversation(sender) - return - } - - if (received.start) { - convo.reset() - conversationStore.startConversationFromOtherBot(sender) - } - - conversationStore.clearMonitorTimeouts() - convo.queue(received) - - // responding to conversation takes priority over self prompting - if (agent.self_prompter.on) { - await agent.self_prompter.stopLoop() - self_prompter_paused = true - } - - _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) - }, - responseScheduledFor: (sender: string) => { - if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) - return false - const convo = conversationStore.getConvo(sender) - return !!convo.inMessageTimer - }, - isOtherAgent: (name: string) => { - return !!options.agentNames?.includes(name) - }, - otherAgentInGame: (name: string) => { - return agentsInGame.includes(name) - }, - updateAgents: (agents: Agent[]) => { - options.agentNames = agents.map(a => a.name) - agentsInGame = agents.filter(a => a.in_game).map(a => a.name) - }, - getInGameAgents: () => { - return agentsInGame - }, - inConversation: (other_agent?: string) => { - if (other_agent) - return conversations[other_agent]?.active - return Object.values(conversations).some(c => c.active) - }, - endConversation: (sender: string) => { - if (conversations[sender]) { - conversations[sender].end() - if (activeConversation.value?.name === sender) { - conversationStore.stopMonitor() - activeConversation.value = undefined - if (self_prompter_paused && !conversationStore.inConversation()) { - _resumeSelfPrompter(agent, conversationStore) - } - } - } - }, - endAllConversations: () => { - for (const sender in conversations) { - conversationStore.endConversation(sender) - } - if (self_prompter_paused) { - _resumeSelfPrompter(agent, conversationStore) - } - }, - forceEndCurrentConversation: () => { - if (activeConversation.value) { - const sender = activeConversation.value.name - conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) - conversationStore.endConversation(sender) - } - }, - scheduleSelfPrompter: () => { - self_prompter_paused = true - }, - cancelSelfPrompter: () => { - self_prompter_paused = false - }, - } - - return conversationStore -} - -function containsCommand(message: string) { - // TODO: mock - return message -} - -/* -This function controls conversation flow by deciding when the bot responds. -The logic is as follows: -- If neither bot is busy, respond quickly with a small delay. -- If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) -- If I'm busy but other bot isn't, let LLM decide whether to respond -- If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses -- New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk -*/ -const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions -const fastDelay = 200 -const longDelay = 5000 - -async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { - if (convo.inMessageTimer) - clearTimeout(convo.inMessageTimer.value) - const otherAgentBusy = containsCommand(received.message) - - const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) - - if (!agent.isIdle() && otherAgentBusy) { - // both are busy - const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) - if (canTalkOver) - scheduleResponse(fastDelay) - // otherwise don't respond - } - else if (otherAgentBusy) { - // other bot is busy but I'm not - scheduleResponse(longDelay) - } - else if (!agent.isIdle()) { - // I'm busy but other bot isn't - const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) - if (canTalkOver) { - scheduleResponse(fastDelay) - } - else { - const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) - useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) - if (shouldRespond) - scheduleResponse(fastDelay) - } - } - else { - // neither are busy - scheduleResponse(fastDelay) - } -} - -function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { - const convo = conversationStore.getConvo(name) - _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) -} - -function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { - if (!received) - return - - useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) - - const convo = conversationStore.getConvo(sender) - convo.active.value = true - - let message = _tagMessage(received.message) - if (received.end) { - conversationStore.endConversation(sender) - message = `Conversation with ${sender} ended with message: "${message}"` - sender = 'system' // bot will respond to system instead of the other bot - } - else if (received.start) { - agent.shut_up = false - } - convo.inMessageTimer.value = undefined - agent.handleMessage(sender, message) -} - -function _tagMessage(message: string) { - return `(FROM OTHER BOT)${message}` -} - -async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { - await new Promise(resolve => setTimeout(resolve, 5000)) - if (self_prompter_paused && !conversationStore.inConversation()) { - self_prompter_paused = false - agent.self_prompter.start() - } -} diff --git a/src/composables/deprecated/conversation.ts b/src/composables/deprecated/conversation.ts new file mode 100644 index 0000000..891fb7a --- /dev/null +++ b/src/composables/deprecated/conversation.ts @@ -0,0 +1,382 @@ +// import { useLogg } from '@guiiai/logg' + +// let self_prompter_paused = false + +// interface ConversationMessage { +// message: string +// start: boolean +// end: boolean +// } + +// function compileInMessages(inQueue: ConversationMessage[]) { +// let pack: ConversationMessage | undefined +// let fullMessage = '' +// while (inQueue.length > 0) { +// pack = inQueue.shift() +// if (!pack) +// continue + +// fullMessage += pack.message +// } +// if (pack) { +// pack.message = fullMessage +// } + +// return pack +// } + +// type Conversation = ReturnType + +// function useConversations(name: string, agent: Agent) { +// const active = { value: false } +// const ignoreUntilStart = { value: false } +// const blocked = { value: false } +// let inQueue: ConversationMessage[] = [] +// const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } + +// function reset() { +// active.value = false +// ignoreUntilStart.value = false +// inQueue = [] +// } + +// function end() { +// active.value = false +// ignoreUntilStart.value = true +// const fullMessage = compileInMessages(inQueue) +// if (!fullMessage) +// return + +// if (fullMessage.message.trim().length > 0) { +// agent.history.add(name, fullMessage.message) +// } + +// if (agent.lastSender === name) { +// agent.lastSender = undefined +// } +// } + +// function queue(message: ConversationMessage) { +// inQueue.push(message) +// } + +// return { +// reset, +// end, +// queue, +// name, +// inMessageTimer, +// blocked, +// active, +// ignoreUntilStart, +// inQueue, +// } +// } + +// const WAIT_TIME_START = 30000 + +// export type ConversationStore = ReturnType + +// export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { +// const conversations: Record = {} +// const activeConversation: { value: Conversation | undefined } = { value: undefined } +// const awaitingResponse = { value: false } +// const waitTimeLimit = { value: WAIT_TIME_START } +// const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } +// const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } +// const agent = options.agent +// let agentsInGame = options.agentNames || [] +// const log = useLogg('ConversationStore').useGlobalConfig() + +// const conversationStore = { +// getConvo: (name: string) => { +// if (!conversations[name]) +// conversations[name] = useConversations(name, agent) +// return conversations[name] +// }, +// startMonitor: () => { +// clearInterval(connectionMonitor.value) +// let waitTime = 0 +// let lastTime = Date.now() +// connectionMonitor.value = setInterval(() => { +// if (!activeConversation.value) { +// conversationStore.stopMonitor() +// return // will clean itself up +// } + +// const delta = Date.now() - lastTime +// lastTime = Date.now() +// const convo_partner = activeConversation.value.name + +// if (awaitingResponse.value && agent.isIdle()) { +// waitTime += delta +// if (waitTime > waitTimeLimit.value) { +// agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) +// waitTime = 0 +// waitTimeLimit.value *= 2 +// } +// } +// else if (!awaitingResponse.value) { +// waitTimeLimit.value = WAIT_TIME_START +// waitTime = 0 +// } + +// if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { +// connectionTimeout.value = setTimeout(() => { +// if (conversationStore.otherAgentInGame(convo_partner)) { +// conversationStore.clearMonitorTimeouts() +// return +// } +// if (!self_prompter_paused) { +// conversationStore.endConversation(convo_partner) +// agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) +// } +// else { +// conversationStore.endConversation(convo_partner) +// } +// }, 10000) +// } +// }, 1000) +// }, +// stopMonitor: () => { +// clearInterval(connectionMonitor.value) +// connectionMonitor.value = undefined +// conversationStore.clearMonitorTimeouts() +// }, +// clearMonitorTimeouts: () => { +// awaitingResponse.value = false +// clearTimeout(connectionTimeout.value) +// connectionTimeout.value = undefined +// }, +// startConversation: (send_to: string, message: string) => { +// const convo = conversationStore.getConvo(send_to) +// convo.reset() + +// if (agent.self_prompter.on) { +// agent.self_prompter.stop() +// self_prompter_paused = true +// } +// if (convo.active.value) +// return + +// convo.active.value = true +// activeConversation.value = convo +// conversationStore.startMonitor() +// conversationStore.sendToBot(send_to, message, true, false) +// }, +// startConversationFromOtherBot: (name: string) => { +// const convo = conversationStore.getConvo(name) +// convo.active.value = true +// activeConversation.value = convo +// conversationStore.startMonitor() +// }, +// sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { +// if (!conversationStore.isOtherAgent(send_to)) { +// console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) +// return +// } +// const convo = conversationStore.getConvo(send_to) + +// if (options.chatBotMessages && open_chat) +// agent.openChat(`(To ${send_to}) ${message}`) + +// if (convo.ignoreUntilStart.value) +// return +// convo.active.value = true + +// const end = message.includes('!endConversation') +// const json = { +// message, +// start, +// end, +// } + +// awaitingResponse.value = true +// // TODO: +// // sendBotChatToServer(send_to, json) +// log.withField('json', json).log(`Sending message to ${send_to}`) +// }, +// receiveFromBot: async (sender: string, received: ConversationMessage) => { +// const convo = conversationStore.getConvo(sender) + +// if (convo.ignoreUntilStart.value && !received.start) +// return + +// // check if any convo is active besides the sender +// if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { +// conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) +// conversationStore.endConversation(sender) +// return +// } + +// if (received.start) { +// convo.reset() +// conversationStore.startConversationFromOtherBot(sender) +// } + +// conversationStore.clearMonitorTimeouts() +// convo.queue(received) + +// // responding to conversation takes priority over self prompting +// if (agent.self_prompter.on) { +// await agent.self_prompter.stopLoop() +// self_prompter_paused = true +// } + +// _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) +// }, +// responseScheduledFor: (sender: string) => { +// if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) +// return false +// const convo = conversationStore.getConvo(sender) +// return !!convo.inMessageTimer +// }, +// isOtherAgent: (name: string) => { +// return !!options.agentNames?.includes(name) +// }, +// otherAgentInGame: (name: string) => { +// return agentsInGame.includes(name) +// }, +// updateAgents: (agents: Agent[]) => { +// options.agentNames = agents.map(a => a.name) +// agentsInGame = agents.filter(a => a.in_game).map(a => a.name) +// }, +// getInGameAgents: () => { +// return agentsInGame +// }, +// inConversation: (other_agent?: string) => { +// if (other_agent) +// return conversations[other_agent]?.active +// return Object.values(conversations).some(c => c.active) +// }, +// endConversation: (sender: string) => { +// if (conversations[sender]) { +// conversations[sender].end() +// if (activeConversation.value?.name === sender) { +// conversationStore.stopMonitor() +// activeConversation.value = undefined +// if (self_prompter_paused && !conversationStore.inConversation()) { +// _resumeSelfPrompter(agent, conversationStore) +// } +// } +// } +// }, +// endAllConversations: () => { +// for (const sender in conversations) { +// conversationStore.endConversation(sender) +// } +// if (self_prompter_paused) { +// _resumeSelfPrompter(agent, conversationStore) +// } +// }, +// forceEndCurrentConversation: () => { +// if (activeConversation.value) { +// const sender = activeConversation.value.name +// conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) +// conversationStore.endConversation(sender) +// } +// }, +// scheduleSelfPrompter: () => { +// self_prompter_paused = true +// }, +// cancelSelfPrompter: () => { +// self_prompter_paused = false +// }, +// } + +// return conversationStore +// } + +// function containsCommand(message: string) { +// // TODO: mock +// return message +// } + +// /* +// This function controls conversation flow by deciding when the bot responds. +// The logic is as follows: +// - If neither bot is busy, respond quickly with a small delay. +// - If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) +// - If I'm busy but other bot isn't, let LLM decide whether to respond +// - If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses +// - New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk +// */ +// const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions +// const fastDelay = 200 +// const longDelay = 5000 + +// async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { +// if (convo.inMessageTimer) +// clearTimeout(convo.inMessageTimer.value) +// const otherAgentBusy = containsCommand(received.message) + +// const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) + +// if (!agent.isIdle() && otherAgentBusy) { +// // both are busy +// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) +// if (canTalkOver) +// scheduleResponse(fastDelay) +// // otherwise don't respond +// } +// else if (otherAgentBusy) { +// // other bot is busy but I'm not +// scheduleResponse(longDelay) +// } +// else if (!agent.isIdle()) { +// // I'm busy but other bot isn't +// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) +// if (canTalkOver) { +// scheduleResponse(fastDelay) +// } +// else { +// const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) +// useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) +// if (shouldRespond) +// scheduleResponse(fastDelay) +// } +// } +// else { +// // neither are busy +// scheduleResponse(fastDelay) +// } +// } + +// function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { +// const convo = conversationStore.getConvo(name) +// _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) +// } + +// function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { +// if (!received) +// return + +// useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) + +// const convo = conversationStore.getConvo(sender) +// convo.active.value = true + +// let message = _tagMessage(received.message) +// if (received.end) { +// conversationStore.endConversation(sender) +// message = `Conversation with ${sender} ended with message: "${message}"` +// sender = 'system' // bot will respond to system instead of the other bot +// } +// else if (received.start) { +// agent.shut_up = false +// } +// convo.inMessageTimer.value = undefined +// agent.handleMessage(sender, message) +// } + +// function _tagMessage(message: string) { +// return `(FROM OTHER BOT)${message}` +// } + +// async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { +// await new Promise(resolve => setTimeout(resolve, 5000)) +// if (self_prompter_paused && !conversationStore.inConversation()) { +// self_prompter_paused = false +// agent.self_prompter.start() +// } +// } diff --git a/src/libs/mineflayer/core.ts b/src/libs/mineflayer/core.ts index 446ac47..4bc9e5c 100644 --- a/src/libs/mineflayer/core.ts +++ b/src/libs/mineflayer/core.ts @@ -8,7 +8,7 @@ import { parseCommand } from './command' import { Components } from './components' import { Health } from './health' import { Memory } from './memory' -import { formBotChat } from './message' +import { ChatMessageHandler } from './message' import { Status } from './status' import { Ticker, type TickEvents, type TickEventsHandler } from './ticker' @@ -189,7 +189,7 @@ export class Mineflayer extends EventEmitter { } private handleCommand() { - return formBotChat(this.username, (sender, message) => { + return new ChatMessageHandler(this.username).handleChat((sender, message) => { const { isCommand, command, args } = parseCommand(sender, message) if (!isCommand) diff --git a/src/libs/mineflayer/core/base-agent.ts b/src/libs/mineflayer/core/base-agent.ts index b05a73b..c884034 100644 --- a/src/libs/mineflayer/core/base-agent.ts +++ b/src/libs/mineflayer/core/base-agent.ts @@ -63,9 +63,9 @@ export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { this.emit('chat', message) } - public clearBotLogs(): void { - // Implement if needed - } + // public clearBotLogs(): void { + // // Implement if needed + // } public requestInterrupt(): void { this.emit('interrupt') diff --git a/src/libs/mineflayer/index.ts b/src/libs/mineflayer/index.ts index d8989b0..1bea770 100644 --- a/src/libs/mineflayer/index.ts +++ b/src/libs/mineflayer/index.ts @@ -3,7 +3,6 @@ export * from './command' export * from './components' export * from './core' export * from './health' -export * from './interfaces' export * from './memory' export * from './message' export * from './plugin' diff --git a/src/libs/mineflayer/interfaces.ts b/src/libs/mineflayer/interfaces.ts deleted file mode 100644 index b55ec74..0000000 --- a/src/libs/mineflayer/interfaces.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface OneLinerable { - toOneLiner: () => string -} diff --git a/src/libs/mineflayer/message.ts b/src/libs/mineflayer/message.ts index aa35fc7..5ab42fb 100644 --- a/src/libs/mineflayer/message.ts +++ b/src/libs/mineflayer/message.ts @@ -1,30 +1,45 @@ import type { Entity } from 'prismarine-entity' -// TODO: need to be refactored -interface ChatBotContext { - fromUsername?: string - fromEntity?: Entity - fromMessage?: string - - isBot: () => boolean - isCommand: () => boolean +// Represents the context of a chat message in the Minecraft world +interface ChatMessage { + readonly sender: { + username: string + entity: Entity | null + } + readonly content: string } -export function newChatBotContext(entity: Entity, botUsername: string, username: string, message: string): ChatBotContext { - return { - fromUsername: username, - fromEntity: entity, - fromMessage: message, - isBot: () => username === botUsername, - isCommand: () => message.startsWith('#'), +// Handles chat message validation and processing +export class ChatMessageHandler { + constructor(private readonly botUsername: string) {} + + // Creates a new chat message context with validation + createMessageContext(entity: Entity | null, username: string, content: string): ChatMessage { + return { + sender: { + username, + entity, + }, + content, + } } -} -export function formBotChat(botUsername: string, cb: (username: string, message: string) => void) { - return (username: string, message: string) => { - if (botUsername === username) - return + // Checks if a message is from the bot itself + isBotMessage(username: string): boolean { + return username === this.botUsername + } + + // Checks if a message is a command + isCommand(content: string): boolean { + return content.startsWith('#') + } - cb(username, message) + // Processes chat messages, filtering out bot's own messages + handleChat(callback: (username: string, message: string) => void): (username: string, message: string) => void { + return (username: string, message: string) => { + if (!this.isBotMessage(username)) { + callback(username, message) + } + } } } diff --git a/src/libs/mineflayer/status.ts b/src/libs/mineflayer/status.ts index 6b5c01a..d39aadc 100644 --- a/src/libs/mineflayer/status.ts +++ b/src/libs/mineflayer/status.ts @@ -1,5 +1,5 @@ import type { Mineflayer } from './core' -import type { OneLinerable } from './interfaces' +import type { OneLinerable } from './types' export class Status implements OneLinerable { public position: string diff --git a/src/libs/mineflayer/types.ts b/src/libs/mineflayer/types.ts index 861b5a6..7f8a3f6 100644 --- a/src/libs/mineflayer/types.ts +++ b/src/libs/mineflayer/types.ts @@ -17,3 +17,7 @@ export interface EventHandlers { export type Events = keyof EventHandlers export type EventsHandler = EventHandlers[K] export type Handler = (ctx: Context) => void | Promise + +export interface OneLinerable { + toOneLiner: () => string +} diff --git a/src/main.ts b/src/main.ts index 138d1b1..70a3598 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,7 @@ import { pathfinder as MineflayerPathfinder } from 'mineflayer-pathfinder' import { plugin as MineflayerPVP } from 'mineflayer-pvp' import { plugin as MineflayerTool } from 'mineflayer-tool' -import { initAgent } from './agents/openai' +import { initAgent } from './agents/action/openai' import { initBot } from './composables/bot' import { botConfig, initEnv } from './composables/config' import { wrapPlugin } from './libs/mineflayer/plugin' diff --git a/src/mineflayer/echo.ts b/src/mineflayer/echo.ts index cf03105..f067b2a 100644 --- a/src/mineflayer/echo.ts +++ b/src/mineflayer/echo.ts @@ -1,14 +1,14 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' -import { formBotChat } from '../libs/mineflayer/message' +import { ChatMessageHandler } from '../libs/mineflayer/message' export function Echo(): MineflayerPlugin { const logger = useLogg('Echo').useGlobalConfig() return { spawned(mineflayer) { - const onChatHandler = formBotChat(mineflayer.username, (username, message) => { + const onChatHandler = new ChatMessageHandler(mineflayer.username).handleChat((username, message) => { logger.withFields({ username, message }).log('Chat message received') mineflayer.bot.chat(message) }) diff --git a/src/mineflayer/index.ts b/src/mineflayer/index.ts deleted file mode 100644 index 9f09fed..0000000 --- a/src/mineflayer/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './echo' -export * from './llm-agent' diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts index 25f4fc5..d75bcd4 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/mineflayer/llm-agent.ts @@ -4,11 +4,11 @@ import type { ChatCompletion } from 'neuri/openai' import type { Mineflayer } from '../libs/mineflayer' import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/interfaces/agents' import type { MineflayerPlugin } from '../libs/mineflayer/plugin' - import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' + import { createAppContainer } from '../container' -import { formBotChat } from '../libs/mineflayer/message' +import { ChatMessageHandler } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' import { toRetriable } from '../utils/reliability' @@ -169,7 +169,7 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) // 设置消息处理 - const onChat = formBotChat(bot.username, (username, message) => + const onChat = new ChatMessageHandler(bot.username).handleChat((username, message) => handleChatMessage(username, message, botWithAgents, options.agent, logger)) options.airiClient.onEvent('input:text:voice', event => From 4f8f37bb5d6401cc73d977b5c136987b85f30055 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 18 Jan 2025 18:53:50 +0800 Subject: [PATCH 07/22] refactor: agent/llm --- src/agents/action/index.ts | 6 +-- .../action/{openai.test.ts => llm.test.ts} | 10 ++-- src/agents/action/{openai.ts => llm.ts} | 2 +- .../action/{actions.test.ts => tools.test.ts} | 4 +- src/agents/action/{actions.ts => tools.ts} | 0 src/agents/chat/index.ts | 6 +-- src/agents/chat/llm.ts | 25 +-------- src/agents/memory/index.ts | 4 +- src/agents/planning/factory.ts | 6 +-- src/agents/planning/index.ts | 8 +-- src/agents/planning/llm.ts | 2 +- src/libs/mineflayer/{core => }/base-agent.ts | 54 ++++++++++++++++++- src/libs/mineflayer/interfaces/agents.ts | 53 ------------------ src/main.ts | 9 ++-- src/{mineflayer => plugins}/echo.ts | 0 src/{mineflayer => plugins}/follow.ts | 0 src/{mineflayer => plugins}/llm-agent.ts | 6 +-- src/{mineflayer => plugins}/pathfinder.ts | 0 src/{mineflayer => plugins}/status.ts | 0 src/utils/mcdata.ts | 2 - src/{prompts/agent.ts => utils/prompt.ts} | 22 ++++++++ tsconfig.json | 1 - 22 files changed, 107 insertions(+), 113 deletions(-) rename src/agents/action/{openai.test.ts => llm.test.ts} (77%) rename src/agents/action/{openai.ts => llm.ts} (97%) rename src/agents/action/{actions.test.ts => tools.test.ts} (97%) rename src/agents/action/{actions.ts => tools.ts} (100%) rename src/libs/mineflayer/{core => }/base-agent.ts (59%) delete mode 100644 src/libs/mineflayer/interfaces/agents.ts rename src/{mineflayer => plugins}/echo.ts (100%) rename src/{mineflayer => plugins}/follow.ts (100%) rename src/{mineflayer => plugins}/llm-agent.ts (98%) rename src/{mineflayer => plugins}/pathfinder.ts (100%) rename src/{mineflayer => plugins}/status.ts (100%) rename src/{prompts/agent.ts => utils/prompt.ts} (78%) diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 3868c6f..483c01c 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -1,10 +1,10 @@ import type { Mineflayer } from '../../libs/mineflayer' import type { Action } from '../../libs/mineflayer/action' -import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/interfaces/agents' +import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' import { ActionManager } from '../../composables/action' import { useBot } from '../../composables/bot' -import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' -import { actionsList } from './actions' +import { AbstractAgent } from '../../libs/mineflayer/base-agent' +import { actionsList } from './tools' interface ActionState { executing: boolean diff --git a/src/agents/action/openai.test.ts b/src/agents/action/llm.test.ts similarity index 77% rename from src/agents/action/openai.test.ts rename to src/agents/action/llm.test.ts index 4f7547e..978b248 100644 --- a/src/agents/action/openai.test.ts +++ b/src/agents/action/llm.test.ts @@ -1,10 +1,10 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' -import { initBot, useBot } from '../composables/bot' -import { botConfig, initEnv } from '../composables/config' -import { genSystemBasicPrompt } from '../prompts/agent' -import { initLogger } from '../utils/logger' -import { initAgent } from './openai' +import { initBot, useBot } from '../../composables/bot' +import { botConfig, initEnv } from '../../composables/config' +import { initLogger } from '../../utils/logger' +import { genSystemBasicPrompt } from '../../utils/prompt' +import { initAgent } from './llm' describe('openAI agent', { timeout: 0 }, () => { beforeAll(() => { diff --git a/src/agents/action/openai.ts b/src/agents/action/llm.ts similarity index 97% rename from src/agents/action/openai.ts rename to src/agents/action/llm.ts index f8fa327..bd44673 100644 --- a/src/agents/action/openai.ts +++ b/src/agents/action/llm.ts @@ -3,7 +3,7 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { agent, neuri } from 'neuri' import { openaiConfig } from '../../composables/config' -import { actionsList } from './actions' +import { actionsList } from './tools' let neuriAgent: Neuri | undefined const agents = new Set>() diff --git a/src/agents/action/actions.test.ts b/src/agents/action/tools.test.ts similarity index 97% rename from src/agents/action/actions.test.ts rename to src/agents/action/tools.test.ts index 5e255f3..ef5ddb2 100644 --- a/src/agents/action/actions.test.ts +++ b/src/agents/action/tools.test.ts @@ -2,10 +2,10 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' -import { genActionAgentPrompt, genQueryAgentPrompt } from '../../prompts/agent' import { sleep } from '../../utils/helper' import { initLogger } from '../../utils/logger' -import { initAgent } from '../openai' +import { genActionAgentPrompt, genQueryAgentPrompt } from '../../utils/prompt' +import { initAgent } from './llm' describe('actions agent', { timeout: 0 }, () => { beforeAll(() => { diff --git a/src/agents/action/actions.ts b/src/agents/action/tools.ts similarity index 100% rename from src/agents/action/actions.ts rename to src/agents/action/tools.ts diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts index c4bbd13..d092b5c 100644 --- a/src/agents/chat/index.ts +++ b/src/agents/chat/index.ts @@ -1,6 +1,6 @@ -import type { ChatAgent } from '../../libs/mineflayer/interfaces/agents' +import type { ChatAgent } from '../../libs/mineflayer/base-agent' import type { ChatAgentConfig, ChatContext } from './types' -import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { generateChatResponse } from './llm' export class ChatAgentImpl extends AbstractAgent implements ChatAgent { @@ -21,12 +21,10 @@ export class ChatAgentImpl extends AbstractAgent implements ChatAgent { protected async initializeAgent(): Promise { this.logger.log('Initializing chat agent') - // 设置事件监听 this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) - // 设置空闲超时检查 setInterval(() => { this.checkIdleChats() }, 60 * 1000) diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index 2875a8b..14e7f4f 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -2,6 +2,7 @@ import type { Neuri } from 'neuri' import type { ChatHistory } from './types' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' +import { genChatAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' const logger = useLogg('chat-llm').useGlobalConfig() @@ -19,7 +20,7 @@ export async function generateChatResponse( history: ChatHistory[], config: LLMChatConfig, ): Promise { - const systemPrompt = generateSystemPrompt() + const systemPrompt = genChatAgentPrompt() const chatHistory = formatChatHistory(history, config.maxContextLength ?? 10) const userPrompt = message @@ -63,28 +64,6 @@ export async function generateChatResponse( return content } -function generateSystemPrompt(): string { - return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. - -Guidelines: -1. Be friendly and helpful -2. Keep responses concise but informative -3. Use game-appropriate language -4. Acknowledge player's emotions and intentions -5. Ask for clarification when needed -6. Remember context from previous messages -7. Be proactive in suggesting helpful actions - -You can: -- Answer questions about the game -- Help with tasks and crafting -- Give directions and suggestions -- Engage in casual conversation -- Coordinate with other bots - -Remember that you're operating in a Minecraft world and should maintain that context in your responses.` -} - function formatChatHistory( history: ChatHistory[], maxLength: number, diff --git a/src/agents/memory/index.ts b/src/agents/memory/index.ts index ce53ead..db09257 100644 --- a/src/agents/memory/index.ts +++ b/src/agents/memory/index.ts @@ -1,6 +1,6 @@ import type { Message } from 'neuri/openai' -import type { Action } from 'src/libs/mineflayer' -import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/interfaces/agents' +import type { Action } from '../../libs/mineflayer' +import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/base-agent' import { useLogg } from '@guiiai/logg' import { Memory } from '../../libs/mineflayer/memory' diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts index f652626..4593f4c 100644 --- a/src/agents/planning/factory.ts +++ b/src/agents/planning/factory.ts @@ -20,7 +20,7 @@ export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin async created(mineflayer: Mineflayer) { logger.log('Initializing planning plugin') - // 创建容器并获取所需的服务 + // Get the container const container = createAppContainer({ neuri: options.agent, model: options.model, @@ -28,11 +28,11 @@ export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin const actionAgent = container.resolve('actionAgent') const planningAgent = container.resolve('planningAgent') - // 初始化 agents + // Initialize agents await actionAgent.init() await planningAgent.init() - // 添加到 bot + // Add to bot ;(mineflayer as MineflayerWithPlanning).planning = planningAgent }, diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 29eaa24..404c4c7 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -1,7 +1,7 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' -import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/interfaces/agents' -import { AbstractAgent } from '../../libs/mineflayer/core/base-agent' +import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/base-agent' +import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { ActionAgentImpl } from '../action' import { generatePlanWithLLM } from './llm' @@ -54,14 +54,14 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { protected async initializeAgent(): Promise { this.logger.log('Initializing planning agent') - // 直接创建 action agent + // Create action agent directly this.actionAgent = new ActionAgentImpl({ id: 'action', type: 'action', }) await this.actionAgent.init() - // 设置事件监听 + // Set event listener this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index c1187b0..52fd120 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -2,7 +2,7 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' -import { genPlanningAgentPrompt } from '../../prompts/agent' +import { genPlanningAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' const logger = useLogg('planning-llm').useGlobalConfig() diff --git a/src/libs/mineflayer/core/base-agent.ts b/src/libs/mineflayer/base-agent.ts similarity index 59% rename from src/libs/mineflayer/core/base-agent.ts rename to src/libs/mineflayer/base-agent.ts index c884034..b023a84 100644 --- a/src/libs/mineflayer/core/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -1,7 +1,59 @@ -import type { AgentConfig, BaseAgent } from '../interfaces/agents' +import type { Action } from './action' import { useLogg } from '@guiiai/logg' import EventEmitter3 from 'eventemitter3' +export type AgentType = 'action' | 'memory' | 'planning' | 'chat' + +export interface AgentConfig { + id: string + type: AgentType +} + +export interface BaseAgent { + readonly id: string + readonly type: AgentType + init: () => Promise + destroy: () => Promise +} + +export interface ActionAgent extends BaseAgent { + type: 'action' + performAction: (name: string, params: unknown[]) => Promise + getAvailableActions: () => Action[] +} + +export interface MemoryAgent extends BaseAgent { + type: 'memory' + remember: (key: string, value: unknown) => void + recall: (key: string) => T | undefined + forget: (key: string) => void + getMemorySnapshot: () => Record +} + +export interface Plan { + goal: string + steps: Array<{ + action: string + params: unknown[] + }> + status: 'pending' | 'in_progress' | 'completed' | 'failed' + requiresAction: boolean +} + +export interface PlanningAgent extends BaseAgent { + type: 'planning' + createPlan: (goal: string) => Promise + executePlan: (plan: Plan) => Promise + adjustPlan: (plan: Plan, feedback: string) => Promise +} + +export interface ChatAgent extends BaseAgent { + type: 'chat' + processMessage: (message: string, sender: string) => Promise + startConversation: (player: string) => void + endConversation: (player: string) => void +} + export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { public readonly id: string public readonly type: AgentConfig['type'] diff --git a/src/libs/mineflayer/interfaces/agents.ts b/src/libs/mineflayer/interfaces/agents.ts deleted file mode 100644 index 62140a5..0000000 --- a/src/libs/mineflayer/interfaces/agents.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { Action } from '../action' - -export type AgentType = 'action' | 'memory' | 'planning' | 'chat' - -export interface AgentConfig { - id: string - type: AgentType -} - -export interface BaseAgent { - readonly id: string - readonly type: AgentType - init: () => Promise - destroy: () => Promise -} - -export interface ActionAgent extends BaseAgent { - type: 'action' - performAction: (name: string, params: unknown[]) => Promise - getAvailableActions: () => Action[] -} - -export interface MemoryAgent extends BaseAgent { - type: 'memory' - remember: (key: string, value: unknown) => void - recall: (key: string) => T | undefined - forget: (key: string) => void - getMemorySnapshot: () => Record -} - -export interface Plan { - goal: string - steps: Array<{ - action: string - params: unknown[] - }> - status: 'pending' | 'in_progress' | 'completed' | 'failed' - requiresAction: boolean -} - -export interface PlanningAgent extends BaseAgent { - type: 'planning' - createPlan: (goal: string) => Promise - executePlan: (plan: Plan) => Promise - adjustPlan: (plan: Plan, feedback: string) => Promise -} - -export interface ChatAgent extends BaseAgent { - type: 'chat' - processMessage: (message: string, sender: string) => Promise - startConversation: (player: string) => void - endConversation: (player: string) => void -} diff --git a/src/main.ts b/src/main.ts index 70a3598..5026eeb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,5 +1,4 @@ import process, { exit } from 'node:process' - import { useLogg } from '@guiiai/logg' import { Client } from '@proj-airi/server-sdk' import MineflayerArmorManager from 'mineflayer-armor-manager' @@ -9,11 +8,11 @@ import { pathfinder as MineflayerPathfinder } from 'mineflayer-pathfinder' import { plugin as MineflayerPVP } from 'mineflayer-pvp' import { plugin as MineflayerTool } from 'mineflayer-tool' -import { initAgent } from './agents/action/openai' +import { initAgent } from './agents/action/llm' import { initBot } from './composables/bot' import { botConfig, initEnv } from './composables/config' -import { wrapPlugin } from './libs/mineflayer/plugin' -import { LLMAgent } from './mineflayer/llm-agent' +import { wrapPlugin } from './libs/mineflayer' +import { LLMAgent } from './plugins/llm-agent' import { initLogger } from './utils/logger' const logger = useLogg('main').useGlobalConfig() @@ -36,7 +35,7 @@ async function main() { const airiClient = new Client({ name: 'minecraft-bot', url: 'ws://localhost:6121/ws' }) - // Dynamically load LLMAgent after bot is initialized + // Dynamically load LLMAgent after the bot is initialized const agent = await initAgent(bot) await bot.loadPlugin(LLMAgent({ agent, airiClient })) diff --git a/src/mineflayer/echo.ts b/src/plugins/echo.ts similarity index 100% rename from src/mineflayer/echo.ts rename to src/plugins/echo.ts diff --git a/src/mineflayer/follow.ts b/src/plugins/follow.ts similarity index 100% rename from src/mineflayer/follow.ts rename to src/plugins/follow.ts diff --git a/src/mineflayer/llm-agent.ts b/src/plugins/llm-agent.ts similarity index 98% rename from src/mineflayer/llm-agent.ts rename to src/plugins/llm-agent.ts index d75bcd4..7c4afd0 100644 --- a/src/mineflayer/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -2,14 +2,14 @@ import type { Client } from '@proj-airi/server-sdk' import type { Neuri, NeuriContext } from 'neuri' import type { ChatCompletion } from 'neuri/openai' import type { Mineflayer } from '../libs/mineflayer' -import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/interfaces/agents' +import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/base-agent' + import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' - import { createAppContainer } from '../container' import { ChatMessageHandler } from '../libs/mineflayer/message' -import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' +import { genActionAgentPrompt, genStatusPrompt } from '../utils/prompt' import { toRetriable } from '../utils/reliability' interface MineflayerWithAgents extends Mineflayer { diff --git a/src/mineflayer/pathfinder.ts b/src/plugins/pathfinder.ts similarity index 100% rename from src/mineflayer/pathfinder.ts rename to src/plugins/pathfinder.ts diff --git a/src/mineflayer/status.ts b/src/plugins/status.ts similarity index 100% rename from src/mineflayer/status.ts rename to src/plugins/status.ts diff --git a/src/utils/mcdata.ts b/src/utils/mcdata.ts index 8dc9630..40ed7b3 100644 --- a/src/utils/mcdata.ts +++ b/src/utils/mcdata.ts @@ -1,5 +1,3 @@ -// src/utils/minecraftData.ts - import type { Bot } from 'mineflayer' import type { Entity } from 'prismarine-entity' import minecraftData, { diff --git a/src/prompts/agent.ts b/src/utils/prompt.ts similarity index 78% rename from src/prompts/agent.ts rename to src/utils/prompt.ts index 6e52276..f22f5d9 100644 --- a/src/prompts/agent.ts +++ b/src/utils/prompt.ts @@ -88,3 +88,25 @@ Example response: } ]` } + +export function genChatAgentPrompt(): string { + return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. + +Guidelines: +1. Be friendly and helpful +2. Keep responses concise but informative +3. Use game-appropriate language +4. Acknowledge player's emotions and intentions +5. Ask for clarification when needed +6. Remember context from previous messages +7. Be proactive in suggesting helpful actions + +You can: +- Answer questions about the game +- Help with tasks and crafting +- Give directions and suggestions +- Engage in casual conversation +- Coordinate with other bots + +Remember that you're operating in a Minecraft world and should maintain that context in your responses.` +} diff --git a/tsconfig.json b/tsconfig.json index 2c7d05d..b7208af 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,6 @@ "ESNext" ], "moduleDetection": "auto", - "baseUrl": ".", "module": "ESNext", "moduleResolution": "bundler", "resolveJsonModule": true, From fbdff4e5d3e4b248312abaa8c2c89e2f71a49e06 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 18 Jan 2025 19:05:20 +0800 Subject: [PATCH 08/22] style: import --- eslint.config.mjs | 13 +++++++++++++ src/agents/action/index.ts | 1 + src/agents/action/llm.test.ts | 1 + src/agents/action/llm.ts | 2 ++ src/agents/action/tools.test.ts | 1 + src/agents/action/tools.ts | 2 ++ src/agents/chat/index.ts | 1 + src/agents/chat/llm.ts | 2 ++ src/agents/memory/index.ts | 2 ++ src/agents/planning/factory.ts | 2 ++ src/agents/planning/index.ts | 1 + src/agents/planning/llm.ts | 2 ++ src/composables/action.ts | 1 + src/composables/config.ts | 1 + src/composables/world.ts | 2 ++ src/container.ts | 2 ++ src/libs/mineflayer/base-agent.ts | 1 + src/libs/mineflayer/core.ts | 2 ++ src/plugins/echo.ts | 1 + src/plugins/follow.ts | 1 + src/plugins/llm-agent.ts | 3 ++- src/plugins/status.ts | 1 + src/skills/actions/collect-block.ts | 2 ++ src/skills/actions/ensure.ts | 2 ++ src/skills/actions/gather-wood.ts | 2 ++ src/skills/actions/inventory.ts | 1 + src/skills/actions/world-interactions.ts | 2 ++ src/skills/base.ts | 1 + src/skills/blocks.ts | 1 + src/skills/inventory.ts | 1 + src/skills/movement.ts | 1 + src/utils/mcdata.ts | 1 + src/utils/prompt.ts | 1 + 33 files changed, 59 insertions(+), 1 deletion(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 97ecee2..4e9cff5 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -2,4 +2,17 @@ import antfu from '@antfu/eslint-config' export default antfu({ formatters: true, + rules: { + 'import/order': [ + 'error', + { + 'groups': [ + ['type'], // types 最优先 + ['builtin', 'external'], // 然后是内置模块和外部包 + ['parent', 'sibling', 'index'], // 最后是内部引用 + ], + 'newlines-between': 'always', + }, + ], + }, }) diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 483c01c..fd5f2e4 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -1,6 +1,7 @@ import type { Mineflayer } from '../../libs/mineflayer' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' + import { ActionManager } from '../../composables/action' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' diff --git a/src/agents/action/llm.test.ts b/src/agents/action/llm.test.ts index 978b248..a816e14 100644 --- a/src/agents/action/llm.test.ts +++ b/src/agents/action/llm.test.ts @@ -1,5 +1,6 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' + import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' import { initLogger } from '../../utils/logger' diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index bd44673..141af37 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -1,7 +1,9 @@ import type { Agent, Neuri } from 'neuri' import type { Mineflayer } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' import { agent, neuri } from 'neuri' + import { openaiConfig } from '../../composables/config' import { actionsList } from './tools' diff --git a/src/agents/action/tools.test.ts b/src/agents/action/tools.test.ts index ef5ddb2..2f515a1 100644 --- a/src/agents/action/tools.test.ts +++ b/src/agents/action/tools.test.ts @@ -1,5 +1,6 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' + import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' import { sleep } from '../../utils/helper' diff --git a/src/agents/action/tools.ts b/src/agents/action/tools.ts index 3d9e2fb..3ba659a 100644 --- a/src/agents/action/tools.ts +++ b/src/agents/action/tools.ts @@ -1,6 +1,8 @@ import type { Action } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' import { z } from 'zod' + import * as world from '../../composables/world' import * as skills from '../../skills' import { collectBlock } from '../../skills/actions/collect-block' diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts index d092b5c..5def0e0 100644 --- a/src/agents/chat/index.ts +++ b/src/agents/chat/index.ts @@ -1,5 +1,6 @@ import type { ChatAgent } from '../../libs/mineflayer/base-agent' import type { ChatAgentConfig, ChatContext } from './types' + import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { generateChatResponse } from './llm' diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index 14e7f4f..cc1f0f8 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -1,7 +1,9 @@ import type { Neuri } from 'neuri' import type { ChatHistory } from './types' + import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' + import { genChatAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' diff --git a/src/agents/memory/index.ts b/src/agents/memory/index.ts index db09257..8cf7c41 100644 --- a/src/agents/memory/index.ts +++ b/src/agents/memory/index.ts @@ -1,7 +1,9 @@ import type { Message } from 'neuri/openai' import type { Action } from '../../libs/mineflayer' import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/base-agent' + import { useLogg } from '@guiiai/logg' + import { Memory } from '../../libs/mineflayer/memory' const logger = useLogg('memory-agent').useGlobalConfig() diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts index 4593f4c..48fa35c 100644 --- a/src/agents/planning/factory.ts +++ b/src/agents/planning/factory.ts @@ -1,7 +1,9 @@ import type { Neuri } from 'neuri' import type { Mineflayer } from '../../libs/mineflayer' import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' + import { useLogg } from '@guiiai/logg' + import { createAppContainer } from '../../container' interface PlanningPluginOptions { diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 404c4c7..5c0767c 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -1,6 +1,7 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/base-agent' + import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { ActionAgentImpl } from '../action' import { generatePlanWithLLM } from './llm' diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index 52fd120..b084538 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -1,7 +1,9 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' + import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' + import { genPlanningAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' diff --git a/src/composables/action.ts b/src/composables/action.ts index 96eeaed..94843c9 100644 --- a/src/composables/action.ts +++ b/src/composables/action.ts @@ -1,4 +1,5 @@ import type { Mineflayer } from '../libs/mineflayer/core' + import { useLogg } from '@guiiai/logg' import EventEmitter from 'eventemitter3' diff --git a/src/composables/config.ts b/src/composables/config.ts index 03fbe52..bae9c66 100644 --- a/src/composables/config.ts +++ b/src/composables/config.ts @@ -1,4 +1,5 @@ import type { BotOptions } from 'mineflayer' + import { env } from 'node:process' import { useLogg } from '@guiiai/logg' diff --git a/src/composables/world.ts b/src/composables/world.ts index db4719c..fd7e950 100644 --- a/src/composables/world.ts +++ b/src/composables/world.ts @@ -3,7 +3,9 @@ import type { Entity } from 'prismarine-entity' import type { Item } from 'prismarine-item' import type { Vec3 } from 'vec3' import type { Mineflayer } from '../libs/mineflayer' + import pf from 'mineflayer-pathfinder' + import * as mc from '../utils/mcdata' export function getNearestFreeSpace( diff --git a/src/container.ts b/src/container.ts index 18911f9..678fa57 100644 --- a/src/container.ts +++ b/src/container.ts @@ -1,6 +1,8 @@ import type { Neuri } from 'neuri' + import { useLogg } from '@guiiai/logg' import { asClass, asFunction, createContainer, InjectionMode } from 'awilix' + import { ActionAgentImpl } from './agents/action' import { ChatAgentImpl } from './agents/chat' import { PlanningAgentImpl } from './agents/planning' diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts index b023a84..3e71576 100644 --- a/src/libs/mineflayer/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -1,4 +1,5 @@ import type { Action } from './action' + import { useLogg } from '@guiiai/logg' import EventEmitter3 from 'eventemitter3' diff --git a/src/libs/mineflayer/core.ts b/src/libs/mineflayer/core.ts index 4bc9e5c..6e019a2 100644 --- a/src/libs/mineflayer/core.ts +++ b/src/libs/mineflayer/core.ts @@ -1,9 +1,11 @@ import type { Bot, BotOptions } from 'mineflayer' import type { MineflayerPlugin } from './plugin' import type { EventHandlers, EventsHandler } from './types' + import { type Logg, useLogg } from '@guiiai/logg' import EventEmitter from 'eventemitter3' import mineflayer from 'mineflayer' + import { parseCommand } from './command' import { Components } from './components' import { Health } from './health' diff --git a/src/plugins/echo.ts b/src/plugins/echo.ts index f067b2a..67b4932 100644 --- a/src/plugins/echo.ts +++ b/src/plugins/echo.ts @@ -1,6 +1,7 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' + import { ChatMessageHandler } from '../libs/mineflayer/message' export function Echo(): MineflayerPlugin { diff --git a/src/plugins/follow.ts b/src/plugins/follow.ts index 7b5c5d8..851b9bf 100644 --- a/src/plugins/follow.ts +++ b/src/plugins/follow.ts @@ -1,4 +1,5 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' + import { useLogg } from '@guiiai/logg' import pathfinderModel from 'mineflayer-pathfinder' diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts index 7c4afd0..442943b 100644 --- a/src/plugins/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -3,10 +3,11 @@ import type { Neuri, NeuriContext } from 'neuri' import type { ChatCompletion } from 'neuri/openai' import type { Mineflayer } from '../libs/mineflayer' import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/base-agent' - import type { MineflayerPlugin } from '../libs/mineflayer/plugin' + import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' + import { createAppContainer } from '../container' import { ChatMessageHandler } from '../libs/mineflayer/message' import { genActionAgentPrompt, genStatusPrompt } from '../utils/prompt' diff --git a/src/plugins/status.ts b/src/plugins/status.ts index 2500835..b1401f6 100644 --- a/src/plugins/status.ts +++ b/src/plugins/status.ts @@ -1,4 +1,5 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' + import { useLogg } from '@guiiai/logg' export function Status(): MineflayerPlugin { diff --git a/src/skills/actions/collect-block.ts b/src/skills/actions/collect-block.ts index 35c539c..333d91d 100644 --- a/src/skills/actions/collect-block.ts +++ b/src/skills/actions/collect-block.ts @@ -1,7 +1,9 @@ import type { Block } from 'prismarine-block' import type { Mineflayer } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' import pathfinder from 'mineflayer-pathfinder' + import { getNearestBlocks } from '../../composables/world' import { breakBlockAt } from '../blocks' import { ensurePickaxe } from './ensure' diff --git a/src/skills/actions/ensure.ts b/src/skills/actions/ensure.ts index dfed7c7..cf22b9d 100644 --- a/src/skills/actions/ensure.ts +++ b/src/skills/actions/ensure.ts @@ -1,5 +1,7 @@ import type { Mineflayer } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' + import { getItemId } from '../../utils/mcdata' import { craftRecipe } from '../crafting' import { moveAway } from '../movement' diff --git a/src/skills/actions/gather-wood.ts b/src/skills/actions/gather-wood.ts index 678468c..91479c1 100644 --- a/src/skills/actions/gather-wood.ts +++ b/src/skills/actions/gather-wood.ts @@ -1,5 +1,7 @@ import type { Mineflayer } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' + import { getNearestBlocks } from '../../composables/world' import { sleep } from '../../utils/helper' import { breakBlockAt } from '../blocks' diff --git a/src/skills/actions/inventory.ts b/src/skills/actions/inventory.ts index a63d068..0fa75ff 100644 --- a/src/skills/actions/inventory.ts +++ b/src/skills/actions/inventory.ts @@ -2,6 +2,7 @@ import type { Item } from 'prismarine-item' import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' + import { getNearestBlock } from '../../composables/world' import { goToPlayer, goToPosition } from '../movement' diff --git a/src/skills/actions/world-interactions.ts b/src/skills/actions/world-interactions.ts index c4d1b77..fbfb5e1 100644 --- a/src/skills/actions/world-interactions.ts +++ b/src/skills/actions/world-interactions.ts @@ -1,9 +1,11 @@ import type { Bot } from 'mineflayer' import type { Block } from 'prismarine-block' import type { Mineflayer } from '../../libs/mineflayer' + import { useLogg } from '@guiiai/logg' import pathfinder from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' + import { sleep } from '../../utils/helper' import { getNearestBlock, makeItem } from '../../utils/mcdata' import { goToPosition } from '../movement' diff --git a/src/skills/base.ts b/src/skills/base.ts index 7991de9..a2721f1 100644 --- a/src/skills/base.ts +++ b/src/skills/base.ts @@ -1,4 +1,5 @@ import type { Mineflayer } from '../libs/mineflayer' + import { useLogg } from '@guiiai/logg' const logger = useLogg('skills').useGlobalConfig() diff --git a/src/skills/blocks.ts b/src/skills/blocks.ts index 4d9f2b9..78e6cff 100644 --- a/src/skills/blocks.ts +++ b/src/skills/blocks.ts @@ -3,6 +3,7 @@ import type { BlockFace } from './base' import pathfinderModel, { type SafeBlock } from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' + import { getNearestBlock, getNearestBlocks, getPosition, shouldPlaceTorch } from '../composables/world' import { getBlockId, makeItem } from '../utils/mcdata' import { log } from './base' diff --git a/src/skills/inventory.ts b/src/skills/inventory.ts index 4d3623a..91c3c9c 100644 --- a/src/skills/inventory.ts +++ b/src/skills/inventory.ts @@ -1,4 +1,5 @@ import type { Mineflayer } from '../libs/mineflayer' + import { getNearestBlock } from '../composables/world' import { log } from './base' import { goToPlayer, goToPosition } from './movement' diff --git a/src/skills/movement.ts b/src/skills/movement.ts index 857c08a..a48ae85 100644 --- a/src/skills/movement.ts +++ b/src/skills/movement.ts @@ -5,6 +5,7 @@ import { useLogg } from '@guiiai/logg' import { randomInt } from 'es-toolkit' import pathfinder from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' + import { getNearestBlock, getNearestEntityWhere } from '../composables/world' import { sleep } from '../utils/helper' import { log } from './base' diff --git a/src/utils/mcdata.ts b/src/utils/mcdata.ts index 40ed7b3..48f093f 100644 --- a/src/utils/mcdata.ts +++ b/src/utils/mcdata.ts @@ -1,5 +1,6 @@ import type { Bot } from 'mineflayer' import type { Entity } from 'prismarine-entity' + import minecraftData, { type Biome, type ShapedRecipe, diff --git a/src/utils/prompt.ts b/src/utils/prompt.ts index f22f5d9..82f5e60 100644 --- a/src/utils/prompt.ts +++ b/src/utils/prompt.ts @@ -1,4 +1,5 @@ import type { Action, Mineflayer } from '../libs/mineflayer' + import { listInventory } from '../skills/actions/inventory' export function genSystemBasicPrompt(botName: string): string { From 35494a5b237c4803020b061ae3274f8e1eb0dd79 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 01:42:07 +0800 Subject: [PATCH 09/22] refactor: llm handler --- src/agents/action/llm-handler.ts | 19 +++++++++ src/agents/chat/llm-handler.ts | 46 ++++++++++++++++++++++ src/agents/planning/llm-handler.ts | 63 ++++++++++++++++++++++++++++++ src/composables/action.ts | 56 ++++++++++++++------------ src/libs/llm/base.ts | 42 ++++++++++++++++++++ src/libs/llm/types.ts | 14 +++++++ 6 files changed, 215 insertions(+), 25 deletions(-) create mode 100644 src/agents/action/llm-handler.ts create mode 100644 src/agents/chat/llm-handler.ts create mode 100644 src/agents/planning/llm-handler.ts create mode 100644 src/libs/llm/base.ts create mode 100644 src/libs/llm/types.ts diff --git a/src/agents/action/llm-handler.ts b/src/agents/action/llm-handler.ts new file mode 100644 index 0000000..44ac118 --- /dev/null +++ b/src/agents/action/llm-handler.ts @@ -0,0 +1,19 @@ +import { BaseLLMHandler } from '../../libs/llm/base' + +export class ActionLLMHandler extends BaseLLMHandler { + public async handleAction(messages: any[]): Promise { + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Processing action...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to process action') + } + + return result + } +} diff --git a/src/agents/chat/llm-handler.ts b/src/agents/chat/llm-handler.ts new file mode 100644 index 0000000..a95a85f --- /dev/null +++ b/src/agents/chat/llm-handler.ts @@ -0,0 +1,46 @@ +import type { ChatHistory } from './types' + +import { system, user } from 'neuri/openai' + +import { BaseLLMHandler } from '../../libs/llm/base' +import { genChatAgentPrompt } from '../../utils/prompt' + +export class ChatLLMHandler extends BaseLLMHandler { + public async generateResponse( + message: string, + history: ChatHistory[], + ): Promise { + const systemPrompt = genChatAgentPrompt() + const chatHistory = this.formatChatHistory(history, this.config.maxContextLength ?? 10) + const messages = [ + system(systemPrompt), + ...chatHistory, + user(message), + ] + + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Generating response...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'chat', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to generate response') + } + + return result + } + + private formatChatHistory( + history: ChatHistory[], + maxLength: number, + ): Array<{ role: 'user' | 'assistant', content: string }> { + const recentHistory = history.slice(-maxLength) + return recentHistory.map(entry => ({ + role: entry.sender === 'bot' ? 'assistant' : 'user', + content: entry.message, + })) + } +} diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts new file mode 100644 index 0000000..18a636d --- /dev/null +++ b/src/agents/planning/llm-handler.ts @@ -0,0 +1,63 @@ +import type { Action } from '../../libs/mineflayer/action' + +import { system, user } from 'neuri/openai' + +import { BaseLLMHandler } from '../../libs/llm/base' +import { genPlanningAgentPrompt } from '../../utils/prompt' + +export class PlanningLLMHandler extends BaseLLMHandler { + public async generatePlan( + goal: string, + availableActions: Action[], + feedback?: string, + ): Promise> { + const systemPrompt = genPlanningAgentPrompt(availableActions) + const userPrompt = this.generateUserPrompt(goal, feedback) + const messages = [system(systemPrompt), user(userPrompt)] + + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Generating plan...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to generate plan') + } + + return this.parsePlanContent(result) + } + + private generateUserPrompt(goal: string, feedback?: string): string { + let prompt = `Create a plan to: ${goal}` + if (feedback) { + prompt += `\nPrevious attempt feedback: ${feedback}` + } + return prompt + } + + private parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { + try { + const match = content.match(/\[[\s\S]*\]/) + if (!match) { + throw new Error('No plan found in response') + } + + const plan = JSON.parse(match[0]) + if (!Array.isArray(plan)) { + throw new TypeError('Invalid plan format') + } + + return plan.map(step => ({ + action: step.action, + params: step.params, + })) + } + catch (error) { + this.logger.withError(error).error('Failed to parse plan') + throw error + } + } +} diff --git a/src/composables/action.ts b/src/composables/action.ts index 94843c9..7f78c0b 100644 --- a/src/composables/action.ts +++ b/src/composables/action.ts @@ -17,6 +17,8 @@ interface QueuedAction { fn: ActionFn timeout: number resume: boolean + resolve: (result: ActionResult) => void + reject: (error: Error) => void } export class ActionManager extends EventEmitter { @@ -75,24 +77,17 @@ export class ActionManager extends EventEmitter { this.state.resume.name = undefined } - private async queueAction(action: QueuedAction): Promise { - // Add action to queue - this.actionQueue.push(action) + private async queueAction(action: Omit): Promise { + return new Promise((resolve, reject) => { + this.actionQueue.push({ + ...action, + resolve, + reject, + }) - // If not executing, start processing queue - if (!this.state.executing) { - return this.processQueue() - } - - // Return a promise that will resolve when the action is executed - return new Promise((resolve) => { - const checkQueue = setInterval(() => { - const index = this.actionQueue.findIndex(a => a === action) - if (index === -1) { - clearInterval(checkQueue) - resolve({ success: true, message: 'success', timedout: false }) - } - }, 100) + if (!this.state.executing) { + this.processQueue().catch(reject) + } }) } @@ -100,17 +95,28 @@ export class ActionManager extends EventEmitter { while (this.actionQueue.length > 0) { const action = this.actionQueue[0] - const result = action.resume - ? await this.executeResume(action.label, action.fn, action.timeout) - : await this.executeAction(action.label, action.fn, action.timeout) + try { + const result = action.resume + ? await this.executeResume(action.label, action.fn, action.timeout) + : await this.executeAction(action.label, action.fn, action.timeout) - // Remove completed action from queue - this.actionQueue.shift() + this.actionQueue.shift()?.resolve(result) - if (!result.success) { - // Clear queue on failure + if (!result.success) { + this.actionQueue.forEach(pendingAction => + pendingAction.reject(new Error('Queue cleared due to action failure')), + ) + this.actionQueue = [] + return result + } + } + catch (error) { + this.actionQueue.shift()?.reject(error as Error) + this.actionQueue.forEach(pendingAction => + pendingAction.reject(new Error('Queue cleared due to error')), + ) this.actionQueue = [] - return result + throw error } } diff --git a/src/libs/llm/base.ts b/src/libs/llm/base.ts new file mode 100644 index 0000000..274db0a --- /dev/null +++ b/src/libs/llm/base.ts @@ -0,0 +1,42 @@ +import type { LLMConfig, LLMResponse } from './types' + +import { useLogg } from '@guiiai/logg' + +import { toRetriable } from '../../utils/reliability' + +export abstract class BaseLLMHandler { + protected logger = useLogg('llm-handler').useGlobalConfig() + + constructor(protected config: LLMConfig) {} + + protected async handleCompletion( + context: any, + route: string, + messages: any[], + ): Promise { + const completion = await context.reroute(route, messages, { + model: this.config.model ?? 'openai/gpt-4-mini', + }) + + if (!completion || 'error' in completion) { + this.logger.withFields(context).error('Completion failed') + throw new Error(completion?.error?.message ?? 'Unknown error') + } + + const content = await completion.firstContent() + this.logger.withFields({ usage: completion.usage, content }).log('Generated content') + + return { + content, + usage: completion.usage, + } + } + + protected createRetryHandler(handler: (context: any) => Promise) { + return toRetriable( + this.config.retryLimit ?? 3, + this.config.delayInterval ?? 1000, + handler, + ) + } +} diff --git a/src/libs/llm/types.ts b/src/libs/llm/types.ts new file mode 100644 index 0000000..a21fd17 --- /dev/null +++ b/src/libs/llm/types.ts @@ -0,0 +1,14 @@ +import type { Neuri } from 'neuri' + +export interface LLMConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number + maxContextLength?: number +} + +export interface LLMResponse { + content: string + usage?: any +} From b98985102a4a6de8fc5adda1a39dab2eee7b934d Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 05:24:28 +0800 Subject: [PATCH 10/22] refactor: split prompt --- src/agents/action/llm.test.ts | 4 +- src/agents/action/llm.ts | 2 +- src/agents/action/tools.test.ts | 34 +------- src/agents/chat/llm-handler.ts | 2 +- src/agents/chat/llm.ts | 2 +- src/agents/planning/llm-handler.ts | 14 +--- src/agents/planning/llm.ts | 14 +--- src/agents/prompt/chat.ts | 21 +++++ src/agents/prompt/llm-agent.plugin.ts | 51 ++++++++++++ src/agents/prompt/planning.ts | 47 +++++++++++ src/libs/mineflayer/base-agent.ts | 4 +- src/plugins/llm-agent.ts | 29 ++++--- src/utils/prompt.ts | 113 -------------------------- 13 files changed, 149 insertions(+), 188 deletions(-) create mode 100644 src/agents/prompt/chat.ts create mode 100644 src/agents/prompt/llm-agent.plugin.ts create mode 100644 src/agents/prompt/planning.ts delete mode 100644 src/utils/prompt.ts diff --git a/src/agents/action/llm.test.ts b/src/agents/action/llm.test.ts index a816e14..502fc88 100644 --- a/src/agents/action/llm.test.ts +++ b/src/agents/action/llm.test.ts @@ -4,7 +4,7 @@ import { beforeAll, describe, expect, it } from 'vitest' import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' import { initLogger } from '../../utils/logger' -import { genSystemBasicPrompt } from '../../utils/prompt' +import { generateSystemBasicPrompt } from '../prompt/llm-agent.plugin' import { initAgent } from './llm' describe('openAI agent', { timeout: 0 }, () => { @@ -22,7 +22,7 @@ describe('openAI agent', { timeout: 0 }, () => { bot.bot.once('spawn', async () => { const text = await agent.handle( messages( - system(genSystemBasicPrompt('airi')), + system(generateSystemBasicPrompt('airi')), user('Hello, who are you?'), ), async (c) => { diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index 141af37..bd6005b 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -10,7 +10,7 @@ import { actionsList } from './tools' let neuriAgent: Neuri | undefined const agents = new Set>() -const logger = useLogg('openai').useGlobalConfig() +const logger = useLogg('action-llm').useGlobalConfig() export async function initAgent(mineflayer: Mineflayer): Promise { logger.log('Initializing agent') diff --git a/src/agents/action/tools.test.ts b/src/agents/action/tools.test.ts index 2f515a1..b02cc22 100644 --- a/src/agents/action/tools.test.ts +++ b/src/agents/action/tools.test.ts @@ -5,7 +5,7 @@ import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' import { sleep } from '../../utils/helper' import { initLogger } from '../../utils/logger' -import { genActionAgentPrompt, genQueryAgentPrompt } from '../../utils/prompt' +import { generateActionAgentPrompt } from '../prompt/llm-agent.plugin' import { initAgent } from './llm' describe('actions agent', { timeout: 0 }, () => { @@ -22,7 +22,7 @@ describe('actions agent', { timeout: 0 }, () => { await new Promise((resolve) => { bot.bot.once('spawn', async () => { const text = await agent.handle(messages( - system(genQueryAgentPrompt(bot)), + system(generateActionAgentPrompt(bot)), user('What\'s your status?'), ), async (c) => { const completion = await c.reroute('query', c.messages, { model: 'openai/gpt-4o-mini' }) @@ -43,7 +43,7 @@ describe('actions agent', { timeout: 0 }, () => { await new Promise((resolve) => { bot.bot.on('spawn', async () => { const text = await agent.handle(messages( - system(genActionAgentPrompt(bot)), + system(generateActionAgentPrompt(bot)), user('goToPlayer: luoling8192'), ), async (c) => { const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) @@ -58,32 +58,4 @@ describe('actions agent', { timeout: 0 }, () => { }) }) }) - - // it('should split question into actions', async () => { - // const { ctx } = useBot() - // const agent = await initAgent(ctx) - - // function testFn() { - // return new Promise((resolve) => { - // ctx.bot.on('spawn', async () => { - // const text = await agent.handle(messages( - // system(genActionAgentPrompt(ctx)), - // user('Help me to cut down the tree'), - // ), async (c) => { - // const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - - // console.log(completion) - - // return await completion?.firstContent() - // }) - - // console.log(text) - - // resolve() - // }) - // }) - // } - - // await testFn() - // }) }) diff --git a/src/agents/chat/llm-handler.ts b/src/agents/chat/llm-handler.ts index a95a85f..7743a8f 100644 --- a/src/agents/chat/llm-handler.ts +++ b/src/agents/chat/llm-handler.ts @@ -3,7 +3,7 @@ import type { ChatHistory } from './types' import { system, user } from 'neuri/openai' import { BaseLLMHandler } from '../../libs/llm/base' -import { genChatAgentPrompt } from '../../utils/prompt' +import { genChatAgentPrompt } from '../prompt/chat' export class ChatLLMHandler extends BaseLLMHandler { public async generateResponse( diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index cc1f0f8..5e7f39c 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -4,8 +4,8 @@ import type { ChatHistory } from './types' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' -import { genChatAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' +import { genChatAgentPrompt } from '../prompt/chat' const logger = useLogg('chat-llm').useGlobalConfig() diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index 18a636d..a17f153 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -3,7 +3,7 @@ import type { Action } from '../../libs/mineflayer/action' import { system, user } from 'neuri/openai' import { BaseLLMHandler } from '../../libs/llm/base' -import { genPlanningAgentPrompt } from '../../utils/prompt' +import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' export class PlanningLLMHandler extends BaseLLMHandler { public async generatePlan( @@ -11,8 +11,8 @@ export class PlanningLLMHandler extends BaseLLMHandler { availableActions: Action[], feedback?: string, ): Promise> { - const systemPrompt = genPlanningAgentPrompt(availableActions) - const userPrompt = this.generateUserPrompt(goal, feedback) + const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) + const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) const messages = [system(systemPrompt), user(userPrompt)] const result = await this.config.agent.handleStateless(messages, async (context) => { @@ -30,14 +30,6 @@ export class PlanningLLMHandler extends BaseLLMHandler { return this.parsePlanContent(result) } - private generateUserPrompt(goal: string, feedback?: string): string { - let prompt = `Create a plan to: ${goal}` - if (feedback) { - prompt += `\nPrevious attempt feedback: ${feedback}` - } - return prompt - } - private parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { try { const match = content.match(/\[[\s\S]*\]/) diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index b084538..1370756 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -4,8 +4,8 @@ import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' -import { genPlanningAgentPrompt } from '../../utils/prompt' import { toRetriable } from '../../utils/reliability' +import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' const logger = useLogg('planning-llm').useGlobalConfig() @@ -22,8 +22,8 @@ export async function generatePlanWithLLM( config: LLMPlanningConfig, feedback?: string, ): Promise> { - const systemPrompt = genPlanningAgentPrompt(availableActions) - const userPrompt = generateUserPrompt(goal, feedback) + const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) + const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) const messages = [ system(systemPrompt), @@ -64,14 +64,6 @@ export async function generatePlanWithLLM( return parsePlanContent(content) } -function generateUserPrompt(goal: string, feedback?: string): string { - let prompt = `Create a plan to: ${goal}` - if (feedback) { - prompt += `\nPrevious attempt feedback: ${feedback}` - } - return prompt -} - function parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { try { // Find JSON array in the content diff --git a/src/agents/prompt/chat.ts b/src/agents/prompt/chat.ts new file mode 100644 index 0000000..a07f41c --- /dev/null +++ b/src/agents/prompt/chat.ts @@ -0,0 +1,21 @@ +export function genChatAgentPrompt(): string { + return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. + +Guidelines: +1. Be friendly and helpful +2. Keep responses concise but informative +3. Use game-appropriate language +4. Acknowledge player's emotions and intentions +5. Ask for clarification when needed +6. Remember context from previous messages +7. Be proactive in suggesting helpful actions + +You can: +- Answer questions about the game +- Help with tasks and crafting +- Give directions and suggestions +- Engage in casual conversation +- Coordinate with other bots + +Remember that you're operating in a Minecraft world and should maintain that context in your responses.` +} diff --git a/src/agents/prompt/llm-agent.plugin.ts b/src/agents/prompt/llm-agent.plugin.ts new file mode 100644 index 0000000..d81fb0a --- /dev/null +++ b/src/agents/prompt/llm-agent.plugin.ts @@ -0,0 +1,51 @@ +import type { Mineflayer } from '../../libs/mineflayer' + +import { listInventory } from '../../skills/actions/inventory' + +export function generateSystemBasicPrompt(botName: string): string { + return `You are a playful Minecraft bot named ${botName} that can converse with players, see, move, +mine, build, and interact with the world by using commands.` +} + +export function generateActionAgentPrompt(mineflayer: Mineflayer): string { + // ${ctx.prompt.selfPrompt} + + return `${generateSystemBasicPrompt(mineflayer.username)} + +Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in +your responses, don't apologize constantly, don't give instructions or make lists unless +asked, and don't refuse requests. + +Do not use any emojis. Just call the function given you if needed. + +- If I command you 'stop', then call the 'stop' function. +- If I require you to find something, then call the 'nearbyBlocks' function first, then call the 'searchForBlock' function. +` +} + +export async function generateStatusPrompt(mineflayer: Mineflayer): Promise { + const inventory = await listInventory(mineflayer) + if (inventory.length === 0) { + return `I will give you the following information: +${mineflayer.status.toOneLiner()} + +Inventory: +[Empty] + +Item in hand: +[Empty] +` + } + const inventoryStr = inventory.map(item => `${item.name} x ${item.count}`).join(', ') + const itemInHand = `${inventory[0].name} x ${inventory[0].count}` // TODO: mock + + return `I will give you the following information: +${mineflayer.status.toOneLiner()} + +Inventory: +${inventoryStr} + +Item in hand: +${itemInHand} +` +} diff --git a/src/agents/prompt/planning.ts b/src/agents/prompt/planning.ts new file mode 100644 index 0000000..6c3bf21 --- /dev/null +++ b/src/agents/prompt/planning.ts @@ -0,0 +1,47 @@ +import type { Action } from '../../libs/mineflayer/action' + +export function generatePlanningAgentSystemPrompt(availableActions: Action[]): string { + const actionsList = availableActions + .map(action => `- ${action.name}: ${action.description}`) + .join('\n') + + return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. +Available actions: +${actionsList} + +Respond with a Valid JSON array of steps, where each step has: +- action: The name of the action to perform +- params: Array of parameters for the action + +DO NOT contains any \`\`\` or explation, otherwise agent will be interrupted. + +Example response: +[ + { + "action": "searchForBlock", + "params": ["log", 64] + }, + { + "action": "collectBlocks", + "params": ["log", 1] + } + ]` +} + +export function generatePlanningAgentUserPrompt(goal: string, feedback?: string): string { + let prompt = `Create a detailed plan to: ${goal} + +Consider the following aspects: +1. Required materials and their quantities +2. Required tools and their availability +3. Necessary crafting steps +4. Block placement requirements +5. Current inventory status + +Please generate steps that handle these requirements in the correct order.` + + if (feedback) { + prompt += `\nPrevious attempt feedback: ${feedback}` + } + return prompt +} diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts index 3e71576..f535afd 100644 --- a/src/libs/mineflayer/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -67,9 +67,9 @@ export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { constructor(config: AgentConfig) { super() - this.id = config.id + this.id = config.id // TODO: use uuid, is it needed? this.type = config.type - this.name = `${this.type}-${this.id}` + this.name = `${this.type}-agent` this.initialized = false this.logger = useLogg(this.name).useGlobalConfig() diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts index 442943b..3daae44 100644 --- a/src/plugins/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -8,9 +8,9 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' import { assistant, system, user } from 'neuri/openai' +import { generateActionAgentPrompt, generateStatusPrompt } from '../agents/prompt/llm-agent.plugin' import { createAppContainer } from '../container' import { ChatMessageHandler } from '../libs/mineflayer/message' -import { genActionAgentPrompt, genStatusPrompt } from '../utils/prompt' import { toRetriable } from '../utils/reliability' interface MineflayerWithAgents extends Mineflayer { @@ -51,26 +51,25 @@ async function handleChatMessage(username: string, message: string, bot: Minefla logger.log('thinking...') try { - // 创建并执行计划 + // Create and execute plan const plan = await bot.planning.createPlan(message) logger.withFields({ plan }).log('Plan created') await bot.planning.executePlan(plan) logger.log('Plan executed successfully') - // 生成回复 - const statusPrompt = await genStatusPrompt(bot) - const retryHandler = toRetriable( - 3, - 1000, - ctx => handleLLMCompletion(ctx, bot, logger), - { onError: err => logger.withError(err).log('error occurred') }, - ) - + // Generate response + // TODO: use chat agent and conversion manager + const statusPrompt = await generateStatusPrompt(bot) const content = await agent.handleStateless( [...bot.memory.chatHistory, system(statusPrompt)], async (c: NeuriContext) => { - logger.log('handling...') - return retryHandler(c) + logger.log('handling response...') + return toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + { onError: err => logger.withError(err).log('error occurred') }, + )(c) }, ) @@ -97,7 +96,7 @@ async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Ne }) .log('Chat message received') - const statusPrompt = await genStatusPrompt(bot) + const statusPrompt = await generateStatusPrompt(bot) bot.memory.chatHistory.push(system(statusPrompt)) bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) @@ -167,7 +166,7 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { botWithAgents.chat = chatAgent // 初始化系统提示 - bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) + bot.memory.chatHistory.push(system(generateActionAgentPrompt(bot))) // 设置消息处理 const onChat = new ChatMessageHandler(bot.username).handleChat((username, message) => diff --git a/src/utils/prompt.ts b/src/utils/prompt.ts deleted file mode 100644 index 82f5e60..0000000 --- a/src/utils/prompt.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { Action, Mineflayer } from '../libs/mineflayer' - -import { listInventory } from '../skills/actions/inventory' - -export function genSystemBasicPrompt(botName: string): string { - return `You are a playful Minecraft bot named ${botName} that can converse with players, see, move, -mine, build, and interact with the world by using commands.` -} - -export function genActionAgentPrompt(mineflayer: Mineflayer): string { - // ${ctx.prompt.selfPrompt} - - return `${genSystemBasicPrompt(mineflayer.username)} - -Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in -your responses, don't apologize constantly, don't give instructions or make lists unless -asked, and don't refuse requests. - -Do not use any emojis. Just call the function given you if needed. - -- If I command you 'stop', then call the 'stop' function. -- If I require you to find something, then call the 'nearbyBlocks' function first, then call the 'searchForBlock' function. -` -} - -export async function genStatusPrompt(mineflayer: Mineflayer): Promise { - const inventory = await listInventory(mineflayer) - if (inventory.length === 0) { - return `I will give you the following information: -${mineflayer.status.toOneLiner()} - -Inventory: -[Empty] - -Item in hand: -[Empty] -` - } - const inventoryStr = inventory.map(item => `${item.name} x ${item.count}`).join(', ') - const itemInHand = `${inventory[0].name} x ${inventory[0].count}` // TODO: mock - - return `I will give you the following information: -${mineflayer.status.toOneLiner()} - -Inventory: -${inventoryStr} - -Item in hand: -${itemInHand} -` -} - -export function genQueryAgentPrompt(mineflayer: Mineflayer): string { - const prompt = `You are a helpful assistant that asks questions to help me decide the next immediate -task to do in Minecraft. My ultimate goal is to discover as many things as possible, -accomplish as many tasks as possible and become the best Minecraft player in the world. - -I will give you the following information: -${mineflayer.status.toOneLiner()} -` - - return prompt -} - -export function genPlanningAgentPrompt(availableActions: Action[]): string { - const actionsList = availableActions - .map(action => `- ${action.name}: ${action.description}`) - .join('\n') - - return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. -Available actions: -${actionsList} - -Respond with a Valid JSON array of steps, where each step has: -- action: The name of the action to perform -- params: Array of parameters for the action - -DO NOT contains any \`\`\` or explation, otherwise agent will be interrupted. - -Example response: -[ - { - "action": "searchForBlock", - "params": ["log", 64] - }, - { - "action": "collectBlocks", - "params": ["log", 1] - } - ]` -} - -export function genChatAgentPrompt(): string { - return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. - -Guidelines: -1. Be friendly and helpful -2. Keep responses concise but informative -3. Use game-appropriate language -4. Acknowledge player's emotions and intentions -5. Ask for clarification when needed -6. Remember context from previous messages -7. Be proactive in suggesting helpful actions - -You can: -- Answer questions about the game -- Help with tasks and crafting -- Give directions and suggestions -- Engage in casual conversation -- Coordinate with other bots - -Remember that you're operating in a Minecraft world and should maintain that context in your responses.` -} From 6ee654d478edd2448831b19cc3217361205cab4c Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 05:24:44 +0800 Subject: [PATCH 11/22] chore: strict container --- src/agents/planning/factory.ts | 47 ---------------------------------- src/container.ts | 1 + 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 src/agents/planning/factory.ts diff --git a/src/agents/planning/factory.ts b/src/agents/planning/factory.ts deleted file mode 100644 index 48fa35c..0000000 --- a/src/agents/planning/factory.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { Neuri } from 'neuri' -import type { Mineflayer } from '../../libs/mineflayer' -import type { MineflayerPlugin } from '../../libs/mineflayer/plugin' - -import { useLogg } from '@guiiai/logg' - -import { createAppContainer } from '../../container' - -interface PlanningPluginOptions { - agent: Neuri - model?: string -} - -interface MineflayerWithPlanning extends Mineflayer { - planning: any -} - -const logger = useLogg('planning-factory').useGlobalConfig() - -export function PlanningPlugin(options: PlanningPluginOptions): MineflayerPlugin { - return { - async created(mineflayer: Mineflayer) { - logger.log('Initializing planning plugin') - - // Get the container - const container = createAppContainer({ - neuri: options.agent, - model: options.model, - }) - const actionAgent = container.resolve('actionAgent') - const planningAgent = container.resolve('planningAgent') - - // Initialize agents - await actionAgent.init() - await planningAgent.init() - - // Add to bot - ;(mineflayer as MineflayerWithPlanning).planning = planningAgent - }, - - async beforeCleanup(bot) { - logger.log('Cleaning up planning plugin') - const botWithPlanning = bot as MineflayerWithPlanning - await botWithPlanning.planning?.destroy() - }, - } -} diff --git a/src/container.ts b/src/container.ts index 678fa57..bd8c6e8 100644 --- a/src/container.ts +++ b/src/container.ts @@ -23,6 +23,7 @@ export function createAppContainer(options: { }) { const container = createContainer({ injectionMode: InjectionMode.PROXY, + strict: true, }) // Register services From c2b7b56506a72e034871266d416f59acf1a50582 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 05:26:01 +0800 Subject: [PATCH 12/22] chore: env --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index 1237916..0983d60 100644 --- a/.env +++ b/.env @@ -5,3 +5,4 @@ BOT_USERNAME='' BOT_HOSTNAME='' BOT_PORT='' BOT_PASSWORD='' +BOT_VERSION='' From 8f1196e337122835757f54f6db9428810b6c6004 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 06:10:56 +0800 Subject: [PATCH 13/22] refactor: move world to skills & move manager to folder --- src/agents/action/index.ts | 2 +- src/agents/action/tools.ts | 2 +- src/agents/chat/llm.ts | 2 +- src/agents/planning/llm.ts | 2 +- src/agents/prompt/llm-agent.plugin.ts | 47 +++++++++---------- src/libs/llm/base.ts | 2 +- src/{composables => manager}/action.ts | 0 .../deprecated => manager}/conversation.ts | 0 src/plugins/llm-agent.ts | 2 +- src/skills/actions/collect-block.ts | 2 +- src/skills/actions/gather-wood.ts | 2 +- src/skills/actions/inventory.ts | 2 +- src/skills/blocks.ts | 2 +- src/skills/combat.ts | 2 +- src/skills/crafting.ts | 2 +- src/skills/inventory.ts | 2 +- src/skills/movement.ts | 2 +- src/{composables => skills}/world.ts | 0 src/utils/helper.ts | 38 +++++++++++++++ src/utils/reliability.ts | 39 --------------- 20 files changed, 75 insertions(+), 77 deletions(-) rename src/{composables => manager}/action.ts (100%) rename src/{composables/deprecated => manager}/conversation.ts (100%) rename src/{composables => skills}/world.ts (100%) delete mode 100644 src/utils/reliability.ts diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index fd5f2e4..555493a 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -2,9 +2,9 @@ import type { Mineflayer } from '../../libs/mineflayer' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' -import { ActionManager } from '../../composables/action' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' +import { ActionManager } from '../../manager/action' import { actionsList } from './tools' interface ActionState { diff --git a/src/agents/action/tools.ts b/src/agents/action/tools.ts index 3ba659a..61f89db 100644 --- a/src/agents/action/tools.ts +++ b/src/agents/action/tools.ts @@ -3,11 +3,11 @@ import type { Action } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { z } from 'zod' -import * as world from '../../composables/world' import * as skills from '../../skills' import { collectBlock } from '../../skills/actions/collect-block' import { discard, equip, putInChest, takeFromChest, viewChest } from '../../skills/actions/inventory' import { activateNearestBlock, placeBlock } from '../../skills/actions/world-interactions' +import * as world from '../../skills/world' // Utils const pad = (str: string): string => `\n${str}\n` diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index 5e7f39c..f5ffa08 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -4,7 +4,7 @@ import type { ChatHistory } from './types' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' -import { toRetriable } from '../../utils/reliability' +import { toRetriable } from '../../utils/helper' import { genChatAgentPrompt } from '../prompt/chat' const logger = useLogg('chat-llm').useGlobalConfig() diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index 1370756..2dfb911 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -4,7 +4,7 @@ import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' import { system, user } from 'neuri/openai' -import { toRetriable } from '../../utils/reliability' +import { toRetriable } from '../../utils/helper' import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' const logger = useLogg('planning-llm').useGlobalConfig() diff --git a/src/agents/prompt/llm-agent.plugin.ts b/src/agents/prompt/llm-agent.plugin.ts index d81fb0a..273e74d 100644 --- a/src/agents/prompt/llm-agent.plugin.ts +++ b/src/agents/prompt/llm-agent.plugin.ts @@ -3,13 +3,12 @@ import type { Mineflayer } from '../../libs/mineflayer' import { listInventory } from '../../skills/actions/inventory' export function generateSystemBasicPrompt(botName: string): string { + // ${ctx.prompt.selfPrompt} return `You are a playful Minecraft bot named ${botName} that can converse with players, see, move, mine, build, and interact with the world by using commands.` } export function generateActionAgentPrompt(mineflayer: Mineflayer): string { - // ${ctx.prompt.selfPrompt} - return `${generateSystemBasicPrompt(mineflayer.username)} Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in @@ -24,28 +23,28 @@ Do not use any emojis. Just call the function given you if needed. } export async function generateStatusPrompt(mineflayer: Mineflayer): Promise { + // Get inventory items const inventory = await listInventory(mineflayer) - if (inventory.length === 0) { - return `I will give you the following information: -${mineflayer.status.toOneLiner()} -Inventory: -[Empty] - -Item in hand: -[Empty] -` - } - const inventoryStr = inventory.map(item => `${item.name} x ${item.count}`).join(', ') - const itemInHand = `${inventory[0].name} x ${inventory[0].count}` // TODO: mock - - return `I will give you the following information: -${mineflayer.status.toOneLiner()} - -Inventory: -${inventoryStr} - -Item in hand: -${itemInHand} -` + // Format inventory string + const inventoryStr = inventory.length === 0 + ? '[Empty]' + : inventory.map(item => `${item.name} x ${item.count}`).join(', ') + + // Get currently held item + const itemInHand = inventory.length === 0 + ? '[Empty]' + : `${inventory[0].name} x ${inventory[0].count}` // TODO: mock + + // Build status message + return [ + 'I will give you the following information:', + mineflayer.status.toOneLiner(), + '', + 'Inventory:', + inventoryStr, + '', + 'Item in hand:', + itemInHand, + ].join('\n') } diff --git a/src/libs/llm/base.ts b/src/libs/llm/base.ts index 274db0a..f5a516b 100644 --- a/src/libs/llm/base.ts +++ b/src/libs/llm/base.ts @@ -2,7 +2,7 @@ import type { LLMConfig, LLMResponse } from './types' import { useLogg } from '@guiiai/logg' -import { toRetriable } from '../../utils/reliability' +import { toRetriable } from '../../utils/helper' export abstract class BaseLLMHandler { protected logger = useLogg('llm-handler').useGlobalConfig() diff --git a/src/composables/action.ts b/src/manager/action.ts similarity index 100% rename from src/composables/action.ts rename to src/manager/action.ts diff --git a/src/composables/deprecated/conversation.ts b/src/manager/conversation.ts similarity index 100% rename from src/composables/deprecated/conversation.ts rename to src/manager/conversation.ts diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts index 3daae44..2cc0173 100644 --- a/src/plugins/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -11,7 +11,7 @@ import { assistant, system, user } from 'neuri/openai' import { generateActionAgentPrompt, generateStatusPrompt } from '../agents/prompt/llm-agent.plugin' import { createAppContainer } from '../container' import { ChatMessageHandler } from '../libs/mineflayer/message' -import { toRetriable } from '../utils/reliability' +import { toRetriable } from '../utils/helper' interface MineflayerWithAgents extends Mineflayer { planning: PlanningAgent diff --git a/src/skills/actions/collect-block.ts b/src/skills/actions/collect-block.ts index 333d91d..07fe945 100644 --- a/src/skills/actions/collect-block.ts +++ b/src/skills/actions/collect-block.ts @@ -4,8 +4,8 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import pathfinder from 'mineflayer-pathfinder' -import { getNearestBlocks } from '../../composables/world' import { breakBlockAt } from '../blocks' +import { getNearestBlocks } from '../world' import { ensurePickaxe } from './ensure' import { pickupNearbyItems } from './world-interactions' diff --git a/src/skills/actions/gather-wood.ts b/src/skills/actions/gather-wood.ts index 91479c1..d757eb4 100644 --- a/src/skills/actions/gather-wood.ts +++ b/src/skills/actions/gather-wood.ts @@ -2,10 +2,10 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getNearestBlocks } from '../../composables/world' import { sleep } from '../../utils/helper' import { breakBlockAt } from '../blocks' import { goToPosition, moveAway } from '../movement' +import { getNearestBlocks } from '../world' import { pickupNearbyItems } from './world-interactions' const logger = useLogg('Action:GatherWood').useGlobalConfig() diff --git a/src/skills/actions/inventory.ts b/src/skills/actions/inventory.ts index 0fa75ff..1614d9d 100644 --- a/src/skills/actions/inventory.ts +++ b/src/skills/actions/inventory.ts @@ -3,8 +3,8 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getNearestBlock } from '../../composables/world' import { goToPlayer, goToPosition } from '../movement' +import { getNearestBlock } from '../world' const logger = useLogg('Action:Inventory').useGlobalConfig() diff --git a/src/skills/blocks.ts b/src/skills/blocks.ts index 78e6cff..383c49b 100644 --- a/src/skills/blocks.ts +++ b/src/skills/blocks.ts @@ -4,10 +4,10 @@ import type { BlockFace } from './base' import pathfinderModel, { type SafeBlock } from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' -import { getNearestBlock, getNearestBlocks, getPosition, shouldPlaceTorch } from '../composables/world' import { getBlockId, makeItem } from '../utils/mcdata' import { log } from './base' import { goToPosition } from './movement' +import { getNearestBlock, getNearestBlocks, getPosition, shouldPlaceTorch } from './world' const { goals, Movements } = pathfinderModel diff --git a/src/skills/combat.ts b/src/skills/combat.ts index eae3d10..ff98a0b 100644 --- a/src/skills/combat.ts +++ b/src/skills/combat.ts @@ -4,10 +4,10 @@ import type { Mineflayer } from '../libs/mineflayer' import pathfinderModel from 'mineflayer-pathfinder' -import { getNearbyEntities, getNearestEntityWhere } from '../composables/world' import { sleep } from '../utils/helper' import { isHostile } from '../utils/mcdata' import { log } from './base' +import { getNearbyEntities, getNearestEntityWhere } from './world' const { goals } = pathfinderModel diff --git a/src/skills/crafting.ts b/src/skills/crafting.ts index 11375df..8ca4397 100644 --- a/src/skills/crafting.ts +++ b/src/skills/crafting.ts @@ -5,11 +5,11 @@ import type { Mineflayer } from '../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getInventoryCounts, getNearestBlock, getNearestFreeSpace } from '../composables/world' import { getItemId, getItemName } from '../utils/mcdata' import { ensureCraftingTable } from './actions/ensure' import { collectBlock, placeBlock } from './blocks' import { goToNearestBlock, goToPosition, moveAway } from './movement' +import { getInventoryCounts, getNearestBlock, getNearestFreeSpace } from './world' const logger = useLogg('Skill:Crafting').useGlobalConfig() diff --git a/src/skills/inventory.ts b/src/skills/inventory.ts index 91c3c9c..ec6c4ef 100644 --- a/src/skills/inventory.ts +++ b/src/skills/inventory.ts @@ -1,8 +1,8 @@ import type { Mineflayer } from '../libs/mineflayer' -import { getNearestBlock } from '../composables/world' import { log } from './base' import { goToPlayer, goToPosition } from './movement' +import { getNearestBlock } from './world' export async function equip(mineflayer: Mineflayer, itemName: string): Promise { const item = mineflayer.bot.inventory.slots.find(slot => slot && slot.name === itemName) diff --git a/src/skills/movement.ts b/src/skills/movement.ts index a48ae85..f73a5ba 100644 --- a/src/skills/movement.ts +++ b/src/skills/movement.ts @@ -6,9 +6,9 @@ import { randomInt } from 'es-toolkit' import pathfinder from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' -import { getNearestBlock, getNearestEntityWhere } from '../composables/world' import { sleep } from '../utils/helper' import { log } from './base' +import { getNearestBlock, getNearestEntityWhere } from './world' const logger = useLogg('Skill:Movement').useGlobalConfig() const { goals, Movements } = pathfinder diff --git a/src/composables/world.ts b/src/skills/world.ts similarity index 100% rename from src/composables/world.ts rename to src/skills/world.ts diff --git a/src/utils/helper.ts b/src/utils/helper.ts index c1eb515..e0600bf 100644 --- a/src/utils/helper.ts +++ b/src/utils/helper.ts @@ -1 +1,39 @@ export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * Returns a retirable anonymous function with configured retryLimit and delayInterval + * + * @param retryLimit Number of retry attempts + * @param delayInterval Delay between retries in milliseconds + * @param func Function to be called + * @returns A wrapped function with the same signature as func + */ +export function toRetriable( + retryLimit: number, + delayInterval: number, + func: (...args: A[]) => Promise, + hooks?: { + onError?: (err: unknown) => void + }, +): (...args: A[]) => Promise { + let retryCount = 0 + return async function (args: A): Promise { + try { + return await func(args) + } + catch (err) { + if (hooks?.onError) { + hooks.onError(err) + } + + if (retryCount < retryLimit) { + retryCount++ + await sleep(delayInterval) + return await toRetriable(retryLimit - retryCount, delayInterval, func)(args) + } + else { + throw err + } + } + } +} diff --git a/src/utils/reliability.ts b/src/utils/reliability.ts deleted file mode 100644 index 3d277f5..0000000 --- a/src/utils/reliability.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { sleep } from './helper' - -/** - * Returns a retirable anonymous function with configured retryLimit and delayInterval - * - * @param retryLimit Number of retry attempts - * @param delayInterval Delay between retries in milliseconds - * @param func Function to be called - * @returns A wrapped function with the same signature as func - */ -export function toRetriable( - retryLimit: number, - delayInterval: number, - func: (...args: A[]) => Promise, - hooks?: { - onError?: (err: unknown) => void - }, -): (...args: A[]) => Promise { - let retryCount = 0 - return async function (args: A): Promise { - try { - return await func(args) - } - catch (err) { - if (hooks?.onError) { - hooks.onError(err) - } - - if (retryCount < retryLimit) { - retryCount++ - await sleep(delayInterval) - return await toRetriable(retryLimit - retryCount, delayInterval, func)(args) - } - else { - throw err - } - } - } -} From 9d9ba91450f8f5be9bbd884c331c0a042185158d Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 06:20:23 +0800 Subject: [PATCH 14/22] refactor: split init agent to composable/neuri --- src/agents/action/llm.ts | 36 +++--------------------------------- src/agents/planning/llm.ts | 4 ++-- src/composables/neuri.ts | 38 ++++++++++++++++++++++++++++++++++++++ src/main.ts | 4 ++-- 4 files changed, 45 insertions(+), 37 deletions(-) create mode 100644 src/composables/neuri.ts diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index bd6005b..162571a 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -1,43 +1,13 @@ -import type { Agent, Neuri } from 'neuri' +import type { Agent } from 'neuri' import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { agent, neuri } from 'neuri' +import { agent } from 'neuri' -import { openaiConfig } from '../../composables/config' import { actionsList } from './tools' -let neuriAgent: Neuri | undefined -const agents = new Set>() - -const logger = useLogg('action-llm').useGlobalConfig() - -export async function initAgent(mineflayer: Mineflayer): Promise { - logger.log('Initializing agent') - let n = neuri() - - agents.add(initActionAgent(mineflayer)) - - agents.forEach(agent => n = n.agent(agent)) - - neuriAgent = await n.build({ - provider: { - apiKey: openaiConfig.apiKey, - baseURL: openaiConfig.baseUrl, - }, - }) - - return neuriAgent -} - -export function getAgent(): Neuri { - if (!neuriAgent) { - throw new Error('Agent not initialized') - } - return neuriAgent -} - export async function initActionAgent(mineflayer: Mineflayer): Promise { + const logger = useLogg('action-llm').useGlobalConfig() logger.log('Initializing action agent') let actionAgent = agent('action') diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index 2dfb911..0d61129 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -1,4 +1,4 @@ -import type { Neuri } from 'neuri' +import type { Neuri, NeuriContext } from 'neuri' import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' @@ -33,7 +33,7 @@ export async function generatePlanWithLLM( const content = await config.agent.handleStateless(messages, async (c) => { logger.log('Generating plan...') - const handleCompletion = async (c: any): Promise => { + const handleCompletion = async (c: NeuriContext): Promise => { const completion = await c.reroute('action', c.messages, { model: config.model ?? 'openai/gpt-4o-mini', }) diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts new file mode 100644 index 0000000..85fdbcf --- /dev/null +++ b/src/composables/neuri.ts @@ -0,0 +1,38 @@ +import type { Agent, Neuri } from 'neuri' +import type { Mineflayer } from '../libs/mineflayer' + +import { useLogg } from '@guiiai/logg' +import { neuri } from 'neuri' + +import { initActionAgent } from '../agents/action/llm' +import { openaiConfig } from './config' + +let neuriAgent: Neuri | undefined +const agents = new Set>() + +const logger = useLogg('action-llm').useGlobalConfig() + +export async function initNeuriAgent(mineflayer: Mineflayer): Promise { + logger.log('Initializing agent') + let n = neuri() + + agents.add(initActionAgent(mineflayer)) + + agents.forEach(agent => n = n.agent(agent)) + + neuriAgent = await n.build({ + provider: { + apiKey: openaiConfig.apiKey, + baseURL: openaiConfig.baseUrl, + }, + }) + + return neuriAgent +} + +export function getAgent(): Neuri { + if (!neuriAgent) { + throw new Error('Agent not initialized') + } + return neuriAgent +} diff --git a/src/main.ts b/src/main.ts index 5026eeb..006f490 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,9 +8,9 @@ import { pathfinder as MineflayerPathfinder } from 'mineflayer-pathfinder' import { plugin as MineflayerPVP } from 'mineflayer-pvp' import { plugin as MineflayerTool } from 'mineflayer-tool' -import { initAgent } from './agents/action/llm' import { initBot } from './composables/bot' import { botConfig, initEnv } from './composables/config' +import { initNeuriAgent } from './composables/neuri' import { wrapPlugin } from './libs/mineflayer' import { LLMAgent } from './plugins/llm-agent' import { initLogger } from './utils/logger' @@ -36,7 +36,7 @@ async function main() { const airiClient = new Client({ name: 'minecraft-bot', url: 'ws://localhost:6121/ws' }) // Dynamically load LLMAgent after the bot is initialized - const agent = await initAgent(bot) + const agent = await initNeuriAgent(bot) await bot.loadPlugin(LLMAgent({ agent, airiClient })) process.on('SIGINT', () => { From e2ce9478b15b55e5e0e8bc3d8125a912413f6f64 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 06:30:43 +0800 Subject: [PATCH 15/22] chore: neuri agents --- src/agents/action/llm.ts | 2 +- src/agents/chat/llm.ts | 7 ++++++- src/agents/planning/llm-handler.ts | 2 +- src/agents/planning/llm.ts | 9 +++++++-- src/composables/neuri.ts | 8 ++++++-- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index 162571a..32308b5 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -6,7 +6,7 @@ import { agent } from 'neuri' import { actionsList } from './tools' -export async function initActionAgent(mineflayer: Mineflayer): Promise { +export async function initActionNeuriAgent(mineflayer: Mineflayer): Promise { const logger = useLogg('action-llm').useGlobalConfig() logger.log('Initializing action agent') let actionAgent = agent('action') diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index f5ffa08..e95d640 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -1,7 +1,8 @@ -import type { Neuri } from 'neuri' +import type { Agent, Neuri } from 'neuri' import type { ChatHistory } from './types' import { useLogg } from '@guiiai/logg' +import { agent } from 'neuri' import { system, user } from 'neuri/openai' import { toRetriable } from '../../utils/helper' @@ -17,6 +18,10 @@ interface LLMChatConfig { maxContextLength?: number } +export async function initChatNeuriAgent(): Promise { + return agent('chat').build() +} + export async function generateChatResponse( message: string, history: ChatHistory[], diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index a17f153..e96edaa 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -18,7 +18,7 @@ export class PlanningLLMHandler extends BaseLLMHandler { const result = await this.config.agent.handleStateless(messages, async (context) => { this.logger.log('Generating plan...') const retryHandler = this.createRetryHandler( - async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content, + async ctx => (await this.handleCompletion(ctx, 'planning', ctx.messages)).content, ) return await retryHandler(context) }) diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index 0d61129..db4963a 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -1,7 +1,8 @@ -import type { Neuri, NeuriContext } from 'neuri' +import type { Agent, Neuri, NeuriContext } from 'neuri' import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' +import { agent } from 'neuri' import { system, user } from 'neuri/openai' import { toRetriable } from '../../utils/helper' @@ -16,6 +17,10 @@ interface LLMPlanningConfig { delayInterval?: number } +export async function initPlanningNeuriAgent(): Promise { + return agent('planning').build() +} + export async function generatePlanWithLLM( goal: string, availableActions: Action[], @@ -34,7 +39,7 @@ export async function generatePlanWithLLM( logger.log('Generating plan...') const handleCompletion = async (c: NeuriContext): Promise => { - const completion = await c.reroute('action', c.messages, { + const completion = await c.reroute('planning', c.messages, { model: config.model ?? 'openai/gpt-4o-mini', }) diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts index 85fdbcf..a2a1d7f 100644 --- a/src/composables/neuri.ts +++ b/src/composables/neuri.ts @@ -4,7 +4,9 @@ import type { Mineflayer } from '../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { neuri } from 'neuri' -import { initActionAgent } from '../agents/action/llm' +import { initActionNeuriAgent } from '../agents/action/llm' +import { initChatNeuriAgent } from '../agents/chat/llm' +import { initPlanningNeuriAgent } from '../agents/planning/llm' import { openaiConfig } from './config' let neuriAgent: Neuri | undefined @@ -16,7 +18,9 @@ export async function initNeuriAgent(mineflayer: Mineflayer): Promise { logger.log('Initializing agent') let n = neuri() - agents.add(initActionAgent(mineflayer)) + agents.add(initPlanningNeuriAgent()) + agents.add(initActionNeuriAgent(mineflayer)) + agents.add(initChatNeuriAgent()) agents.forEach(agent => n = n.agent(agent)) From 1b06bffaa83ca64d4c62fa40c00885d2ba384dec Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 06:35:35 +0800 Subject: [PATCH 16/22] chore: lint fix --- src/agents/action/llm.test.ts | 4 ++-- src/agents/action/llm.ts | 2 +- src/agents/action/tools.test.ts | 6 +++--- src/agents/chat/llm.ts | 2 +- src/agents/planning/llm.ts | 8 ++++---- src/composables/neuri.ts | 16 ++++++++-------- src/main.ts | 4 ++-- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/agents/action/llm.test.ts b/src/agents/action/llm.test.ts index 502fc88..56e9f6d 100644 --- a/src/agents/action/llm.test.ts +++ b/src/agents/action/llm.test.ts @@ -3,9 +3,9 @@ import { beforeAll, describe, expect, it } from 'vitest' import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' +import { createNeuriAgent } from '../../composables/neuri' import { initLogger } from '../../utils/logger' import { generateSystemBasicPrompt } from '../prompt/llm-agent.plugin' -import { initAgent } from './llm' describe('openAI agent', { timeout: 0 }, () => { beforeAll(() => { @@ -16,7 +16,7 @@ describe('openAI agent', { timeout: 0 }, () => { it('should initialize the agent', async () => { const { bot } = useBot() - const agent = await initAgent(bot) + const agent = await createNeuriAgent(bot) await new Promise((resolve) => { bot.bot.once('spawn', async () => { diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index 32308b5..f5a3495 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -6,7 +6,7 @@ import { agent } from 'neuri' import { actionsList } from './tools' -export async function initActionNeuriAgent(mineflayer: Mineflayer): Promise { +export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { const logger = useLogg('action-llm').useGlobalConfig() logger.log('Initializing action agent') let actionAgent = agent('action') diff --git a/src/agents/action/tools.test.ts b/src/agents/action/tools.test.ts index b02cc22..a3aac85 100644 --- a/src/agents/action/tools.test.ts +++ b/src/agents/action/tools.test.ts @@ -3,10 +3,10 @@ import { beforeAll, describe, expect, it } from 'vitest' import { initBot, useBot } from '../../composables/bot' import { botConfig, initEnv } from '../../composables/config' +import { createNeuriAgent } from '../../composables/neuri' import { sleep } from '../../utils/helper' import { initLogger } from '../../utils/logger' import { generateActionAgentPrompt } from '../prompt/llm-agent.plugin' -import { initAgent } from './llm' describe('actions agent', { timeout: 0 }, () => { beforeAll(() => { @@ -17,7 +17,7 @@ describe('actions agent', { timeout: 0 }, () => { it('should choose right query command', async () => { const { bot } = useBot() - const agent = await initAgent(bot) + const agent = await createNeuriAgent(bot) await new Promise((resolve) => { bot.bot.once('spawn', async () => { @@ -38,7 +38,7 @@ describe('actions agent', { timeout: 0 }, () => { it('should choose right action command', async () => { const { bot } = useBot() - const agent = await initAgent(bot) + const agent = await createNeuriAgent(bot) await new Promise((resolve) => { bot.bot.on('spawn', async () => { diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index e95d640..195d6ab 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -18,7 +18,7 @@ interface LLMChatConfig { maxContextLength?: number } -export async function initChatNeuriAgent(): Promise { +export async function createChatNeuriAgent(): Promise { return agent('chat').build() } diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts index db4963a..873475d 100644 --- a/src/agents/planning/llm.ts +++ b/src/agents/planning/llm.ts @@ -3,7 +3,7 @@ import type { Action } from '../../libs/mineflayer/action' import { useLogg } from '@guiiai/logg' import { agent } from 'neuri' -import { system, user } from 'neuri/openai' +import { type ChatCompletion, system, user } from 'neuri/openai' import { toRetriable } from '../../utils/helper' import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' @@ -17,7 +17,7 @@ interface LLMPlanningConfig { delayInterval?: number } -export async function initPlanningNeuriAgent(): Promise { +export async function createPlanningNeuriAgent(): Promise { return agent('planning').build() } @@ -41,7 +41,7 @@ export async function generatePlanWithLLM( const handleCompletion = async (c: NeuriContext): Promise => { const completion = await c.reroute('planning', c.messages, { model: config.model ?? 'openai/gpt-4o-mini', - }) + }) as ChatCompletion | ChatCompletion & { error: { message: string } } if (!completion || 'error' in completion) { logger.withFields(c).error('Completion failed') @@ -53,7 +53,7 @@ export async function generatePlanWithLLM( return content } - const retirableHandler = toRetriable( + const retirableHandler = toRetriable( config.retryLimit ?? 3, config.delayInterval ?? 1000, handleCompletion, diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts index a2a1d7f..afd2286 100644 --- a/src/composables/neuri.ts +++ b/src/composables/neuri.ts @@ -4,9 +4,9 @@ import type { Mineflayer } from '../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { neuri } from 'neuri' -import { initActionNeuriAgent } from '../agents/action/llm' -import { initChatNeuriAgent } from '../agents/chat/llm' -import { initPlanningNeuriAgent } from '../agents/planning/llm' +import { createActionNeuriAgent } from '../agents/action/llm' +import { createChatNeuriAgent } from '../agents/chat/llm' +import { createPlanningNeuriAgent } from '../agents/planning/llm' import { openaiConfig } from './config' let neuriAgent: Neuri | undefined @@ -14,13 +14,13 @@ const agents = new Set>() const logger = useLogg('action-llm').useGlobalConfig() -export async function initNeuriAgent(mineflayer: Mineflayer): Promise { +export async function createNeuriAgent(mineflayer: Mineflayer): Promise { logger.log('Initializing agent') let n = neuri() - agents.add(initPlanningNeuriAgent()) - agents.add(initActionNeuriAgent(mineflayer)) - agents.add(initChatNeuriAgent()) + agents.add(createPlanningNeuriAgent()) + agents.add(createActionNeuriAgent(mineflayer)) + agents.add(createChatNeuriAgent()) agents.forEach(agent => n = n.agent(agent)) @@ -34,7 +34,7 @@ export async function initNeuriAgent(mineflayer: Mineflayer): Promise { return neuriAgent } -export function getAgent(): Neuri { +export function useNeuriAgent(): Neuri { if (!neuriAgent) { throw new Error('Agent not initialized') } diff --git a/src/main.ts b/src/main.ts index 006f490..076471b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,7 +10,7 @@ import { plugin as MineflayerTool } from 'mineflayer-tool' import { initBot } from './composables/bot' import { botConfig, initEnv } from './composables/config' -import { initNeuriAgent } from './composables/neuri' +import { createNeuriAgent } from './composables/neuri' import { wrapPlugin } from './libs/mineflayer' import { LLMAgent } from './plugins/llm-agent' import { initLogger } from './utils/logger' @@ -36,7 +36,7 @@ async function main() { const airiClient = new Client({ name: 'minecraft-bot', url: 'ws://localhost:6121/ws' }) // Dynamically load LLMAgent after the bot is initialized - const agent = await initNeuriAgent(bot) + const agent = await createNeuriAgent(bot) await bot.loadPlugin(LLMAgent({ agent, airiClient })) process.on('SIGINT', () => { From b2c6a95ce565f2622c9f2e4953c59f644959dcf5 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 06:40:39 +0800 Subject: [PATCH 17/22] chore: debug log --- src/agents/action/llm.ts | 2 +- src/composables/neuri.ts | 4 ++-- src/libs/mineflayer/base-agent.ts | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts index f5a3495..030a6f5 100644 --- a/src/agents/action/llm.ts +++ b/src/agents/action/llm.ts @@ -7,7 +7,7 @@ import { agent } from 'neuri' import { actionsList } from './tools' export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { - const logger = useLogg('action-llm').useGlobalConfig() + const logger = useLogg('action-neuri').useGlobalConfig() logger.log('Initializing action agent') let actionAgent = agent('action') diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts index afd2286..7a4ae64 100644 --- a/src/composables/neuri.ts +++ b/src/composables/neuri.ts @@ -12,10 +12,10 @@ import { openaiConfig } from './config' let neuriAgent: Neuri | undefined const agents = new Set>() -const logger = useLogg('action-llm').useGlobalConfig() +const logger = useLogg('neuri').useGlobalConfig() export async function createNeuriAgent(mineflayer: Mineflayer): Promise { - logger.log('Initializing agent') + logger.log('Initializing neuri agent') let n = neuri() agents.add(createPlanningNeuriAgent()) diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts index f535afd..fc573a5 100644 --- a/src/libs/mineflayer/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -86,7 +86,6 @@ export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { return } - this.logger.log('Initializing agent') await this.initializeAgent() this.initialized = true } From cee5a24da36f905ee3b51d574f1d86a9e97b0985 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 07:02:10 +0800 Subject: [PATCH 18/22] refactor: use llm handler --- src/agents/action/index.ts | 1 + .../{llm.test.ts => llm-handler.test.ts} | 0 src/agents/action/llm-handler.ts | 32 ++++++- src/agents/action/llm.ts | 29 ------ src/agents/chat/llm.ts | 2 +- src/agents/planning/index.ts | 18 ++-- src/agents/planning/llm-handler.ts | 6 ++ src/agents/planning/llm.ts | 94 ------------------- src/composables/config.ts | 3 + src/composables/neuri.ts | 4 +- src/libs/llm/base.ts | 15 +-- src/plugins/llm-agent.ts | 2 +- 12 files changed, 62 insertions(+), 144 deletions(-) rename src/agents/action/{llm.test.ts => llm-handler.test.ts} (100%) delete mode 100644 src/agents/action/llm.ts delete mode 100644 src/agents/planning/llm.ts diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 555493a..7815b63 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -41,6 +41,7 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { actionsList.forEach(action => this.actions.set(action.name, action)) // Set up event listeners + // todo: nothing to call here this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) diff --git a/src/agents/action/llm.test.ts b/src/agents/action/llm-handler.test.ts similarity index 100% rename from src/agents/action/llm.test.ts rename to src/agents/action/llm-handler.test.ts diff --git a/src/agents/action/llm-handler.ts b/src/agents/action/llm-handler.ts index 44ac118..82e3aa1 100644 --- a/src/agents/action/llm-handler.ts +++ b/src/agents/action/llm-handler.ts @@ -1,7 +1,37 @@ +import type { Agent } from 'neuri' +import type { Message } from 'neuri/openai' +import type { Mineflayer } from '../../libs/mineflayer' + +import { useLogg } from '@guiiai/logg' +import { agent } from 'neuri' + import { BaseLLMHandler } from '../../libs/llm/base' +import { actionsList } from './tools' + +export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { + const logger = useLogg('action-neuri').useGlobalConfig() + logger.log('Initializing action agent') + let actionAgent = agent('action') + + Object.values(actionsList).forEach((action) => { + actionAgent = actionAgent.tool( + action.name, + action.schema, + async ({ parameters }) => { + logger.withFields({ name: action.name, parameters }).log('Calling action') + mineflayer.memory.actions.push(action) + const fn = action.perform(mineflayer) + return await fn(...Object.values(parameters)) + }, + { description: action.description }, + ) + }) + + return actionAgent.build() +} export class ActionLLMHandler extends BaseLLMHandler { - public async handleAction(messages: any[]): Promise { + public async handleAction(messages: Message[]): Promise { const result = await this.config.agent.handleStateless(messages, async (context) => { this.logger.log('Processing action...') const retryHandler = this.createRetryHandler( diff --git a/src/agents/action/llm.ts b/src/agents/action/llm.ts deleted file mode 100644 index 030a6f5..0000000 --- a/src/agents/action/llm.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { Agent } from 'neuri' -import type { Mineflayer } from '../../libs/mineflayer' - -import { useLogg } from '@guiiai/logg' -import { agent } from 'neuri' - -import { actionsList } from './tools' - -export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { - const logger = useLogg('action-neuri').useGlobalConfig() - logger.log('Initializing action agent') - let actionAgent = agent('action') - - Object.values(actionsList).forEach((action) => { - actionAgent = actionAgent.tool( - action.name, - action.schema, - async ({ parameters }) => { - logger.withFields({ name: action.name, parameters }).log('Calling action') - mineflayer.memory.actions.push(action) - const fn = action.perform(mineflayer) - return await fn(...Object.values(parameters)) - }, - { description: action.description }, - ) - }) - - return actionAgent.build() -} diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts index 195d6ab..6659db1 100644 --- a/src/agents/chat/llm.ts +++ b/src/agents/chat/llm.ts @@ -42,7 +42,7 @@ export async function generateChatResponse( const handleCompletion = async (c: any): Promise => { const completion = await c.reroute('chat', c.messages, { - model: config.model ?? 'openai/gpt-4-mini', + model: config.model ?? 'openai/gpt-4o-mini', }) if (!completion || 'error' in completion) { diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 5c0767c..929a670 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -4,7 +4,7 @@ import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { ActionAgentImpl } from '../action' -import { generatePlanWithLLM } from './llm' +import { PlanningLLMHandler } from './llm-handler' interface PlanContext { goal: string @@ -44,12 +44,17 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { private memoryAgent: MemoryAgent | null = null private planTemplates: Map private llmConfig: PlanningAgentConfig['llm'] + private llmHandler: PlanningLLMHandler constructor(config: PlanningAgentConfig) { super(config) this.planTemplates = new Map() this.llmConfig = config.llm this.initializePlanTemplates() + this.llmHandler = new PlanningLLMHandler({ + agent: this.llmConfig.agent, + model: this.llmConfig.model, + }) } protected async initializeAgent(): Promise { @@ -304,13 +309,9 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { let currentChunk = 1 while (true) { - const steps = await generatePlanWithLLM( + const steps = await this.llmHandler.generatePlan( goal, availableActions, - { - agent: this.llmConfig.agent, - model: this.llmConfig.model, - }, `Generate steps ${currentChunk * chunkSize - 2} to ${currentChunk * chunkSize}`, ) @@ -575,10 +576,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { // If no template matches, use LLM to generate plan this.logger.log('Generating plan using LLM') - return await generatePlanWithLLM(goal, availableActions, { - agent: this.llmConfig.agent, - model: this.llmConfig.model, - }, feedback) + return await this.llmHandler.generatePlan(goal, availableActions, feedback) } private findMatchingTemplate(goal: string): PlanTemplate | undefined { diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index e96edaa..175a60e 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -1,10 +1,16 @@ +import type { Agent } from 'neuri' import type { Action } from '../../libs/mineflayer/action' +import { agent } from 'neuri' import { system, user } from 'neuri/openai' import { BaseLLMHandler } from '../../libs/llm/base' import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' +export async function createPlanningNeuriAgent(): Promise { + return agent('planning').build() +} + export class PlanningLLMHandler extends BaseLLMHandler { public async generatePlan( goal: string, diff --git a/src/agents/planning/llm.ts b/src/agents/planning/llm.ts deleted file mode 100644 index 873475d..0000000 --- a/src/agents/planning/llm.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { Agent, Neuri, NeuriContext } from 'neuri' -import type { Action } from '../../libs/mineflayer/action' - -import { useLogg } from '@guiiai/logg' -import { agent } from 'neuri' -import { type ChatCompletion, system, user } from 'neuri/openai' - -import { toRetriable } from '../../utils/helper' -import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' - -const logger = useLogg('planning-llm').useGlobalConfig() - -interface LLMPlanningConfig { - agent: Neuri - model?: string - retryLimit?: number - delayInterval?: number -} - -export async function createPlanningNeuriAgent(): Promise { - return agent('planning').build() -} - -export async function generatePlanWithLLM( - goal: string, - availableActions: Action[], - config: LLMPlanningConfig, - feedback?: string, -): Promise> { - const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) - const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) - - const messages = [ - system(systemPrompt), - user(userPrompt), - ] - - const content = await config.agent.handleStateless(messages, async (c) => { - logger.log('Generating plan...') - - const handleCompletion = async (c: NeuriContext): Promise => { - const completion = await c.reroute('planning', c.messages, { - model: config.model ?? 'openai/gpt-4o-mini', - }) as ChatCompletion | ChatCompletion & { error: { message: string } } - - if (!completion || 'error' in completion) { - logger.withFields(c).error('Completion failed') - throw new Error(completion?.error?.message ?? 'Unknown error') - } - - const content = await completion.firstContent() - logger.withFields({ usage: completion.usage, content }).log('Plan generated') - return content - } - - const retirableHandler = toRetriable( - config.retryLimit ?? 3, - config.delayInterval ?? 1000, - handleCompletion, - ) - - return await retirableHandler(c) - }) - - if (!content) { - throw new Error('Failed to generate plan') - } - - return parsePlanContent(content) -} - -function parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { - try { - // Find JSON array in the content - const match = content.match(/\[[\s\S]*\]/) - if (!match) { - throw new Error('No plan found in response') - } - - const plan = JSON.parse(match[0]) - if (!Array.isArray(plan)) { - throw new TypeError('Invalid plan format') - } - - return plan.map(step => ({ - action: step.action, - params: step.params, - })) - } - catch (error) { - logger.withError(error).error('Failed to parse plan') - throw error - } -} diff --git a/src/composables/config.ts b/src/composables/config.ts index bae9c66..6076dc5 100644 --- a/src/composables/config.ts +++ b/src/composables/config.ts @@ -9,6 +9,7 @@ const logger = useLogg('config').useGlobalConfig() interface OpenAIConfig { apiKey: string baseUrl: string + model: string } interface EnvConfig { @@ -21,6 +22,7 @@ const defaultConfig: EnvConfig = { openai: { apiKey: '', baseUrl: '', + model: 'openai/gpt-4o-mini', }, bot: { username: '', @@ -43,6 +45,7 @@ export function initEnv(): void { openai: { apiKey: env.OPENAI_API_KEY || defaultConfig.openai.apiKey, baseUrl: env.OPENAI_API_BASEURL || defaultConfig.openai.baseUrl, + model: env.OPENAI_MODEL || defaultConfig.openai.model, }, bot: { username: env.BOT_USERNAME || defaultConfig.bot.username, diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts index 7a4ae64..1c62160 100644 --- a/src/composables/neuri.ts +++ b/src/composables/neuri.ts @@ -4,9 +4,9 @@ import type { Mineflayer } from '../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { neuri } from 'neuri' -import { createActionNeuriAgent } from '../agents/action/llm' +import { createActionNeuriAgent } from '../agents/action/llm-handler' import { createChatNeuriAgent } from '../agents/chat/llm' -import { createPlanningNeuriAgent } from '../agents/planning/llm' +import { createPlanningNeuriAgent } from '../agents/planning/llm-handler' import { openaiConfig } from './config' let neuriAgent: Neuri | undefined diff --git a/src/libs/llm/base.ts b/src/libs/llm/base.ts index f5a516b..7e8df46 100644 --- a/src/libs/llm/base.ts +++ b/src/libs/llm/base.ts @@ -1,7 +1,10 @@ +import type { NeuriContext } from 'neuri' +import type { ChatCompletion, Message } from 'neuri/openai' import type { LLMConfig, LLMResponse } from './types' import { useLogg } from '@guiiai/logg' +import { openaiConfig } from '../../composables/config' import { toRetriable } from '../../utils/helper' export abstract class BaseLLMHandler { @@ -10,13 +13,13 @@ export abstract class BaseLLMHandler { constructor(protected config: LLMConfig) {} protected async handleCompletion( - context: any, + context: NeuriContext, route: string, - messages: any[], + messages: Message[], ): Promise { const completion = await context.reroute(route, messages, { - model: this.config.model ?? 'openai/gpt-4-mini', - }) + model: this.config.model ?? openaiConfig.model, + }) as ChatCompletion | ChatCompletion & { error: { message: string } } if (!completion || 'error' in completion) { this.logger.withFields(context).error('Completion failed') @@ -32,8 +35,8 @@ export abstract class BaseLLMHandler { } } - protected createRetryHandler(handler: (context: any) => Promise) { - return toRetriable( + protected createRetryHandler(handler: (context: NeuriContext) => Promise) { + return toRetriable( this.config.retryLimit ?? 3, this.config.delayInterval ?? 1000, handler, diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts index 2cc0173..3532d64 100644 --- a/src/plugins/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -145,7 +145,7 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { // 创建容器并获取所需的服务 const container = createAppContainer({ neuri: options.agent, - model: 'openai/gpt-4-mini', + model: 'openai/gpt-4o-mini', maxHistoryLength: 50, idleTimeout: 5 * 60 * 1000, }) From 571e054b8b50170e6e410314f1faa96b7f2a7f3b Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 07:21:54 +0800 Subject: [PATCH 19/22] refactor: planning agent return action name instead of valid json --- src/agents/action/index.ts | 249 +++++++++++++++++++---------- src/agents/action/llm-handler.ts | 35 ++++ src/agents/planning/index.ts | 163 +++++++------------ src/agents/planning/llm-handler.ts | 46 +++--- src/agents/prompt/planning.ts | 54 +++---- src/libs/mineflayer/base-agent.ts | 8 +- src/manager/plan.ts | 98 ++++++++++++ 7 files changed, 414 insertions(+), 239 deletions(-) create mode 100644 src/manager/plan.ts diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 7815b63..15348de 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -1,6 +1,9 @@ import type { Mineflayer } from '../../libs/mineflayer' import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' +import type { PlanStep } from '../planning/llm-handler' + +import { z } from 'zod' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' @@ -13,6 +16,15 @@ interface ActionState { startTime: number } +interface ActionTemplate { + description: string + tool: string + parameterExtractors: { + [key: string]: (step: PlanStep) => unknown + } + conditions?: Array<(step: PlanStep) => boolean> +} + /** * ActionAgentImpl implements the ActionAgent interface to handle action execution * Manages action lifecycle, state tracking and error handling @@ -20,6 +32,7 @@ interface ActionState { export class ActionAgentImpl extends AbstractAgent implements ActionAgent { public readonly type = 'action' as const private actions: Map + private actionTemplates: Map private actionManager: ActionManager private mineflayer: Mineflayer private currentActionState: ActionState @@ -27,6 +40,7 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { constructor(config: AgentConfig) { super(config) this.actions = new Map() + this.actionTemplates = new Map() this.mineflayer = useBot().bot this.actionManager = new ActionManager(this.mineflayer) this.currentActionState = { @@ -34,6 +48,7 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { label: '', startTime: 0, } + this.initializeActionTemplates() } protected async initializeAgent(): Promise { @@ -41,102 +56,178 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { actionsList.forEach(action => this.actions.set(action.name, action)) // Set up event listeners - // todo: nothing to call here this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) }) } + private initializeActionTemplates(): void { + // 搜索方块的模板 + this.addActionTemplate('searchForBlock', { + description: 'Search for a specific block type', + tool: 'searchForBlock', + parameterExtractors: { + blockType: (step) => { + // 从描述中提取方块类型 + const match = step.description.match(/(?:search for|find|locate) (?:a |an |the )?(\w+)/i) + return match?.[1] || 'log' + }, + range: (step) => { + // 从描述中提取搜索范围 + const match = step.description.match(/within (\d+) blocks/i) + return match ? Number.parseInt(match[1]) : 64 + }, + }, + conditions: [ + step => step.description.toLowerCase().includes('search') + || step.description.toLowerCase().includes('find') + || step.description.toLowerCase().includes('locate'), + ], + }) + + // 收集方块的模板 + this.addActionTemplate('collectBlocks', { + description: 'Collect blocks of a specific type', + tool: 'collectBlocks', + parameterExtractors: { + blockType: (step) => { + // 从描述中提取方块类型 + const match = step.description.match(/collect (?:some )?(\w+)/i) + return match?.[1] || 'log' + }, + count: (step) => { + // 从描述中提取数量 + const match = step.description.match(/collect (\d+)/i) + return match ? Number.parseInt(match[1]) : 1 + }, + }, + conditions: [ + step => step.description.toLowerCase().includes('collect') + || step.description.toLowerCase().includes('gather') + || step.description.toLowerCase().includes('mine'), + ], + }) + + // 移动的模板 + this.addActionTemplate('moveAway', { + description: 'Move away from current position', + tool: 'moveAway', + parameterExtractors: { + distance: (step) => { + // 从描述中提取距离 + const match = step.description.match(/move (?:away |back |forward )?(\d+)/i) + return match ? Number.parseInt(match[1]) : 5 + }, + }, + conditions: [ + step => step.description.toLowerCase().includes('move away') + || step.description.toLowerCase().includes('step back'), + ], + }) + + // 装备物品的模板 + this.addActionTemplate('equip', { + description: 'Equip a specific item', + tool: 'equip', + parameterExtractors: { + item: (step) => { + // 从描述中提取物品名称 + const match = step.description.match(/equip (?:the )?(\w+)/i) + return match?.[1] || '' + }, + }, + conditions: [ + step => step.description.toLowerCase().includes('equip') + || step.description.toLowerCase().includes('hold') + || step.description.toLowerCase().includes('use'), + ], + }) + } + + private addActionTemplate(actionName: string, template: ActionTemplate): void { + const templates = this.actionTemplates.get(actionName) || [] + templates.push(template) + this.actionTemplates.set(actionName, templates) + } + + private findMatchingTemplate(step: PlanStep): ActionTemplate | null { + const templates = this.actionTemplates.get(step.tool) || [] + return templates.find(template => + template.conditions?.every(condition => condition(step)) ?? true, + ) || null + } + + /** + * Extract parameters from step description and reasoning using templates + */ + private async extractParameters(step: PlanStep, action: Action): Promise { + // First try to use a template if available + const template = this.findMatchingTemplate(step) + if (template) { + this.logger.log('Using action template for parameter extraction') + const shape = action.schema.shape as Record + return Object.keys(shape).map((key) => { + const extractor = template.parameterExtractors[key] + return extractor ? extractor(step) : this.getDefaultValue(shape[key]) + }) + } + + // Fallback to default values if no template matches + this.logger.log('No matching template found, using default values') + const shape = action.schema.shape as Record + return Object.values(shape).map(field => this.getDefaultValue(field)) + } + + private getDefaultValue(field: z.ZodTypeAny): unknown { + if (field instanceof z.ZodString) + return '' + if (field instanceof z.ZodNumber) + return 0 + if (field instanceof z.ZodBoolean) + return false + if (field instanceof z.ZodArray) + return [] + return null + } + protected async destroyAgent(): Promise { - await this.actionManager.stop() - this.actionManager.cancelResume() this.actions.clear() this.removeAllListeners() - this.currentActionState = { - executing: false, - label: '', - startTime: 0, - } } - public async performAction( - name: string, - params: unknown[], - options: { timeout?: number, resume?: boolean } = {}, - ): Promise { + public async performAction(step: PlanStep): Promise { if (!this.initialized) { throw new Error('Action agent not initialized') } - const action = this.actions.get(name) + const action = this.actions.get(step.tool) if (!action) { - throw new Error(`Action not found: ${name}`) + throw new Error(`Unknown action: ${step.tool}`) } - try { - this.updateActionState(true, name) - this.logger.withFields({ name, params }).log('Performing action') - - const result = await this.actionManager.runAction( - name, - async () => { - const fn = action.perform(this.mineflayer) - return await fn(...params) - }, - { - timeout: options.timeout ?? 60, - resume: options.resume ?? false, - }, - ) - - if (!result.success) { - throw new Error(result.message ?? 'Action failed') - } + this.logger.withFields({ + action: step.tool, + description: step.description, + reasoning: step.reasoning, + }).log('Performing action') - return this.formatActionOutput({ - message: result.message, - timedout: result.timedout, - interrupted: false, - }) - } - catch (error) { - this.logger.withFields({ name, params, error }).error('Failed to perform action') - throw error - } - finally { - this.updateActionState(false) - } - } + // Extract parameters from the step description and reasoning + const params = await this.extractParameters(step, action) - public async resumeAction(name: string, params: unknown[]): Promise { - const action = this.actions.get(name) - if (!action) { - throw new Error(`Action not found: ${name}`) - } + // Update action state + this.updateActionState(true, step.description) try { - this.updateActionState(true, name) - const result = await this.actionManager.resumeAction( - name, - async () => { - const fn = action.perform(this.mineflayer) - return await fn(...params) - }, - 60, - ) - - if (!result.success) { - throw new Error(result.message ?? 'Action failed') - } - + // Execute action with extracted parameters + const result = await action.perform(this.mineflayer)(...params) return this.formatActionOutput({ - message: result.message, - timedout: result.timedout, + message: result, + timedout: false, interrupted: false, }) } catch (error) { - this.logger.withFields({ name, params, error }).error('Failed to resume action') + this.logger.withError(error).error('Action failed') throw error } finally { @@ -149,13 +240,10 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { } private async handleAgentMessage(sender: string, message: string): Promise { - if (sender === 'system') { - if (message.includes('interrupt')) { - await this.actionManager.stop() - } - } - else { - this.logger.withFields({ sender, message }).log('Processing agent message') + if (sender === 'system' && message.includes('interrupt') && this.currentActionState.executing) { + // Handle interruption + this.logger.log('Received interrupt request') + // Additional interrupt handling logic here } } @@ -163,18 +251,17 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { this.currentActionState = { executing, label, - startTime: executing ? Date.now() : 0, + startTime: executing ? Date.now() : this.currentActionState.startTime, } - this.emit('actionStateChanged', this.currentActionState) } private formatActionOutput(result: { message: string | null, timedout: boolean, interrupted: boolean }): string { if (result.timedout) { - return `Action timed out: ${result.message}` + return 'Action timed out' } if (result.interrupted) { return 'Action was interrupted' } - return result.message ?? '' + return result.message || 'Action completed successfully' } } diff --git a/src/agents/action/llm-handler.ts b/src/agents/action/llm-handler.ts index 82e3aa1..db59997 100644 --- a/src/agents/action/llm-handler.ts +++ b/src/agents/action/llm-handler.ts @@ -1,9 +1,11 @@ import type { Agent } from 'neuri' import type { Message } from 'neuri/openai' import type { Mineflayer } from '../../libs/mineflayer' +import type { PlanStep } from '../planning/llm-handler' import { useLogg } from '@guiiai/logg' import { agent } from 'neuri' +import { system, user } from 'neuri/openai' import { BaseLLMHandler } from '../../libs/llm/base' import { actionsList } from './tools' @@ -31,6 +33,39 @@ export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { + const systemPrompt = this.generateActionSystemPrompt() + const userPrompt = this.generateActionUserPrompt(step) + const messages = [system(systemPrompt), user(userPrompt)] + + const result = await this.handleAction(messages) + return result + } + + private generateActionSystemPrompt(): string { + return `You are a Minecraft bot action executor. Your task is to execute a given step using available tools. +You have access to various tools that can help you accomplish tasks. +When using a tool: +1. Choose the most appropriate tool for the task +2. Determine the correct parameters based on the context +3. Handle any errors or unexpected situations + +Remember to: +- Be precise with tool parameters +- Consider the current state of the bot +- Handle failures gracefully` + } + + private generateActionUserPrompt(step: PlanStep): string { + return `Execute this step: ${step.description} + +Suggested tool: ${step.tool} +Context and reasoning: ${step.reasoning} + +Please use the appropriate tool with the correct parameters to accomplish this step. +If the suggested tool is not appropriate, you may choose a different one.` + } + public async handleAction(messages: Message[]): Promise { const result = await this.config.agent.handleStateless(messages, async (context) => { this.logger.log('Processing action...') diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 929a670..6a893cf 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -4,7 +4,7 @@ import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { ActionAgentImpl } from '../action' -import { PlanningLLMHandler } from './llm-handler' +import { PlanningLLMHandler, type PlanStep } from './llm-handler' interface PlanContext { goal: string @@ -13,20 +13,7 @@ interface PlanContext { lastUpdate: number retryCount: number isGenerating: boolean - pendingSteps: Array<{ - action: string - params: unknown[] - }> -} - -interface PlanTemplate { - goal: string - conditions: string[] - steps: Array<{ - action: string - params: unknown[] - }> - requiresAction: boolean + pendingSteps: PlanStep[] } export interface PlanningAgentConfig extends AgentConfig { @@ -42,15 +29,12 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { private context: PlanContext | null = null private actionAgent: ActionAgent | null = null private memoryAgent: MemoryAgent | null = null - private planTemplates: Map private llmConfig: PlanningAgentConfig['llm'] private llmHandler: PlanningLLMHandler constructor(config: PlanningAgentConfig) { super(config) - this.planTemplates = new Map() this.llmConfig = config.llm - this.initializePlanTemplates() this.llmHandler = new PlanningLLMHandler({ agent: this.llmConfig.agent, model: this.llmConfig.model, @@ -82,7 +66,6 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { this.context = null this.actionAgent = null this.memoryAgent = null - this.planTemplates.clear() this.removeAllListeners() } @@ -259,7 +242,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { try { this.logger.withField('step', step).log('Executing step') - await this.actionAgent.performAction(step.action, step.params) + await this.actionAgent.performAction(step) this.context.lastUpdate = Date.now() this.context.currentStep++ } @@ -294,16 +277,8 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { private async *createStepGenerator( goal: string, availableActions: Action[], - ): AsyncGenerator, void, unknown> { - // First, try to find a matching template - const template = this.findMatchingTemplate(goal) - if (template) { - this.logger.log('Using plan template') - yield template.steps - return - } - - // If no template matches, use LLM to generate plan in chunks + ): AsyncGenerator { + // Use LLM to generate plan in chunks this.logger.log('Generating plan using LLM') const chunkSize = 3 // Generate 3 steps at a time let currentChunk = 1 @@ -409,59 +384,93 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - private generateGatheringSteps(items: string[]): Array<{ action: string, params: unknown[] }> { - const steps: Array<{ action: string, params: unknown[] }> = [] + private generateGatheringSteps(items: string[]): PlanStep[] { + const steps: PlanStep[] = [] for (const item of items) { steps.push( - { action: 'searchForBlock', params: [item, 64] }, - { action: 'collectBlocks', params: [item, 1] }, + { + description: `Search for ${item} in the surrounding area`, + tool: 'searchForBlock', + reasoning: `We need to locate ${item} before we can collect it`, + }, + { + description: `Collect ${item} from the found location`, + tool: 'collectBlocks', + reasoning: `We need to gather ${item} for our inventory`, + }, ) } return steps } - private generateMovementSteps(location: { x?: number, y?: number, z?: number }): Array<{ action: string, params: unknown[] }> { + private generateMovementSteps(location: { x?: number, y?: number, z?: number }): PlanStep[] { if (location.x !== undefined && location.y !== undefined && location.z !== undefined) { return [{ - action: 'goToCoordinates', - params: [location.x, location.y, location.z, 1], + description: `Move to coordinates (${location.x}, ${location.y}, ${location.z})`, + tool: 'goToCoordinates', + reasoning: 'We need to reach the specified location', }] } return [] } - private generateInteractionSteps(target: string): Array<{ action: string, params: unknown[] }> { + private generateInteractionSteps(target: string): PlanStep[] { return [{ - action: 'activate', - params: [target], + description: `Interact with ${target}`, + tool: 'activate', + reasoning: `We need to use ${target} to proceed`, }] } - private generateRecoverySteps(feedback: string): Array<{ action: string, params: unknown[] }> { - const steps: Array<{ action: string, params: unknown[] }> = [] + private generateRecoverySteps(feedback: string): PlanStep[] { + const steps: PlanStep[] = [] if (feedback.includes('not found')) { - steps.push({ action: 'searchForBlock', params: ['any', 128] }) + steps.push({ + description: 'Search in a wider area', + tool: 'searchForBlock', + reasoning: 'The target was not found in the immediate vicinity', + }) } if (feedback.includes('inventory full')) { - steps.push({ action: 'discard', params: ['cobblestone', 64] }) + steps.push({ + description: 'Clear inventory space', + tool: 'discard', + reasoning: 'We need to make room in our inventory', + }) } if (feedback.includes('blocked') || feedback.includes('cannot reach')) { - steps.push({ action: 'moveAway', params: [5] }) + steps.push({ + description: 'Move away from obstacles', + tool: 'moveAway', + reasoning: 'We need to find a clear path', + }) } if (feedback.includes('too far')) { - steps.push({ action: 'moveAway', params: [-3] }) // Move closer + steps.push({ + description: 'Move closer to target', + tool: 'moveAway', + reasoning: 'We need to get within range', + }) } if (feedback.includes('need tool')) { steps.push( - { action: 'craftRecipe', params: ['wooden_pickaxe', 1] }, - { action: 'equip', params: ['wooden_pickaxe'] }, + { + description: 'Craft a wooden pickaxe', + tool: 'craftRecipe', + reasoning: 'We need the appropriate tool for this task', + }, + { + description: 'Equip the wooden pickaxe', + tool: 'equip', + reasoning: 'We need to use the tool we just crafted', + }, ) } @@ -491,44 +500,6 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return true } - private initializePlanTemplates(): void { - // Add common plan templates - this.planTemplates.set('collect wood', { - goal: 'collect wood', - conditions: ['needs_axe', 'near_trees'], - steps: [ - { action: 'searchForBlock', params: ['log', 64] }, - { action: 'collectBlocks', params: ['log', 1] }, - ], - requiresAction: true, - }) - - this.planTemplates.set('find shelter', { - goal: 'find shelter', - conditions: ['is_night', 'unsafe'], - steps: [ - { action: 'searchForBlock', params: ['bed', 64] }, - { action: 'goToBed', params: [] }, - ], - requiresAction: true, - }) - - // Add templates for non-action goals - this.planTemplates.set('hello', { - goal: 'hello', - conditions: [], - steps: [], - requiresAction: false, - }) - - this.planTemplates.set('how are you', { - goal: 'how are you', - conditions: [], - steps: [], - requiresAction: false, - }) - } - private async handleAgentMessage(sender: string, message: string): Promise { if (sender === 'system') { if (message.includes('interrupt')) { @@ -566,28 +537,12 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { goal: string, availableActions: Action[], feedback?: string, - ): Promise> { - // First, try to find a matching template - const template = this.findMatchingTemplate(goal) - if (template) { - this.logger.log('Using plan template') - return template.steps - } - - // If no template matches, use LLM to generate plan + ): Promise { + // Use LLM to generate plan this.logger.log('Generating plan using LLM') return await this.llmHandler.generatePlan(goal, availableActions, feedback) } - private findMatchingTemplate(goal: string): PlanTemplate | undefined { - for (const [pattern, template] of this.planTemplates.entries()) { - if (goal.toLowerCase().includes(pattern.toLowerCase())) { - return template - } - } - return undefined - } - private parseGoalRequirements(goal: string): { needsItems: boolean items?: string[] diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index 175a60e..ccd4fc5 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -11,12 +11,18 @@ export async function createPlanningNeuriAgent(): Promise { return agent('planning').build() } +export interface PlanStep { + description: string + tool: string + reasoning: string +} + export class PlanningLLMHandler extends BaseLLMHandler { public async generatePlan( goal: string, availableActions: Action[], feedback?: string, - ): Promise> { + ): Promise { const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) const messages = [system(systemPrompt), user(userPrompt)] @@ -36,26 +42,26 @@ export class PlanningLLMHandler extends BaseLLMHandler { return this.parsePlanContent(result) } - private parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { - try { - const match = content.match(/\[[\s\S]*\]/) - if (!match) { - throw new Error('No plan found in response') - } + private parsePlanContent(content: string): PlanStep[] { + // Split content into steps (numbered list) + const steps = content.split(/\d+\./).filter(step => step.trim().length > 0) - const plan = JSON.parse(match[0]) - if (!Array.isArray(plan)) { - throw new TypeError('Invalid plan format') - } + return steps.map((step) => { + const lines = step.trim().split('\n') + const description = lines[0] - return plan.map(step => ({ - action: step.action, - params: step.params, - })) - } - catch (error) { - this.logger.withError(error).error('Failed to parse plan') - throw error - } + // Extract tool name from the content (usually in single quotes) + const toolMatch = step.match(/'([^']+)'/) + const tool = toolMatch ? toolMatch[1] : '' + + // Everything else is considered reasoning + const reasoning = lines.slice(1).join('\n').trim() + + return { + description, + tool, + reasoning, + } + }) } } diff --git a/src/agents/prompt/planning.ts b/src/agents/prompt/planning.ts index 6c3bf21..f4a6e10 100644 --- a/src/agents/prompt/planning.ts +++ b/src/agents/prompt/planning.ts @@ -5,43 +5,39 @@ export function generatePlanningAgentSystemPrompt(availableActions: Action[]): s .map(action => `- ${action.name}: ${action.description}`) .join('\n') - return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. -Available actions: + return `You are a Minecraft bot planner. Break down goals into simple action steps. + +Available tools: ${actionsList} -Respond with a Valid JSON array of steps, where each step has: -- action: The name of the action to perform -- params: Array of parameters for the action - -DO NOT contains any \`\`\` or explation, otherwise agent will be interrupted. - -Example response: -[ - { - "action": "searchForBlock", - "params": ["log", 64] - }, - { - "action": "collectBlocks", - "params": ["log", 1] - } - ]` +Format each step as: +1. Action description (short, direct command) +2. Tool name to use +3. Brief context + +Example: +1. Find oak log + Tool: searchForBlock + Context: need wood + +2. Mine the log + Tool: collectBlocks + Context: get resource + +Keep steps: +- Short and direct +- Action-focused +- No explanations needed` } export function generatePlanningAgentUserPrompt(goal: string, feedback?: string): string { - let prompt = `Create a detailed plan to: ${goal} - -Consider the following aspects: -1. Required materials and their quantities -2. Required tools and their availability -3. Necessary crafting steps -4. Block placement requirements -5. Current inventory status + let prompt = `Goal: ${goal} -Please generate steps that handle these requirements in the correct order.` +Generate minimal steps to complete this task. +Focus on actions only, no explanations needed.` if (feedback) { - prompt += `\nPrevious attempt feedback: ${feedback}` + prompt += `\n\nPrevious attempt failed: ${feedback}` } return prompt } diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts index fc573a5..12f74d0 100644 --- a/src/libs/mineflayer/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -1,3 +1,4 @@ +import type { PlanStep } from '../../agents/planning/llm-handler' import type { Action } from './action' import { useLogg } from '@guiiai/logg' @@ -19,7 +20,7 @@ export interface BaseAgent { export interface ActionAgent extends BaseAgent { type: 'action' - performAction: (name: string, params: unknown[]) => Promise + performAction: (step: PlanStep) => Promise getAvailableActions: () => Action[] } @@ -33,10 +34,7 @@ export interface MemoryAgent extends BaseAgent { export interface Plan { goal: string - steps: Array<{ - action: string - params: unknown[] - }> + steps: PlanStep[] status: 'pending' | 'in_progress' | 'completed' | 'failed' requiresAction: boolean } diff --git a/src/manager/plan.ts b/src/manager/plan.ts new file mode 100644 index 0000000..55c1692 --- /dev/null +++ b/src/manager/plan.ts @@ -0,0 +1,98 @@ +// import type { ActionLLMHandler } from '../agents/action/llm-handler' +// import type { PlanningLLMHandler } from '../agents/planning/llm-handler' +// import type { Mineflayer } from '../libs/mineflayer/core' + +// import { useLogg } from '@guiiai/logg' +// import EventEmitter from 'eventemitter3' + +// import { actionsList } from '../agents/action/tools' + +// interface PlanExecutionResult { +// success: boolean +// message: string +// step: number +// totalSteps: number +// } + +// export class PlanManager extends EventEmitter { +// private logger = useLogg('PlanManager').useGlobalConfig() +// private mineflayer: Mineflayer +// private planningHandler: PlanningLLMHandler +// private actionHandler: ActionLLMHandler + +// constructor( +// mineflayer: Mineflayer, +// planningHandler: PlanningLLMHandler, +// actionHandler: ActionLLMHandler, +// ) { +// super() +// this.mineflayer = mineflayer +// this.planningHandler = planningHandler +// this.actionHandler = actionHandler +// } + +// /** +// * Execute a plan to achieve a goal +// * @param goal The goal to achieve +// * @returns The result of the plan execution +// */ +// public async executePlan(goal: string): Promise { +// try { +// // Generate plan +// this.logger.log('Generating plan for goal:', goal) +// const plan = await this.planningHandler.generatePlan(goal, Object.values(actionsList)) + +// // Execute each step +// let currentStep = 0 +// for (const step of plan) { +// currentStep++ +// this.logger.log(`Executing step ${currentStep}/${plan.length}:`, step.description) + +// try { +// const result = await this.actionHandler.executeStep(step) +// this.logger.log('Step result:', result) +// } +// catch (error) { +// // If a step fails, try to regenerate the plan with feedback +// this.logger.error('Step failed:', error) +// const feedback = `Failed at step ${currentStep}: ${error.message}` +// return this.retryWithFeedback(goal, feedback) +// } +// } + +// return { +// success: true, +// message: 'Plan executed successfully', +// step: plan.length, +// totalSteps: plan.length, +// } +// } +// catch (error) { +// this.logger.error('Plan execution failed:', error) +// return { +// success: false, +// message: error.message, +// step: 0, +// totalSteps: 0, +// } +// } +// } + +// /** +// * Retry executing the plan with feedback from previous failure +// */ +// private async retryWithFeedback( +// goal: string, +// feedback: string, +// ): Promise { +// this.logger.log('Retrying with feedback:', feedback) +// const plan = await this.planningHandler.generatePlan( +// goal, +// Object.values(actionsList), +// feedback, +// ) + +// // Execute the new plan +// return this.executePlan(goal) +// } +// } From 853c5f8e5d37e8062eadef6824c497d59a540fd3 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sun, 19 Jan 2025 21:16:24 +0800 Subject: [PATCH 20/22] fix: it works --- src/agents/action/index.ts | 152 +---------------------------- src/agents/planning/llm-handler.ts | 49 ++++++++-- src/agents/prompt/planning.ts | 26 +++-- 3 files changed, 61 insertions(+), 166 deletions(-) diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index 15348de..e5d3906 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -3,8 +3,6 @@ import type { Action } from '../../libs/mineflayer/action' import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' import type { PlanStep } from '../planning/llm-handler' -import { z } from 'zod' - import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { ActionManager } from '../../manager/action' @@ -16,15 +14,6 @@ interface ActionState { startTime: number } -interface ActionTemplate { - description: string - tool: string - parameterExtractors: { - [key: string]: (step: PlanStep) => unknown - } - conditions?: Array<(step: PlanStep) => boolean> -} - /** * ActionAgentImpl implements the ActionAgent interface to handle action execution * Manages action lifecycle, state tracking and error handling @@ -32,7 +21,6 @@ interface ActionTemplate { export class ActionAgentImpl extends AbstractAgent implements ActionAgent { public readonly type = 'action' as const private actions: Map - private actionTemplates: Map private actionManager: ActionManager private mineflayer: Mineflayer private currentActionState: ActionState @@ -40,7 +28,6 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { constructor(config: AgentConfig) { super(config) this.actions = new Map() - this.actionTemplates = new Map() this.mineflayer = useBot().bot this.actionManager = new ActionManager(this.mineflayer) this.currentActionState = { @@ -48,7 +35,6 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { label: '', startTime: 0, } - this.initializeActionTemplates() } protected async initializeAgent(): Promise { @@ -61,135 +47,6 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { }) } - private initializeActionTemplates(): void { - // 搜索方块的模板 - this.addActionTemplate('searchForBlock', { - description: 'Search for a specific block type', - tool: 'searchForBlock', - parameterExtractors: { - blockType: (step) => { - // 从描述中提取方块类型 - const match = step.description.match(/(?:search for|find|locate) (?:a |an |the )?(\w+)/i) - return match?.[1] || 'log' - }, - range: (step) => { - // 从描述中提取搜索范围 - const match = step.description.match(/within (\d+) blocks/i) - return match ? Number.parseInt(match[1]) : 64 - }, - }, - conditions: [ - step => step.description.toLowerCase().includes('search') - || step.description.toLowerCase().includes('find') - || step.description.toLowerCase().includes('locate'), - ], - }) - - // 收集方块的模板 - this.addActionTemplate('collectBlocks', { - description: 'Collect blocks of a specific type', - tool: 'collectBlocks', - parameterExtractors: { - blockType: (step) => { - // 从描述中提取方块类型 - const match = step.description.match(/collect (?:some )?(\w+)/i) - return match?.[1] || 'log' - }, - count: (step) => { - // 从描述中提取数量 - const match = step.description.match(/collect (\d+)/i) - return match ? Number.parseInt(match[1]) : 1 - }, - }, - conditions: [ - step => step.description.toLowerCase().includes('collect') - || step.description.toLowerCase().includes('gather') - || step.description.toLowerCase().includes('mine'), - ], - }) - - // 移动的模板 - this.addActionTemplate('moveAway', { - description: 'Move away from current position', - tool: 'moveAway', - parameterExtractors: { - distance: (step) => { - // 从描述中提取距离 - const match = step.description.match(/move (?:away |back |forward )?(\d+)/i) - return match ? Number.parseInt(match[1]) : 5 - }, - }, - conditions: [ - step => step.description.toLowerCase().includes('move away') - || step.description.toLowerCase().includes('step back'), - ], - }) - - // 装备物品的模板 - this.addActionTemplate('equip', { - description: 'Equip a specific item', - tool: 'equip', - parameterExtractors: { - item: (step) => { - // 从描述中提取物品名称 - const match = step.description.match(/equip (?:the )?(\w+)/i) - return match?.[1] || '' - }, - }, - conditions: [ - step => step.description.toLowerCase().includes('equip') - || step.description.toLowerCase().includes('hold') - || step.description.toLowerCase().includes('use'), - ], - }) - } - - private addActionTemplate(actionName: string, template: ActionTemplate): void { - const templates = this.actionTemplates.get(actionName) || [] - templates.push(template) - this.actionTemplates.set(actionName, templates) - } - - private findMatchingTemplate(step: PlanStep): ActionTemplate | null { - const templates = this.actionTemplates.get(step.tool) || [] - return templates.find(template => - template.conditions?.every(condition => condition(step)) ?? true, - ) || null - } - - /** - * Extract parameters from step description and reasoning using templates - */ - private async extractParameters(step: PlanStep, action: Action): Promise { - // First try to use a template if available - const template = this.findMatchingTemplate(step) - if (template) { - this.logger.log('Using action template for parameter extraction') - const shape = action.schema.shape as Record - return Object.keys(shape).map((key) => { - const extractor = template.parameterExtractors[key] - return extractor ? extractor(step) : this.getDefaultValue(shape[key]) - }) - } - - // Fallback to default values if no template matches - this.logger.log('No matching template found, using default values') - const shape = action.schema.shape as Record - return Object.values(shape).map(field => this.getDefaultValue(field)) - } - - private getDefaultValue(field: z.ZodTypeAny): unknown { - if (field instanceof z.ZodString) - return '' - if (field instanceof z.ZodNumber) - return 0 - if (field instanceof z.ZodBoolean) - return false - if (field instanceof z.ZodArray) - return [] - return null - } - protected async destroyAgent(): Promise { this.actions.clear() this.removeAllListeners() @@ -208,18 +65,15 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { this.logger.withFields({ action: step.tool, description: step.description, - reasoning: step.reasoning, + params: step.params, }).log('Performing action') - // Extract parameters from the step description and reasoning - const params = await this.extractParameters(step, action) - // Update action state this.updateActionState(true, step.description) try { - // Execute action with extracted parameters - const result = await action.perform(this.mineflayer)(...params) + // Execute action with provided parameters + const result = await action.perform(this.mineflayer)(...Object.values(step.params)) return this.formatActionOutput({ message: result, timedout: false, diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index ccd4fc5..f3501f1 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -14,7 +14,7 @@ export async function createPlanningNeuriAgent(): Promise { export interface PlanStep { description: string tool: string - reasoning: string + params: Record } export class PlanningLLMHandler extends BaseLLMHandler { @@ -48,19 +48,52 @@ export class PlanningLLMHandler extends BaseLLMHandler { return steps.map((step) => { const lines = step.trim().split('\n') - const description = lines[0] + const description = lines[0].trim() - // Extract tool name from the content (usually in single quotes) - const toolMatch = step.match(/'([^']+)'/) - const tool = toolMatch ? toolMatch[1] : '' + // Extract tool name and parameters + let tool = '' + const params: Record = {} - // Everything else is considered reasoning - const reasoning = lines.slice(1).join('\n').trim() + for (const line of lines) { + const trimmed = line.trim() + + // Extract tool name + if (trimmed.startsWith('Tool:')) { + tool = trimmed.split(':')[1].trim() + continue + } + + // Extract parameters + if (trimmed === 'Params:') { + let i = lines.indexOf(line) + 1 + while (i < lines.length) { + const paramLine = lines[i].trim() + if (paramLine === '') + break + + const paramMatch = paramLine.match(/(\w+):\s*(.+)/) + if (paramMatch) { + const [, key, value] = paramMatch + // Try to parse numbers and booleans + if (value === 'true') + params[key] = true + else if (value === 'false') + params[key] = false + else if (/^\d+$/.test(value)) + params[key] = Number.parseInt(value) + else if (/^\d*\.\d+$/.test(value)) + params[key] = Number.parseFloat(value) + else params[key] = value.trim() + } + i++ + } + } + } return { description, tool, - reasoning, + params, } }) } diff --git a/src/agents/prompt/planning.ts b/src/agents/prompt/planning.ts index f4a6e10..35077bf 100644 --- a/src/agents/prompt/planning.ts +++ b/src/agents/prompt/planning.ts @@ -2,8 +2,13 @@ import type { Action } from '../../libs/mineflayer/action' export function generatePlanningAgentSystemPrompt(availableActions: Action[]): string { const actionsList = availableActions - .map(action => `- ${action.name}: ${action.description}`) - .join('\n') + .map((action) => { + const params = Object.entries(action.schema.shape as Record) + .map(([name, type]) => ` - ${name}: ${type._def.typeName}`) + .join('\n') + return `- ${action.name}: ${action.description}\n Parameters:\n${params}` + }) + .join('\n\n') return `You are a Minecraft bot planner. Break down goals into simple action steps. @@ -12,29 +17,32 @@ ${actionsList} Format each step as: 1. Action description (short, direct command) -2. Tool name to use -3. Brief context +2. Tool name +3. Required parameters Example: 1. Find oak log Tool: searchForBlock - Context: need wood + Params: + blockType: oak_log + range: 64 2. Mine the log Tool: collectBlocks - Context: get resource + Params: + blockType: oak_log + count: 1 Keep steps: - Short and direct - Action-focused -- No explanations needed` +- Parameters precise` } export function generatePlanningAgentUserPrompt(goal: string, feedback?: string): string { let prompt = `Goal: ${goal} -Generate minimal steps to complete this task. -Focus on actions only, no explanations needed.` +Generate minimal steps with exact parameters.` if (feedback) { prompt += `\n\nPrevious attempt failed: ${feedback}` From c28db69f2e6c7cee8418f4d47cf3cd0ad1258b46 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Mon, 20 Jan 2025 10:36:39 +0800 Subject: [PATCH 21/22] chore: it works --- src/agents/action/index.ts | 3 - src/agents/action/llm-handler.ts | 2 +- src/agents/planning/index.ts | 128 ++++++---- src/agents/planning/llm-handler.ts | 3 +- src/agents/prompt/planning.ts | 28 +-- src/libs/mineflayer/base-agent.ts | 2 +- src/manager/action.ts | 203 --------------- src/manager/conversation.ts | 382 ----------------------------- src/manager/plan.ts | 98 -------- 9 files changed, 94 insertions(+), 755 deletions(-) delete mode 100644 src/manager/action.ts delete mode 100644 src/manager/conversation.ts delete mode 100644 src/manager/plan.ts diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts index e5d3906..19109b0 100644 --- a/src/agents/action/index.ts +++ b/src/agents/action/index.ts @@ -5,7 +5,6 @@ import type { PlanStep } from '../planning/llm-handler' import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' -import { ActionManager } from '../../manager/action' import { actionsList } from './tools' interface ActionState { @@ -21,7 +20,6 @@ interface ActionState { export class ActionAgentImpl extends AbstractAgent implements ActionAgent { public readonly type = 'action' as const private actions: Map - private actionManager: ActionManager private mineflayer: Mineflayer private currentActionState: ActionState @@ -29,7 +27,6 @@ export class ActionAgentImpl extends AbstractAgent implements ActionAgent { super(config) this.actions = new Map() this.mineflayer = useBot().bot - this.actionManager = new ActionManager(this.mineflayer) this.currentActionState = { executing: false, label: '', diff --git a/src/agents/action/llm-handler.ts b/src/agents/action/llm-handler.ts index db59997..1cdf747 100644 --- a/src/agents/action/llm-handler.ts +++ b/src/agents/action/llm-handler.ts @@ -60,7 +60,7 @@ Remember to: return `Execute this step: ${step.description} Suggested tool: ${step.tool} -Context and reasoning: ${step.reasoning} +Params: ${JSON.stringify(step.params)} Please use the appropriate tool with the correct parameters to accomplish this step. If the suggested tool is not appropriate, you may choose a different one.` diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 6a893cf..33d68af 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -103,7 +103,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } // Create plan steps based on available actions and goal - const steps = await this.generatePlanSteps(goal, availableActions) + const steps = await this.generatePlanSteps(goal, availableActions, 'system') // Create new plan const plan: Plan = { @@ -155,8 +155,31 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { plan.status = 'in_progress' this.currentPlan = plan - // Start generating and executing steps in parallel - await this.generateAndExecutePlanSteps(plan) + // Execute each step + for (const step of plan.steps) { + try { + this.logger.withField('step', step).log('Executing step') + await this.actionAgent.performAction(step) + } + catch (stepError) { + this.logger.withError(stepError).error('Failed to execute step') + + // Attempt to adjust plan and retry + if (this.context && this.context.retryCount < 3) { + this.context.retryCount++ + // Adjust plan and restart + const adjustedPlan = await this.adjustPlan( + plan, + stepError instanceof Error ? stepError.message : 'Unknown error', + 'system', + ) + await this.executePlan(adjustedPlan) + return + } + + throw stepError + } + } plan.status = 'completed' } @@ -169,48 +192,25 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - private async generateAndExecutePlanSteps(plan: Plan): Promise { - if (!this.context || !this.actionAgent) { - return - } - - // Initialize step generation - this.context.isGenerating = true - this.context.pendingSteps = [] - - // Get available actions - const availableActions = this.actionAgent.getAvailableActions() - - // Start step generation - const generationPromise = this.generateStepsStream(plan.goal, availableActions) - - // Start step execution - const executionPromise = this.executeStepsStream() - - // Wait for both generation and execution to complete - await Promise.all([generationPromise, executionPromise]) - } - private async generateStepsStream( goal: string, availableActions: Action[], + sender: string, ): Promise { if (!this.context) { return } try { - // Generate steps in chunks - const generator = this.createStepGenerator(goal, availableActions) - for await (const steps of generator) { - if (!this.context.isGenerating) { - break - } - - // Add generated steps to pending queue - this.context.pendingSteps.push(...steps) - this.logger.withField('steps', steps).log('Generated new steps') + // Generate all steps at once + const steps = await this.llmHandler.generatePlan(goal, availableActions, sender) + if (!this.context.isGenerating) { + return } + + // Add all steps to pending queue + this.context.pendingSteps.push(...steps) + this.logger.withField('steps', steps).log('Generated steps') } catch (error) { this.logger.withError(error).error('Failed to generate steps') @@ -259,6 +259,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { const adjustedPlan = await this.adjustPlan( this.currentPlan!, stepError instanceof Error ? stepError.message : 'Unknown error', + 'system', ) await this.executePlan(adjustedPlan) return @@ -341,7 +342,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return true } - public async adjustPlan(plan: Plan, feedback: string): Promise { + public async adjustPlan(plan: Plan, feedback: string, sender: string): Promise { if (!this.initialized) { throw new Error('Planning agent not initialized') } @@ -358,7 +359,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { const recoverySteps = this.generateRecoverySteps(feedback) // Generate new steps from the current point - const newSteps = await this.generatePlanSteps(plan.goal, availableActions, feedback) + const newSteps = await this.generatePlanSteps(plan.goal, availableActions, sender, feedback) // Create adjusted plan const adjustedPlan: Plan = { @@ -392,12 +393,18 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { { description: `Search for ${item} in the surrounding area`, tool: 'searchForBlock', - reasoning: `We need to locate ${item} before we can collect it`, + params: { + blockType: item, + range: 64, + }, }, { description: `Collect ${item} from the found location`, tool: 'collectBlocks', - reasoning: `We need to gather ${item} for our inventory`, + params: { + blockType: item, + count: 1, + }, }, ) } @@ -410,7 +417,11 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return [{ description: `Move to coordinates (${location.x}, ${location.y}, ${location.z})`, tool: 'goToCoordinates', - reasoning: 'We need to reach the specified location', + params: { + x: location.x, + y: location.y, + z: location.z, + }, }] } return [] @@ -420,7 +431,9 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return [{ description: `Interact with ${target}`, tool: 'activate', - reasoning: `We need to use ${target} to proceed`, + params: { + target, + }, }] } @@ -431,7 +444,10 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { steps.push({ description: 'Search in a wider area', tool: 'searchForBlock', - reasoning: 'The target was not found in the immediate vicinity', + params: { + blockType: 'oak_log', + range: 64, + }, }) } @@ -439,7 +455,10 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { steps.push({ description: 'Clear inventory space', tool: 'discard', - reasoning: 'We need to make room in our inventory', + params: { + blockType: 'oak_log', + count: 1, + }, }) } @@ -447,7 +466,9 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { steps.push({ description: 'Move away from obstacles', tool: 'moveAway', - reasoning: 'We need to find a clear path', + params: { + range: 64, + }, }) } @@ -455,7 +476,9 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { steps.push({ description: 'Move closer to target', tool: 'moveAway', - reasoning: 'We need to get within range', + params: { + range: 64, + }, }) } @@ -464,12 +487,16 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { { description: 'Craft a wooden pickaxe', tool: 'craftRecipe', - reasoning: 'We need the appropriate tool for this task', + params: { + recipe: 'oak_pickaxe', + }, }, { description: 'Equip the wooden pickaxe', tool: 'equip', - reasoning: 'We need to use the tool we just crafted', + params: { + item: 'oak_pickaxe', + }, }, ) } @@ -512,7 +539,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { // If there's a current plan, try to adjust it based on the message if (this.currentPlan) { - await this.adjustPlan(this.currentPlan, message) + await this.adjustPlan(this.currentPlan, message, sender) } } } @@ -536,11 +563,12 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { private async generatePlanSteps( goal: string, availableActions: Action[], + sender: string, feedback?: string, ): Promise { - // Use LLM to generate plan + // Generate all steps at once this.logger.log('Generating plan using LLM') - return await this.llmHandler.generatePlan(goal, availableActions, feedback) + return await this.llmHandler.generatePlan(goal, availableActions, sender, feedback) } private parseGoalRequirements(goal: string): { diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts index f3501f1..81d9a4e 100644 --- a/src/agents/planning/llm-handler.ts +++ b/src/agents/planning/llm-handler.ts @@ -21,10 +21,11 @@ export class PlanningLLMHandler extends BaseLLMHandler { public async generatePlan( goal: string, availableActions: Action[], + sender: string, feedback?: string, ): Promise { const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) - const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) + const userPrompt = generatePlanningAgentUserPrompt(goal, sender, feedback) const messages = [system(systemPrompt), user(userPrompt)] const result = await this.config.agent.handleStateless(messages, async (context) => { diff --git a/src/agents/prompt/planning.ts b/src/agents/prompt/planning.ts index 35077bf..87928c7 100644 --- a/src/agents/prompt/planning.ts +++ b/src/agents/prompt/planning.ts @@ -3,8 +3,8 @@ import type { Action } from '../../libs/mineflayer/action' export function generatePlanningAgentSystemPrompt(availableActions: Action[]): string { const actionsList = availableActions .map((action) => { - const params = Object.entries(action.schema.shape as Record) - .map(([name, type]) => ` - ${name}: ${type._def.typeName}`) + const params = Object.keys(action.schema.shape) + .map(name => ` - ${name}`) .join('\n') return `- ${action.name}: ${action.description}\n Parameters:\n${params}` }) @@ -21,28 +21,24 @@ Format each step as: 3. Required parameters Example: -1. Find oak log - Tool: searchForBlock +1. Follow player + Tool: followPlayer Params: - blockType: oak_log - range: 64 - -2. Mine the log - Tool: collectBlocks - Params: - blockType: oak_log - count: 1 + player: luoling8192 + follow_dist: 3 Keep steps: - Short and direct - Action-focused -- Parameters precise` +- Parameters precise +- Generate all steps at once` } -export function generatePlanningAgentUserPrompt(goal: string, feedback?: string): string { - let prompt = `Goal: ${goal} +export function generatePlanningAgentUserPrompt(goal: string, sender: string, feedback?: string): string { + let prompt = `${sender}: ${goal} -Generate minimal steps with exact parameters.` +Generate minimal steps with exact parameters. +Use the sender's name (${sender}) for player-related parameters.` if (feedback) { prompt += `\n\nPrevious attempt failed: ${feedback}` diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts index 12f74d0..132ca98 100644 --- a/src/libs/mineflayer/base-agent.ts +++ b/src/libs/mineflayer/base-agent.ts @@ -43,7 +43,7 @@ export interface PlanningAgent extends BaseAgent { type: 'planning' createPlan: (goal: string) => Promise executePlan: (plan: Plan) => Promise - adjustPlan: (plan: Plan, feedback: string) => Promise + adjustPlan: (plan: Plan, feedback: string, sender: string) => Promise } export interface ChatAgent extends BaseAgent { diff --git a/src/manager/action.ts b/src/manager/action.ts deleted file mode 100644 index 7f78c0b..0000000 --- a/src/manager/action.ts +++ /dev/null @@ -1,203 +0,0 @@ -import type { Mineflayer } from '../libs/mineflayer/core' - -import { useLogg } from '@guiiai/logg' -import EventEmitter from 'eventemitter3' - -// Types and interfaces -type ActionFn = (...args: any[]) => void - -interface ActionResult { - success: boolean - message: string | null - timedout: boolean -} - -interface QueuedAction { - label: string - fn: ActionFn - timeout: number - resume: boolean - resolve: (result: ActionResult) => void - reject: (error: Error) => void -} - -export class ActionManager extends EventEmitter { - private state = { - executing: false, - currentActionLabel: '', - currentActionFn: undefined as ActionFn | undefined, - timedout: false, - resume: { - func: undefined as ActionFn | undefined, - name: undefined as string | undefined, - }, - } - - // Action queue to store pending actions - private actionQueue: QueuedAction[] = [] - - private logger = useLogg('ActionManager').useGlobalConfig() - private mineflayer: Mineflayer - - constructor(mineflayer: Mineflayer) { - super() - this.mineflayer = mineflayer - } - - public async resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { - return this.queueAction({ - label: actionLabel, - fn: actionFn, - timeout, - resume: true, - }) - } - - public async runAction( - actionLabel: string, - actionFn: ActionFn, - options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }, - ): Promise { - return this.queueAction({ - label: actionLabel, - fn: actionFn, - timeout: options.timeout, - resume: options.resume, - }) - } - - public async stop(): Promise { - this.mineflayer.emit('interrupt') - // Clear the action queue when stopping - this.actionQueue = [] - } - - public cancelResume(): void { - this.state.resume.func = undefined - this.state.resume.name = undefined - } - - private async queueAction(action: Omit): Promise { - return new Promise((resolve, reject) => { - this.actionQueue.push({ - ...action, - resolve, - reject, - }) - - if (!this.state.executing) { - this.processQueue().catch(reject) - } - }) - } - - private async processQueue(): Promise { - while (this.actionQueue.length > 0) { - const action = this.actionQueue[0] - - try { - const result = action.resume - ? await this.executeResume(action.label, action.fn, action.timeout) - : await this.executeAction(action.label, action.fn, action.timeout) - - this.actionQueue.shift()?.resolve(result) - - if (!result.success) { - this.actionQueue.forEach(pendingAction => - pendingAction.reject(new Error('Queue cleared due to action failure')), - ) - this.actionQueue = [] - return result - } - } - catch (error) { - this.actionQueue.shift()?.reject(error as Error) - this.actionQueue.forEach(pendingAction => - pendingAction.reject(new Error('Queue cleared due to error')), - ) - this.actionQueue = [] - throw error - } - } - - return { success: true, message: 'success', timedout: false } - } - - private async executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { - const isNewResume = actionFn != null - - if (isNewResume) { - if (!actionLabel) { - throw new Error('actionLabel is required for new resume') - } - this.state.resume.func = actionFn - this.state.resume.name = actionLabel - } - - const canExecute = this.state.resume.func != null && isNewResume - - if (!canExecute) { - return { success: false, message: null, timedout: false } - } - - this.state.currentActionLabel = this.state.resume.name || '' - const result = await this.executeAction(this.state.resume.name || '', this.state.resume.func, timeout) - this.state.currentActionLabel = '' - return result - } - - private async executeAction(actionLabel: string, actionFn?: ActionFn, timeout = 10): Promise { - let timeoutHandle: NodeJS.Timeout | undefined - - try { - this.logger.log('executing action...\n') - - if (this.state.executing) { - this.logger.log(`action "${actionLabel}" trying to interrupt current action "${this.state.currentActionLabel}"`) - } - - await this.stop() - - // Set execution state - this.state.executing = true - this.state.currentActionLabel = actionLabel - this.state.currentActionFn = actionFn - - if (timeout > 0) { - timeoutHandle = this.startTimeout(timeout) - } - - await actionFn?.() - - // Reset state after successful execution - this.resetExecutionState(timeoutHandle) - - return { success: true, message: 'success', timedout: false } - } - catch (err) { - this.resetExecutionState(timeoutHandle) - this.cancelResume() - this.logger.withError(err).error('Code execution triggered catch') - await this.stop() - - return { success: false, message: 'failed', timedout: false } - } - } - - private resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { - this.state.executing = false - this.state.currentActionLabel = '' - this.state.currentActionFn = undefined - if (timeoutHandle) - clearTimeout(timeoutHandle) - } - - private startTimeout(timeoutMins = 10): NodeJS.Timeout { - return setTimeout(async () => { - this.logger.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - this.state.timedout = true - this.emit('timeout', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) - await this.stop() - }, timeoutMins * 60 * 1000) - } -} diff --git a/src/manager/conversation.ts b/src/manager/conversation.ts deleted file mode 100644 index 891fb7a..0000000 --- a/src/manager/conversation.ts +++ /dev/null @@ -1,382 +0,0 @@ -// import { useLogg } from '@guiiai/logg' - -// let self_prompter_paused = false - -// interface ConversationMessage { -// message: string -// start: boolean -// end: boolean -// } - -// function compileInMessages(inQueue: ConversationMessage[]) { -// let pack: ConversationMessage | undefined -// let fullMessage = '' -// while (inQueue.length > 0) { -// pack = inQueue.shift() -// if (!pack) -// continue - -// fullMessage += pack.message -// } -// if (pack) { -// pack.message = fullMessage -// } - -// return pack -// } - -// type Conversation = ReturnType - -// function useConversations(name: string, agent: Agent) { -// const active = { value: false } -// const ignoreUntilStart = { value: false } -// const blocked = { value: false } -// let inQueue: ConversationMessage[] = [] -// const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } - -// function reset() { -// active.value = false -// ignoreUntilStart.value = false -// inQueue = [] -// } - -// function end() { -// active.value = false -// ignoreUntilStart.value = true -// const fullMessage = compileInMessages(inQueue) -// if (!fullMessage) -// return - -// if (fullMessage.message.trim().length > 0) { -// agent.history.add(name, fullMessage.message) -// } - -// if (agent.lastSender === name) { -// agent.lastSender = undefined -// } -// } - -// function queue(message: ConversationMessage) { -// inQueue.push(message) -// } - -// return { -// reset, -// end, -// queue, -// name, -// inMessageTimer, -// blocked, -// active, -// ignoreUntilStart, -// inQueue, -// } -// } - -// const WAIT_TIME_START = 30000 - -// export type ConversationStore = ReturnType - -// export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { -// const conversations: Record = {} -// const activeConversation: { value: Conversation | undefined } = { value: undefined } -// const awaitingResponse = { value: false } -// const waitTimeLimit = { value: WAIT_TIME_START } -// const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } -// const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } -// const agent = options.agent -// let agentsInGame = options.agentNames || [] -// const log = useLogg('ConversationStore').useGlobalConfig() - -// const conversationStore = { -// getConvo: (name: string) => { -// if (!conversations[name]) -// conversations[name] = useConversations(name, agent) -// return conversations[name] -// }, -// startMonitor: () => { -// clearInterval(connectionMonitor.value) -// let waitTime = 0 -// let lastTime = Date.now() -// connectionMonitor.value = setInterval(() => { -// if (!activeConversation.value) { -// conversationStore.stopMonitor() -// return // will clean itself up -// } - -// const delta = Date.now() - lastTime -// lastTime = Date.now() -// const convo_partner = activeConversation.value.name - -// if (awaitingResponse.value && agent.isIdle()) { -// waitTime += delta -// if (waitTime > waitTimeLimit.value) { -// agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) -// waitTime = 0 -// waitTimeLimit.value *= 2 -// } -// } -// else if (!awaitingResponse.value) { -// waitTimeLimit.value = WAIT_TIME_START -// waitTime = 0 -// } - -// if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { -// connectionTimeout.value = setTimeout(() => { -// if (conversationStore.otherAgentInGame(convo_partner)) { -// conversationStore.clearMonitorTimeouts() -// return -// } -// if (!self_prompter_paused) { -// conversationStore.endConversation(convo_partner) -// agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) -// } -// else { -// conversationStore.endConversation(convo_partner) -// } -// }, 10000) -// } -// }, 1000) -// }, -// stopMonitor: () => { -// clearInterval(connectionMonitor.value) -// connectionMonitor.value = undefined -// conversationStore.clearMonitorTimeouts() -// }, -// clearMonitorTimeouts: () => { -// awaitingResponse.value = false -// clearTimeout(connectionTimeout.value) -// connectionTimeout.value = undefined -// }, -// startConversation: (send_to: string, message: string) => { -// const convo = conversationStore.getConvo(send_to) -// convo.reset() - -// if (agent.self_prompter.on) { -// agent.self_prompter.stop() -// self_prompter_paused = true -// } -// if (convo.active.value) -// return - -// convo.active.value = true -// activeConversation.value = convo -// conversationStore.startMonitor() -// conversationStore.sendToBot(send_to, message, true, false) -// }, -// startConversationFromOtherBot: (name: string) => { -// const convo = conversationStore.getConvo(name) -// convo.active.value = true -// activeConversation.value = convo -// conversationStore.startMonitor() -// }, -// sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { -// if (!conversationStore.isOtherAgent(send_to)) { -// console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) -// return -// } -// const convo = conversationStore.getConvo(send_to) - -// if (options.chatBotMessages && open_chat) -// agent.openChat(`(To ${send_to}) ${message}`) - -// if (convo.ignoreUntilStart.value) -// return -// convo.active.value = true - -// const end = message.includes('!endConversation') -// const json = { -// message, -// start, -// end, -// } - -// awaitingResponse.value = true -// // TODO: -// // sendBotChatToServer(send_to, json) -// log.withField('json', json).log(`Sending message to ${send_to}`) -// }, -// receiveFromBot: async (sender: string, received: ConversationMessage) => { -// const convo = conversationStore.getConvo(sender) - -// if (convo.ignoreUntilStart.value && !received.start) -// return - -// // check if any convo is active besides the sender -// if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { -// conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) -// conversationStore.endConversation(sender) -// return -// } - -// if (received.start) { -// convo.reset() -// conversationStore.startConversationFromOtherBot(sender) -// } - -// conversationStore.clearMonitorTimeouts() -// convo.queue(received) - -// // responding to conversation takes priority over self prompting -// if (agent.self_prompter.on) { -// await agent.self_prompter.stopLoop() -// self_prompter_paused = true -// } - -// _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) -// }, -// responseScheduledFor: (sender: string) => { -// if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) -// return false -// const convo = conversationStore.getConvo(sender) -// return !!convo.inMessageTimer -// }, -// isOtherAgent: (name: string) => { -// return !!options.agentNames?.includes(name) -// }, -// otherAgentInGame: (name: string) => { -// return agentsInGame.includes(name) -// }, -// updateAgents: (agents: Agent[]) => { -// options.agentNames = agents.map(a => a.name) -// agentsInGame = agents.filter(a => a.in_game).map(a => a.name) -// }, -// getInGameAgents: () => { -// return agentsInGame -// }, -// inConversation: (other_agent?: string) => { -// if (other_agent) -// return conversations[other_agent]?.active -// return Object.values(conversations).some(c => c.active) -// }, -// endConversation: (sender: string) => { -// if (conversations[sender]) { -// conversations[sender].end() -// if (activeConversation.value?.name === sender) { -// conversationStore.stopMonitor() -// activeConversation.value = undefined -// if (self_prompter_paused && !conversationStore.inConversation()) { -// _resumeSelfPrompter(agent, conversationStore) -// } -// } -// } -// }, -// endAllConversations: () => { -// for (const sender in conversations) { -// conversationStore.endConversation(sender) -// } -// if (self_prompter_paused) { -// _resumeSelfPrompter(agent, conversationStore) -// } -// }, -// forceEndCurrentConversation: () => { -// if (activeConversation.value) { -// const sender = activeConversation.value.name -// conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) -// conversationStore.endConversation(sender) -// } -// }, -// scheduleSelfPrompter: () => { -// self_prompter_paused = true -// }, -// cancelSelfPrompter: () => { -// self_prompter_paused = false -// }, -// } - -// return conversationStore -// } - -// function containsCommand(message: string) { -// // TODO: mock -// return message -// } - -// /* -// This function controls conversation flow by deciding when the bot responds. -// The logic is as follows: -// - If neither bot is busy, respond quickly with a small delay. -// - If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) -// - If I'm busy but other bot isn't, let LLM decide whether to respond -// - If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses -// - New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk -// */ -// const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions -// const fastDelay = 200 -// const longDelay = 5000 - -// async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { -// if (convo.inMessageTimer) -// clearTimeout(convo.inMessageTimer.value) -// const otherAgentBusy = containsCommand(received.message) - -// const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) - -// if (!agent.isIdle() && otherAgentBusy) { -// // both are busy -// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) -// if (canTalkOver) -// scheduleResponse(fastDelay) -// // otherwise don't respond -// } -// else if (otherAgentBusy) { -// // other bot is busy but I'm not -// scheduleResponse(longDelay) -// } -// else if (!agent.isIdle()) { -// // I'm busy but other bot isn't -// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) -// if (canTalkOver) { -// scheduleResponse(fastDelay) -// } -// else { -// const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) -// useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) -// if (shouldRespond) -// scheduleResponse(fastDelay) -// } -// } -// else { -// // neither are busy -// scheduleResponse(fastDelay) -// } -// } - -// function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { -// const convo = conversationStore.getConvo(name) -// _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) -// } - -// function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { -// if (!received) -// return - -// useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) - -// const convo = conversationStore.getConvo(sender) -// convo.active.value = true - -// let message = _tagMessage(received.message) -// if (received.end) { -// conversationStore.endConversation(sender) -// message = `Conversation with ${sender} ended with message: "${message}"` -// sender = 'system' // bot will respond to system instead of the other bot -// } -// else if (received.start) { -// agent.shut_up = false -// } -// convo.inMessageTimer.value = undefined -// agent.handleMessage(sender, message) -// } - -// function _tagMessage(message: string) { -// return `(FROM OTHER BOT)${message}` -// } - -// async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { -// await new Promise(resolve => setTimeout(resolve, 5000)) -// if (self_prompter_paused && !conversationStore.inConversation()) { -// self_prompter_paused = false -// agent.self_prompter.start() -// } -// } diff --git a/src/manager/plan.ts b/src/manager/plan.ts deleted file mode 100644 index 55c1692..0000000 --- a/src/manager/plan.ts +++ /dev/null @@ -1,98 +0,0 @@ -// import type { ActionLLMHandler } from '../agents/action/llm-handler' -// import type { PlanningLLMHandler } from '../agents/planning/llm-handler' -// import type { Mineflayer } from '../libs/mineflayer/core' - -// import { useLogg } from '@guiiai/logg' -// import EventEmitter from 'eventemitter3' - -// import { actionsList } from '../agents/action/tools' - -// interface PlanExecutionResult { -// success: boolean -// message: string -// step: number -// totalSteps: number -// } - -// export class PlanManager extends EventEmitter { -// private logger = useLogg('PlanManager').useGlobalConfig() -// private mineflayer: Mineflayer -// private planningHandler: PlanningLLMHandler -// private actionHandler: ActionLLMHandler - -// constructor( -// mineflayer: Mineflayer, -// planningHandler: PlanningLLMHandler, -// actionHandler: ActionLLMHandler, -// ) { -// super() -// this.mineflayer = mineflayer -// this.planningHandler = planningHandler -// this.actionHandler = actionHandler -// } - -// /** -// * Execute a plan to achieve a goal -// * @param goal The goal to achieve -// * @returns The result of the plan execution -// */ -// public async executePlan(goal: string): Promise { -// try { -// // Generate plan -// this.logger.log('Generating plan for goal:', goal) -// const plan = await this.planningHandler.generatePlan(goal, Object.values(actionsList)) - -// // Execute each step -// let currentStep = 0 -// for (const step of plan) { -// currentStep++ -// this.logger.log(`Executing step ${currentStep}/${plan.length}:`, step.description) - -// try { -// const result = await this.actionHandler.executeStep(step) -// this.logger.log('Step result:', result) -// } -// catch (error) { -// // If a step fails, try to regenerate the plan with feedback -// this.logger.error('Step failed:', error) -// const feedback = `Failed at step ${currentStep}: ${error.message}` -// return this.retryWithFeedback(goal, feedback) -// } -// } - -// return { -// success: true, -// message: 'Plan executed successfully', -// step: plan.length, -// totalSteps: plan.length, -// } -// } -// catch (error) { -// this.logger.error('Plan execution failed:', error) -// return { -// success: false, -// message: error.message, -// step: 0, -// totalSteps: 0, -// } -// } -// } - -// /** -// * Retry executing the plan with feedback from previous failure -// */ -// private async retryWithFeedback( -// goal: string, -// feedback: string, -// ): Promise { -// this.logger.log('Retrying with feedback:', feedback) -// const plan = await this.planningHandler.generatePlan( -// goal, -// Object.values(actionsList), -// feedback, -// ) - -// // Execute the new plan -// return this.executePlan(goal) -// } -// } From 7d99a461afef1f7466f73e6b5374b921d16cfbd0 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Wed, 22 Jan 2025 00:27:35 +0800 Subject: [PATCH 22/22] chore: lint --- src/agents/planning/index.ts | 400 +++++++++++++++++------------------ src/plugins/llm-agent.ts | 14 +- 2 files changed, 207 insertions(+), 207 deletions(-) diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts index 33d68af..61d1c71 100644 --- a/src/agents/planning/index.ts +++ b/src/agents/planning/index.ts @@ -192,155 +192,155 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - private async generateStepsStream( - goal: string, - availableActions: Action[], - sender: string, - ): Promise { - if (!this.context) { - return - } - - try { - // Generate all steps at once - const steps = await this.llmHandler.generatePlan(goal, availableActions, sender) - if (!this.context.isGenerating) { - return - } - - // Add all steps to pending queue - this.context.pendingSteps.push(...steps) - this.logger.withField('steps', steps).log('Generated steps') - } - catch (error) { - this.logger.withError(error).error('Failed to generate steps') - throw error - } - finally { - this.context.isGenerating = false - } - } - - private async executeStepsStream(): Promise { - if (!this.context || !this.actionAgent) { - return - } - - try { - while (this.context.isGenerating || this.context.pendingSteps.length > 0) { - // Wait for steps to be available - if (this.context.pendingSteps.length === 0) { - await new Promise(resolve => setTimeout(resolve, 100)) - continue - } - - // Execute next step - const step = this.context.pendingSteps.shift() - if (!step) { - continue - } - - try { - this.logger.withField('step', step).log('Executing step') - await this.actionAgent.performAction(step) - this.context.lastUpdate = Date.now() - this.context.currentStep++ - } - catch (stepError) { - this.logger.withError(stepError).error('Failed to execute step') - - // Attempt to adjust plan and retry - if (this.context.retryCount < 3) { - this.context.retryCount++ - // Stop current generation - this.context.isGenerating = false - this.context.pendingSteps = [] - // Adjust plan and restart - const adjustedPlan = await this.adjustPlan( - this.currentPlan!, - stepError instanceof Error ? stepError.message : 'Unknown error', - 'system', - ) - await this.executePlan(adjustedPlan) - return - } - - throw stepError - } - } - } - catch (error) { - this.logger.withError(error).error('Failed to execute steps') - throw error - } - } - - private async *createStepGenerator( - goal: string, - availableActions: Action[], - ): AsyncGenerator { - // Use LLM to generate plan in chunks - this.logger.log('Generating plan using LLM') - const chunkSize = 3 // Generate 3 steps at a time - let currentChunk = 1 - - while (true) { - const steps = await this.llmHandler.generatePlan( - goal, - availableActions, - `Generate steps ${currentChunk * chunkSize - 2} to ${currentChunk * chunkSize}`, - ) - - if (steps.length === 0) { - break - } - - yield steps - currentChunk++ - - // Check if we've generated enough steps or if the goal is achieved - if (steps.length < chunkSize || await this.isGoalAchieved(goal)) { - break - } - } - } - - private async isGoalAchieved(goal: string): Promise { - if (!this.context || !this.actionAgent) { - return false - } - - const requirements = this.parseGoalRequirements(goal) - - // Check inventory for required items - if (requirements.needsItems && requirements.items) { - const inventorySteps = this.generateGatheringSteps(requirements.items) - if (inventorySteps.length > 0) { - this.context.pendingSteps.push(...inventorySteps) - return false - } - } - - // Check location requirements - if (requirements.needsMovement && requirements.location) { - const movementSteps = this.generateMovementSteps(requirements.location) - if (movementSteps.length > 0) { - this.context.pendingSteps.push(...movementSteps) - return false - } - } - - // Check interaction requirements - if (requirements.needsInteraction && requirements.target) { - const interactionSteps = this.generateInteractionSteps(requirements.target) - if (interactionSteps.length > 0) { - this.context.pendingSteps.push(...interactionSteps) - return false - } - } - - return true - } + // private async generateStepsStream( + // goal: string, + // availableActions: Action[], + // sender: string, + // ): Promise { + // if (!this.context) { + // return + // } + + // try { + // // Generate all steps at once + // const steps = await this.llmHandler.generatePlan(goal, availableActions, sender) + // if (!this.context.isGenerating) { + // return + // } + + // // Add all steps to pending queue + // this.context.pendingSteps.push(...steps) + // this.logger.withField('steps', steps).log('Generated steps') + // } + // catch (error) { + // this.logger.withError(error).error('Failed to generate steps') + // throw error + // } + // finally { + // this.context.isGenerating = false + // } + // } + + // private async executeStepsStream(): Promise { + // if (!this.context || !this.actionAgent) { + // return + // } + + // try { + // while (this.context.isGenerating || this.context.pendingSteps.length > 0) { + // // Wait for steps to be available + // if (this.context.pendingSteps.length === 0) { + // await new Promise(resolve => setTimeout(resolve, 100)) + // continue + // } + + // // Execute next step + // const step = this.context.pendingSteps.shift() + // if (!step) { + // continue + // } + + // try { + // this.logger.withField('step', step).log('Executing step') + // await this.actionAgent.performAction(step) + // this.context.lastUpdate = Date.now() + // this.context.currentStep++ + // } + // catch (stepError) { + // this.logger.withError(stepError).error('Failed to execute step') + + // // Attempt to adjust plan and retry + // if (this.context.retryCount < 3) { + // this.context.retryCount++ + // // Stop current generation + // this.context.isGenerating = false + // this.context.pendingSteps = [] + // // Adjust plan and restart + // const adjustedPlan = await this.adjustPlan( + // this.currentPlan!, + // stepError instanceof Error ? stepError.message : 'Unknown error', + // 'system', + // ) + // await this.executePlan(adjustedPlan) + // return + // } + + // throw stepError + // } + // } + // } + // catch (error) { + // this.logger.withError(error).error('Failed to execute steps') + // throw error + // } + // } + + // private async *createStepGenerator( + // goal: string, + // availableActions: Action[], + // ): AsyncGenerator { + // // Use LLM to generate plan in chunks + // this.logger.log('Generating plan using LLM') + // const chunkSize = 3 // Generate 3 steps at a time + // let currentChunk = 1 + + // while (true) { + // const steps = await this.llmHandler.generatePlan( + // goal, + // availableActions, + // `Generate steps ${currentChunk * chunkSize - 2} to ${currentChunk * chunkSize}`, + // ) + + // if (steps.length === 0) { + // break + // } + + // yield steps + // currentChunk++ + + // // Check if we've generated enough steps or if the goal is achieved + // if (steps.length < chunkSize || await this.isGoalAchieved(goal)) { + // break + // } + // } + // } + + // private async isGoalAchieved(goal: string): Promise { + // if (!this.context || !this.actionAgent) { + // return false + // } + + // const requirements = this.parseGoalRequirements(goal) + + // // Check inventory for required items + // if (requirements.needsItems && requirements.items) { + // const inventorySteps = this.generateGatheringSteps(requirements.items) + // if (inventorySteps.length > 0) { + // this.context.pendingSteps.push(...inventorySteps) + // return false + // } + // } + + // // Check location requirements + // if (requirements.needsMovement && requirements.location) { + // const movementSteps = this.generateMovementSteps(requirements.location) + // if (movementSteps.length > 0) { + // this.context.pendingSteps.push(...movementSteps) + // return false + // } + // } + + // // Check interaction requirements + // if (requirements.needsInteraction && requirements.target) { + // const interactionSteps = this.generateInteractionSteps(requirements.target) + // if (interactionSteps.length > 0) { + // this.context.pendingSteps.push(...interactionSteps) + // return false + // } + // } + + // return true + // } public async adjustPlan(plan: Plan, feedback: string, sender: string): Promise { if (!this.initialized) { @@ -385,57 +385,57 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - private generateGatheringSteps(items: string[]): PlanStep[] { - const steps: PlanStep[] = [] - - for (const item of items) { - steps.push( - { - description: `Search for ${item} in the surrounding area`, - tool: 'searchForBlock', - params: { - blockType: item, - range: 64, - }, - }, - { - description: `Collect ${item} from the found location`, - tool: 'collectBlocks', - params: { - blockType: item, - count: 1, - }, - }, - ) - } - - return steps - } - - private generateMovementSteps(location: { x?: number, y?: number, z?: number }): PlanStep[] { - if (location.x !== undefined && location.y !== undefined && location.z !== undefined) { - return [{ - description: `Move to coordinates (${location.x}, ${location.y}, ${location.z})`, - tool: 'goToCoordinates', - params: { - x: location.x, - y: location.y, - z: location.z, - }, - }] - } - return [] - } - - private generateInteractionSteps(target: string): PlanStep[] { - return [{ - description: `Interact with ${target}`, - tool: 'activate', - params: { - target, - }, - }] - } + // private generateGatheringSteps(items: string[]): PlanStep[] { + // const steps: PlanStep[] = [] + + // for (const item of items) { + // steps.push( + // { + // description: `Search for ${item} in the surrounding area`, + // tool: 'searchForBlock', + // params: { + // blockType: item, + // range: 64, + // }, + // }, + // { + // description: `Collect ${item} from the found location`, + // tool: 'collectBlocks', + // params: { + // blockType: item, + // count: 1, + // }, + // }, + // ) + // } + + // return steps + // } + + // private generateMovementSteps(location: { x?: number, y?: number, z?: number }): PlanStep[] { + // if (location.x !== undefined && location.y !== undefined && location.z !== undefined) { + // return [{ + // description: `Move to coordinates (${location.x}, ${location.y}, ${location.z})`, + // tool: 'goToCoordinates', + // params: { + // x: location.x, + // y: location.y, + // z: location.z, + // }, + // }] + // } + // return [] + // } + + // private generateInteractionSteps(target: string): PlanStep[] { + // return [{ + // description: `Interact with ${target}`, + // tool: 'activate', + // params: { + // target, + // }, + // }] + // } private generateRecoverySteps(feedback: string): PlanStep[] { const steps: PlanStep[] = [] diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts index 3532d64..e05bde2 100644 --- a/src/plugins/llm-agent.ts +++ b/src/plugins/llm-agent.ts @@ -101,13 +101,13 @@ async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Ne bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) try { - // 创建并执行计划 + // Create and execute plan const plan = await bot.planning.createPlan(event.data.transcription) logger.withFields({ plan }).log('Plan created') await bot.planning.executePlan(plan) logger.log('Plan executed successfully') - // 生成回复 + // Generate response const retryHandler = toRetriable( 3, 1000, @@ -142,7 +142,7 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { async created(bot) { const logger = useLogg('LLMAgent').useGlobalConfig() - // 创建容器并获取所需的服务 + // Create container and get required services const container = createAppContainer({ neuri: options.agent, model: 'openai/gpt-4o-mini', @@ -154,21 +154,21 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { const planningAgent = container.resolve('planningAgent') const chatAgent = container.resolve('chatAgent') - // 初始化 agents + // Initialize agents await actionAgent.init() await planningAgent.init() await chatAgent.init() - // 类型转换 + // Type conversion const botWithAgents = bot as unknown as MineflayerWithAgents botWithAgents.action = actionAgent botWithAgents.planning = planningAgent botWithAgents.chat = chatAgent - // 初始化系统提示 + // Initialize system prompt bot.memory.chatHistory.push(system(generateActionAgentPrompt(bot))) - // 设置消息处理 + // Set message handling const onChat = new ChatMessageHandler(bot.username).handleChat((username, message) => handleChatMessage(username, message, botWithAgents, options.agent, logger))