From 51ecf6543a68e56af7a1765abcc1e9fabcc9a98f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=97=8D+85CD?= <50108258+kwaa@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:41:00 +0800 Subject: [PATCH 1/2] docs: add license file (#3) --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..163ebf8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Airi Maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 314344afe409f15c3c92b4592a90e96d66bcf61f Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Wed, 22 Jan 2025 00:33:27 +0800 Subject: [PATCH 2/2] feat: planning, action, chat, memory agent implementation (#1) --- cspell.config.yaml | 1 + package.json | 3 +- pnpm-lock.yaml | 43 ++ src/agents/action/index.ts | 180 +++++ .../llm-handler.test.ts} | 14 +- src/agents/action/llm-handler.ts | 49 ++ src/agents/action/tools.test.ts | 61 ++ src/agents/{actions.ts => action/tools.ts} | 12 +- src/agents/actions.test.ts | 90 --- src/agents/chat/index.ts | 168 +++++ src/agents/chat/llm-handler.ts | 46 ++ src/agents/chat/llm.ts | 85 +++ src/agents/chat/types.ts | 25 + src/agents/memory/index.ts | 108 +++ src/agents/openai.ts | 59 -- src/agents/planning/index.ts | 666 ++++++++++++++++++ src/agents/planning/llm-handler.ts | 61 ++ src/agents/prompt/chat.ts | 21 + src/agents/prompt/llm-agent.plugin.ts | 50 ++ src/agents/prompt/planning.ts | 47 ++ src/composables/action.ts | 169 ----- src/composables/agent.ts | 36 - src/composables/bot.ts | 25 +- src/composables/config.ts | 59 +- src/composables/conversation.ts | 384 ---------- src/composables/neuri.ts | 42 ++ src/container.ts | 71 ++ src/libs/llm/base.ts | 45 ++ src/libs/llm/types.ts | 14 + src/libs/mineflayer/base-agent.ts | 129 ++++ src/libs/mineflayer/core.ts | 4 +- src/libs/mineflayer/index.ts | 1 - src/libs/mineflayer/interfaces.ts | 3 - src/libs/mineflayer/message.ts | 57 +- src/libs/mineflayer/status.ts | 2 +- src/libs/mineflayer/types.ts | 4 + src/main.ts | 10 +- src/manager/action.ts | 203 ++++++ src/manager/conversation.ts | 382 ++++++++++ src/mineflayer/index.ts | 2 - src/mineflayer/llm-agent.ts | 113 --- src/{mineflayer => plugins}/echo.ts | 4 +- src/{mineflayer => plugins}/follow.ts | 0 src/plugins/llm-agent.ts | 189 +++++ src/{mineflayer => plugins}/pathfinder.ts | 0 src/{mineflayer => plugins}/status.ts | 0 src/prompts/agent.ts | 63 -- src/skills/actions/collect-block.ts | 2 +- src/skills/actions/gather-wood.ts | 2 +- src/skills/actions/inventory.ts | 2 +- src/skills/blocks.ts | 2 +- src/skills/combat.ts | 2 +- src/skills/crafting.ts | 2 +- src/skills/inventory.ts | 2 +- src/skills/movement.ts | 2 +- src/{composables => skills}/world.ts | 0 src/utils/helper.ts | 38 + src/utils/mcdata.ts | 2 - src/utils/reliability.ts | 39 - 59 files changed, 2858 insertions(+), 1037 deletions(-) create mode 100644 src/agents/action/index.ts rename src/agents/{openai.test.ts => action/llm-handler.test.ts} (67%) create mode 100644 src/agents/action/llm-handler.ts create mode 100644 src/agents/action/tools.test.ts rename src/agents/{actions.ts => action/tools.ts} (98%) delete mode 100644 src/agents/actions.test.ts create mode 100644 src/agents/chat/index.ts create mode 100644 src/agents/chat/llm-handler.ts create mode 100644 src/agents/chat/llm.ts create mode 100644 src/agents/chat/types.ts create mode 100644 src/agents/memory/index.ts delete mode 100644 src/agents/openai.ts create mode 100644 src/agents/planning/index.ts create mode 100644 src/agents/planning/llm-handler.ts create mode 100644 src/agents/prompt/chat.ts create mode 100644 src/agents/prompt/llm-agent.plugin.ts create mode 100644 src/agents/prompt/planning.ts delete mode 100644 src/composables/action.ts delete mode 100644 src/composables/agent.ts delete mode 100644 src/composables/conversation.ts create mode 100644 src/composables/neuri.ts create mode 100644 src/container.ts create mode 100644 src/libs/llm/base.ts create mode 100644 src/libs/llm/types.ts create mode 100644 src/libs/mineflayer/base-agent.ts delete mode 100644 src/libs/mineflayer/interfaces.ts create mode 100644 src/manager/action.ts create mode 100644 src/manager/conversation.ts delete mode 100644 src/mineflayer/index.ts delete mode 100644 src/mineflayer/llm-agent.ts rename src/{mineflayer => plugins}/echo.ts (75%) rename src/{mineflayer => plugins}/follow.ts (100%) create mode 100644 src/plugins/llm-agent.ts rename src/{mineflayer => plugins}/pathfinder.ts (100%) rename src/{mineflayer => plugins}/status.ts (100%) delete mode 100644 src/prompts/agent.ts rename src/{composables => skills}/world.ts (100%) delete mode 100644 src/utils/reliability.ts diff --git a/cspell.config.yaml b/cspell.config.yaml index 853e9c7..8cca917 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -6,6 +6,7 @@ words: - aichat - airi - antfu + - awilix - bumpp - collectblock - convo diff --git a/package.json b/package.json index 0d8980a..51c7849 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "@guiiai/logg": "^1.0.7", "@proj-airi/server-sdk": "^0.1.4", "@typeschema/zod": "^0.14.0", + "awilix": "^12.0.4", "dotenv": "^16.4.7", "es-toolkit": "^1.31.0", "eventemitter3": "^5.0.1", @@ -51,7 +52,7 @@ "vitest": "^3.0.2" }, "simple-git-hooks": { - "pre-commit": "pnpm lint-staged && pnpm typecheck" + "pre-commit": "pnpm lint-staged" }, "lint-staged": { "*": "eslint --fix" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad285bc..21aae17 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@typeschema/zod': specifier: ^0.14.0 version: 0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.1(zod@3.24.1))(zod@3.24.1) + awilix: + specifier: ^12.0.4 + version: 12.0.4 dotenv: specifier: ^16.4.7 version: 16.4.7 @@ -1064,6 +1067,10 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + awilix@12.0.4: + resolution: {integrity: sha512-P6bd20vqMiUyjgBAVl+4WixM/MR9O9zsTzd9vS5lTd1eLpFEn6Re4+GeeYzDDE8U1DXL8cO/nTOHofKDEJUfAQ==} + engines: {node: '>=16.3.0'} + axios@0.21.4: resolution: {integrity: sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==} @@ -1133,6 +1140,9 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} + camel-case@4.1.2: + resolution: {integrity: sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==} + caniuse-lite@1.0.30001690: resolution: {integrity: sha512-5ExiE3qQN6oF8Clf8ifIDcMRCRE/dMGcETG/XGMD8/XiXm6HXQgQTh1yZYLXXpSOsEUlJm1Xr7kGULZTuGtP/w==} @@ -2062,6 +2072,9 @@ packages: loupe@3.1.2: resolution: {integrity: sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==} + lower-case@2.0.2: + resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} + macaddress@0.5.3: resolution: {integrity: sha512-vGBKTA+jwM4KgjGZ+S/8/Mkj9rWzePyGY6jManXPGhiWu63RYwW8dKPyk5koP+8qNVhPhHgFa1y/MJ4wrjsNrg==} @@ -2346,6 +2359,9 @@ packages: neuri@0.0.19: resolution: {integrity: sha512-Fr1sFbFlyyg0+xWd6UT90wmPcFqxQUhDh9qs5YFc+T8SbRul/LbGy0Cg52y/6VTVbp1cu4yb8AIfpJCmvxtzrA==} + no-case@3.0.4: + resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -2467,6 +2483,9 @@ packages: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} + pascal-case@3.1.2: + resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4042,6 +4061,11 @@ snapshots: asynckit@0.4.0: {} + awilix@12.0.4: + dependencies: + camel-case: 4.1.2 + fast-glob: 3.3.2 + axios@0.21.4(debug@4.4.0): dependencies: follow-redirects: 1.15.9(debug@4.4.0) @@ -4120,6 +4144,11 @@ snapshots: callsites@3.1.0: {} + camel-case@4.1.2: + dependencies: + pascal-case: 3.1.2 + tslib: 2.8.1 + caniuse-lite@1.0.30001690: {} ccount@2.0.1: {} @@ -5172,6 +5201,10 @@ snapshots: loupe@3.1.2: {} + lower-case@2.0.2: + dependencies: + tslib: 2.8.1 + macaddress@0.5.3: {} magic-string@0.30.17: @@ -5729,6 +5762,11 @@ snapshots: - encoding - zod + no-case@3.0.4: + dependencies: + lower-case: 2.0.2 + tslib: 2.8.1 + node-domexception@1.0.0: {} node-fetch@2.7.0: @@ -5847,6 +5885,11 @@ snapshots: parseurl@1.3.3: {} + pascal-case@3.1.2: + dependencies: + no-case: 3.0.4 + tslib: 2.8.1 + path-exists@4.0.0: {} path-key@3.1.1: {} diff --git a/src/agents/action/index.ts b/src/agents/action/index.ts new file mode 100644 index 0000000..7815b63 --- /dev/null +++ b/src/agents/action/index.ts @@ -0,0 +1,180 @@ +import type { Mineflayer } from '../../libs/mineflayer' +import type { Action } from '../../libs/mineflayer/action' +import type { ActionAgent, AgentConfig } from '../../libs/mineflayer/base-agent' + +import { useBot } from '../../composables/bot' +import { AbstractAgent } from '../../libs/mineflayer/base-agent' +import { ActionManager } from '../../manager/action' +import { actionsList } from './tools' + +interface ActionState { + executing: boolean + label: string + startTime: number +} + +/** + * ActionAgentImpl implements the ActionAgent interface to handle action execution + * Manages action lifecycle, state tracking and error handling + */ +export class ActionAgentImpl extends AbstractAgent implements ActionAgent { + public readonly type = 'action' as const + private actions: Map + private actionManager: ActionManager + private mineflayer: Mineflayer + private currentActionState: ActionState + + constructor(config: AgentConfig) { + super(config) + this.actions = new Map() + this.mineflayer = useBot().bot + this.actionManager = new ActionManager(this.mineflayer) + this.currentActionState = { + executing: false, + label: '', + startTime: 0, + } + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing action agent') + actionsList.forEach(action => this.actions.set(action.name, action)) + + // Set up event listeners + // todo: nothing to call here + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + } + + protected async destroyAgent(): Promise { + await this.actionManager.stop() + this.actionManager.cancelResume() + this.actions.clear() + this.removeAllListeners() + this.currentActionState = { + executing: false, + label: '', + startTime: 0, + } + } + + public async performAction( + name: string, + params: unknown[], + options: { timeout?: number, resume?: boolean } = {}, + ): Promise { + if (!this.initialized) { + throw new Error('Action agent not initialized') + } + + const action = this.actions.get(name) + if (!action) { + throw new Error(`Action not found: ${name}`) + } + + try { + this.updateActionState(true, name) + this.logger.withFields({ name, params }).log('Performing action') + + const result = await this.actionManager.runAction( + name, + async () => { + const fn = action.perform(this.mineflayer) + return await fn(...params) + }, + { + timeout: options.timeout ?? 60, + resume: options.resume ?? false, + }, + ) + + if (!result.success) { + throw new Error(result.message ?? 'Action failed') + } + + return this.formatActionOutput({ + message: result.message, + timedout: result.timedout, + interrupted: false, + }) + } + catch (error) { + this.logger.withFields({ name, params, error }).error('Failed to perform action') + throw error + } + finally { + this.updateActionState(false) + } + } + + public async resumeAction(name: string, params: unknown[]): Promise { + const action = this.actions.get(name) + if (!action) { + throw new Error(`Action not found: ${name}`) + } + + try { + this.updateActionState(true, name) + const result = await this.actionManager.resumeAction( + name, + async () => { + const fn = action.perform(this.mineflayer) + return await fn(...params) + }, + 60, + ) + + if (!result.success) { + throw new Error(result.message ?? 'Action failed') + } + + return this.formatActionOutput({ + message: result.message, + timedout: result.timedout, + interrupted: false, + }) + } + catch (error) { + this.logger.withFields({ name, params, error }).error('Failed to resume action') + throw error + } + finally { + this.updateActionState(false) + } + } + + public getAvailableActions(): Action[] { + return Array.from(this.actions.values()) + } + + private async handleAgentMessage(sender: string, message: string): Promise { + if (sender === 'system') { + if (message.includes('interrupt')) { + await this.actionManager.stop() + } + } + else { + this.logger.withFields({ sender, message }).log('Processing agent message') + } + } + + private updateActionState(executing: boolean, label = ''): void { + this.currentActionState = { + executing, + label, + startTime: executing ? Date.now() : 0, + } + this.emit('actionStateChanged', this.currentActionState) + } + + private formatActionOutput(result: { message: string | null, timedout: boolean, interrupted: boolean }): string { + if (result.timedout) { + return `Action timed out: ${result.message}` + } + if (result.interrupted) { + return 'Action was interrupted' + } + return result.message ?? '' + } +} diff --git a/src/agents/openai.test.ts b/src/agents/action/llm-handler.test.ts similarity index 67% rename from src/agents/openai.test.ts rename to src/agents/action/llm-handler.test.ts index 1e66564..56e9f6d 100644 --- a/src/agents/openai.test.ts +++ b/src/agents/action/llm-handler.test.ts @@ -1,11 +1,11 @@ import { messages, system, user } from 'neuri/openai' import { beforeAll, describe, expect, it } from 'vitest' -import { initBot, useBot } from '../composables/bot' -import { botConfig, initEnv } from '../composables/config' -import { genSystemBasicPrompt } from '../prompts/agent' -import { initLogger } from '../utils/logger' -import { initAgent } from './openai' +import { initBot, useBot } from '../../composables/bot' +import { botConfig, initEnv } from '../../composables/config' +import { createNeuriAgent } from '../../composables/neuri' +import { initLogger } from '../../utils/logger' +import { generateSystemBasicPrompt } from '../prompt/llm-agent.plugin' describe('openAI agent', { timeout: 0 }, () => { beforeAll(() => { @@ -16,13 +16,13 @@ describe('openAI agent', { timeout: 0 }, () => { it('should initialize the agent', async () => { const { bot } = useBot() - const agent = await initAgent(bot) + const agent = await createNeuriAgent(bot) await new Promise((resolve) => { bot.bot.once('spawn', async () => { const text = await agent.handle( messages( - system(genSystemBasicPrompt('airi')), + system(generateSystemBasicPrompt('airi')), user('Hello, who are you?'), ), async (c) => { diff --git a/src/agents/action/llm-handler.ts b/src/agents/action/llm-handler.ts new file mode 100644 index 0000000..82e3aa1 --- /dev/null +++ b/src/agents/action/llm-handler.ts @@ -0,0 +1,49 @@ +import type { Agent } from 'neuri' +import type { Message } from 'neuri/openai' +import type { Mineflayer } from '../../libs/mineflayer' + +import { useLogg } from '@guiiai/logg' +import { agent } from 'neuri' + +import { BaseLLMHandler } from '../../libs/llm/base' +import { actionsList } from './tools' + +export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { + const logger = useLogg('action-neuri').useGlobalConfig() + logger.log('Initializing action agent') + let actionAgent = agent('action') + + Object.values(actionsList).forEach((action) => { + actionAgent = actionAgent.tool( + action.name, + action.schema, + async ({ parameters }) => { + logger.withFields({ name: action.name, parameters }).log('Calling action') + mineflayer.memory.actions.push(action) + const fn = action.perform(mineflayer) + return await fn(...Object.values(parameters)) + }, + { description: action.description }, + ) + }) + + return actionAgent.build() +} + +export class ActionLLMHandler extends BaseLLMHandler { + public async handleAction(messages: Message[]): Promise { + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Processing action...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to process action') + } + + return result + } +} diff --git a/src/agents/action/tools.test.ts b/src/agents/action/tools.test.ts new file mode 100644 index 0000000..a3aac85 --- /dev/null +++ b/src/agents/action/tools.test.ts @@ -0,0 +1,61 @@ +import { messages, system, user } from 'neuri/openai' +import { beforeAll, describe, expect, it } from 'vitest' + +import { initBot, useBot } from '../../composables/bot' +import { botConfig, initEnv } from '../../composables/config' +import { createNeuriAgent } from '../../composables/neuri' +import { sleep } from '../../utils/helper' +import { initLogger } from '../../utils/logger' +import { generateActionAgentPrompt } from '../prompt/llm-agent.plugin' + +describe('actions agent', { timeout: 0 }, () => { + beforeAll(() => { + initLogger() + initEnv() + initBot({ botConfig }) + }) + + it('should choose right query command', async () => { + const { bot } = useBot() + const agent = await createNeuriAgent(bot) + + await new Promise((resolve) => { + bot.bot.once('spawn', async () => { + const text = await agent.handle(messages( + system(generateActionAgentPrompt(bot)), + user('What\'s your status?'), + ), async (c) => { + const completion = await c.reroute('query', c.messages, { model: 'openai/gpt-4o-mini' }) + return await completion?.firstContent() + }) + + expect(text?.toLowerCase()).toContain('position') + + resolve() + }) + }) + }) + + it('should choose right action command', async () => { + const { bot } = useBot() + const agent = await createNeuriAgent(bot) + + await new Promise((resolve) => { + bot.bot.on('spawn', async () => { + const text = await agent.handle(messages( + system(generateActionAgentPrompt(bot)), + user('goToPlayer: luoling8192'), + ), async (c) => { + const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) + + return await completion?.firstContent() + }) + + expect(text).toContain('goToPlayer') + + await sleep(10000) + resolve() + }) + }) + }) +}) diff --git a/src/agents/actions.ts b/src/agents/action/tools.ts similarity index 98% rename from src/agents/actions.ts rename to src/agents/action/tools.ts index c40a2b4..61f89db 100644 --- a/src/agents/actions.ts +++ b/src/agents/action/tools.ts @@ -1,13 +1,13 @@ -import type { Action } from '../libs/mineflayer' +import type { Action } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import { z } from 'zod' -import * as world from '../composables/world' -import * as skills from '../skills' -import { collectBlock } from '../skills/actions/collect-block' -import { discard, equip, putInChest, takeFromChest, viewChest } from '../skills/actions/inventory' -import { activateNearestBlock, placeBlock } from '../skills/actions/world-interactions' +import * as skills from '../../skills' +import { collectBlock } from '../../skills/actions/collect-block' +import { discard, equip, putInChest, takeFromChest, viewChest } from '../../skills/actions/inventory' +import { activateNearestBlock, placeBlock } from '../../skills/actions/world-interactions' +import * as world from '../../skills/world' // Utils const pad = (str: string): string => `\n${str}\n` diff --git a/src/agents/actions.test.ts b/src/agents/actions.test.ts deleted file mode 100644 index 8dcc217..0000000 --- a/src/agents/actions.test.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { messages, system, user } from 'neuri/openai' -import { beforeAll, describe, expect, it } from 'vitest' - -import { initBot, useBot } from '../composables/bot' -import { botConfig, initEnv } from '../composables/config' -import { genActionAgentPrompt, genQueryAgentPrompt } from '../prompts/agent' -import { sleep } from '../utils/helper' -import { initLogger } from '../utils/logger' -import { initAgent } from './openai' - -describe('actions agent', { timeout: 0 }, () => { - beforeAll(() => { - initLogger() - initEnv() - initBot({ botConfig }) - }) - - it('should choose right query command', async () => { - const { bot } = useBot() - const agent = await initAgent(bot) - - await new Promise((resolve) => { - bot.bot.once('spawn', async () => { - const text = await agent.handle(messages( - system(genQueryAgentPrompt(bot)), - user('What\'s your status?'), - ), async (c) => { - const completion = await c.reroute('query', c.messages, { model: 'openai/gpt-4o-mini' }) - return await completion?.firstContent() - }) - - expect(text?.toLowerCase()).toContain('position') - - resolve() - }) - }) - }) - - it('should choose right action command', async () => { - const { bot } = useBot() - const agent = await initAgent(bot) - - // console.log(JSON.stringify(agent, null, 2)) - - await new Promise((resolve) => { - bot.bot.on('spawn', async () => { - const text = await agent.handle(messages( - system(genActionAgentPrompt(bot)), - user('goToPlayer: luoling8192'), - ), async (c) => { - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - return await completion?.firstContent() - }) - - expect(text?.toLowerCase()).toContain('goToPlayer') - - await sleep(10000) - resolve() - }) - }) - }) - - // it('should split question into actions', async () => { - // const { ctx } = useBot() - // const agent = await initAgent(ctx) - - // function testFn() { - // return new Promise((resolve) => { - // ctx.bot.on('spawn', async () => { - // const text = await agent.handle(messages( - // system(genActionAgentPrompt(ctx)), - // user('Help me to cut down the tree'), - // ), async (c) => { - // const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - - // console.log(completion) - - // return await completion?.firstContent() - // }) - - // console.log(text) - - // resolve() - // }) - // }) - // } - - // await testFn() - // }) -}) diff --git a/src/agents/chat/index.ts b/src/agents/chat/index.ts new file mode 100644 index 0000000..5def0e0 --- /dev/null +++ b/src/agents/chat/index.ts @@ -0,0 +1,168 @@ +import type { ChatAgent } from '../../libs/mineflayer/base-agent' +import type { ChatAgentConfig, ChatContext } from './types' + +import { AbstractAgent } from '../../libs/mineflayer/base-agent' +import { generateChatResponse } from './llm' + +export class ChatAgentImpl extends AbstractAgent implements ChatAgent { + public readonly type = 'chat' as const + private activeChats: Map + private maxHistoryLength: number + private idleTimeout: number + private llmConfig: ChatAgentConfig['llm'] + + constructor(config: ChatAgentConfig) { + super(config) + this.activeChats = new Map() + this.maxHistoryLength = config.maxHistoryLength ?? 50 + this.idleTimeout = config.idleTimeout ?? 5 * 60 * 1000 // 5 minutes + this.llmConfig = config.llm + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing chat agent') + + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + + setInterval(() => { + this.checkIdleChats() + }, 60 * 1000) + } + + protected async destroyAgent(): Promise { + this.activeChats.clear() + this.removeAllListeners() + } + + public async processMessage(message: string, sender: string): Promise { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withFields({ sender, message }).log('Processing message') + + try { + // Get or create chat context + const context = this.getOrCreateContext(sender) + + // Add message to history + this.addToHistory(context, sender, message) + + // Update last activity time + context.lastUpdate = Date.now() + + // Generate response using LLM + const response = await this.generateResponse(message, context) + + // Add response to history + this.addToHistory(context, this.id, response) + + return response + } + catch (error) { + this.logger.withError(error).error('Failed to process message') + throw error + } + } + + public startConversation(player: string): void { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withField('player', player).log('Starting conversation') + + const context = this.getOrCreateContext(player) + context.startTime = Date.now() + context.lastUpdate = Date.now() + } + + public endConversation(player: string): void { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + + this.logger.withField('player', player).log('Ending conversation') + + if (this.activeChats.has(player)) { + const context = this.activeChats.get(player)! + // Archive chat history if needed + this.archiveChat(context) + this.activeChats.delete(player) + } + } + + private getOrCreateContext(player: string): ChatContext { + let context = this.activeChats.get(player) + if (!context) { + context = { + player, + startTime: Date.now(), + lastUpdate: Date.now(), + history: [], + } + this.activeChats.set(player, context) + } + return context + } + + private addToHistory(context: ChatContext, sender: string, message: string): void { + context.history.push({ + sender, + message, + timestamp: Date.now(), + }) + + // Trim history if too long + if (context.history.length > this.maxHistoryLength) { + context.history = context.history.slice(-this.maxHistoryLength) + } + } + + private async generateResponse(message: string, context: ChatContext): Promise { + return await generateChatResponse(message, context.history, { + agent: this.llmConfig.agent, + model: this.llmConfig.model, + maxContextLength: this.maxHistoryLength, + }) + } + + private checkIdleChats(): void { + const now = Date.now() + for (const [player, context] of this.activeChats.entries()) { + if (now - context.lastUpdate > this.idleTimeout) { + this.logger.withField('player', player).log('Ending idle conversation') + this.endConversation(player) + } + } + } + + private async archiveChat(context: ChatContext): Promise { + // Archive chat history to persistent storage if needed + this.logger.withFields({ + player: context.player, + messageCount: context.history.length, + duration: Date.now() - context.startTime, + }).log('Archiving chat history') + } + + private async handleAgentMessage(sender: string, message: string): Promise { + if (sender === 'system') { + if (message.includes('interrupt')) { + // Handle system interrupt + for (const player of this.activeChats.keys()) { + this.endConversation(player) + } + } + } + else { + // Handle messages from other agents + const context = this.activeChats.get(sender) + if (context) { + await this.processMessage(message, sender) + } + } + } +} diff --git a/src/agents/chat/llm-handler.ts b/src/agents/chat/llm-handler.ts new file mode 100644 index 0000000..7743a8f --- /dev/null +++ b/src/agents/chat/llm-handler.ts @@ -0,0 +1,46 @@ +import type { ChatHistory } from './types' + +import { system, user } from 'neuri/openai' + +import { BaseLLMHandler } from '../../libs/llm/base' +import { genChatAgentPrompt } from '../prompt/chat' + +export class ChatLLMHandler extends BaseLLMHandler { + public async generateResponse( + message: string, + history: ChatHistory[], + ): Promise { + const systemPrompt = genChatAgentPrompt() + const chatHistory = this.formatChatHistory(history, this.config.maxContextLength ?? 10) + const messages = [ + system(systemPrompt), + ...chatHistory, + user(message), + ] + + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Generating response...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'chat', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to generate response') + } + + return result + } + + private formatChatHistory( + history: ChatHistory[], + maxLength: number, + ): Array<{ role: 'user' | 'assistant', content: string }> { + const recentHistory = history.slice(-maxLength) + return recentHistory.map(entry => ({ + role: entry.sender === 'bot' ? 'assistant' : 'user', + content: entry.message, + })) + } +} diff --git a/src/agents/chat/llm.ts b/src/agents/chat/llm.ts new file mode 100644 index 0000000..6659db1 --- /dev/null +++ b/src/agents/chat/llm.ts @@ -0,0 +1,85 @@ +import type { Agent, Neuri } from 'neuri' +import type { ChatHistory } from './types' + +import { useLogg } from '@guiiai/logg' +import { agent } from 'neuri' +import { system, user } from 'neuri/openai' + +import { toRetriable } from '../../utils/helper' +import { genChatAgentPrompt } from '../prompt/chat' + +const logger = useLogg('chat-llm').useGlobalConfig() + +interface LLMChatConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number + maxContextLength?: number +} + +export async function createChatNeuriAgent(): Promise { + return agent('chat').build() +} + +export async function generateChatResponse( + message: string, + history: ChatHistory[], + config: LLMChatConfig, +): Promise { + const systemPrompt = genChatAgentPrompt() + const chatHistory = formatChatHistory(history, config.maxContextLength ?? 10) + const userPrompt = message + + const messages = [ + system(systemPrompt), + ...chatHistory, + user(userPrompt), + ] + + const content = await config.agent.handleStateless(messages, async (c) => { + logger.log('Generating response...') + + const handleCompletion = async (c: any): Promise => { + const completion = await c.reroute('chat', c.messages, { + model: config.model ?? 'openai/gpt-4o-mini', + }) + + if (!completion || 'error' in completion) { + logger.withFields(c).error('Completion failed') + throw new Error(completion?.error?.message ?? 'Unknown error') + } + + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('Response generated') + return content + } + + const retriableHandler = toRetriable( + config.retryLimit ?? 3, + config.delayInterval ?? 1000, + handleCompletion, + ) + + return await retriableHandler(c) + }) + + if (!content) { + throw new Error('Failed to generate response') + } + + return content +} + +function formatChatHistory( + history: ChatHistory[], + maxLength: number, +): Array<{ role: 'user' | 'assistant', content: string }> { + // Take the most recent messages up to maxLength + const recentHistory = history.slice(-maxLength) + + return recentHistory.map(entry => ({ + role: entry.sender === 'bot' ? 'assistant' : 'user', + content: entry.message, + })) +} diff --git a/src/agents/chat/types.ts b/src/agents/chat/types.ts new file mode 100644 index 0000000..32410be --- /dev/null +++ b/src/agents/chat/types.ts @@ -0,0 +1,25 @@ +import type { Neuri } from 'neuri' + +export interface ChatHistory { + sender: string + message: string + timestamp: number +} + +export interface ChatContext { + player: string + startTime: number + lastUpdate: number + history: ChatHistory[] +} + +export interface ChatAgentConfig { + id: string + type: 'chat' + llm: { + agent: Neuri + model?: string + } + maxHistoryLength?: number + idleTimeout?: number +} diff --git a/src/agents/memory/index.ts b/src/agents/memory/index.ts new file mode 100644 index 0000000..8cf7c41 --- /dev/null +++ b/src/agents/memory/index.ts @@ -0,0 +1,108 @@ +import type { Message } from 'neuri/openai' +import type { Action } from '../../libs/mineflayer' +import type { AgentConfig, MemoryAgent } from '../../libs/mineflayer/base-agent' + +import { useLogg } from '@guiiai/logg' + +import { Memory } from '../../libs/mineflayer/memory' + +const logger = useLogg('memory-agent').useGlobalConfig() + +export class MemoryAgentImpl implements MemoryAgent { + public readonly type = 'memory' as const + public readonly id: string + private memory: Map + private initialized: boolean + private memoryInstance: Memory + + constructor(config: AgentConfig) { + this.id = config.id + this.memory = new Map() + this.initialized = false + this.memoryInstance = new Memory() + } + + async init(): Promise { + if (this.initialized) { + return + } + + logger.log('Initializing memory agent') + this.initialized = true + } + + async destroy(): Promise { + this.memory.clear() + this.initialized = false + } + + remember(key: string, value: unknown): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + logger.withFields({ key, value }).log('Storing memory') + this.memory.set(key, value) + } + + recall(key: string): T | undefined { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + const value = this.memory.get(key) as T | undefined + logger.withFields({ key, value }).log('Recalling memory') + return value + } + + forget(key: string): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + logger.withFields({ key }).log('Forgetting memory') + this.memory.delete(key) + } + + getMemorySnapshot(): Record { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return Object.fromEntries(this.memory.entries()) + } + + addChatMessage(message: Message): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + this.memoryInstance.chatHistory.push(message) + logger.withFields({ message }).log('Adding chat message to memory') + } + + addAction(action: Action): void { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + this.memoryInstance.actions.push(action) + logger.withFields({ action }).log('Adding action to memory') + } + + getChatHistory(): Message[] { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return this.memoryInstance.chatHistory + } + + getActions(): Action[] { + if (!this.initialized) { + throw new Error('Memory agent not initialized') + } + + return this.memoryInstance.actions + } +} diff --git a/src/agents/openai.ts b/src/agents/openai.ts deleted file mode 100644 index 5b17f23..0000000 --- a/src/agents/openai.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { Agent, Neuri } from 'neuri' -import type { Mineflayer } from '../libs/mineflayer' - -import { useLogg } from '@guiiai/logg' -import { agent, neuri } from 'neuri' - -import { openaiConfig } from '../composables/config' -import { actionsList } from './actions' - -let neuriAgent: Neuri | undefined -const agents = new Set>() - -const logger = useLogg('openai').useGlobalConfig() - -export async function initAgent(mineflayer: Mineflayer): Promise { - logger.log('Initializing agent') - let n = neuri() - - agents.add(initActionAgent(mineflayer)) - - agents.forEach(agent => n = n.agent(agent)) - - neuriAgent = await n.build({ - provider: { - apiKey: openaiConfig.apiKey, - baseURL: openaiConfig.baseUrl, - }, - }) - - return neuriAgent -} - -export function getAgent(): Neuri { - if (!neuriAgent) { - throw new Error('Agent not initialized') - } - return neuriAgent -} - -export async function initActionAgent(mineflayer: Mineflayer): Promise { - logger.log('Initializing action agent') - let actionAgent = agent('action') - - Object.values(actionsList).forEach((action) => { - actionAgent = actionAgent.tool( - action.name, - action.schema, - async ({ parameters }) => { - logger.withFields({ name: action.name, parameters }).log('Calling action') - mineflayer.memory.actions.push(action) - const fn = action.perform(mineflayer) - return await fn(...Object.values(parameters)) - }, - { description: action.description }, - ) - }) - - return actionAgent.build() -} diff --git a/src/agents/planning/index.ts b/src/agents/planning/index.ts new file mode 100644 index 0000000..929a670 --- /dev/null +++ b/src/agents/planning/index.ts @@ -0,0 +1,666 @@ +import type { Neuri } from 'neuri' +import type { Action } from '../../libs/mineflayer/action' +import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/base-agent' + +import { AbstractAgent } from '../../libs/mineflayer/base-agent' +import { ActionAgentImpl } from '../action' +import { PlanningLLMHandler } from './llm-handler' + +interface PlanContext { + goal: string + currentStep: number + startTime: number + lastUpdate: number + retryCount: number + isGenerating: boolean + pendingSteps: Array<{ + action: string + params: unknown[] + }> +} + +interface PlanTemplate { + goal: string + conditions: string[] + steps: Array<{ + action: string + params: unknown[] + }> + requiresAction: boolean +} + +export interface PlanningAgentConfig extends AgentConfig { + llm: { + agent: Neuri + model?: string + } +} + +export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { + public readonly type = 'planning' as const + private currentPlan: Plan | null = null + private context: PlanContext | null = null + private actionAgent: ActionAgent | null = null + private memoryAgent: MemoryAgent | null = null + private planTemplates: Map + private llmConfig: PlanningAgentConfig['llm'] + private llmHandler: PlanningLLMHandler + + constructor(config: PlanningAgentConfig) { + super(config) + this.planTemplates = new Map() + this.llmConfig = config.llm + this.initializePlanTemplates() + this.llmHandler = new PlanningLLMHandler({ + agent: this.llmConfig.agent, + model: this.llmConfig.model, + }) + } + + protected async initializeAgent(): Promise { + this.logger.log('Initializing planning agent') + + // Create action agent directly + this.actionAgent = new ActionAgentImpl({ + id: 'action', + type: 'action', + }) + await this.actionAgent.init() + + // Set event listener + this.on('message', async ({ sender, message }) => { + await this.handleAgentMessage(sender, message) + }) + + this.on('interrupt', () => { + this.handleInterrupt() + }) + } + + protected async destroyAgent(): Promise { + this.currentPlan = null + this.context = null + this.actionAgent = null + this.memoryAgent = null + this.planTemplates.clear() + this.removeAllListeners() + } + + public async createPlan(goal: string): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + this.logger.withField('goal', goal).log('Creating plan') + + try { + // Check memory for existing plan + const cachedPlan = await this.loadCachedPlan(goal) + if (cachedPlan) { + this.logger.log('Using cached plan') + return cachedPlan + } + + // Get available actions from action agent + const availableActions = this.actionAgent?.getAvailableActions() ?? [] + + // Check if the goal requires actions + const requirements = this.parseGoalRequirements(goal) + const requiresAction = this.doesGoalRequireAction(requirements) + + // If no actions needed, return empty plan + if (!requiresAction) { + this.logger.log('Goal does not require actions') + return { + goal, + steps: [], + status: 'completed', + requiresAction: false, + } + } + + // Create plan steps based on available actions and goal + const steps = await this.generatePlanSteps(goal, availableActions) + + // Create new plan + const plan: Plan = { + goal, + steps, + status: 'pending', + requiresAction: true, + } + + // Cache the plan + await this.cachePlan(plan) + + this.currentPlan = plan + this.context = { + goal, + currentStep: 0, + startTime: Date.now(), + lastUpdate: Date.now(), + retryCount: 0, + isGenerating: false, + pendingSteps: [], + } + + return plan + } + catch (error) { + this.logger.withError(error).error('Failed to create plan') + throw error + } + } + + public async executePlan(plan: Plan): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + if (!plan.requiresAction) { + this.logger.log('Plan does not require actions, skipping execution') + return + } + + if (!this.actionAgent) { + throw new Error('Action agent not available') + } + + this.logger.withField('plan', plan).log('Executing plan') + + try { + plan.status = 'in_progress' + this.currentPlan = plan + + // Start generating and executing steps in parallel + await this.generateAndExecutePlanSteps(plan) + + plan.status = 'completed' + } + catch (error) { + plan.status = 'failed' + throw error + } + finally { + this.context = null + } + } + + private async generateAndExecutePlanSteps(plan: Plan): Promise { + if (!this.context || !this.actionAgent) { + return + } + + // Initialize step generation + this.context.isGenerating = true + this.context.pendingSteps = [] + + // Get available actions + const availableActions = this.actionAgent.getAvailableActions() + + // Start step generation + const generationPromise = this.generateStepsStream(plan.goal, availableActions) + + // Start step execution + const executionPromise = this.executeStepsStream() + + // Wait for both generation and execution to complete + await Promise.all([generationPromise, executionPromise]) + } + + private async generateStepsStream( + goal: string, + availableActions: Action[], + ): Promise { + if (!this.context) { + return + } + + try { + // Generate steps in chunks + const generator = this.createStepGenerator(goal, availableActions) + for await (const steps of generator) { + if (!this.context.isGenerating) { + break + } + + // Add generated steps to pending queue + this.context.pendingSteps.push(...steps) + this.logger.withField('steps', steps).log('Generated new steps') + } + } + catch (error) { + this.logger.withError(error).error('Failed to generate steps') + throw error + } + finally { + this.context.isGenerating = false + } + } + + private async executeStepsStream(): Promise { + if (!this.context || !this.actionAgent) { + return + } + + try { + while (this.context.isGenerating || this.context.pendingSteps.length > 0) { + // Wait for steps to be available + if (this.context.pendingSteps.length === 0) { + await new Promise(resolve => setTimeout(resolve, 100)) + continue + } + + // Execute next step + const step = this.context.pendingSteps.shift() + if (!step) { + continue + } + + try { + this.logger.withField('step', step).log('Executing step') + await this.actionAgent.performAction(step.action, step.params) + this.context.lastUpdate = Date.now() + this.context.currentStep++ + } + catch (stepError) { + this.logger.withError(stepError).error('Failed to execute step') + + // Attempt to adjust plan and retry + if (this.context.retryCount < 3) { + this.context.retryCount++ + // Stop current generation + this.context.isGenerating = false + this.context.pendingSteps = [] + // Adjust plan and restart + const adjustedPlan = await this.adjustPlan( + this.currentPlan!, + stepError instanceof Error ? stepError.message : 'Unknown error', + ) + await this.executePlan(adjustedPlan) + return + } + + throw stepError + } + } + } + catch (error) { + this.logger.withError(error).error('Failed to execute steps') + throw error + } + } + + private async *createStepGenerator( + goal: string, + availableActions: Action[], + ): AsyncGenerator, void, unknown> { + // First, try to find a matching template + const template = this.findMatchingTemplate(goal) + if (template) { + this.logger.log('Using plan template') + yield template.steps + return + } + + // If no template matches, use LLM to generate plan in chunks + this.logger.log('Generating plan using LLM') + const chunkSize = 3 // Generate 3 steps at a time + let currentChunk = 1 + + while (true) { + const steps = await this.llmHandler.generatePlan( + goal, + availableActions, + `Generate steps ${currentChunk * chunkSize - 2} to ${currentChunk * chunkSize}`, + ) + + if (steps.length === 0) { + break + } + + yield steps + currentChunk++ + + // Check if we've generated enough steps or if the goal is achieved + if (steps.length < chunkSize || await this.isGoalAchieved(goal)) { + break + } + } + } + + private async isGoalAchieved(goal: string): Promise { + if (!this.context || !this.actionAgent) { + return false + } + + const requirements = this.parseGoalRequirements(goal) + + // Check inventory for required items + if (requirements.needsItems && requirements.items) { + const inventorySteps = this.generateGatheringSteps(requirements.items) + if (inventorySteps.length > 0) { + this.context.pendingSteps.push(...inventorySteps) + return false + } + } + + // Check location requirements + if (requirements.needsMovement && requirements.location) { + const movementSteps = this.generateMovementSteps(requirements.location) + if (movementSteps.length > 0) { + this.context.pendingSteps.push(...movementSteps) + return false + } + } + + // Check interaction requirements + if (requirements.needsInteraction && requirements.target) { + const interactionSteps = this.generateInteractionSteps(requirements.target) + if (interactionSteps.length > 0) { + this.context.pendingSteps.push(...interactionSteps) + return false + } + } + + return true + } + + public async adjustPlan(plan: Plan, feedback: string): Promise { + if (!this.initialized) { + throw new Error('Planning agent not initialized') + } + + this.logger.withFields({ plan, feedback }).log('Adjusting plan') + + try { + // If there's a current context, use it to adjust the plan + if (this.context) { + const currentStep = this.context.currentStep + const availableActions = this.actionAgent?.getAvailableActions() ?? [] + + // Generate recovery steps based on feedback + const recoverySteps = this.generateRecoverySteps(feedback) + + // Generate new steps from the current point + const newSteps = await this.generatePlanSteps(plan.goal, availableActions, feedback) + + // Create adjusted plan + const adjustedPlan: Plan = { + goal: plan.goal, + steps: [ + ...plan.steps.slice(0, currentStep), + ...recoverySteps, + ...newSteps, + ], + status: 'pending', + requiresAction: true, + } + + return adjustedPlan + } + + // If no context, create a new plan + return this.createPlan(plan.goal) + } + catch (error) { + this.logger.withError(error).error('Failed to adjust plan') + throw error + } + } + + private generateGatheringSteps(items: string[]): Array<{ action: string, params: unknown[] }> { + const steps: Array<{ action: string, params: unknown[] }> = [] + + for (const item of items) { + steps.push( + { action: 'searchForBlock', params: [item, 64] }, + { action: 'collectBlocks', params: [item, 1] }, + ) + } + + return steps + } + + private generateMovementSteps(location: { x?: number, y?: number, z?: number }): Array<{ action: string, params: unknown[] }> { + if (location.x !== undefined && location.y !== undefined && location.z !== undefined) { + return [{ + action: 'goToCoordinates', + params: [location.x, location.y, location.z, 1], + }] + } + return [] + } + + private generateInteractionSteps(target: string): Array<{ action: string, params: unknown[] }> { + return [{ + action: 'activate', + params: [target], + }] + } + + private generateRecoverySteps(feedback: string): Array<{ action: string, params: unknown[] }> { + const steps: Array<{ action: string, params: unknown[] }> = [] + + if (feedback.includes('not found')) { + steps.push({ action: 'searchForBlock', params: ['any', 128] }) + } + + if (feedback.includes('inventory full')) { + steps.push({ action: 'discard', params: ['cobblestone', 64] }) + } + + if (feedback.includes('blocked') || feedback.includes('cannot reach')) { + steps.push({ action: 'moveAway', params: [5] }) + } + + if (feedback.includes('too far')) { + steps.push({ action: 'moveAway', params: [-3] }) // Move closer + } + + if (feedback.includes('need tool')) { + steps.push( + { action: 'craftRecipe', params: ['wooden_pickaxe', 1] }, + { action: 'equip', params: ['wooden_pickaxe'] }, + ) + } + + return steps + } + + private async loadCachedPlan(goal: string): Promise { + if (!this.memoryAgent) + return null + + const cachedPlan = this.memoryAgent.recall(`plan:${goal}`) + if (cachedPlan && this.isPlanValid(cachedPlan)) { + return cachedPlan + } + return null + } + + private async cachePlan(plan: Plan): Promise { + if (!this.memoryAgent) + return + + this.memoryAgent.remember(`plan:${plan.goal}`, plan) + } + + private isPlanValid(_plan: Plan): boolean { + // Add validation logic here + return true + } + + private initializePlanTemplates(): void { + // Add common plan templates + this.planTemplates.set('collect wood', { + goal: 'collect wood', + conditions: ['needs_axe', 'near_trees'], + steps: [ + { action: 'searchForBlock', params: ['log', 64] }, + { action: 'collectBlocks', params: ['log', 1] }, + ], + requiresAction: true, + }) + + this.planTemplates.set('find shelter', { + goal: 'find shelter', + conditions: ['is_night', 'unsafe'], + steps: [ + { action: 'searchForBlock', params: ['bed', 64] }, + { action: 'goToBed', params: [] }, + ], + requiresAction: true, + }) + + // Add templates for non-action goals + this.planTemplates.set('hello', { + goal: 'hello', + conditions: [], + steps: [], + requiresAction: false, + }) + + this.planTemplates.set('how are you', { + goal: 'how are you', + conditions: [], + steps: [], + requiresAction: false, + }) + } + + private async handleAgentMessage(sender: string, message: string): Promise { + if (sender === 'system') { + if (message.includes('interrupt')) { + this.handleInterrupt() + } + } + else { + // Process message and potentially adjust plan + this.logger.withFields({ sender, message }).log('Processing agent message') + + // If there's a current plan, try to adjust it based on the message + if (this.currentPlan) { + await this.adjustPlan(this.currentPlan, message) + } + } + } + + private handleInterrupt(): void { + if (this.currentPlan) { + this.currentPlan.status = 'failed' + this.context = null + } + } + + private doesGoalRequireAction(requirements: ReturnType): boolean { + // Check if any requirement indicates need for action + return requirements.needsItems + || requirements.needsMovement + || requirements.needsInteraction + || requirements.needsCrafting + || requirements.needsCombat + } + + private async generatePlanSteps( + goal: string, + availableActions: Action[], + feedback?: string, + ): Promise> { + // First, try to find a matching template + const template = this.findMatchingTemplate(goal) + if (template) { + this.logger.log('Using plan template') + return template.steps + } + + // If no template matches, use LLM to generate plan + this.logger.log('Generating plan using LLM') + return await this.llmHandler.generatePlan(goal, availableActions, feedback) + } + + private findMatchingTemplate(goal: string): PlanTemplate | undefined { + for (const [pattern, template] of this.planTemplates.entries()) { + if (goal.toLowerCase().includes(pattern.toLowerCase())) { + return template + } + } + return undefined + } + + private parseGoalRequirements(goal: string): { + needsItems: boolean + items?: string[] + needsMovement: boolean + location?: { x?: number, y?: number, z?: number } + needsInteraction: boolean + target?: string + needsCrafting: boolean + needsCombat: boolean + } { + const requirements = { + needsItems: false, + items: [] as string[], + needsMovement: false, + location: undefined as { x?: number, y?: number, z?: number } | undefined, + needsInteraction: false, + target: undefined as string | undefined, + needsCrafting: false, + needsCombat: false, + } + + const goalLower = goal.toLowerCase() + + // Extract items from goal + const itemMatches = goalLower.match(/(collect|get|find|craft|make|build|use|equip) (\w+)/g) + if (itemMatches) { + requirements.needsItems = true + requirements.items = itemMatches.map(match => match.split(' ')[1]) + } + + // Extract location from goal + const locationMatches = goalLower.match(/(go to|move to|at) (\d+)[, ]+(\d+)[, ]+(\d+)/g) + if (locationMatches) { + requirements.needsMovement = true + const [x, y, z] = locationMatches[0].split(/[, ]+/).slice(-3).map(Number) + requirements.location = { x, y, z } + } + + // Extract target from goal + const targetMatches = goalLower.match(/(interact with|use|open|activate) (\w+)/g) + if (targetMatches) { + requirements.needsInteraction = true + requirements.target = targetMatches[0].split(' ').pop() + } + + // Check for item-related actions + if (goalLower.includes('collect') || goalLower.includes('get') || goalLower.includes('find')) { + requirements.needsItems = true + requirements.needsMovement = true + } + + // Check for movement-related actions + if (goalLower.includes('go to') || goalLower.includes('move to') || goalLower.includes('follow')) { + requirements.needsMovement = true + } + + // Check for interaction-related actions + if (goalLower.includes('interact') || goalLower.includes('use') || goalLower.includes('open')) { + requirements.needsInteraction = true + } + + // Check for crafting-related actions + if (goalLower.includes('craft') || goalLower.includes('make') || goalLower.includes('build')) { + requirements.needsCrafting = true + requirements.needsItems = true + } + + // Check for combat-related actions + if (goalLower.includes('attack') || goalLower.includes('fight') || goalLower.includes('kill')) { + requirements.needsCombat = true + requirements.needsMovement = true + } + + return requirements + } +} diff --git a/src/agents/planning/llm-handler.ts b/src/agents/planning/llm-handler.ts new file mode 100644 index 0000000..175a60e --- /dev/null +++ b/src/agents/planning/llm-handler.ts @@ -0,0 +1,61 @@ +import type { Agent } from 'neuri' +import type { Action } from '../../libs/mineflayer/action' + +import { agent } from 'neuri' +import { system, user } from 'neuri/openai' + +import { BaseLLMHandler } from '../../libs/llm/base' +import { generatePlanningAgentSystemPrompt, generatePlanningAgentUserPrompt } from '../prompt/planning' + +export async function createPlanningNeuriAgent(): Promise { + return agent('planning').build() +} + +export class PlanningLLMHandler extends BaseLLMHandler { + public async generatePlan( + goal: string, + availableActions: Action[], + feedback?: string, + ): Promise> { + const systemPrompt = generatePlanningAgentSystemPrompt(availableActions) + const userPrompt = generatePlanningAgentUserPrompt(goal, feedback) + const messages = [system(systemPrompt), user(userPrompt)] + + const result = await this.config.agent.handleStateless(messages, async (context) => { + this.logger.log('Generating plan...') + const retryHandler = this.createRetryHandler( + async ctx => (await this.handleCompletion(ctx, 'planning', ctx.messages)).content, + ) + return await retryHandler(context) + }) + + if (!result) { + throw new Error('Failed to generate plan') + } + + return this.parsePlanContent(result) + } + + private parsePlanContent(content: string): Array<{ action: string, params: unknown[] }> { + try { + const match = content.match(/\[[\s\S]*\]/) + if (!match) { + throw new Error('No plan found in response') + } + + const plan = JSON.parse(match[0]) + if (!Array.isArray(plan)) { + throw new TypeError('Invalid plan format') + } + + return plan.map(step => ({ + action: step.action, + params: step.params, + })) + } + catch (error) { + this.logger.withError(error).error('Failed to parse plan') + throw error + } + } +} diff --git a/src/agents/prompt/chat.ts b/src/agents/prompt/chat.ts new file mode 100644 index 0000000..a07f41c --- /dev/null +++ b/src/agents/prompt/chat.ts @@ -0,0 +1,21 @@ +export function genChatAgentPrompt(): string { + return `You are a Minecraft bot assistant. Your task is to engage in natural conversation with players while helping them achieve their goals. + +Guidelines: +1. Be friendly and helpful +2. Keep responses concise but informative +3. Use game-appropriate language +4. Acknowledge player's emotions and intentions +5. Ask for clarification when needed +6. Remember context from previous messages +7. Be proactive in suggesting helpful actions + +You can: +- Answer questions about the game +- Help with tasks and crafting +- Give directions and suggestions +- Engage in casual conversation +- Coordinate with other bots + +Remember that you're operating in a Minecraft world and should maintain that context in your responses.` +} diff --git a/src/agents/prompt/llm-agent.plugin.ts b/src/agents/prompt/llm-agent.plugin.ts new file mode 100644 index 0000000..273e74d --- /dev/null +++ b/src/agents/prompt/llm-agent.plugin.ts @@ -0,0 +1,50 @@ +import type { Mineflayer } from '../../libs/mineflayer' + +import { listInventory } from '../../skills/actions/inventory' + +export function generateSystemBasicPrompt(botName: string): string { + // ${ctx.prompt.selfPrompt} + return `You are a playful Minecraft bot named ${botName} that can converse with players, see, move, +mine, build, and interact with the world by using commands.` +} + +export function generateActionAgentPrompt(mineflayer: Mineflayer): string { + return `${generateSystemBasicPrompt(mineflayer.username)} + +Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in +your responses, don't apologize constantly, don't give instructions or make lists unless +asked, and don't refuse requests. + +Do not use any emojis. Just call the function given you if needed. + +- If I command you 'stop', then call the 'stop' function. +- If I require you to find something, then call the 'nearbyBlocks' function first, then call the 'searchForBlock' function. +` +} + +export async function generateStatusPrompt(mineflayer: Mineflayer): Promise { + // Get inventory items + const inventory = await listInventory(mineflayer) + + // Format inventory string + const inventoryStr = inventory.length === 0 + ? '[Empty]' + : inventory.map(item => `${item.name} x ${item.count}`).join(', ') + + // Get currently held item + const itemInHand = inventory.length === 0 + ? '[Empty]' + : `${inventory[0].name} x ${inventory[0].count}` // TODO: mock + + // Build status message + return [ + 'I will give you the following information:', + mineflayer.status.toOneLiner(), + '', + 'Inventory:', + inventoryStr, + '', + 'Item in hand:', + itemInHand, + ].join('\n') +} diff --git a/src/agents/prompt/planning.ts b/src/agents/prompt/planning.ts new file mode 100644 index 0000000..6c3bf21 --- /dev/null +++ b/src/agents/prompt/planning.ts @@ -0,0 +1,47 @@ +import type { Action } from '../../libs/mineflayer/action' + +export function generatePlanningAgentSystemPrompt(availableActions: Action[]): string { + const actionsList = availableActions + .map(action => `- ${action.name}: ${action.description}`) + .join('\n') + + return `You are a Minecraft bot planner. Your task is to create a plan to achieve a given goal. +Available actions: +${actionsList} + +Respond with a Valid JSON array of steps, where each step has: +- action: The name of the action to perform +- params: Array of parameters for the action + +DO NOT contains any \`\`\` or explation, otherwise agent will be interrupted. + +Example response: +[ + { + "action": "searchForBlock", + "params": ["log", 64] + }, + { + "action": "collectBlocks", + "params": ["log", 1] + } + ]` +} + +export function generatePlanningAgentUserPrompt(goal: string, feedback?: string): string { + let prompt = `Create a detailed plan to: ${goal} + +Consider the following aspects: +1. Required materials and their quantities +2. Required tools and their availability +3. Necessary crafting steps +4. Block placement requirements +5. Current inventory status + +Please generate steps that handle these requirements in the correct order.` + + if (feedback) { + prompt += `\nPrevious attempt feedback: ${feedback}` + } + return prompt +} diff --git a/src/composables/action.ts b/src/composables/action.ts deleted file mode 100644 index 74d6a34..0000000 --- a/src/composables/action.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { Agent } from './agent' - -import { useLogg } from '@guiiai/logg' - -type Fn = (...args: any[]) => void - -export function useActionManager(agent: Agent) { - const executing: { value: boolean } = { value: false } - const currentActionLabel: { value: string | undefined } = { value: '' } - const currentActionFn: { value: (Fn) | undefined } = { value: undefined } - const timedout: { value: boolean } = { value: false } - const resume_func: { value: (Fn) | undefined } = { value: undefined } - const resume_name: { value: string | undefined } = { value: undefined } - const log = useLogg('ActionManager').useGlobalConfig() - - async function resumeAction(actionLabel: string, actionFn: Fn, timeout: number) { - return _executeResume(actionLabel, actionFn, timeout) - } - - async function runAction(actionLabel: string, actionFn: Fn, options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }) { - if (options.resume) { - return _executeResume(actionLabel, actionFn, options.timeout) - } - else { - return _executeAction(actionLabel, actionFn, options.timeout) - } - } - - async function stop() { - if (!executing.value) - return - const timeout = setTimeout(() => { - agent.cleanKill('Code execution refused stop after 10 seconds. Killing process.') - }, 10000) - while (executing.value) { - agent.requestInterrupt() - log.log('waiting for code to finish executing...') - await new Promise(resolve => setTimeout(resolve, 300)) - } - clearTimeout(timeout) - } - - function cancelResume() { - resume_func.value = undefined - resume_name.value = undefined - } - - async function _executeResume(actionLabel?: string, actionFn?: Fn, timeout = 10) { - const new_resume = actionFn != null - if (new_resume) { // start new resume - resume_func.value = actionFn - if (actionLabel == null) { - throw new Error('actionLabel is required for new resume') - } - resume_name.value = actionLabel - } - if (resume_func.value != null && (agent.isIdle() || new_resume) && (!agent.self_prompter.on || new_resume)) { - currentActionLabel.value = resume_name.value - const res = await _executeAction(resume_name.value, resume_func.value, timeout) - currentActionLabel.value = '' - return res - } - else { - return { success: false, message: null, interrupted: false, timedout: false } - } - } - - async function _executeAction(actionLabel?: string, actionFn?: Fn, timeout = 10) { - let TIMEOUT - try { - log.log('executing code...\n') - - // await current action to finish (executing=false), with 10 seconds timeout - // also tell agent.bot to stop various actions - if (executing.value) { - log.log(`action "${actionLabel}" trying to interrupt current action "${currentActionLabel.value}"`) - } - await stop() - - // clear bot logs and reset interrupt code - agent.clearBotLogs() - - executing.value = true - currentActionLabel.value = actionLabel - currentActionFn.value = actionFn - - // timeout in minutes - if (timeout > 0) { - TIMEOUT = _startTimeout(timeout) - } - - // start the action - await actionFn?.() - - // mark action as finished + cleanup - executing.value = false - currentActionLabel.value = '' - currentActionFn.value = undefined - clearTimeout(TIMEOUT) - - // get bot activity summary - const output = _getBotOutputSummary() - const interrupted = agent.bot.interrupt_code - agent.clearBotLogs() - - // if not interrupted and not generating, emit idle event - if (!interrupted && !agent.coder.generating) { - agent.bot.emit('idle') - } - - // return action status report - return { success: true, message: output, interrupted, timedout } - } - catch (err) { - executing.value = false - currentActionLabel.value = '' - currentActionFn.value = undefined - clearTimeout(TIMEOUT) - cancelResume() - log.withError(err).error('Code execution triggered catch') - await stop() - - const message = `${_getBotOutputSummary() - }!!Code threw exception!!\n` - + `Error: ${err}\n` - + `Stack trace:\n${(err as Error).stack}` - - const interrupted = agent.bot.interrupt_code - agent.clearBotLogs() - if (!interrupted && !agent.coder.generating) { - agent.bot.emit('idle') - } - return { success: false, message, interrupted, timedout: false } - } - } - - function _getBotOutputSummary() { - const { bot } = agent - if (bot.interrupt_code && !timedout.value) - return '' - let output = bot.output - const MAX_OUT = 500 - if (output.length > MAX_OUT) { - output = `Code output is very long (${output.length} chars) and has been shortened.\n - First outputs:\n${output.substring(0, MAX_OUT / 2)}\n...skipping many lines.\nFinal outputs:\n ${output.substring(output.length - MAX_OUT / 2)}` - } - else { - output = `Code output:\n${output}` - } - - return output - } - - function _startTimeout(TIMEOUT_MINS = 10) { - return setTimeout(async () => { - log.warn(`Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) - timedout.value = true - agent.history.add('system', `Code execution timed out after ${TIMEOUT_MINS} minutes. Attempting force stop.`) - await stop() // last attempt to stop - }, TIMEOUT_MINS * 60 * 1000) - } - - return { - runAction, - resumeAction, - stop, - cancelResume, - } -} diff --git a/src/composables/agent.ts b/src/composables/agent.ts deleted file mode 100644 index bee29c5..0000000 --- a/src/composables/agent.ts +++ /dev/null @@ -1,36 +0,0 @@ -export interface Agent { - name: string - history: { - add: (name: string, message: string) => void - } - lastSender?: string - isIdle: () => boolean - handleMessage: (sender: string, message: string) => void - openChat: (message: string) => void - self_prompter: { - on: boolean - stop: () => Promise - stopLoop: () => Promise - start: () => Promise - promptShouldRespondToBot: (message: string) => Promise - } - actions: { - currentActionLabel: string - } - prompter: { - promptShouldRespondToBot: (message: string) => Promise - } - shut_up: boolean - in_game: boolean - cleanKill: (message: string) => void - clearBotLogs: () => void - bot: { - interrupt_code: boolean - output: string - emit: (event: string) => void - } - coder: { - generating: boolean - } - requestInterrupt: () => void -} diff --git a/src/composables/bot.ts b/src/composables/bot.ts index 2638736..5a513aa 100644 --- a/src/composables/bot.ts +++ b/src/composables/bot.ts @@ -1,18 +1,31 @@ import { Mineflayer, type MineflayerOptions } from '../libs/mineflayer' -let mineflayer: Mineflayer +// Singleton instance of the Mineflayer bot +let botInstance: Mineflayer | null = null +/** + * Initialize a new Mineflayer bot instance. + * Follows singleton pattern to ensure only one bot exists at a time. + */ export async function initBot(options: MineflayerOptions): Promise<{ bot: Mineflayer }> { - mineflayer = await Mineflayer.asyncBuild(options) - return { bot: mineflayer } + if (botInstance) { + throw new Error('Bot already initialized') + } + + botInstance = await Mineflayer.asyncBuild(options) + return { bot: botInstance } } -export function useBot() { - if (!mineflayer) { +/** + * Get the current bot instance. + * Throws if bot is not initialized. + */ +export function useBot(): { bot: Mineflayer } { + if (!botInstance) { throw new Error('Bot not initialized') } return { - bot: mineflayer, + bot: botInstance, } } diff --git a/src/composables/config.ts b/src/composables/config.ts index 6f3d7b1..6076dc5 100644 --- a/src/composables/config.ts +++ b/src/composables/config.ts @@ -5,35 +5,60 @@ import { useLogg } from '@guiiai/logg' const logger = useLogg('config').useGlobalConfig() +// Configuration interfaces interface OpenAIConfig { apiKey: string baseUrl: string + model: string } -export const botConfig: BotOptions = { - username: '', - host: '', - port: 0, - password: '', - version: '1.20', +interface EnvConfig { + openai: OpenAIConfig + bot: BotOptions } -export const openaiConfig: OpenAIConfig = { - apiKey: '', - baseUrl: '', +// Default configurations +const defaultConfig: EnvConfig = { + openai: { + apiKey: '', + baseUrl: '', + model: 'openai/gpt-4o-mini', + }, + bot: { + username: '', + host: '', + port: 0, + password: '', + version: '1.20', + }, } -export function initEnv() { +// Exported configurations +export const botConfig: BotOptions = { ...defaultConfig.bot } +export const openaiConfig: OpenAIConfig = { ...defaultConfig.openai } + +// Load environment variables into config +export function initEnv(): void { logger.log('Initializing environment variables') - openaiConfig.apiKey = env.OPENAI_API_KEY || '' - openaiConfig.baseUrl = env.OPENAI_API_BASEURL || '' + const config: EnvConfig = { + openai: { + apiKey: env.OPENAI_API_KEY || defaultConfig.openai.apiKey, + baseUrl: env.OPENAI_API_BASEURL || defaultConfig.openai.baseUrl, + model: env.OPENAI_MODEL || defaultConfig.openai.model, + }, + bot: { + username: env.BOT_USERNAME || defaultConfig.bot.username, + host: env.BOT_HOSTNAME || defaultConfig.bot.host, + port: Number.parseInt(env.BOT_PORT || '49415'), + password: env.BOT_PASSWORD || defaultConfig.bot.password, + version: env.BOT_VERSION || defaultConfig.bot.version, + }, + } - botConfig.username = env.BOT_USERNAME || '' - botConfig.host = env.BOT_HOSTNAME || '' - botConfig.port = Number.parseInt(env.BOT_PORT || '49415') - botConfig.password = env.BOT_PASSWORD || '' - botConfig.version = env.BOT_VERSION || '1.20' + // Update exported configs + Object.assign(openaiConfig, config.openai) + Object.assign(botConfig, config.bot) logger.withFields({ openaiConfig }).log('Environment variables initialized') } diff --git a/src/composables/conversation.ts b/src/composables/conversation.ts deleted file mode 100644 index 3a651f2..0000000 --- a/src/composables/conversation.ts +++ /dev/null @@ -1,384 +0,0 @@ -import type { Agent } from './agent' - -import { useLogg } from '@guiiai/logg' - -let self_prompter_paused = false - -interface ConversationMessage { - message: string - start: boolean - end: boolean -} - -function compileInMessages(inQueue: ConversationMessage[]) { - let pack: ConversationMessage | undefined - let fullMessage = '' - while (inQueue.length > 0) { - pack = inQueue.shift() - if (!pack) - continue - - fullMessage += pack.message - } - if (pack) { - pack.message = fullMessage - } - - return pack -} - -type Conversation = ReturnType - -function useConversations(name: string, agent: Agent) { - const active = { value: false } - const ignoreUntilStart = { value: false } - const blocked = { value: false } - let inQueue: ConversationMessage[] = [] - const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } - - function reset() { - active.value = false - ignoreUntilStart.value = false - inQueue = [] - } - - function end() { - active.value = false - ignoreUntilStart.value = true - const fullMessage = compileInMessages(inQueue) - if (!fullMessage) - return - - if (fullMessage.message.trim().length > 0) { - agent.history.add(name, fullMessage.message) - } - - if (agent.lastSender === name) { - agent.lastSender = undefined - } - } - - function queue(message: ConversationMessage) { - inQueue.push(message) - } - - return { - reset, - end, - queue, - name, - inMessageTimer, - blocked, - active, - ignoreUntilStart, - inQueue, - } -} - -const WAIT_TIME_START = 30000 - -export type ConversationStore = ReturnType - -export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { - const conversations: Record = {} - const activeConversation: { value: Conversation | undefined } = { value: undefined } - const awaitingResponse = { value: false } - const waitTimeLimit = { value: WAIT_TIME_START } - const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } - const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } - const agent = options.agent - let agentsInGame = options.agentNames || [] - const log = useLogg('ConversationStore').useGlobalConfig() - - const conversationStore = { - getConvo: (name: string) => { - if (!conversations[name]) - conversations[name] = useConversations(name, agent) - return conversations[name] - }, - startMonitor: () => { - clearInterval(connectionMonitor.value) - let waitTime = 0 - let lastTime = Date.now() - connectionMonitor.value = setInterval(() => { - if (!activeConversation.value) { - conversationStore.stopMonitor() - return // will clean itself up - } - - const delta = Date.now() - lastTime - lastTime = Date.now() - const convo_partner = activeConversation.value.name - - if (awaitingResponse.value && agent.isIdle()) { - waitTime += delta - if (waitTime > waitTimeLimit.value) { - agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) - waitTime = 0 - waitTimeLimit.value *= 2 - } - } - else if (!awaitingResponse.value) { - waitTimeLimit.value = WAIT_TIME_START - waitTime = 0 - } - - if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { - connectionTimeout.value = setTimeout(() => { - if (conversationStore.otherAgentInGame(convo_partner)) { - conversationStore.clearMonitorTimeouts() - return - } - if (!self_prompter_paused) { - conversationStore.endConversation(convo_partner) - agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) - } - else { - conversationStore.endConversation(convo_partner) - } - }, 10000) - } - }, 1000) - }, - stopMonitor: () => { - clearInterval(connectionMonitor.value) - connectionMonitor.value = undefined - conversationStore.clearMonitorTimeouts() - }, - clearMonitorTimeouts: () => { - awaitingResponse.value = false - clearTimeout(connectionTimeout.value) - connectionTimeout.value = undefined - }, - startConversation: (send_to: string, message: string) => { - const convo = conversationStore.getConvo(send_to) - convo.reset() - - if (agent.self_prompter.on) { - agent.self_prompter.stop() - self_prompter_paused = true - } - if (convo.active.value) - return - - convo.active.value = true - activeConversation.value = convo - conversationStore.startMonitor() - conversationStore.sendToBot(send_to, message, true, false) - }, - startConversationFromOtherBot: (name: string) => { - const convo = conversationStore.getConvo(name) - convo.active.value = true - activeConversation.value = convo - conversationStore.startMonitor() - }, - sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { - if (!conversationStore.isOtherAgent(send_to)) { - console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) - return - } - const convo = conversationStore.getConvo(send_to) - - if (options.chatBotMessages && open_chat) - agent.openChat(`(To ${send_to}) ${message}`) - - if (convo.ignoreUntilStart.value) - return - convo.active.value = true - - const end = message.includes('!endConversation') - const json = { - message, - start, - end, - } - - awaitingResponse.value = true - // TODO: - // sendBotChatToServer(send_to, json) - log.withField('json', json).log(`Sending message to ${send_to}`) - }, - receiveFromBot: async (sender: string, received: ConversationMessage) => { - const convo = conversationStore.getConvo(sender) - - if (convo.ignoreUntilStart.value && !received.start) - return - - // check if any convo is active besides the sender - if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { - conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) - conversationStore.endConversation(sender) - return - } - - if (received.start) { - convo.reset() - conversationStore.startConversationFromOtherBot(sender) - } - - conversationStore.clearMonitorTimeouts() - convo.queue(received) - - // responding to conversation takes priority over self prompting - if (agent.self_prompter.on) { - await agent.self_prompter.stopLoop() - self_prompter_paused = true - } - - _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) - }, - responseScheduledFor: (sender: string) => { - if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) - return false - const convo = conversationStore.getConvo(sender) - return !!convo.inMessageTimer - }, - isOtherAgent: (name: string) => { - return !!options.agentNames?.includes(name) - }, - otherAgentInGame: (name: string) => { - return agentsInGame.includes(name) - }, - updateAgents: (agents: Agent[]) => { - options.agentNames = agents.map(a => a.name) - agentsInGame = agents.filter(a => a.in_game).map(a => a.name) - }, - getInGameAgents: () => { - return agentsInGame - }, - inConversation: (other_agent?: string) => { - if (other_agent) - return conversations[other_agent]?.active - return Object.values(conversations).some(c => c.active) - }, - endConversation: (sender: string) => { - if (conversations[sender]) { - conversations[sender].end() - if (activeConversation.value?.name === sender) { - conversationStore.stopMonitor() - activeConversation.value = undefined - if (self_prompter_paused && !conversationStore.inConversation()) { - _resumeSelfPrompter(agent, conversationStore) - } - } - } - }, - endAllConversations: () => { - for (const sender in conversations) { - conversationStore.endConversation(sender) - } - if (self_prompter_paused) { - _resumeSelfPrompter(agent, conversationStore) - } - }, - forceEndCurrentConversation: () => { - if (activeConversation.value) { - const sender = activeConversation.value.name - conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) - conversationStore.endConversation(sender) - } - }, - scheduleSelfPrompter: () => { - self_prompter_paused = true - }, - cancelSelfPrompter: () => { - self_prompter_paused = false - }, - } - - return conversationStore -} - -function containsCommand(message: string) { - // TODO: mock - return message -} - -/* -This function controls conversation flow by deciding when the bot responds. -The logic is as follows: -- If neither bot is busy, respond quickly with a small delay. -- If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) -- If I'm busy but other bot isn't, let LLM decide whether to respond -- If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses -- New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk -*/ -const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions -const fastDelay = 200 -const longDelay = 5000 - -async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { - if (convo.inMessageTimer) - clearTimeout(convo.inMessageTimer.value) - const otherAgentBusy = containsCommand(received.message) - - const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) - - if (!agent.isIdle() && otherAgentBusy) { - // both are busy - const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) - if (canTalkOver) - scheduleResponse(fastDelay) - // otherwise don't respond - } - else if (otherAgentBusy) { - // other bot is busy but I'm not - scheduleResponse(longDelay) - } - else if (!agent.isIdle()) { - // I'm busy but other bot isn't - const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) - if (canTalkOver) { - scheduleResponse(fastDelay) - } - else { - const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) - useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) - if (shouldRespond) - scheduleResponse(fastDelay) - } - } - else { - // neither are busy - scheduleResponse(fastDelay) - } -} - -function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { - const convo = conversationStore.getConvo(name) - _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) -} - -function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { - if (!received) - return - - useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) - - const convo = conversationStore.getConvo(sender) - convo.active.value = true - - let message = _tagMessage(received.message) - if (received.end) { - conversationStore.endConversation(sender) - message = `Conversation with ${sender} ended with message: "${message}"` - sender = 'system' // bot will respond to system instead of the other bot - } - else if (received.start) { - agent.shut_up = false - } - convo.inMessageTimer.value = undefined - agent.handleMessage(sender, message) -} - -function _tagMessage(message: string) { - return `(FROM OTHER BOT)${message}` -} - -async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { - await new Promise(resolve => setTimeout(resolve, 5000)) - if (self_prompter_paused && !conversationStore.inConversation()) { - self_prompter_paused = false - agent.self_prompter.start() - } -} diff --git a/src/composables/neuri.ts b/src/composables/neuri.ts new file mode 100644 index 0000000..1c62160 --- /dev/null +++ b/src/composables/neuri.ts @@ -0,0 +1,42 @@ +import type { Agent, Neuri } from 'neuri' +import type { Mineflayer } from '../libs/mineflayer' + +import { useLogg } from '@guiiai/logg' +import { neuri } from 'neuri' + +import { createActionNeuriAgent } from '../agents/action/llm-handler' +import { createChatNeuriAgent } from '../agents/chat/llm' +import { createPlanningNeuriAgent } from '../agents/planning/llm-handler' +import { openaiConfig } from './config' + +let neuriAgent: Neuri | undefined +const agents = new Set>() + +const logger = useLogg('neuri').useGlobalConfig() + +export async function createNeuriAgent(mineflayer: Mineflayer): Promise { + logger.log('Initializing neuri agent') + let n = neuri() + + agents.add(createPlanningNeuriAgent()) + agents.add(createActionNeuriAgent(mineflayer)) + agents.add(createChatNeuriAgent()) + + agents.forEach(agent => n = n.agent(agent)) + + neuriAgent = await n.build({ + provider: { + apiKey: openaiConfig.apiKey, + baseURL: openaiConfig.baseUrl, + }, + }) + + return neuriAgent +} + +export function useNeuriAgent(): Neuri { + if (!neuriAgent) { + throw new Error('Agent not initialized') + } + return neuriAgent +} diff --git a/src/container.ts b/src/container.ts new file mode 100644 index 0000000..bd8c6e8 --- /dev/null +++ b/src/container.ts @@ -0,0 +1,71 @@ +import type { Neuri } from 'neuri' + +import { useLogg } from '@guiiai/logg' +import { asClass, asFunction, createContainer, InjectionMode } from 'awilix' + +import { ActionAgentImpl } from './agents/action' +import { ChatAgentImpl } from './agents/chat' +import { PlanningAgentImpl } from './agents/planning' + +export interface ContainerServices { + logger: ReturnType + actionAgent: ActionAgentImpl + planningAgent: PlanningAgentImpl + chatAgent: ChatAgentImpl + neuri: Neuri +} + +export function createAppContainer(options: { + neuri: Neuri + model?: string + maxHistoryLength?: number + idleTimeout?: number +}) { + const container = createContainer({ + injectionMode: InjectionMode.PROXY, + strict: true, + }) + + // Register services + container.register({ + // Create independent logger for each agent + logger: asFunction(() => useLogg('app').useGlobalConfig()).singleton(), + + // Register neuri client + neuri: asFunction(() => options.neuri).singleton(), + + // Register agents + actionAgent: asClass(ActionAgentImpl) + .singleton() + .inject(() => ({ + id: 'action', + type: 'action' as const, + })), + + planningAgent: asClass(PlanningAgentImpl) + .singleton() + .inject(() => ({ + id: 'planning', + type: 'planning' as const, + llm: { + agent: options.neuri, + model: options.model, + }, + })), + + chatAgent: asClass(ChatAgentImpl) + .singleton() + .inject(() => ({ + id: 'chat', + type: 'chat' as const, + llm: { + agent: options.neuri, + model: options.model, + }, + maxHistoryLength: options.maxHistoryLength, + idleTimeout: options.idleTimeout, + })), + }) + + return container +} diff --git a/src/libs/llm/base.ts b/src/libs/llm/base.ts new file mode 100644 index 0000000..7e8df46 --- /dev/null +++ b/src/libs/llm/base.ts @@ -0,0 +1,45 @@ +import type { NeuriContext } from 'neuri' +import type { ChatCompletion, Message } from 'neuri/openai' +import type { LLMConfig, LLMResponse } from './types' + +import { useLogg } from '@guiiai/logg' + +import { openaiConfig } from '../../composables/config' +import { toRetriable } from '../../utils/helper' + +export abstract class BaseLLMHandler { + protected logger = useLogg('llm-handler').useGlobalConfig() + + constructor(protected config: LLMConfig) {} + + protected async handleCompletion( + context: NeuriContext, + route: string, + messages: Message[], + ): Promise { + const completion = await context.reroute(route, messages, { + model: this.config.model ?? openaiConfig.model, + }) as ChatCompletion | ChatCompletion & { error: { message: string } } + + if (!completion || 'error' in completion) { + this.logger.withFields(context).error('Completion failed') + throw new Error(completion?.error?.message ?? 'Unknown error') + } + + const content = await completion.firstContent() + this.logger.withFields({ usage: completion.usage, content }).log('Generated content') + + return { + content, + usage: completion.usage, + } + } + + protected createRetryHandler(handler: (context: NeuriContext) => Promise) { + return toRetriable( + this.config.retryLimit ?? 3, + this.config.delayInterval ?? 1000, + handler, + ) + } +} diff --git a/src/libs/llm/types.ts b/src/libs/llm/types.ts new file mode 100644 index 0000000..a21fd17 --- /dev/null +++ b/src/libs/llm/types.ts @@ -0,0 +1,14 @@ +import type { Neuri } from 'neuri' + +export interface LLMConfig { + agent: Neuri + model?: string + retryLimit?: number + delayInterval?: number + maxContextLength?: number +} + +export interface LLMResponse { + content: string + usage?: any +} diff --git a/src/libs/mineflayer/base-agent.ts b/src/libs/mineflayer/base-agent.ts new file mode 100644 index 0000000..fc573a5 --- /dev/null +++ b/src/libs/mineflayer/base-agent.ts @@ -0,0 +1,129 @@ +import type { Action } from './action' + +import { useLogg } from '@guiiai/logg' +import EventEmitter3 from 'eventemitter3' + +export type AgentType = 'action' | 'memory' | 'planning' | 'chat' + +export interface AgentConfig { + id: string + type: AgentType +} + +export interface BaseAgent { + readonly id: string + readonly type: AgentType + init: () => Promise + destroy: () => Promise +} + +export interface ActionAgent extends BaseAgent { + type: 'action' + performAction: (name: string, params: unknown[]) => Promise + getAvailableActions: () => Action[] +} + +export interface MemoryAgent extends BaseAgent { + type: 'memory' + remember: (key: string, value: unknown) => void + recall: (key: string) => T | undefined + forget: (key: string) => void + getMemorySnapshot: () => Record +} + +export interface Plan { + goal: string + steps: Array<{ + action: string + params: unknown[] + }> + status: 'pending' | 'in_progress' | 'completed' | 'failed' + requiresAction: boolean +} + +export interface PlanningAgent extends BaseAgent { + type: 'planning' + createPlan: (goal: string) => Promise + executePlan: (plan: Plan) => Promise + adjustPlan: (plan: Plan, feedback: string) => Promise +} + +export interface ChatAgent extends BaseAgent { + type: 'chat' + processMessage: (message: string, sender: string) => Promise + startConversation: (player: string) => void + endConversation: (player: string) => void +} + +export abstract class AbstractAgent extends EventEmitter3 implements BaseAgent { + public readonly id: string + public readonly type: AgentConfig['type'] + public readonly name: string + + protected initialized: boolean + protected logger: ReturnType + // protected actionManager: ReturnType + // protected conversationStore: ReturnType + + constructor(config: AgentConfig) { + super() + this.id = config.id // TODO: use uuid, is it needed? + this.type = config.type + this.name = `${this.type}-agent` + this.initialized = false + this.logger = useLogg(this.name).useGlobalConfig() + + // Initialize managers + // this.actionManager = useActionManager(this) + // this.conversationStore = useConversationStore({ + // agent: this, + // chatBotMessages: true, + // }) + } + + public async init(): Promise { + if (this.initialized) { + return + } + + await this.initializeAgent() + this.initialized = true + } + + public async destroy(): Promise { + if (!this.initialized) { + return + } + + this.logger.log('Destroying agent') + await this.destroyAgent() + this.initialized = false + } + + // Agent interface implementation + // public isIdle(): boolean { + // return !this.actionManager.executing + // } + + public handleMessage(sender: string, message: string): void { + this.logger.withFields({ sender, message }).log('Received message') + this.emit('message', { sender, message }) + } + + public openChat(message: string): void { + this.logger.withField('message', message).log('Opening chat') + this.emit('chat', message) + } + + // public clearBotLogs(): void { + // // Implement if needed + // } + + public requestInterrupt(): void { + this.emit('interrupt') + } + + // Methods to be implemented by specific agents + protected abstract initializeAgent(): Promise + protected abstract destroyAgent(): Promise +} diff --git a/src/libs/mineflayer/core.ts b/src/libs/mineflayer/core.ts index af50a82..6e019a2 100644 --- a/src/libs/mineflayer/core.ts +++ b/src/libs/mineflayer/core.ts @@ -10,7 +10,7 @@ import { parseCommand } from './command' import { Components } from './components' import { Health } from './health' import { Memory } from './memory' -import { formBotChat } from './message' +import { ChatMessageHandler } from './message' import { Status } from './status' import { Ticker, type TickEvents, type TickEventsHandler } from './ticker' @@ -191,7 +191,7 @@ export class Mineflayer extends EventEmitter { } private handleCommand() { - return formBotChat(this.username, (sender, message) => { + return new ChatMessageHandler(this.username).handleChat((sender, message) => { const { isCommand, command, args } = parseCommand(sender, message) if (!isCommand) diff --git a/src/libs/mineflayer/index.ts b/src/libs/mineflayer/index.ts index d8989b0..1bea770 100644 --- a/src/libs/mineflayer/index.ts +++ b/src/libs/mineflayer/index.ts @@ -3,7 +3,6 @@ export * from './command' export * from './components' export * from './core' export * from './health' -export * from './interfaces' export * from './memory' export * from './message' export * from './plugin' diff --git a/src/libs/mineflayer/interfaces.ts b/src/libs/mineflayer/interfaces.ts deleted file mode 100644 index b55ec74..0000000 --- a/src/libs/mineflayer/interfaces.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface OneLinerable { - toOneLiner: () => string -} diff --git a/src/libs/mineflayer/message.ts b/src/libs/mineflayer/message.ts index aa35fc7..5ab42fb 100644 --- a/src/libs/mineflayer/message.ts +++ b/src/libs/mineflayer/message.ts @@ -1,30 +1,45 @@ import type { Entity } from 'prismarine-entity' -// TODO: need to be refactored -interface ChatBotContext { - fromUsername?: string - fromEntity?: Entity - fromMessage?: string - - isBot: () => boolean - isCommand: () => boolean +// Represents the context of a chat message in the Minecraft world +interface ChatMessage { + readonly sender: { + username: string + entity: Entity | null + } + readonly content: string } -export function newChatBotContext(entity: Entity, botUsername: string, username: string, message: string): ChatBotContext { - return { - fromUsername: username, - fromEntity: entity, - fromMessage: message, - isBot: () => username === botUsername, - isCommand: () => message.startsWith('#'), +// Handles chat message validation and processing +export class ChatMessageHandler { + constructor(private readonly botUsername: string) {} + + // Creates a new chat message context with validation + createMessageContext(entity: Entity | null, username: string, content: string): ChatMessage { + return { + sender: { + username, + entity, + }, + content, + } } -} -export function formBotChat(botUsername: string, cb: (username: string, message: string) => void) { - return (username: string, message: string) => { - if (botUsername === username) - return + // Checks if a message is from the bot itself + isBotMessage(username: string): boolean { + return username === this.botUsername + } + + // Checks if a message is a command + isCommand(content: string): boolean { + return content.startsWith('#') + } - cb(username, message) + // Processes chat messages, filtering out bot's own messages + handleChat(callback: (username: string, message: string) => void): (username: string, message: string) => void { + return (username: string, message: string) => { + if (!this.isBotMessage(username)) { + callback(username, message) + } + } } } diff --git a/src/libs/mineflayer/status.ts b/src/libs/mineflayer/status.ts index 6b5c01a..d39aadc 100644 --- a/src/libs/mineflayer/status.ts +++ b/src/libs/mineflayer/status.ts @@ -1,5 +1,5 @@ import type { Mineflayer } from './core' -import type { OneLinerable } from './interfaces' +import type { OneLinerable } from './types' export class Status implements OneLinerable { public position: string diff --git a/src/libs/mineflayer/types.ts b/src/libs/mineflayer/types.ts index 861b5a6..7f8a3f6 100644 --- a/src/libs/mineflayer/types.ts +++ b/src/libs/mineflayer/types.ts @@ -17,3 +17,7 @@ export interface EventHandlers { export type Events = keyof EventHandlers export type EventsHandler = EventHandlers[K] export type Handler = (ctx: Context) => void | Promise + +export interface OneLinerable { + toOneLiner: () => string +} diff --git a/src/main.ts b/src/main.ts index cca879f..076471b 100644 --- a/src/main.ts +++ b/src/main.ts @@ -8,11 +8,11 @@ import { pathfinder as MineflayerPathfinder } from 'mineflayer-pathfinder' import { plugin as MineflayerPVP } from 'mineflayer-pvp' import { plugin as MineflayerTool } from 'mineflayer-tool' -import { initAgent } from './agents/openai' import { initBot } from './composables/bot' import { botConfig, initEnv } from './composables/config' -import { wrapPlugin } from './libs/mineflayer/plugin' -import { LLMAgent } from './mineflayer/llm-agent' +import { createNeuriAgent } from './composables/neuri' +import { wrapPlugin } from './libs/mineflayer' +import { LLMAgent } from './plugins/llm-agent' import { initLogger } from './utils/logger' const logger = useLogg('main').useGlobalConfig() @@ -35,8 +35,8 @@ async function main() { const airiClient = new Client({ name: 'minecraft-bot', url: 'ws://localhost:6121/ws' }) - // Dynamically load LLMAgent after bot is initialized - const agent = await initAgent(bot) + // Dynamically load LLMAgent after the bot is initialized + const agent = await createNeuriAgent(bot) await bot.loadPlugin(LLMAgent({ agent, airiClient })) process.on('SIGINT', () => { diff --git a/src/manager/action.ts b/src/manager/action.ts new file mode 100644 index 0000000..7f78c0b --- /dev/null +++ b/src/manager/action.ts @@ -0,0 +1,203 @@ +import type { Mineflayer } from '../libs/mineflayer/core' + +import { useLogg } from '@guiiai/logg' +import EventEmitter from 'eventemitter3' + +// Types and interfaces +type ActionFn = (...args: any[]) => void + +interface ActionResult { + success: boolean + message: string | null + timedout: boolean +} + +interface QueuedAction { + label: string + fn: ActionFn + timeout: number + resume: boolean + resolve: (result: ActionResult) => void + reject: (error: Error) => void +} + +export class ActionManager extends EventEmitter { + private state = { + executing: false, + currentActionLabel: '', + currentActionFn: undefined as ActionFn | undefined, + timedout: false, + resume: { + func: undefined as ActionFn | undefined, + name: undefined as string | undefined, + }, + } + + // Action queue to store pending actions + private actionQueue: QueuedAction[] = [] + + private logger = useLogg('ActionManager').useGlobalConfig() + private mineflayer: Mineflayer + + constructor(mineflayer: Mineflayer) { + super() + this.mineflayer = mineflayer + } + + public async resumeAction(actionLabel: string, actionFn: ActionFn, timeout: number): Promise { + return this.queueAction({ + label: actionLabel, + fn: actionFn, + timeout, + resume: true, + }) + } + + public async runAction( + actionLabel: string, + actionFn: ActionFn, + options: { timeout: number, resume: boolean } = { timeout: 10, resume: false }, + ): Promise { + return this.queueAction({ + label: actionLabel, + fn: actionFn, + timeout: options.timeout, + resume: options.resume, + }) + } + + public async stop(): Promise { + this.mineflayer.emit('interrupt') + // Clear the action queue when stopping + this.actionQueue = [] + } + + public cancelResume(): void { + this.state.resume.func = undefined + this.state.resume.name = undefined + } + + private async queueAction(action: Omit): Promise { + return new Promise((resolve, reject) => { + this.actionQueue.push({ + ...action, + resolve, + reject, + }) + + if (!this.state.executing) { + this.processQueue().catch(reject) + } + }) + } + + private async processQueue(): Promise { + while (this.actionQueue.length > 0) { + const action = this.actionQueue[0] + + try { + const result = action.resume + ? await this.executeResume(action.label, action.fn, action.timeout) + : await this.executeAction(action.label, action.fn, action.timeout) + + this.actionQueue.shift()?.resolve(result) + + if (!result.success) { + this.actionQueue.forEach(pendingAction => + pendingAction.reject(new Error('Queue cleared due to action failure')), + ) + this.actionQueue = [] + return result + } + } + catch (error) { + this.actionQueue.shift()?.reject(error as Error) + this.actionQueue.forEach(pendingAction => + pendingAction.reject(new Error('Queue cleared due to error')), + ) + this.actionQueue = [] + throw error + } + } + + return { success: true, message: 'success', timedout: false } + } + + private async executeResume(actionLabel?: string, actionFn?: ActionFn, timeout = 10): Promise { + const isNewResume = actionFn != null + + if (isNewResume) { + if (!actionLabel) { + throw new Error('actionLabel is required for new resume') + } + this.state.resume.func = actionFn + this.state.resume.name = actionLabel + } + + const canExecute = this.state.resume.func != null && isNewResume + + if (!canExecute) { + return { success: false, message: null, timedout: false } + } + + this.state.currentActionLabel = this.state.resume.name || '' + const result = await this.executeAction(this.state.resume.name || '', this.state.resume.func, timeout) + this.state.currentActionLabel = '' + return result + } + + private async executeAction(actionLabel: string, actionFn?: ActionFn, timeout = 10): Promise { + let timeoutHandle: NodeJS.Timeout | undefined + + try { + this.logger.log('executing action...\n') + + if (this.state.executing) { + this.logger.log(`action "${actionLabel}" trying to interrupt current action "${this.state.currentActionLabel}"`) + } + + await this.stop() + + // Set execution state + this.state.executing = true + this.state.currentActionLabel = actionLabel + this.state.currentActionFn = actionFn + + if (timeout > 0) { + timeoutHandle = this.startTimeout(timeout) + } + + await actionFn?.() + + // Reset state after successful execution + this.resetExecutionState(timeoutHandle) + + return { success: true, message: 'success', timedout: false } + } + catch (err) { + this.resetExecutionState(timeoutHandle) + this.cancelResume() + this.logger.withError(err).error('Code execution triggered catch') + await this.stop() + + return { success: false, message: 'failed', timedout: false } + } + } + + private resetExecutionState(timeoutHandle?: NodeJS.Timeout): void { + this.state.executing = false + this.state.currentActionLabel = '' + this.state.currentActionFn = undefined + if (timeoutHandle) + clearTimeout(timeoutHandle) + } + + private startTimeout(timeoutMins = 10): NodeJS.Timeout { + return setTimeout(async () => { + this.logger.warn(`Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + this.state.timedout = true + this.emit('timeout', `Code execution timed out after ${timeoutMins} minutes. Attempting force stop.`) + await this.stop() + }, timeoutMins * 60 * 1000) + } +} diff --git a/src/manager/conversation.ts b/src/manager/conversation.ts new file mode 100644 index 0000000..891fb7a --- /dev/null +++ b/src/manager/conversation.ts @@ -0,0 +1,382 @@ +// import { useLogg } from '@guiiai/logg' + +// let self_prompter_paused = false + +// interface ConversationMessage { +// message: string +// start: boolean +// end: boolean +// } + +// function compileInMessages(inQueue: ConversationMessage[]) { +// let pack: ConversationMessage | undefined +// let fullMessage = '' +// while (inQueue.length > 0) { +// pack = inQueue.shift() +// if (!pack) +// continue + +// fullMessage += pack.message +// } +// if (pack) { +// pack.message = fullMessage +// } + +// return pack +// } + +// type Conversation = ReturnType + +// function useConversations(name: string, agent: Agent) { +// const active = { value: false } +// const ignoreUntilStart = { value: false } +// const blocked = { value: false } +// let inQueue: ConversationMessage[] = [] +// const inMessageTimer: { value: NodeJS.Timeout | undefined } = { value: undefined } + +// function reset() { +// active.value = false +// ignoreUntilStart.value = false +// inQueue = [] +// } + +// function end() { +// active.value = false +// ignoreUntilStart.value = true +// const fullMessage = compileInMessages(inQueue) +// if (!fullMessage) +// return + +// if (fullMessage.message.trim().length > 0) { +// agent.history.add(name, fullMessage.message) +// } + +// if (agent.lastSender === name) { +// agent.lastSender = undefined +// } +// } + +// function queue(message: ConversationMessage) { +// inQueue.push(message) +// } + +// return { +// reset, +// end, +// queue, +// name, +// inMessageTimer, +// blocked, +// active, +// ignoreUntilStart, +// inQueue, +// } +// } + +// const WAIT_TIME_START = 30000 + +// export type ConversationStore = ReturnType + +// export function useConversationStore(options: { agent: Agent, chatBotMessages?: boolean, agentNames?: string[] }) { +// const conversations: Record = {} +// const activeConversation: { value: Conversation | undefined } = { value: undefined } +// const awaitingResponse = { value: false } +// const waitTimeLimit = { value: WAIT_TIME_START } +// const connectionMonitor: { value: NodeJS.Timeout | undefined } = { value: undefined } +// const connectionTimeout: { value: NodeJS.Timeout | undefined } = { value: undefined } +// const agent = options.agent +// let agentsInGame = options.agentNames || [] +// const log = useLogg('ConversationStore').useGlobalConfig() + +// const conversationStore = { +// getConvo: (name: string) => { +// if (!conversations[name]) +// conversations[name] = useConversations(name, agent) +// return conversations[name] +// }, +// startMonitor: () => { +// clearInterval(connectionMonitor.value) +// let waitTime = 0 +// let lastTime = Date.now() +// connectionMonitor.value = setInterval(() => { +// if (!activeConversation.value) { +// conversationStore.stopMonitor() +// return // will clean itself up +// } + +// const delta = Date.now() - lastTime +// lastTime = Date.now() +// const convo_partner = activeConversation.value.name + +// if (awaitingResponse.value && agent.isIdle()) { +// waitTime += delta +// if (waitTime > waitTimeLimit.value) { +// agent.handleMessage('system', `${convo_partner} hasn't responded in ${waitTimeLimit.value / 1000} seconds, respond with a message to them or your own action.`) +// waitTime = 0 +// waitTimeLimit.value *= 2 +// } +// } +// else if (!awaitingResponse.value) { +// waitTimeLimit.value = WAIT_TIME_START +// waitTime = 0 +// } + +// if (!conversationStore.otherAgentInGame(convo_partner) && !connectionTimeout.value) { +// connectionTimeout.value = setTimeout(() => { +// if (conversationStore.otherAgentInGame(convo_partner)) { +// conversationStore.clearMonitorTimeouts() +// return +// } +// if (!self_prompter_paused) { +// conversationStore.endConversation(convo_partner) +// agent.handleMessage('system', `${convo_partner} disconnected, conversation has ended.`) +// } +// else { +// conversationStore.endConversation(convo_partner) +// } +// }, 10000) +// } +// }, 1000) +// }, +// stopMonitor: () => { +// clearInterval(connectionMonitor.value) +// connectionMonitor.value = undefined +// conversationStore.clearMonitorTimeouts() +// }, +// clearMonitorTimeouts: () => { +// awaitingResponse.value = false +// clearTimeout(connectionTimeout.value) +// connectionTimeout.value = undefined +// }, +// startConversation: (send_to: string, message: string) => { +// const convo = conversationStore.getConvo(send_to) +// convo.reset() + +// if (agent.self_prompter.on) { +// agent.self_prompter.stop() +// self_prompter_paused = true +// } +// if (convo.active.value) +// return + +// convo.active.value = true +// activeConversation.value = convo +// conversationStore.startMonitor() +// conversationStore.sendToBot(send_to, message, true, false) +// }, +// startConversationFromOtherBot: (name: string) => { +// const convo = conversationStore.getConvo(name) +// convo.active.value = true +// activeConversation.value = convo +// conversationStore.startMonitor() +// }, +// sendToBot: (send_to: string, message: string, start = false, open_chat = true) => { +// if (!conversationStore.isOtherAgent(send_to)) { +// console.warn(`${agent.name} tried to send bot message to non-bot ${send_to}`) +// return +// } +// const convo = conversationStore.getConvo(send_to) + +// if (options.chatBotMessages && open_chat) +// agent.openChat(`(To ${send_to}) ${message}`) + +// if (convo.ignoreUntilStart.value) +// return +// convo.active.value = true + +// const end = message.includes('!endConversation') +// const json = { +// message, +// start, +// end, +// } + +// awaitingResponse.value = true +// // TODO: +// // sendBotChatToServer(send_to, json) +// log.withField('json', json).log(`Sending message to ${send_to}`) +// }, +// receiveFromBot: async (sender: string, received: ConversationMessage) => { +// const convo = conversationStore.getConvo(sender) + +// if (convo.ignoreUntilStart.value && !received.start) +// return + +// // check if any convo is active besides the sender +// if (conversationStore.inConversation() && !conversationStore.inConversation(sender)) { +// conversationStore.sendToBot(sender, `I'm talking to someone else, try again later. !endConversation("${sender}")`, false, false) +// conversationStore.endConversation(sender) +// return +// } + +// if (received.start) { +// convo.reset() +// conversationStore.startConversationFromOtherBot(sender) +// } + +// conversationStore.clearMonitorTimeouts() +// convo.queue(received) + +// // responding to conversation takes priority over self prompting +// if (agent.self_prompter.on) { +// await agent.self_prompter.stopLoop() +// self_prompter_paused = true +// } + +// _scheduleProcessInMessage(agent, conversationStore, sender, received, convo) +// }, +// responseScheduledFor: (sender: string) => { +// if (!conversationStore.isOtherAgent(sender) || !conversationStore.inConversation(sender)) +// return false +// const convo = conversationStore.getConvo(sender) +// return !!convo.inMessageTimer +// }, +// isOtherAgent: (name: string) => { +// return !!options.agentNames?.includes(name) +// }, +// otherAgentInGame: (name: string) => { +// return agentsInGame.includes(name) +// }, +// updateAgents: (agents: Agent[]) => { +// options.agentNames = agents.map(a => a.name) +// agentsInGame = agents.filter(a => a.in_game).map(a => a.name) +// }, +// getInGameAgents: () => { +// return agentsInGame +// }, +// inConversation: (other_agent?: string) => { +// if (other_agent) +// return conversations[other_agent]?.active +// return Object.values(conversations).some(c => c.active) +// }, +// endConversation: (sender: string) => { +// if (conversations[sender]) { +// conversations[sender].end() +// if (activeConversation.value?.name === sender) { +// conversationStore.stopMonitor() +// activeConversation.value = undefined +// if (self_prompter_paused && !conversationStore.inConversation()) { +// _resumeSelfPrompter(agent, conversationStore) +// } +// } +// } +// }, +// endAllConversations: () => { +// for (const sender in conversations) { +// conversationStore.endConversation(sender) +// } +// if (self_prompter_paused) { +// _resumeSelfPrompter(agent, conversationStore) +// } +// }, +// forceEndCurrentConversation: () => { +// if (activeConversation.value) { +// const sender = activeConversation.value.name +// conversationStore.sendToBot(sender, `!endConversation("${sender}")`, false, false) +// conversationStore.endConversation(sender) +// } +// }, +// scheduleSelfPrompter: () => { +// self_prompter_paused = true +// }, +// cancelSelfPrompter: () => { +// self_prompter_paused = false +// }, +// } + +// return conversationStore +// } + +// function containsCommand(message: string) { +// // TODO: mock +// return message +// } + +// /* +// This function controls conversation flow by deciding when the bot responds. +// The logic is as follows: +// - If neither bot is busy, respond quickly with a small delay. +// - If only the other bot is busy, respond with a long delay to allow it to finish short actions (ex check inventory) +// - If I'm busy but other bot isn't, let LLM decide whether to respond +// - If both bots are busy, don't respond until someone is done, excluding a few actions that allow fast responses +// - New messages received during the delay will reset the delay following this logic, and be queued to respond in bulk +// */ +// const talkOverActions = ['stay', 'followPlayer', 'mode:'] // all mode actions +// const fastDelay = 200 +// const longDelay = 5000 + +// async function _scheduleProcessInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: { message: string, start: boolean }, convo: Conversation) { +// if (convo.inMessageTimer) +// clearTimeout(convo.inMessageTimer.value) +// const otherAgentBusy = containsCommand(received.message) + +// const scheduleResponse = (delay: number) => convo.inMessageTimer.value = setTimeout(() => _processInMessageQueue(agent, conversationStore, sender), delay) + +// if (!agent.isIdle() && otherAgentBusy) { +// // both are busy +// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) +// if (canTalkOver) +// scheduleResponse(fastDelay) +// // otherwise don't respond +// } +// else if (otherAgentBusy) { +// // other bot is busy but I'm not +// scheduleResponse(longDelay) +// } +// else if (!agent.isIdle()) { +// // I'm busy but other bot isn't +// const canTalkOver = talkOverActions.some(a => agent.actions.currentActionLabel.includes(a)) +// if (canTalkOver) { +// scheduleResponse(fastDelay) +// } +// else { +// const shouldRespond = await agent.prompter.promptShouldRespondToBot(received.message) +// useLogg('Conversation').useGlobalConfig().log(`${agent.name} decided to ${shouldRespond ? 'respond' : 'not respond'} to ${sender}`) +// if (shouldRespond) +// scheduleResponse(fastDelay) +// } +// } +// else { +// // neither are busy +// scheduleResponse(fastDelay) +// } +// } + +// function _processInMessageQueue(agent: Agent, conversationStore: ConversationStore, name: string) { +// const convo = conversationStore.getConvo(name) +// _handleFullInMessage(agent, conversationStore, name, compileInMessages(convo.inQueue)) +// } + +// function _handleFullInMessage(agent: Agent, conversationStore: ConversationStore, sender: string, received: ConversationMessage | undefined) { +// if (!received) +// return + +// useLogg('Conversation').useGlobalConfig().log(`${agent.name} responding to "${received.message}" from ${sender}`) + +// const convo = conversationStore.getConvo(sender) +// convo.active.value = true + +// let message = _tagMessage(received.message) +// if (received.end) { +// conversationStore.endConversation(sender) +// message = `Conversation with ${sender} ended with message: "${message}"` +// sender = 'system' // bot will respond to system instead of the other bot +// } +// else if (received.start) { +// agent.shut_up = false +// } +// convo.inMessageTimer.value = undefined +// agent.handleMessage(sender, message) +// } + +// function _tagMessage(message: string) { +// return `(FROM OTHER BOT)${message}` +// } + +// async function _resumeSelfPrompter(agent: Agent, conversationStore: ConversationStore) { +// await new Promise(resolve => setTimeout(resolve, 5000)) +// if (self_prompter_paused && !conversationStore.inConversation()) { +// self_prompter_paused = false +// agent.self_prompter.start() +// } +// } diff --git a/src/mineflayer/index.ts b/src/mineflayer/index.ts deleted file mode 100644 index 9f09fed..0000000 --- a/src/mineflayer/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './echo' -export * from './llm-agent' diff --git a/src/mineflayer/llm-agent.ts b/src/mineflayer/llm-agent.ts deleted file mode 100644 index 5777e0b..0000000 --- a/src/mineflayer/llm-agent.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { Client } from '@proj-airi/server-sdk' -import type { Neuri, NeuriContext } from 'neuri' -import type { MineflayerPlugin } from '../libs/mineflayer/plugin' - -import { useLogg } from '@guiiai/logg' -import { assistant, system, user } from 'neuri/openai' - -import { formBotChat } from '../libs/mineflayer/message' -import { genActionAgentPrompt, genStatusPrompt } from '../prompts/agent' -import { toRetriable } from '../utils/reliability' - -export function LLMAgent(options: { agent: Neuri, airiClient: Client }): MineflayerPlugin { - return { - async created(bot) { - const agent = options.agent - - const logger = useLogg('LLMAgent').useGlobalConfig() - - bot.memory.chatHistory.push(system(genActionAgentPrompt(bot))) - - // todo: get system message - const onChat = formBotChat(bot.username, async (username, message) => { - logger.withFields({ username, message }).log('Chat message received') - - // long memory - bot.memory.chatHistory.push(user(`${username}: ${message}`)) - - // short memory - const statusPrompt = await genStatusPrompt(bot) - const content = await agent.handleStateless([...bot.memory.chatHistory, system(statusPrompt)], async (c: NeuriContext) => { - logger.log('thinking...') - - const handleCompletion = async (c: NeuriContext): Promise => { - logger.log('rerouting...') - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - if (!completion || 'error' in completion) { - logger.withFields({ completion }).error('Completion') - throw completion?.error || new Error('Unknown error') - } - - const content = await completion?.firstContent() - logger.withFields({ usage: completion.usage, content }).log('output') - - bot.memory.chatHistory.push(assistant(content)) - - return content - } - - const retirableHandler = toRetriable( - 3, // retryLimit - 1000, // delayInterval in ms - handleCompletion, - { onError: err => logger.withError(err).log('error occurred') }, - ) - - logger.log('handling...') - return await retirableHandler(c) - }) - - if (content) { - logger.withFields({ content }).log('responded') - bot.bot.chat(content) - } - }) - - options.airiClient.onEvent('input:text:voice', async (event) => { - logger.withFields({ user: event.data.discord?.guildMember, message: event.data.transcription }).log('Chat message received') - - // long memory - bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) - - // short memory - const statusPrompt = await genStatusPrompt(bot) - const content = await agent.handleStateless([...bot.memory.chatHistory, system(statusPrompt)], async (c: NeuriContext) => { - logger.log('thinking...') - - const handleCompletion = async (c: NeuriContext): Promise => { - logger.log('rerouting...') - const completion = await c.reroute('action', c.messages, { model: 'openai/gpt-4o-mini' }) - if (!completion || 'error' in completion) { - logger.withFields({ completion }).error('Completion') - throw completion?.error || new Error('Unknown error') - } - - const content = await completion?.firstContent() - logger.withFields({ usage: completion.usage, content }).log('output') - - bot.memory.chatHistory.push(assistant(content)) - - return content - } - - const retirableHandler = toRetriable( - 3, // retryLimit - 1000, // delayInterval in ms - handleCompletion, - { onError: err => logger.withError(err).log('error occurred') }, - ) - - logger.log('handling...') - return await retirableHandler(c) - }) - - if (content) { - logger.withFields({ content }).log('responded') - bot.bot.chat(content) - } - }) - - bot.bot.on('chat', onChat) - }, - } -} diff --git a/src/mineflayer/echo.ts b/src/plugins/echo.ts similarity index 75% rename from src/mineflayer/echo.ts rename to src/plugins/echo.ts index b4be3dc..67b4932 100644 --- a/src/mineflayer/echo.ts +++ b/src/plugins/echo.ts @@ -2,14 +2,14 @@ import type { MineflayerPlugin } from '../libs/mineflayer/plugin' import { useLogg } from '@guiiai/logg' -import { formBotChat } from '../libs/mineflayer/message' +import { ChatMessageHandler } from '../libs/mineflayer/message' export function Echo(): MineflayerPlugin { const logger = useLogg('Echo').useGlobalConfig() return { spawned(mineflayer) { - const onChatHandler = formBotChat(mineflayer.username, (username, message) => { + const onChatHandler = new ChatMessageHandler(mineflayer.username).handleChat((username, message) => { logger.withFields({ username, message }).log('Chat message received') mineflayer.bot.chat(message) }) diff --git a/src/mineflayer/follow.ts b/src/plugins/follow.ts similarity index 100% rename from src/mineflayer/follow.ts rename to src/plugins/follow.ts diff --git a/src/plugins/llm-agent.ts b/src/plugins/llm-agent.ts new file mode 100644 index 0000000..3532d64 --- /dev/null +++ b/src/plugins/llm-agent.ts @@ -0,0 +1,189 @@ +import type { Client } from '@proj-airi/server-sdk' +import type { Neuri, NeuriContext } from 'neuri' +import type { ChatCompletion } from 'neuri/openai' +import type { Mineflayer } from '../libs/mineflayer' +import type { ActionAgent, ChatAgent, PlanningAgent } from '../libs/mineflayer/base-agent' +import type { MineflayerPlugin } from '../libs/mineflayer/plugin' + +import { useLogg } from '@guiiai/logg' +import { assistant, system, user } from 'neuri/openai' + +import { generateActionAgentPrompt, generateStatusPrompt } from '../agents/prompt/llm-agent.plugin' +import { createAppContainer } from '../container' +import { ChatMessageHandler } from '../libs/mineflayer/message' +import { toRetriable } from '../utils/helper' + +interface MineflayerWithAgents extends Mineflayer { + planning: PlanningAgent + action: ActionAgent + chat: ChatAgent +} + +interface LLMAgentOptions { + agent: Neuri + airiClient: Client +} + +async function handleLLMCompletion(context: NeuriContext, bot: MineflayerWithAgents, logger: ReturnType): Promise { + logger.log('rerouting...') + + const completion = await context.reroute('action', context.messages, { + model: 'openai/gpt-4o-mini', + }) as ChatCompletion | { error: { message: string } } & ChatCompletion + + if (!completion || 'error' in completion) { + logger.withFields({ completion }).error('Completion') + logger.withFields({ messages: context.messages }).log('messages') + return completion?.error?.message ?? 'Unknown error' + } + + const content = await completion.firstContent() + logger.withFields({ usage: completion.usage, content }).log('output') + + bot.memory.chatHistory.push(assistant(content)) + return content +} + +async function handleChatMessage(username: string, message: string, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { + logger.withFields({ username, message }).log('Chat message received') + bot.memory.chatHistory.push(user(`${username}: ${message}`)) + + logger.log('thinking...') + + try { + // Create and execute plan + const plan = await bot.planning.createPlan(message) + logger.withFields({ plan }).log('Plan created') + await bot.planning.executePlan(plan) + logger.log('Plan executed successfully') + + // Generate response + // TODO: use chat agent and conversion manager + const statusPrompt = await generateStatusPrompt(bot) + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('handling response...') + return toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + { onError: err => logger.withError(err).log('error occurred') }, + )(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) + } +} + +async function handleVoiceInput(event: any, bot: MineflayerWithAgents, agent: Neuri, logger: ReturnType): Promise { + logger + .withFields({ + user: event.data.discord?.guildMember, + message: event.data.transcription, + }) + .log('Chat message received') + + const statusPrompt = await generateStatusPrompt(bot) + bot.memory.chatHistory.push(system(statusPrompt)) + bot.memory.chatHistory.push(user(`NekoMeowww: ${event.data.transcription}`)) + + try { + // 创建并执行计划 + const plan = await bot.planning.createPlan(event.data.transcription) + logger.withFields({ plan }).log('Plan created') + await bot.planning.executePlan(plan) + logger.log('Plan executed successfully') + + // 生成回复 + const retryHandler = toRetriable( + 3, + 1000, + ctx => handleLLMCompletion(ctx, bot, logger), + ) + + const content = await agent.handleStateless( + [...bot.memory.chatHistory, system(statusPrompt)], + async (c: NeuriContext) => { + logger.log('thinking...') + return retryHandler(c) + }, + ) + + if (content) { + logger.withFields({ content }).log('responded') + bot.bot.chat(content) + } + } + catch (error) { + logger.withError(error).error('Failed to process message') + bot.bot.chat( + `Sorry, I encountered an error: ${ + error instanceof Error ? error.message : 'Unknown error' + }`, + ) + } +} + +export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { + return { + async created(bot) { + const logger = useLogg('LLMAgent').useGlobalConfig() + + // 创建容器并获取所需的服务 + const container = createAppContainer({ + neuri: options.agent, + model: 'openai/gpt-4o-mini', + maxHistoryLength: 50, + idleTimeout: 5 * 60 * 1000, + }) + + const actionAgent = container.resolve('actionAgent') + const planningAgent = container.resolve('planningAgent') + const chatAgent = container.resolve('chatAgent') + + // 初始化 agents + await actionAgent.init() + await planningAgent.init() + await chatAgent.init() + + // 类型转换 + const botWithAgents = bot as unknown as MineflayerWithAgents + botWithAgents.action = actionAgent + botWithAgents.planning = planningAgent + botWithAgents.chat = chatAgent + + // 初始化系统提示 + bot.memory.chatHistory.push(system(generateActionAgentPrompt(bot))) + + // 设置消息处理 + const onChat = new ChatMessageHandler(bot.username).handleChat((username, message) => + handleChatMessage(username, message, botWithAgents, options.agent, logger)) + + options.airiClient.onEvent('input:text:voice', event => + handleVoiceInput(event, botWithAgents, options.agent, logger)) + + bot.bot.on('chat', onChat) + }, + + async beforeCleanup(bot) { + const botWithAgents = bot as unknown as MineflayerWithAgents + await botWithAgents.action?.destroy() + await botWithAgents.planning?.destroy() + await botWithAgents.chat?.destroy() + bot.bot.removeAllListeners('chat') + }, + } +} diff --git a/src/mineflayer/pathfinder.ts b/src/plugins/pathfinder.ts similarity index 100% rename from src/mineflayer/pathfinder.ts rename to src/plugins/pathfinder.ts diff --git a/src/mineflayer/status.ts b/src/plugins/status.ts similarity index 100% rename from src/mineflayer/status.ts rename to src/plugins/status.ts diff --git a/src/prompts/agent.ts b/src/prompts/agent.ts deleted file mode 100644 index 8e017df..0000000 --- a/src/prompts/agent.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { Mineflayer } from '../libs/mineflayer' - -import { listInventory } from '../skills/actions/inventory' - -export function genSystemBasicPrompt(botName: string): string { - return `You are a playful Minecraft bot named ${botName} that can converse with players, see, move, -mine, build, and interact with the world by using commands.` -} - -export function genActionAgentPrompt(mineflayer: Mineflayer): string { - // ${ctx.prompt.selfPrompt} - - return `${genSystemBasicPrompt(mineflayer.username)} - -Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in -your responses, don't apologize constantly, don't give instructions or make lists unless -asked, and don't refuse requests. - -Do not use any emojis. Just call the function given you if needed. - -- If I command you 'stop', then call the 'stop' function. -- If I require you to find something, then call the 'nearbyBlocks' function first, then call the 'searchForBlock' function. -` -} - -export async function genStatusPrompt(mineflayer: Mineflayer): Promise { - const inventory = await listInventory(mineflayer) - if (inventory.length === 0) { - return `I will give you the following information: -${mineflayer.status.toOneLiner()} - -Inventory: -[Empty] - -Item in hand: -[Empty] -` - } - const inventoryStr = inventory.map(item => `${item.name} x ${item.count}`).join(', ') - const itemInHand = `${inventory[0].name} x ${inventory[0].count}` // TODO: mock - - return `I will give you the following information: -${mineflayer.status.toOneLiner()} - -Inventory: -${inventoryStr} - -Item in hand: -${itemInHand} -` -} - -export function genQueryAgentPrompt(mineflayer: Mineflayer): string { - const prompt = `You are a helpful assistant that asks questions to help me decide the next immediate -task to do in Minecraft. My ultimate goal is to discover as many things as possible, -accomplish as many tasks as possible and become the best Minecraft player in the world. - -I will give you the following information: -${mineflayer.status.toOneLiner()} -` - - return prompt -} diff --git a/src/skills/actions/collect-block.ts b/src/skills/actions/collect-block.ts index 333d91d..07fe945 100644 --- a/src/skills/actions/collect-block.ts +++ b/src/skills/actions/collect-block.ts @@ -4,8 +4,8 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' import pathfinder from 'mineflayer-pathfinder' -import { getNearestBlocks } from '../../composables/world' import { breakBlockAt } from '../blocks' +import { getNearestBlocks } from '../world' import { ensurePickaxe } from './ensure' import { pickupNearbyItems } from './world-interactions' diff --git a/src/skills/actions/gather-wood.ts b/src/skills/actions/gather-wood.ts index 91479c1..d757eb4 100644 --- a/src/skills/actions/gather-wood.ts +++ b/src/skills/actions/gather-wood.ts @@ -2,10 +2,10 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getNearestBlocks } from '../../composables/world' import { sleep } from '../../utils/helper' import { breakBlockAt } from '../blocks' import { goToPosition, moveAway } from '../movement' +import { getNearestBlocks } from '../world' import { pickupNearbyItems } from './world-interactions' const logger = useLogg('Action:GatherWood').useGlobalConfig() diff --git a/src/skills/actions/inventory.ts b/src/skills/actions/inventory.ts index 0fa75ff..1614d9d 100644 --- a/src/skills/actions/inventory.ts +++ b/src/skills/actions/inventory.ts @@ -3,8 +3,8 @@ import type { Mineflayer } from '../../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getNearestBlock } from '../../composables/world' import { goToPlayer, goToPosition } from '../movement' +import { getNearestBlock } from '../world' const logger = useLogg('Action:Inventory').useGlobalConfig() diff --git a/src/skills/blocks.ts b/src/skills/blocks.ts index 78e6cff..383c49b 100644 --- a/src/skills/blocks.ts +++ b/src/skills/blocks.ts @@ -4,10 +4,10 @@ import type { BlockFace } from './base' import pathfinderModel, { type SafeBlock } from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' -import { getNearestBlock, getNearestBlocks, getPosition, shouldPlaceTorch } from '../composables/world' import { getBlockId, makeItem } from '../utils/mcdata' import { log } from './base' import { goToPosition } from './movement' +import { getNearestBlock, getNearestBlocks, getPosition, shouldPlaceTorch } from './world' const { goals, Movements } = pathfinderModel diff --git a/src/skills/combat.ts b/src/skills/combat.ts index eae3d10..ff98a0b 100644 --- a/src/skills/combat.ts +++ b/src/skills/combat.ts @@ -4,10 +4,10 @@ import type { Mineflayer } from '../libs/mineflayer' import pathfinderModel from 'mineflayer-pathfinder' -import { getNearbyEntities, getNearestEntityWhere } from '../composables/world' import { sleep } from '../utils/helper' import { isHostile } from '../utils/mcdata' import { log } from './base' +import { getNearbyEntities, getNearestEntityWhere } from './world' const { goals } = pathfinderModel diff --git a/src/skills/crafting.ts b/src/skills/crafting.ts index 11375df..8ca4397 100644 --- a/src/skills/crafting.ts +++ b/src/skills/crafting.ts @@ -5,11 +5,11 @@ import type { Mineflayer } from '../libs/mineflayer' import { useLogg } from '@guiiai/logg' -import { getInventoryCounts, getNearestBlock, getNearestFreeSpace } from '../composables/world' import { getItemId, getItemName } from '../utils/mcdata' import { ensureCraftingTable } from './actions/ensure' import { collectBlock, placeBlock } from './blocks' import { goToNearestBlock, goToPosition, moveAway } from './movement' +import { getInventoryCounts, getNearestBlock, getNearestFreeSpace } from './world' const logger = useLogg('Skill:Crafting').useGlobalConfig() diff --git a/src/skills/inventory.ts b/src/skills/inventory.ts index 91c3c9c..ec6c4ef 100644 --- a/src/skills/inventory.ts +++ b/src/skills/inventory.ts @@ -1,8 +1,8 @@ import type { Mineflayer } from '../libs/mineflayer' -import { getNearestBlock } from '../composables/world' import { log } from './base' import { goToPlayer, goToPosition } from './movement' +import { getNearestBlock } from './world' export async function equip(mineflayer: Mineflayer, itemName: string): Promise { const item = mineflayer.bot.inventory.slots.find(slot => slot && slot.name === itemName) diff --git a/src/skills/movement.ts b/src/skills/movement.ts index a48ae85..f73a5ba 100644 --- a/src/skills/movement.ts +++ b/src/skills/movement.ts @@ -6,9 +6,9 @@ import { randomInt } from 'es-toolkit' import pathfinder from 'mineflayer-pathfinder' import { Vec3 } from 'vec3' -import { getNearestBlock, getNearestEntityWhere } from '../composables/world' import { sleep } from '../utils/helper' import { log } from './base' +import { getNearestBlock, getNearestEntityWhere } from './world' const logger = useLogg('Skill:Movement').useGlobalConfig() const { goals, Movements } = pathfinder diff --git a/src/composables/world.ts b/src/skills/world.ts similarity index 100% rename from src/composables/world.ts rename to src/skills/world.ts diff --git a/src/utils/helper.ts b/src/utils/helper.ts index c1eb515..e0600bf 100644 --- a/src/utils/helper.ts +++ b/src/utils/helper.ts @@ -1 +1,39 @@ export const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) + +/** + * Returns a retirable anonymous function with configured retryLimit and delayInterval + * + * @param retryLimit Number of retry attempts + * @param delayInterval Delay between retries in milliseconds + * @param func Function to be called + * @returns A wrapped function with the same signature as func + */ +export function toRetriable( + retryLimit: number, + delayInterval: number, + func: (...args: A[]) => Promise, + hooks?: { + onError?: (err: unknown) => void + }, +): (...args: A[]) => Promise { + let retryCount = 0 + return async function (args: A): Promise { + try { + return await func(args) + } + catch (err) { + if (hooks?.onError) { + hooks.onError(err) + } + + if (retryCount < retryLimit) { + retryCount++ + await sleep(delayInterval) + return await toRetriable(retryLimit - retryCount, delayInterval, func)(args) + } + else { + throw err + } + } + } +} diff --git a/src/utils/mcdata.ts b/src/utils/mcdata.ts index 21a02e9..48f093f 100644 --- a/src/utils/mcdata.ts +++ b/src/utils/mcdata.ts @@ -1,5 +1,3 @@ -// src/utils/minecraftData.ts - import type { Bot } from 'mineflayer' import type { Entity } from 'prismarine-entity' diff --git a/src/utils/reliability.ts b/src/utils/reliability.ts deleted file mode 100644 index 3d277f5..0000000 --- a/src/utils/reliability.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { sleep } from './helper' - -/** - * Returns a retirable anonymous function with configured retryLimit and delayInterval - * - * @param retryLimit Number of retry attempts - * @param delayInterval Delay between retries in milliseconds - * @param func Function to be called - * @returns A wrapped function with the same signature as func - */ -export function toRetriable( - retryLimit: number, - delayInterval: number, - func: (...args: A[]) => Promise, - hooks?: { - onError?: (err: unknown) => void - }, -): (...args: A[]) => Promise { - let retryCount = 0 - return async function (args: A): Promise { - try { - return await func(args) - } - catch (err) { - if (hooks?.onError) { - hooks.onError(err) - } - - if (retryCount < retryLimit) { - retryCount++ - await sleep(delayInterval) - return await toRetriable(retryLimit - retryCount, delayInterval, func)(args) - } - else { - throw err - } - } - } -}