From b8539de7c37454407412c02be72f511a7bd1c119 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 00:12:21 +0800 Subject: [PATCH 01/20] feat: twitter services --- pnpm-lock.yaml | 702 ++++++++++++++++-- services/twitter-services/.env.example | 19 + .../twitter-services/docs/architecture.md | 297 ++++++++ services/twitter-services/package.json | 48 ++ .../src/adapters/airi-adapter.ts | 93 +++ .../src/adapters/browser-adapter.ts | 58 ++ .../src/adapters/browserbase-adapter.ts | 120 +++ .../src/adapters/mcp-adapter.ts | 383 ++++++++++ .../src/browser/browserbase.ts | 200 +++++ services/twitter-services/src/cli.ts | 112 +++ services/twitter-services/src/config/index.ts | 105 +++ services/twitter-services/src/config/types.ts | 96 +++ .../twitter-services/src/core/auth-service.ts | 93 +++ .../src/core/timeline-service.ts | 66 ++ .../src/core/twitter-service.ts | 114 +++ services/twitter-services/src/dev-server.ts | 202 +++++ services/twitter-services/src/index.ts | 150 ++++ .../src/parsers/html-parser.ts | 95 +++ .../src/parsers/profile-parser.ts | 142 ++++ .../src/parsers/tweet-parser.ts | 104 +++ .../twitter-services/src/types/browser.ts | 34 + .../twitter-services/src/types/twitter.ts | 89 +++ services/twitter-services/src/utils/api.ts | 62 ++ services/twitter-services/src/utils/error.ts | 74 ++ services/twitter-services/src/utils/logger.ts | 58 ++ .../src/utils/rate-limiter.ts | 67 ++ .../twitter-services/src/utils/selectors.ts | 35 + services/twitter-services/tsconfig.json | 25 + 28 files changed, 3569 insertions(+), 74 deletions(-) create mode 100644 services/twitter-services/.env.example create mode 100644 services/twitter-services/docs/architecture.md create mode 100644 services/twitter-services/package.json create mode 100644 services/twitter-services/src/adapters/airi-adapter.ts create mode 100644 services/twitter-services/src/adapters/browser-adapter.ts create mode 100644 services/twitter-services/src/adapters/browserbase-adapter.ts create mode 100644 services/twitter-services/src/adapters/mcp-adapter.ts create mode 100644 services/twitter-services/src/browser/browserbase.ts create mode 100644 services/twitter-services/src/cli.ts create mode 100644 services/twitter-services/src/config/index.ts create mode 100644 services/twitter-services/src/config/types.ts create mode 100644 services/twitter-services/src/core/auth-service.ts create mode 100644 services/twitter-services/src/core/timeline-service.ts create mode 100644 services/twitter-services/src/core/twitter-service.ts create mode 100644 services/twitter-services/src/dev-server.ts create mode 100644 services/twitter-services/src/index.ts create mode 100644 services/twitter-services/src/parsers/html-parser.ts create mode 100644 services/twitter-services/src/parsers/profile-parser.ts create mode 100644 services/twitter-services/src/parsers/tweet-parser.ts create mode 100644 services/twitter-services/src/types/browser.ts create mode 100644 services/twitter-services/src/types/twitter.ts create mode 100644 services/twitter-services/src/utils/api.ts create mode 100644 services/twitter-services/src/utils/error.ts create mode 100644 services/twitter-services/src/utils/logger.ts create mode 100644 services/twitter-services/src/utils/rate-limiter.ts create mode 100644 services/twitter-services/src/utils/selectors.ts create mode 100644 services/twitter-services/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2cd364e2..69d4a0a7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -378,7 +378,7 @@ importers: version: 2.3.0 '@intlify/unplugin-vue-i18n': specifier: ^6.0.3 - version: 6.0.3(@vue/compiler-dom@3.5.13)(eslint@9.21.0(jiti@2.4.2))(rollup@4.34.9)(typescript@5.8.2)(vue-i18n@11.1.1(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) + version: 6.0.3(@vue/compiler-dom@3.5.13)(eslint@9.21.0(jiti@2.4.2))(rollup@2.79.1)(typescript@5.8.2)(vue-i18n@11.1.1(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) '@proj-airi/elevenlabs': specifier: workspace:^ version: link:../../packages/elevenlabs @@ -405,10 +405,10 @@ importers: version: 5.2.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) '@vue-macros/volar': specifier: ^0.30.15 - version: 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) '@vueuse/motion': specifier: ^2.2.6 - version: 2.2.6(magicast@0.3.5)(rollup@4.34.9)(vue@3.5.13(typescript@5.8.2)) + version: 2.2.6(magicast@0.3.5)(rollup@2.79.1)(vue@3.5.13(typescript@5.8.2)) electron: specifier: ^34.3.0 version: 34.3.0 @@ -426,28 +426,28 @@ importers: version: 3.2.0(unocss@66.1.0-beta.3(postcss@8.5.3)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2))) unplugin-auto-import: specifier: ^19.1.1 - version: 19.1.1(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(@vueuse/core@12.7.0(typescript@5.8.2)) + version: 19.1.1(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(@vueuse/core@12.7.0(typescript@5.8.2)) unplugin-vue-components: specifier: ^28.4.1 - version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(vue@3.5.13(typescript@5.8.2)) + version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-macros: specifier: ^2.14.5 - version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-markdown: specifier: ^28.3.1 version: 28.3.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-router: specifier: ^0.11.2 - version: 0.11.2(rollup@4.34.9)(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) + version: 0.11.2(rollup@2.79.1)(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) vite-bundle-visualizer: specifier: ^1.2.1 - version: 1.2.1(rollup@4.34.9) + version: 1.2.1(rollup@2.79.1) vite-plugin-pwa: specifier: ^0.21.1 - version: 0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0)(workbox-window@7.3.0) + version: 0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0(@types/babel__core@7.20.5))(workbox-window@7.3.0) vite-plugin-vue-devtools: specifier: ^7.7.2 - version: 7.7.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(rollup@4.34.9)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) + version: 7.7.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(rollup@2.79.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) vite-plugin-vue-layouts: specifier: ^0.11.0 version: 0.11.0(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) @@ -685,7 +685,7 @@ importers: version: 2.3.0 '@intlify/unplugin-vue-i18n': specifier: ^6.0.3 - version: 6.0.3(@vue/compiler-dom@3.5.13)(eslint@9.21.0(jiti@2.4.2))(rollup@2.79.1)(typescript@5.8.2)(vue-i18n@11.1.1(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) + version: 6.0.3(@vue/compiler-dom@3.5.13)(eslint@9.21.0(jiti@2.4.2))(rollup@4.34.9)(typescript@5.8.2)(vue-i18n@11.1.1(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) '@proj-airi/drizzle-duckdb-wasm': specifier: workspace:^ version: link:../../packages/drizzle-duckdb-wasm @@ -721,10 +721,10 @@ importers: version: 5.2.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) '@vue-macros/volar': specifier: ^0.30.15 - version: 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) '@vueuse/motion': specifier: ^2.2.6 - version: 2.2.6(magicast@0.3.5)(rollup@2.79.1)(vue@3.5.13(typescript@5.8.2)) + version: 2.2.6(magicast@0.3.5)(rollup@4.34.9)(vue@3.5.13(typescript@5.8.2)) hfup: specifier: workspace:^ version: link:../../packages/hfup @@ -733,28 +733,28 @@ importers: version: 4.0.1 unplugin-auto-import: specifier: ^19.1.1 - version: 19.1.1(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(@vueuse/core@12.7.0(typescript@5.8.2)) + version: 19.1.1(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(@vueuse/core@12.7.0(typescript@5.8.2)) unplugin-vue-components: specifier: ^28.4.1 - version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(vue@3.5.13(typescript@5.8.2)) + version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-macros: specifier: ^2.14.5 - version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-markdown: specifier: ^28.3.1 version: 28.3.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-router: specifier: ^0.11.2 - version: 0.11.2(rollup@2.79.1)(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) + version: 0.11.2(rollup@4.34.9)(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) vite-bundle-visualizer: specifier: ^1.2.1 - version: 1.2.1(rollup@2.79.1) + version: 1.2.1(rollup@4.34.9) vite-plugin-pwa: specifier: ^0.21.1 - version: 0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0)(workbox-window@7.3.0) + version: 0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0(@types/babel__core@7.20.5))(workbox-window@7.3.0) vite-plugin-vue-devtools: specifier: ^7.7.2 - version: 7.7.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(rollup@2.79.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) + version: 7.7.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(rollup@4.34.9)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.2)) vite-plugin-vue-layouts: specifier: ^0.11.0 version: 0.11.0(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-router@4.5.0(vue@3.5.13(typescript@5.8.2)))(vue@3.5.13(typescript@5.8.2)) @@ -1297,6 +1297,52 @@ importers: specifier: ^4.19.3 version: 4.19.3 + services/twitter-services: + dependencies: + '@guiiai/logg': + specifier: ^1.0.0 + version: 1.0.7 + '@modelcontextprotocol/sdk': + specifier: ^1.6.1 + version: 1.6.1 + '@proj-airi/server-sdk': + specifier: ^0.1.0 + version: 0.1.4 + '@types/hast': + specifier: ^3.0.1 + version: 3.0.4 + dotenv: + specifier: ^16.0.3 + version: 16.4.7 + h3: + specifier: ^1.11.0 + version: 1.15.1 + listhen: + specifier: ^1.6.0 + version: 1.9.0 + rehype-parse: + specifier: ^9.0.0 + version: 9.0.1 + unified: + specifier: ^11.0.3 + version: 11.0.5 + unist-util-visit: + specifier: ^5.0.0 + version: 5.0.0 + zod: + specifier: ^3.22.2 + version: 3.24.2 + devDependencies: + '@types/node': + specifier: ^18.16.3 + version: 18.19.76 + tsx: + specifier: ^4.19.0 + version: 4.19.3 + typescript: + specifier: ^5.0.4 + version: 5.8.2 + packages: 7zip-bin@5.2.0: @@ -3258,6 +3304,10 @@ packages: '@mdx-js/mdx@3.1.0': resolution: {integrity: sha512-/QxEhPAvGwbQmy1Px8F899L5Uc2KZ6JtXwlCgJmjSTBedwOZkByYcBG4GceIGPXRDsmfxhHazuS+hlOShRLeDw==} + '@modelcontextprotocol/sdk@1.6.1': + resolution: {integrity: sha512-oxzMzYCkZHMntzuyerehK3fV6A2Kwh5BD6CGEJSVDU2QNEhfLOptf2X7esQgaHZXHZY0oHmMsOtIDLP71UJXgA==} + engines: {node: '>=18'} + '@mswjs/interceptors@0.37.6': resolution: {integrity: sha512-wK+5pLK5XFmgtH3aQ2YVvA3HohS3xqV/OxuVOdNx9Wpnz7VE/fnC+e1A7ln6LFYeck7gOJ/dsZV6OLplOtAJ2w==} engines: {node: '>=18'} @@ -3864,9 +3914,15 @@ packages: '@polka/url@1.0.0-next.24': resolution: {integrity: sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==} + '@proj-airi/server-sdk@0.1.4': + resolution: {integrity: sha512-iyOtnFcQJL0xs+BZ8flDbxkCdrS5kMZrLBmQI//fmYj3yV0ny7TpVwvvPsJaqtcDfrYDOjmQ+JOu4R/YntHVxg==} + '@proj-airi/server-sdk@0.3.6': resolution: {integrity: sha512-jpUxVoqrEA0qt7Z24PpjHg9R7J6PzB/GAgRgjaHOzEF3MHwg6FqQludnmbqUo+gkhn1GIynVMzoVTOujsPI2tg==} + '@proj-airi/server-shared@0.1.4': + resolution: {integrity: sha512-cVqVqLqvC9n8HcXQMm6zKA8W0eTGpcyhdn5finIZr9fyHo6pdhAZN62ER+Nq8FP4uWwbf4uiDnWdVHN+ZwZJHQ==} + '@proj-airi/server-shared@0.3.6': resolution: {integrity: sha512-VVGhVHDrZojH2RQIC6w4znmA/QlWrxFL3UI1N/pQU20vpRN23r4NaG/XvxdlzxQOYuXkyHvdc6/gn1b+qJnBpA==} @@ -4233,6 +4289,18 @@ packages: '@types/audioworklet@0.0.71': resolution: {integrity: sha512-bi5UKn9UXQaDPN9C3gVIT7D1CbPRlJyUPqfTqpLOUVh2cRoPdCscrVPuBkXiJV2JvHH/4bXvmYDSg7SmC7wuXA==} + '@types/babel__core@7.20.5': + resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} + + '@types/babel__generator@7.6.8': + resolution: {integrity: sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==} + + '@types/babel__template@7.4.4': + resolution: {integrity: sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==} + + '@types/babel__traverse@7.20.6': + resolution: {integrity: sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==} + '@types/cacheable-request@6.0.3': resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} @@ -5172,9 +5240,15 @@ packages: '@xsai/providers@0.1.0-beta.5': resolution: {integrity: sha512-jx630mUblMSUcDcXa+BNiFB7deQy1lp0XcJHQokfo/+QvYOpgaPrIRH6oUcjwLio0+8gJXqQmQfEY3+fl9Tjew==} + '@xsai/shared-chat@0.1.0': + resolution: {integrity: sha512-LGF11CBDEWGs5NsCtQ85k9q/jHPzEEFFtScZR9gd7Nxpui5g/TTrgcDMW9Tz0oLHQBPGK31sqQcLYbC7KXjkOw==} + '@xsai/shared-chat@0.1.0-beta.9': resolution: {integrity: sha512-ctb5LV0oig/8ZsjVVSSYcR64CMXtsSYZ3gnVwLGGkBhKpD3UB5a55FFPaHoZA9Iqucnkw/AhWz3xwgWhOJbvqA==} + '@xsai/shared@0.1.0': + resolution: {integrity: sha512-Amr/+IAFqdWei2ec0uy4a8vpSl3QA3J+dvC7nX+oENokU2+204l7WIkT8l2iUabw0mJjjQ8SlMMh/7YSe0vVxg==} + '@xsai/shared@0.1.0-beta.9': resolution: {integrity: sha512-1QIL042SdDcuZt+tJQmK9VbW62KOHKKC3N9aYNE3ZXu+3dRiYgMUewpW0os7u/H1RPb9BzLLB/5r0cN5G8ijNA==} @@ -5198,6 +5272,10 @@ packages: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -5519,6 +5597,10 @@ packages: resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + body-parser@2.1.0: + resolution: {integrity: sha512-/hPxh61E+ll0Ujp24Ilm64cykicul1ypfwjVttduAiEdtnJFvLePSrIPk+HMImtNv5270wOGCb1Tns2rybMkoQ==} + engines: {node: '>=18'} + boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} @@ -5628,10 +5710,18 @@ packages: resolution: {integrity: sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==} engines: {node: '>=8'} + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + call-bind@1.0.7: resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==} engines: {node: '>= 0.4'} + call-bound@1.0.3: + resolution: {integrity: sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==} + engines: {node: '>= 0.4'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -5911,6 +6001,10 @@ packages: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} + content-disposition@1.0.0: + resolution: {integrity: sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg==} + engines: {node: '>= 0.6'} + content-type@1.0.5: resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} engines: {node: '>= 0.6'} @@ -5927,6 +6021,10 @@ packages: cookie-signature@1.0.6: resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + cookie@0.7.1: resolution: {integrity: sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==} engines: {node: '>= 0.6'} @@ -6202,6 +6300,15 @@ packages: supports-color: optional: true + debug@4.3.6: + resolution: {integrity: sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} engines: {node: '>=6.0'} @@ -6493,6 +6600,10 @@ packages: resolution: {integrity: sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==} engines: {node: '>=4'} + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} @@ -6512,6 +6623,11 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + ejs@3.1.10: + resolution: {integrity: sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==} + engines: {node: '>=0.10.0'} + hasBin: true + ejs@3.1.9: resolution: {integrity: sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==} engines: {node: '>=0.10.0'} @@ -6640,6 +6756,10 @@ packages: resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==} engines: {node: '>= 0.4'} + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + es-errors@1.3.0: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} @@ -6647,6 +6767,10 @@ packages: es-module-lexer@1.6.0: resolution: {integrity: sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==} + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + es-set-tostringtag@2.0.1: resolution: {integrity: sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==} engines: {node: '>= 0.4'} @@ -6997,6 +7121,14 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + eventsource-parser@3.0.0: + resolution: {integrity: sha512-T1C0XCUimhxVQzW4zFipdx0SficT651NnkR0ZSH3yQwh+mFMdLfgjABVi4YtMTtaL4s168593DaoaRLMqryavA==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.5: + resolution: {integrity: sha512-LT/5J605bx5SNyE+ITBDiM3FxffBiq9un7Vx0EwMDM3vg8sWKx/tO2zC+LMqZ+smAM0F2hblaDZUVZF0te2pSw==} + engines: {node: '>=18.0.0'} + execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -7013,10 +7145,20 @@ packages: resolution: {integrity: sha512-bFi65yM+xZgk+u/KRIpekdSYkTB5W1pEf0Lt8Q8Msh7b+eQ7LXVtIB1Bkm4fvclDEL1b2CZkMhv2mOeF8tMdkA==} engines: {node: '>=12.0.0'} + express-rate-limit@7.5.0: + resolution: {integrity: sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==} + engines: {node: '>= 16'} + peerDependencies: + express: ^4.11 || 5 || ^5.0.0-beta.1 + express@4.21.2: resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==} engines: {node: '>= 0.10.0'} + express@5.0.1: + resolution: {integrity: sha512-ORF7g6qGnD+YtUG9yx4DFoqCShNMmUKiXuT5oWMHiOvt/4WFbHC6yCwQMTSBMno7AqntNCAzzcnnjowRkTL9eQ==} + engines: {node: '>= 18'} + expressive-code@0.40.1: resolution: {integrity: sha512-jBsTRX+MPsqiqYQsE9vRXMiAkUafU11j2zuWAaOX9vubLutNB0er8c0FJWeudVDH5D52V4Lf4vTIqbOE54PUcQ==} @@ -7113,6 +7255,10 @@ packages: resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} engines: {node: '>= 0.8'} + finalhandler@2.0.0: + resolution: {integrity: sha512-MX6Zo2adDViYh+GcxxB1dpO43eypOGUOL12rLCOTMQv/DfIbpSJUy4oQIIZhVZkH9e+bZWKMon0XHFEju16tkQ==} + engines: {node: '>= 0.8'} + find-cache-dir@3.3.2: resolution: {integrity: sha512-wXZV5emFEjrridIgED11OoUKLxiYjAcqot/NJdAkOhlJ+vGzwhOAfcG5OX1jP+S0PcjEn8bdMJv+g2jwQ3Onig==} engines: {node: '>=8'} @@ -7214,6 +7360,10 @@ packages: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -7286,12 +7436,20 @@ packages: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + get-own-enumerable-property-symbols@3.0.2: resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} get-port-please@3.1.2: resolution: {integrity: sha512-Gxc29eLs1fbn6LQ4jSU4vXjlwyZhF5HsGuMAa7gqBP4Rw4yxxltyDUuF5MBclFzDTXO+ACchGQoeela4DSfzdQ==} + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + get-stream@5.2.0: resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} engines: {node: '>=8'} @@ -7394,6 +7552,10 @@ packages: gopd@1.0.1: resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + got@11.8.6: resolution: {integrity: sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==} engines: {node: '>=10.19.0'} @@ -7444,6 +7606,10 @@ packages: resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} engines: {node: '>= 0.4'} + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + has-tostringtag@1.0.0: resolution: {integrity: sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==} engines: {node: '>= 0.4'} @@ -7462,6 +7628,10 @@ packages: resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} engines: {node: '>= 0.4'} + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + hast-util-embedded@3.0.0: resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} @@ -7630,6 +7800,10 @@ packages: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} + iconv-lite@0.5.2: + resolution: {integrity: sha512-kERHXvpSaB4aU3eANwidg79K8FlrN77m8G9V+0vOR3HYaRifrlwMEpT7ZBJqLSEIHnEgJTHcWK82wwLwwKwtag==} + engines: {node: '>=0.10.0'} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -7824,6 +7998,9 @@ packages: is-potential-custom-element-name@1.0.1: resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + is-promise@4.0.0: + resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + is-reference@1.2.1: resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==} @@ -8308,6 +8485,10 @@ packages: resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} engines: {node: '>=10'} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + mdast-util-definitions@6.0.0: resolution: {integrity: sha512-scTllyX6pnYNZH/AIp/0ePz6s4cZtARxImwoPJ7kS42n+MnVsI4XbnG6d4ibehRIldYMWM2LD7ImQblVhUejVQ==} @@ -8378,9 +8559,17 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + engines: {node: '>= 0.8'} + merge-descriptors@1.0.3: resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + merge-stream@2.0.0: resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} @@ -8511,10 +8700,18 @@ packages: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} + mime-db@1.53.0: + resolution: {integrity: sha512-oHlN/w+3MQ3rba9rqFr6V/ypF10LSkdwUysQL7GkXoTgIWeV+tcXGA852TBxH+gsh8UWoyhR1hKcoMJTuWflpg==} + engines: {node: '>= 0.6'} + mime-types@2.1.35: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + mime-types@3.0.0: + resolution: {integrity: sha512-XqoSHeCGjVClAmoGFG3lVFqQFRIrTVw2OH3axRqAcfaw+gHWIfnASS92AV+Rl/mk0MupgZTRHQOjxY6YVnzK5w==} + engines: {node: '>= 0.6'} + mime@1.6.0: resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} engines: {node: '>=4'} @@ -8681,6 +8878,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -8735,6 +8935,10 @@ packages: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + neotraverse@0.6.18: resolution: {integrity: sha512-Z4SmBUweYa09+o6pG+eASabEpP6QkQ70yHj351pQoEXIs8uHbaU2DWVmzBANKgflPa47A50PtB2+NgRpQvr7vA==} engines: {node: '>= 10'} @@ -9108,6 +9312,10 @@ packages: path-to-regexp@6.3.0: resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} + path-to-regexp@8.2.0: + resolution: {integrity: sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==} + engines: {node: '>=16'} + path-type@5.0.0: resolution: {integrity: sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==} engines: {node: '>=12'} @@ -9224,6 +9432,10 @@ packages: '@pixi/sprite': ^6 '@pixi/utils': ^6 + pkce-challenge@4.1.0: + resolution: {integrity: sha512-ZBmhE1C9LcPoH9XZSdwiPtbPHZROwAnMy+kIFQVrnMCxY4Cudlz3gBOpzilgc0jOgRaiT3sIWfpMomW2ar2orQ==} + engines: {node: '>=16.20.0'} + pkg-dir@4.2.0: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} @@ -9663,6 +9875,10 @@ packages: resolution: {integrity: sha512-EJPeIn0CYrGu+hli1xilKAPXODtJ12T0sP63Ijx2/khC2JtuaN3JyNIpvmnkmaEtha9ocbG4A4cMcr+TvqvwQg==} engines: {node: '>=0.6'} + qs@6.14.0: + resolution: {integrity: sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w==} + engines: {node: '>=0.6'} + quansync@0.2.6: resolution: {integrity: sha512-u3TuxVTuJtkTxKGk5oZ7K2/o+l0/cC6J8SOyaaSnrnroqvcVy7xBxtvBUyd+Xa8cGoCr87XmQj4NR6W+zbqH8w==} @@ -9702,6 +9918,10 @@ packages: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} + raw-body@3.0.0: + resolution: {integrity: sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==} + engines: {node: '>= 0.8'} + rc9@2.1.2: resolution: {integrity: sha512-btXCnMmRIBINM2LDZoEmOogIZU7Qe7zn4BpomSKZ/ykbLObuBdvG+mFq11DL6fjH1DRwHhrlgtYWG96bJiC7Cg==} @@ -9977,6 +10197,10 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + router@2.1.0: + resolution: {integrity: sha512-/m/NSLxeYEgWNtyC+WtNHCF7jbGxOibVWKnn+1Psff4dJGOfoXP+MuC/f2CwSmyiHdOIzYnYFp4W6GxWfekaLA==} + engines: {node: '>= 18'} + rrweb-cssom@0.7.1: resolution: {integrity: sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==} @@ -10050,6 +10274,10 @@ packages: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} + send@1.1.0: + resolution: {integrity: sha512-v67WcEouB5GxbTWL/4NeToqcZiAWEq90N888fczVArY8A79J0L4FD7vj5hm3eUMua5EpoQ59wa/oovY6TLvRUA==} + engines: {node: '>= 18'} + serialize-error@7.0.1: resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} engines: {node: '>=10'} @@ -10061,6 +10289,10 @@ packages: resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} engines: {node: '>= 0.8.0'} + serve-static@2.1.0: + resolution: {integrity: sha512-A3We5UfEjG8Z7VkDv6uItWw6HY2bBSBJT1KtVESn6EOoOr2jAxNhxWCLY3jDE2WcuHXByWju74ck3ZgLwL8xmA==} + engines: {node: '>= 18'} + set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} @@ -10100,10 +10332,26 @@ packages: shiki@3.1.0: resolution: {integrity: sha512-LdTNyWQlC5zdCaHdcp1zPA1OVA2ivb+KjGOOnGcy02tGaF5ja+dGibWFH7Ar8YlngUgK/scDqworK18Ys9cbYA==} + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + side-channel@1.0.6: resolution: {integrity: sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==} engines: {node: '>= 0.4'} + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} @@ -10658,6 +10906,10 @@ packages: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} + type-is@2.0.0: + resolution: {integrity: sha512-gd0sGezQYCbWSbkZr75mln4YBidWUN60+devscpLF5mtRDUpiaTvKpBNrdaCvel1NdR2k6vclXybU5fBd2i+nw==} + engines: {node: '>= 0.6'} + type@2.7.3: resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} @@ -11526,8 +11778,8 @@ packages: resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} engines: {node: '>=0.4.0'} - xsschema@0.1.0-beta.9: - resolution: {integrity: sha512-x8NFpT6DvT3al+WrdEvK6sOI1igbtYi7Kx11rCT4iuOfbH+q/l57nFYkaFcfiO6cHhPuCMzCIQ3L7pICHbn++Q==} + xsschema@0.1.0: + resolution: {integrity: sha512-AZY7cxEBJHq4YioVsRQnvcAdUQ5it6ZoqEUaY4TqcuKtY5VEsdx4pX0/XFiMyx0eWdpfGt3TQCPiV9TaQM3EAw==} peerDependencies: '@valibot/to-json-schema': ^1.0.0-rc.0 arktype: ^2.0.4 @@ -13709,6 +13961,20 @@ snapshots: - acorn - supports-color + '@modelcontextprotocol/sdk@1.6.1': + dependencies: + content-type: 1.0.5 + cors: 2.8.5 + eventsource: 3.0.5 + express: 5.0.1 + express-rate-limit: 7.5.0(express@5.0.1) + pkce-challenge: 4.1.0 + raw-body: 3.0.0 + zod: 3.24.2 + zod-to-json-schema: 3.24.3(zod@3.24.2) + transitivePeerDependencies: + - supports-color + '@mswjs/interceptors@0.37.6': dependencies: '@open-draft/deferred-promise': 2.2.0 @@ -14316,12 +14582,22 @@ snapshots: '@polka/url@1.0.0-next.24': {} + '@proj-airi/server-sdk@0.1.4': + dependencies: + '@proj-airi/server-shared': 0.1.4 + crossws: 0.3.4 + defu: 6.1.4 + '@proj-airi/server-sdk@0.3.6': dependencies: '@proj-airi/server-shared': 0.3.6 crossws: 0.3.4 defu: 6.1.4 + '@proj-airi/server-shared@0.1.4': + dependencies: + crossws: 0.3.4 + '@proj-airi/server-shared@0.3.6': dependencies: crossws: 0.3.4 @@ -14357,12 +14633,14 @@ snapshots: optionalDependencies: rollup: 4.34.9 - '@rollup/plugin-babel@5.3.1(@babel/core@7.26.0)(rollup@2.79.1)': + '@rollup/plugin-babel@5.3.1(@babel/core@7.26.0)(@types/babel__core@7.20.5)(rollup@2.79.1)': dependencies: '@babel/core': 7.26.0 '@babel/helper-module-imports': 7.25.9 '@rollup/pluginutils': 3.1.0(rollup@2.79.1) rollup: 2.79.1 + optionalDependencies: + '@types/babel__core': 7.20.5 transitivePeerDependencies: - supports-color @@ -14614,7 +14892,7 @@ snapshots: '@surma/rollup-plugin-off-main-thread@2.2.3': dependencies: - ejs: 3.1.9 + ejs: 3.1.10 json5: 2.2.3 magic-string: 0.25.9 string.prototype.matchall: 4.0.8 @@ -14695,6 +14973,31 @@ snapshots: '@types/audioworklet@0.0.71': {} + '@types/babel__core@7.20.5': + dependencies: + '@babel/parser': 7.26.7 + '@babel/types': 7.26.7 + '@types/babel__generator': 7.6.8 + '@types/babel__template': 7.4.4 + '@types/babel__traverse': 7.20.6 + optional: true + + '@types/babel__generator@7.6.8': + dependencies: + '@babel/types': 7.26.7 + optional: true + + '@types/babel__template@7.4.4': + dependencies: + '@babel/parser': 7.26.7 + '@babel/types': 7.26.7 + optional: true + + '@types/babel__traverse@7.20.6': + dependencies: + '@babel/types': 7.26.7 + optional: true + '@types/cacheable-request@6.0.3': dependencies: '@types/http-cache-semantics': 4.0.4 @@ -15011,7 +15314,7 @@ snapshots: dependencies: '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) optionalDependencies: - '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) + '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) '@typeschema/zod': 0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.3(zod@3.24.2))(zod@3.24.2) transitivePeerDependencies: - '@types/json-schema' @@ -15941,47 +16244,53 @@ snapshots: '@xsai-ext/shared-providers@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 '@xsai/embed@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 '@xsai/generate-speech@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 '@xsai/generate-text@0.1.0-beta.9': dependencies: - '@xsai/shared-chat': 0.1.0-beta.9 + '@xsai/shared-chat': 0.1.0 '@xsai/generate-transcription@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 '@xsai/model@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 '@xsai/providers@0.1.0-beta.5': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 + + '@xsai/shared-chat@0.1.0': + dependencies: + '@xsai/shared': 0.1.0 '@xsai/shared-chat@0.1.0-beta.9': dependencies: - '@xsai/shared': 0.1.0-beta.9 + '@xsai/shared': 0.1.0 + + '@xsai/shared@0.1.0': {} '@xsai/shared@0.1.0-beta.9': {} '@xsai/stream-text@0.1.0-beta.9': dependencies: - '@xsai/shared-chat': 0.1.0-beta.9 + '@xsai/shared-chat': 0.1.0 '@xsai/tool@0.1.0-beta.9(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)))(zod-to-json-schema@3.24.3(zod@3.24.2))': dependencies: - '@xsai/shared': 0.1.0-beta.9 - '@xsai/shared-chat': 0.1.0-beta.9 - xsschema: 0.1.0-beta.9(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)))(zod-to-json-schema@3.24.3(zod@3.24.2)) + '@xsai/shared': 0.1.0 + '@xsai/shared-chat': 0.1.0 + xsschema: 0.1.0(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)))(zod-to-json-schema@3.24.3(zod@3.24.2)) transitivePeerDependencies: - '@valibot/to-json-schema' - arktype @@ -15989,7 +16298,7 @@ snapshots: '@xsai/utils-chat@0.1.0-beta.9': dependencies: - '@xsai/shared-chat': 0.1.0-beta.9 + '@xsai/shared-chat': 0.1.0 abbrev@1.1.1: optional: true @@ -16003,6 +16312,11 @@ snapshots: mime-types: 2.1.35 negotiator: 0.6.3 + accepts@2.0.0: + dependencies: + mime-types: 3.0.0 + negotiator: 1.0.0 + acorn-jsx@5.3.2(acorn@8.14.0): dependencies: acorn: 8.14.0 @@ -16235,7 +16549,7 @@ snapshots: call-bind: 1.0.7 define-properties: 1.2.0 es-abstract: 1.22.2 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 is-array-buffer: 3.0.2 is-shared-array-buffer: 1.0.2 @@ -16491,6 +16805,20 @@ snapshots: transitivePeerDependencies: - supports-color + body-parser@2.1.0: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 4.4.0 + http-errors: 2.0.0 + iconv-lite: 0.5.2 + on-finished: 2.4.1 + qs: 6.14.0 + raw-body: 3.0.0 + type-is: 2.0.0 + transitivePeerDependencies: + - supports-color + boolbase@1.0.0: {} boolean@3.2.0: @@ -16666,6 +16994,11 @@ snapshots: normalize-url: 6.1.0 responselike: 2.0.1 + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + call-bind@1.0.7: dependencies: es-define-property: 1.0.0 @@ -16674,6 +17007,11 @@ snapshots: get-intrinsic: 1.2.4 set-function-length: 1.2.2 + call-bound@1.0.3: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + callsites@3.1.0: {} camel-case@4.1.2: @@ -16984,6 +17322,10 @@ snapshots: dependencies: safe-buffer: 5.2.1 + content-disposition@1.0.0: + dependencies: + safe-buffer: 5.2.1 + content-type@1.0.5: {} convert-gitmoji@0.1.5: {} @@ -16994,6 +17336,8 @@ snapshots: cookie-signature@1.0.6: {} + cookie-signature@1.2.2: {} + cookie@0.7.1: {} cookie@0.7.2: {} @@ -17312,6 +17656,10 @@ snapshots: dependencies: ms: 2.1.3 + debug@4.3.6: + dependencies: + ms: 2.1.2 + debug@4.3.7: dependencies: ms: 2.1.3 @@ -17522,6 +17870,12 @@ snapshots: dset@3.1.4: {} + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + duplexer@0.1.2: {} earcut@2.2.4: {} @@ -17541,6 +17895,10 @@ snapshots: ee-first@1.1.1: {} + ejs@3.1.10: + dependencies: + jake: 10.8.6 + ejs@3.1.9: dependencies: jake: 10.8.6 @@ -17715,14 +18073,14 @@ snapshots: es-set-tostringtag: 2.0.1 es-to-primitive: 1.2.1 function.prototype.name: 1.1.6 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 get-symbol-description: 1.0.0 globalthis: 1.0.3 - gopd: 1.0.1 + gopd: 1.2.0 has: 1.0.3 has-property-descriptors: 1.0.2 has-proto: 1.0.1 - has-symbols: 1.0.3 + has-symbols: 1.1.0 internal-slot: 1.0.5 is-array-buffer: 3.0.2 is-callable: 1.2.7 @@ -17752,13 +18110,19 @@ snapshots: dependencies: get-intrinsic: 1.2.4 + es-define-property@1.0.1: {} + es-errors@1.3.0: {} es-module-lexer@1.6.0: {} + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + es-set-tostringtag@2.0.1: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 has: 1.0.3 has-tostringtag: 1.0.0 @@ -18345,6 +18709,12 @@ snapshots: events@3.3.0: {} + eventsource-parser@3.0.0: {} + + eventsource@3.0.5: + dependencies: + eventsource-parser: 3.0.0 + execa@5.1.1: dependencies: cross-spawn: 7.0.6 @@ -18386,6 +18756,10 @@ snapshots: expect-type@1.1.0: {} + express-rate-limit@7.5.0(express@5.0.1): + dependencies: + express: 5.0.1 + express@4.21.2: dependencies: accepts: 1.3.8 @@ -18422,6 +18796,43 @@ snapshots: transitivePeerDependencies: - supports-color + express@5.0.1: + dependencies: + accepts: 2.0.0 + body-parser: 2.1.0 + content-disposition: 1.0.0 + content-type: 1.0.5 + cookie: 0.7.1 + cookie-signature: 1.2.2 + debug: 4.3.6 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.0.0 + fresh: 2.0.0 + http-errors: 2.0.0 + merge-descriptors: 2.0.0 + methods: 1.1.2 + mime-types: 3.0.0 + on-finished: 2.4.1 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.7 + qs: 6.13.0 + range-parser: 1.2.1 + router: 2.1.0 + safe-buffer: 5.2.1 + send: 1.1.0 + serve-static: 2.1.0 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 2.0.0 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + expressive-code@0.40.1: dependencies: '@expressive-code/core': 0.40.1 @@ -18540,6 +18951,18 @@ snapshots: transitivePeerDependencies: - supports-color + finalhandler@2.0.0: + dependencies: + debug: 2.6.9 + encodeurl: 1.0.2 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + find-cache-dir@3.3.2: dependencies: commondir: 1.0.1 @@ -18624,6 +19047,8 @@ snapshots: fresh@0.5.2: {} + fresh@2.0.0: {} + fs-constants@1.0.0: {} fs-extra@10.1.0: @@ -18712,10 +19137,28 @@ snapshots: has-symbols: 1.0.3 hasown: 2.0.0 + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + get-own-enumerable-property-symbols@3.0.2: {} get-port-please@3.1.2: {} + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + get-stream@5.2.0: dependencies: pump: 3.0.2 @@ -18732,7 +19175,7 @@ snapshots: get-symbol-description@1.0.0: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 get-tsconfig@4.8.1: dependencies: @@ -18852,6 +19295,8 @@ snapshots: dependencies: get-intrinsic: 1.2.4 + gopd@1.2.0: {} + got@11.8.6: dependencies: '@sindresorhus/is': 4.6.0 @@ -18919,9 +19364,11 @@ snapshots: has-symbols@1.0.3: {} + has-symbols@1.1.0: {} + has-tostringtag@1.0.0: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 has-unicode@2.0.1: optional: true @@ -18937,6 +19384,10 @@ snapshots: dependencies: function-bind: 1.1.2 + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + hast-util-embedded@3.0.0: dependencies: '@types/hast': 3.0.4 @@ -19248,6 +19699,10 @@ snapshots: dependencies: safer-buffer: 2.1.2 + iconv-lite@0.5.2: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -19297,9 +19752,9 @@ snapshots: internal-slot@1.0.5: dependencies: - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 has: 1.0.3 - side-channel: 1.0.6 + side-channel: 1.1.0 internmap@2.0.3: {} @@ -19322,7 +19777,7 @@ snapshots: is-array-buffer@3.0.2: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 is-typed-array: 1.1.12 is-arrayish@0.3.2: {} @@ -19405,6 +19860,8 @@ snapshots: is-potential-custom-element-name@1.0.1: optional: true + is-promise@4.0.0: {} + is-reference@1.2.1: dependencies: '@types/estree': 1.0.6 @@ -19436,7 +19893,7 @@ snapshots: is-symbol@1.0.4: dependencies: - has-symbols: 1.0.3 + has-symbols: 1.1.0 is-typed-array@1.1.12: dependencies: @@ -19922,6 +20379,8 @@ snapshots: escape-string-regexp: 4.0.0 optional: true + math-intrinsics@1.1.0: {} + mdast-util-definitions@6.0.0: dependencies: '@types/mdast': 4.0.4 @@ -20115,8 +20574,12 @@ snapshots: media-typer@0.3.0: {} + media-typer@1.1.0: {} + merge-descriptors@1.0.3: {} + merge-descriptors@2.0.0: {} + merge-stream@2.0.0: {} merge2@1.4.1: {} @@ -20408,10 +20871,16 @@ snapshots: mime-db@1.52.0: {} + mime-db@1.53.0: {} + mime-types@2.1.35: dependencies: mime-db: 1.52.0 + mime-types@3.0.0: + dependencies: + mime-db: 1.53.0 + mime@1.6.0: {} mime@2.6.0: {} @@ -20647,6 +21116,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} msw@2.7.3(@types/node@22.13.8)(typescript@5.8.2): @@ -20703,6 +21174,8 @@ snapshots: negotiator@0.6.4: {} + negotiator@1.0.0: {} + neotraverse@0.6.18: {} neuri@0.0.22(@types/json-schema@7.0.15)(@typeschema/valibot@0.14.0(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)))(@typeschema/zod@0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.3(zod@3.24.2))(zod@3.24.2))(encoding@0.1.13)(vitest@3.0.7)(ws@8.18.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod-to-json-schema@3.24.3(zod@3.24.2))(zod@3.24.2): @@ -20872,7 +21345,7 @@ snapshots: dependencies: call-bind: 1.0.7 define-properties: 1.2.0 - has-symbols: 1.0.3 + has-symbols: 1.1.0 object-keys: 1.1.1 obuf@1.1.2: {} @@ -21148,6 +21621,8 @@ snapshots: path-to-regexp@6.3.0: {} + path-to-regexp@8.2.0: {} + path-type@5.0.0: optional: true @@ -21286,6 +21761,8 @@ snapshots: '@pixi/utils': 6.5.10(@pixi/constants@6.5.10)(@pixi/settings@6.5.10(@pixi/constants@6.5.10)) gh-pages: 4.0.0 + pkce-challenge@4.1.0: {} + pkg-dir@4.2.0: dependencies: find-up: 4.1.0 @@ -21787,6 +22264,10 @@ snapshots: dependencies: side-channel: 1.0.6 + qs@6.14.0: + dependencies: + side-channel: 1.1.0 + quansync@0.2.6: {} querystringify@2.2.0: {} @@ -21834,6 +22315,13 @@ snapshots: iconv-lite: 0.4.24 unpipe: 1.0.0 + raw-body@3.0.0: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.6.3 + unpipe: 1.0.0 + rc9@2.1.2: dependencies: defu: 6.1.4 @@ -22261,6 +22749,12 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.34.9 fsevents: 2.3.3 + router@2.1.0: + dependencies: + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.2.0 + rrweb-cssom@0.7.1: optional: true @@ -22280,8 +22774,8 @@ snapshots: safe-array-concat@1.0.1: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 - has-symbols: 1.0.3 + get-intrinsic: 1.3.0 + has-symbols: 1.1.0 isarray: 2.0.5 safe-buffer@5.1.2: {} @@ -22291,7 +22785,7 @@ snapshots: safe-regex-test@1.0.0: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 is-regex: 1.1.4 safer-buffer@2.1.2: {} @@ -22348,6 +22842,23 @@ snapshots: transitivePeerDependencies: - supports-color + send@1.1.0: + dependencies: + debug: 4.4.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime-types: 2.1.35 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + serialize-error@7.0.1: dependencies: type-fest: 0.13.1 @@ -22366,6 +22877,15 @@ snapshots: transitivePeerDependencies: - supports-color + serve-static@2.1.0: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.1.0 + transitivePeerDependencies: + - supports-color + set-blocking@2.0.0: optional: true @@ -22444,6 +22964,26 @@ snapshots: '@shikijs/vscode-textmate': 10.0.2 '@types/hast': 3.0.4 + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.3 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.3 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.3 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.3 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.3 + side-channel-map: 1.0.1 + side-channel@1.0.6: dependencies: call-bind: 1.0.7 @@ -22451,6 +22991,14 @@ snapshots: get-intrinsic: 1.2.4 object-inspect: 1.13.3 + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.3 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + siginfo@2.0.0: {} signal-exit@3.0.7: {} @@ -22659,11 +23207,11 @@ snapshots: call-bind: 1.0.7 define-properties: 1.2.0 es-abstract: 1.22.2 - get-intrinsic: 1.2.4 - has-symbols: 1.0.3 + get-intrinsic: 1.3.0 + has-symbols: 1.1.0 internal-slot: 1.0.5 regexp.prototype.flags: 1.5.1 - side-channel: 1.0.6 + side-channel: 1.1.0 string.prototype.trim@1.2.8: dependencies: @@ -23064,12 +23612,18 @@ snapshots: media-typer: 0.3.0 mime-types: 2.1.35 + type-is@2.0.0: + dependencies: + content-type: 1.0.5 + media-typer: 1.1.0 + mime-types: 3.0.0 + type@2.7.3: {} typed-array-buffer@1.0.0: dependencies: call-bind: 1.0.7 - get-intrinsic: 1.2.4 + get-intrinsic: 1.3.0 is-typed-array: 1.1.12 typed-array-byte-length@1.0.0: @@ -23128,7 +23682,7 @@ snapshots: dependencies: call-bind: 1.0.7 has-bigints: 1.0.2 - has-symbols: 1.0.3 + has-symbols: 1.1.0 which-boxed-primitive: 1.0.2 unbuild@3.0.0-rc.11(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2)): @@ -23402,17 +23956,17 @@ snapshots: '@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.34.9) '@vueuse/core': 12.7.0(typescript@5.8.2) - unplugin-combine@1.2.0(esbuild@0.19.12)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: esbuild: 0.19.12 - rollup: 2.79.1 + rollup: 4.34.9 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) - unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: esbuild: 0.25.0 - rollup: 4.34.9 + rollup: 2.79.1 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) @@ -23463,7 +24017,7 @@ snapshots: transitivePeerDependencies: - vue - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -23493,9 +24047,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.19.12)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: @@ -23509,7 +24063,7 @@ snapshots: - vue-tsc - webpack - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -23539,9 +24093,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: @@ -23850,13 +24404,13 @@ snapshots: transitivePeerDependencies: - supports-color - vite-plugin-pwa@0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0)(workbox-window@7.3.0): + vite-plugin-pwa@0.21.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(workbox-build@7.3.0(@types/babel__core@7.20.5))(workbox-window@7.3.0): dependencies: debug: 4.4.0 pretty-bytes: 6.1.1 tinyglobby: 0.2.12 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) - workbox-build: 7.3.0 + workbox-build: 7.3.0(@types/babel__core@7.20.5) workbox-window: 7.3.0 transitivePeerDependencies: - supports-color @@ -24103,7 +24657,7 @@ snapshots: available-typed-arrays: 1.0.5 call-bind: 1.0.7 for-each: 0.3.3 - gopd: 1.0.1 + gopd: 1.2.0 has-tostringtag: 1.0.0 which@2.0.2: @@ -24139,13 +24693,13 @@ snapshots: dependencies: workbox-core: 7.3.0 - workbox-build@7.3.0: + workbox-build@7.3.0(@types/babel__core@7.20.5): dependencies: '@apideck/better-ajv-errors': 0.3.6(ajv@8.12.0) '@babel/core': 7.26.0 '@babel/preset-env': 7.26.0(@babel/core@7.26.0) '@babel/runtime': 7.26.7 - '@rollup/plugin-babel': 5.3.1(@babel/core@7.26.0)(rollup@2.79.1) + '@rollup/plugin-babel': 5.3.1(@babel/core@7.26.0)(@types/babel__core@7.20.5)(rollup@2.79.1) '@rollup/plugin-node-resolve': 15.3.0(rollup@2.79.1) '@rollup/plugin-replace': 2.4.2(rollup@2.79.1) '@rollup/plugin-terser': 0.4.4(rollup@2.79.1) @@ -24297,7 +24851,7 @@ snapshots: xmlhttprequest-ssl@2.1.2: {} - xsschema@0.1.0-beta.9(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)))(zod-to-json-schema@3.24.3(zod@3.24.2)): + xsschema@0.1.0(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)))(zod-to-json-schema@3.24.3(zod@3.24.2)): optionalDependencies: '@valibot/to-json-schema': 1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.2)) zod-to-json-schema: 3.24.3(zod@3.24.2) diff --git a/services/twitter-services/.env.example b/services/twitter-services/.env.example new file mode 100644 index 00000000..99f77fdf --- /dev/null +++ b/services/twitter-services/.env.example @@ -0,0 +1,19 @@ +# BrowserBase 配置 +BROWSERBASE_API_KEY=your_api_key_here + +# Twitter 账号配置 +TWITTER_USERNAME=your_twitter_username +TWITTER_PASSWORD=your_twitter_password + +# 适配器配置 +ENABLE_AIRI=false +AIRI_URL=http://localhost:3000 +AIRI_TOKEN=your_airi_token + +ENABLE_MCP=true +MCP_PORT=8080 + +# 系统配置 +LOG_LEVEL=info # 可选: error, warn, info, verbose, debug +LOG_FORMAT=pretty # 可选: json, pretty +CONCURRENCY=1 diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md new file mode 100644 index 00000000..8aa2de4c --- /dev/null +++ b/services/twitter-services/docs/architecture.md @@ -0,0 +1,297 @@ +# Twitter 服务架构文档 + +## 1. 项目概述 + +Twitter 服务是一个基于 BrowserBase 的 Web 自动化服务，提供结构化的 Twitter 数据访问和交互能力。它采用分层架构设计，支持多种适配器以便与不同的应用集成。 + +## 2. 设计目标 + +- **可靠性**：稳定处理 Twitter 页面的变化和限制 +- **可扩展性**：易于添加新功能和支持不同接入方式 +- **性能优化**：智能管理请求频率和浏览器会话 +- **数据结构化**：提供规范、类型化的数据模型 + +## 3. 架构总览 + +``` +┌─────────────────────────────────────────────┐ +│ 应用层/消费者层 │ +│ │ +│ ┌────────────┐ ┌─────────────┐ │ +│ │ │ │ │ │ +│ │ Airi Core │ │ 其他 LLM 应用 │ │ +│ │ │ │ │ │ +│ └──────┬─────┘ └──────┬──────┘ │ +└──────────┼─────────────────────┼────────────┘ + │ │ +┌──────────▼─────────────────────▼────────────┐ +│ 适配器层 │ +│ │ +│ ┌────────────┐ ┌─────────────┐ │ +│ │Airi Adapter│ │ MCP Adapter │ │ +│ │(@server-sdk)│ │ (HTTP/JSON) │ │ +│ └──────┬─────┘ └──────┬──────┘ │ +└──────────┼─────────────────────┼────────────┘ + │ │ +┌──────────▼─────────────────────▼────────────┐ +│ 核心服务层 │ +│ │ +│ ┌──────────────────────────────────┐ │ +│ │ Twitter Services │ │ +│ │ │ │ +│ │ ┌────────┐ ┌────────────┐ │ │ +│ │ │ Auth │ │ Timeline │ │ │ +│ │ │ Service│ │ Service │ │ │ +│ │ └────────┘ └────────────┘ │ │ +│ │ │ │ +│ └──────────────────┬────────────────┘ │ +└──────────────────────┼────────────────────────┘ + │ + ┌───────────▼────────────┐ + │ 浏览器适配层 │ + │ (BrowserAdapter) │ + └───────────┬────────────┘ + │ + ┌───────────▼────────────┐ + │ BrowserBase API │ + └──────────────────────────┘ +``` + +## 4. 技术栈与依赖 + +- **核心库**: TypeScript, Node.js +- **浏览器自动化**: BrowserBase API +- **HTML解析**: unified, rehype-parse, unist-util-visit +- **API服务器**: H3.js, listhen +- **适配器**: Airi Server SDK, MCP SDK +- **日志系统**: @guiiai/logg +- **工具库**: zod（类型验证） + +## 5. 关键组件 + +### 5.1 适配器层 + +#### 5.1.1 Airi 适配器 + +提供与 Airi LLM 平台的集成，处理事件驱动的通信。 + +#### 5.1.2 MCP 适配器 + +实现 Model Context Protocol 接口，提供基于 HTTP 的通信。现使用官方 MCP SDK 实现，通过 H3.js 提供高性能 HTTP 服务器和 SSE 通信。 + +#### 5.1.3 开发服务器 + +使用 listhen 提供优化的开发体验，包括自动打开浏览器、实时日志和调试工具。 + +### 5.2 核心服务层 + +#### 5.2.1 认证服务 (Auth Service) + +处理 Twitter 登录和会话维护。 + +#### 5.2.2 时间线服务 (Timeline Service) + +获取和处理 Twitter 时间线内容。 + +#### 5.2.3 其他服务 + +包括搜索服务、互动服务、用户资料服务等（部分未在 MVP 中实现）。 + +### 5.3 解析器和工具 + +#### 5.3.1 Tweet 解析器 + +从 HTML 中提取推文结构化数据。 + +#### 5.3.2 频率限制器 + +控制请求频率，避免触发 Twitter 的限制。 + +## 6. 数据流 + +1. **请求流**：应用层 → 适配器 → 核心服务 → 浏览器适配层 → BrowserBase API → Twitter +2. **响应流**：Twitter → BrowserBase API → 浏览器适配层 → 核心服务 → 数据解析 → 适配器 → 应用层 + +## 7. 配置系统 + +配置分为以下几个主要部分： + +```typescript +interface Config { + // BrowserBase 配置 + browserbase: { + apiKey: string + endpoint?: string + } + + // 浏览器配置 + browser: BrowserConfig + + // Twitter 配置 + twitter: { + credentials?: TwitterCredentials + defaultOptions?: { + timeline?: TimelineOptions + search?: SearchOptions + } + } + + // 适配器配置 + adapters: { + airi?: { + url?: string + token?: string + enabled: boolean + } + mcp?: { + port?: number + enabled: boolean + } + } + + // 系统配置 + system: { + logLevel: string + concurrency: number + } +} +``` + +## 8. 开发与测试 + +### 8.1 开发环境设置 + +```bash +# 安装依赖 +npm install + +# 设置环境变量 +cp .env.example .env +# 编辑 .env 添加 BrowserBase API 密钥和 Twitter 凭据 + +# 开发模式启动 +npm run dev # 标准模式 +npm run dev:mcp # MCP 开发服务器模式 +``` + +### 8.2 测试策略 + +- **单元测试**：测试解析器、工具类和业务逻辑 +- **集成测试**：测试服务和适配器的交互 +- **端到端测试**：模拟完整的使用场景 + +## 9. 集成示例 + +### 9.1 从其他 Node.js 应用集成 + +```typescript +import { BrowserBaseMCPAdapter, TwitterService } from 'twitter-services' + +async function main() { + // 初始化浏览器 + const browser = new BrowserBaseMCPAdapter('your-api-key') + await browser.initialize({ headless: true }) + + // 创建 Twitter 服务 + const twitter = new TwitterService(browser) + + // 登录 + await twitter.login({ + username: 'your-username', + password: 'your-password' + }) + + // 获取时间线 + const tweets = await twitter.getTimeline({ count: 10 }) + console.log(tweets) + + // 释放资源 + await browser.close() +} +``` + +### 9.2 作为 Airi 模块集成 + +```typescript +import { AiriAdapter, BrowserBaseMCPAdapter, TwitterService } from 'twitter-services' + +async function startAiriModule() { + const browser = new BrowserBaseMCPAdapter(process.env.BROWSERBASE_API_KEY) + await browser.initialize({ headless: true }) + + const twitter = new TwitterService(browser) + + // 创建 Airi 适配器 + const airiAdapter = new AiriAdapter(twitter, { + url: process.env.AIRI_URL, + token: process.env.AIRI_TOKEN, + credentials: { + username: process.env.TWITTER_USERNAME, + password: process.env.TWITTER_PASSWORD + } + }) + + // 启动适配器 + await airiAdapter.start() + + console.log('Twitter service running as Airi module') +} +``` + +### 9.3 使用 MCP 进行集成 + +```typescript +// 使用 MCP SDK 与 Twitter 服务交互 +import { McpClient } from '@modelcontextprotocol/sdk/client/mcp.js' +import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js' + +async function connectToTwitterService() { + // 创建 SSE 传输 + const transport = new SSEClientTransport('http://localhost:8080/sse', 'http://localhost:8080/messages') + + // 创建客户端 + const client = new McpClient() + await client.connect(transport) + + // 获取时间线 + const timeline = await client.get('twitter://timeline/10') + console.log('Timeline:', timeline.contents) + + // 使用工具发送推文 + const result = await client.useTool('post-tweet', { content: 'Hello from MCP!' }) + console.log('Result:', result.content) + + return client +} +``` + +## 10. 扩展指南 + +### 10.1 添加新功能 + +例如添加"获取特定用户发布的推文"功能： + +1. 在 `src/types/twitter.ts` 中扩展接口 +2. 在 `src/core/twitter-service.ts` 中实现方法 +3. 在适配器中添加对应的处理逻辑 +4. 如果是 MCP 适配器，在 `configureServer()` 中添加相应的资源或工具 + +### 10.2 支持新的适配器 + +1. 创建新的适配器类 +2. 实现与目标系统的通信逻辑 +3. 在入口文件中添加配置支持 + +## 11. 维护建议 + +- **自动化测试**：编写单元测试和集成测试 +- **监控与告警**：监控服务状态和 Twitter 的访问限制 +- **选择器更新**：定期验证和更新选择器配置 +- **会话管理**：优化会话管理以提高稳定性 + +## 12. 项目路线图 + +- MVP 阶段：实现核心功能（认证、浏览时间线） +- 阶段二：完善互动功能（点赞、评论、转发） +- 阶段三：高级功能（搜索、高级过滤、数据分析） +- 阶段四：性能优化和稳定性提升 diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json new file mode 100644 index 00000000..b6d5a210 --- /dev/null +++ b/services/twitter-services/package.json @@ -0,0 +1,48 @@ +{ + "name": "twitter-services", + "version": "0.1.0", + "description": "Twitter 服务 - 提供结构化的 Twitter 数据访问", + "author": "Your Name", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/yourusername/twitter-services.git" + }, + "keywords": [ + "twitter", + "api", + "automation", + "scraping", + "web" + ], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "engines": { + "node": ">=14.0.0" + }, + "scripts": { + "build": "tsc", + "start": "tsx dist/index.js", + "dev": "tsx src/index.ts", + "dev:mcp": "tsx src/dev-server.ts" + }, + "dependencies": { + "@guiiai/logg": "^1.0.0", + "@modelcontextprotocol/sdk": "^1.6.1", + "@proj-airi/server-sdk": "^0.1.0", + "@types/hast": "^3.0.1", + "dotenv": "^16.0.3", + "h3": "^1.11.0", + "listhen": "^1.6.0", + "ofetch": "^1.3.3", + "rehype-parse": "^9.0.0", + "unified": "^11.0.3", + "unist-util-visit": "^5.0.0", + "zod": "^3.22.2" + }, + "devDependencies": { + "@types/node": "^18.16.3", + "tsx": "^4.19.0", + "typescript": "^5.0.4" + } +} diff --git a/services/twitter-services/src/adapters/airi-adapter.ts b/services/twitter-services/src/adapters/airi-adapter.ts new file mode 100644 index 00000000..45d1733f --- /dev/null +++ b/services/twitter-services/src/adapters/airi-adapter.ts @@ -0,0 +1,93 @@ +import type { TimelineOptions, TwitterCredentials, TwitterService } from '../types/twitter' + +import { Client } from '@proj-airi/server-sdk' + +/** + * Airi 适配器 + * 将 Twitter 服务适配为 Airi 模块 + */ +export class AiriAdapter { + private client: Client + private twitterService: TwitterService + private credentials: TwitterCredentials + + constructor(twitterService: TwitterService, options: { + url?: string + token?: string + credentials: TwitterCredentials + }) { + this.twitterService = twitterService + this.credentials = options.credentials + + this.client = new Client({ + url: options.url, + name: 'twitter-module', + token: options.token, + possibleEvents: [ + // 定义此模块可以处理的事件类型 + 'twitter:login', + 'twitter:getTimeline', + 'twitter:getTweetDetails', + 'twitter:searchTweets', + 'twitter:getUserProfile', + 'twitter:followUser', + 'twitter:likeTweet', + 'twitter:retweet', + 'twitter:postTweet', + ], + }) + + this.setupEventHandlers() + } + + /** + * 设置事件处理器 + */ + private setupEventHandlers(): void { + // 登录处理 + this.client.onEvent('twitter:login', async (event) => { + try { + const credentials = event.data.credentials as TwitterCredentials || this.credentials + const success = await this.twitterService.login(credentials) + this.client.send({ + type: 'twitter:loginResult', + data: { success }, + }) + } + catch (error) { + this.client.send({ + type: 'twitter:error', + data: { error: error.message, operation: 'login' }, + }) + } + }) + + // 获取时间线处理 + this.client.onEvent('twitter:getTimeline', async (event) => { + try { + const options = event.data.options as TimelineOptions || {} + const tweets = await this.twitterService.getTimeline(options) + this.client.send({ + type: 'twitter:timelineResult', + data: { tweets }, + }) + } + catch (error) { + this.client.send({ + type: 'twitter:error', + data: { error: error.message, operation: 'getTimeline' }, + }) + } + }) + + // 其他事件处理... + } + + /** + * 启动适配器 + */ + async start(): Promise { + // 可以在这里添加初始化逻辑 + logger.airi.log('Airi Twitter adapter started') + } +} diff --git a/services/twitter-services/src/adapters/browser-adapter.ts b/services/twitter-services/src/adapters/browser-adapter.ts new file mode 100644 index 00000000..e43a6622 --- /dev/null +++ b/services/twitter-services/src/adapters/browser-adapter.ts @@ -0,0 +1,58 @@ +import type { Buffer } from 'node:buffer' +import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' + +/** + * 浏览器操作的通用接口 + * 定义了与不同浏览器后端交互所需的基本操作 + */ +export interface BrowserAdapter { + /** + * 初始化浏览器会话 + */ + initialize: (config: BrowserConfig) => Promise + + /** + * 导航到指定 URL + */ + navigate: (url: string) => Promise + + /** + * 执行 JavaScript 脚本 + */ + executeScript: (script: string) => Promise + + /** + * 等待元素出现 + */ + waitForSelector: (selector: string, options?: WaitOptions) => Promise + + /** + * 点击元素 + */ + click: (selector: string) => Promise + + /** + * 向输入框输入文本 + */ + type: (selector: string, text: string) => Promise + + /** + * 获取元素文本内容 + */ + getText: (selector: string) => Promise + + /** + * 获取多个元素的句柄 + */ + getElements: (selector: string) => Promise + + /** + * 获取屏幕截图 + */ + getScreenshot: () => Promise + + /** + * 关闭浏览器会话 + */ + close: () => Promise +} diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts new file mode 100644 index 00000000..4ebd3a27 --- /dev/null +++ b/services/twitter-services/src/adapters/browserbase-adapter.ts @@ -0,0 +1,120 @@ +import type { Buffer } from 'node:buffer' +import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' +import type { BrowserAdapter } from './browser-adapter' + +import { BrowserBaseClient } from '../browser/browserbase' +import { errorToMessage } from '../utils/error' +import { logger } from '../utils/logger' + +/** + * BrowserBase 元素句柄实现 + */ +class BrowserBaseElementHandle implements ElementHandle { + private client: BrowserBaseClient + private selector: string + + constructor(client: BrowserBaseClient, selector: string) { + this.client = client + this.selector = selector + } + + async getText(): Promise { + return this.client.executeScript(` + document.querySelector('${this.selector}').textContent.trim() + `) + } + + async getAttribute(name: string): Promise { + return this.client.executeScript(` + document.querySelector('${this.selector}').getAttribute('${name}') + `) + } + + async click(): Promise { + await this.client.click(this.selector) + } + + async type(text: string): Promise { + await this.client.type(this.selector, text) + } +} + +/** + * BrowserBase 适配器实现 + * 将 BrowserBase API 适配为通用浏览器接口 + */ +export class BrowserBaseMCPAdapter implements BrowserAdapter { + private client: BrowserBaseClient + + constructor(apiKey: string, baseUrl?: string, options: Partial = {}) { + this.client = new BrowserBaseClient({ + apiKey, + baseUrl, + ...options, + }) + } + + async initialize(config: BrowserConfig): Promise { + try { + await this.client.createSession({ + headless: config.headless, + userAgent: config.userAgent, + viewport: config.viewport, + proxyUrl: config.proxy, + }) + logger.browser.log('浏览器会话已创建', { headless: config.headless }) + } + catch (error) { + logger.browser.errorWithError('浏览器初始化失败', error) + throw new Error(`无法初始化浏览器: ${errorToMessage(error)}`) + } + } + + async navigate(url: string): Promise { + await this.client.navigate(url) + } + + async executeScript(script: string): Promise { + return this.client.executeScript(script) + } + + async waitForSelector(selector: string, options?: WaitOptions): Promise { + await this.client.waitForSelector(selector, { + timeout: options?.timeout, + }) + } + + async click(selector: string): Promise { + await this.client.click(selector) + } + + async type(selector: string, text: string): Promise { + await this.client.type(selector, text) + } + + async getText(selector: string): Promise { + return this.client.getText(selector) + } + + async getElements(selector: string): Promise { + // 获取所有匹配元素的选择器 + const selectors = await this.executeScript(` + Array.from(document.querySelectorAll('${selector}')).map((el, i) => { + const uniqueId = 'browserbase-' + Date.now() + '-' + i; + el.setAttribute('data-browserbase-id', uniqueId); + return '[data-browserbase-id="' + uniqueId + '"]'; + }) + `) + + // 为每个匹配的元素创建一个 ElementHandle + return selectors.map(selector => new BrowserBaseElementHandle(this.client, selector)) + } + + async getScreenshot(): Promise { + return this.client.getScreenshot() + } + + async close(): Promise { + await this.client.closeSession() + } +} diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts new file mode 100644 index 00000000..9c4f65a5 --- /dev/null +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -0,0 +1,383 @@ +import type { TwitterService } from '../types/twitter' + +import { Buffer } from 'node:buffer' +import { createServer } from 'node:http' +import { McpServer, ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js' +import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js' +import { createApp, createRouter, defineEventHandler, toNodeListener } from 'h3' +import { z } from 'zod' + +import { errorToMessage } from '../utils/error' +import { logger } from '../utils/logger' + +/** + * MCP 协议适配器 + * 使用官方 MCP SDK 将 Twitter 服务适配为 MCP 协议服务 + * 基于 H3.js 实现 HTTP 服务器 + */ +export class MCPAdapter { + private twitterService: TwitterService + private mcpServer: McpServer + private app: ReturnType + private server: ReturnType + private port: number + private activeTransports: SSEServerTransport[] = [] + + constructor(twitterService: TwitterService, port: number = 8080) { + this.twitterService = twitterService + this.port = port + + // 创建 MCP 服务器 + this.mcpServer = new McpServer({ + name: 'Twitter Service', + version: '1.0.0', + }) + + // 创建 H3 应用 + this.app = createApp() + + // 配置资源和工具 + this.configureServer() + + // 设置 H3 路由 + this.setupRoutes() + } + + /** + * 配置 MCP 服务器的资源和工具 + */ + private configureServer(): void { + // 添加时间线资源 + this.mcpServer.resource( + 'timeline', + new ResourceTemplate('twitter://timeline/{count}', { list: 'twitter://timeline' }), + async (uri, { count }) => { + try { + const tweets = await this.twitterService.getTimeline({ + count: count ? Number.parseInt(count) : undefined, + }) + + return { + contents: tweets.map(tweet => ({ + uri: `twitter://tweet/${tweet.id}`, + text: `Tweet by @${tweet.author.username} (${tweet.author.displayName}):\n${tweet.text}`, + })), + } + } + catch (error) { + logger.mcp.error('获取时间线错误:', error) + return { contents: [] } + } + }, + ) + + // 添加推文详情资源 + this.mcpServer.resource( + 'tweet', + new ResourceTemplate('twitter://tweet/{id}', { list: undefined }), + async (uri, { id }) => { + try { + const tweet = await this.twitterService.getTweetDetails(id) + + return { + contents: [{ + uri: uri.href, + text: `Tweet by @${tweet.author.username} (${tweet.author.displayName}):\n${tweet.text}`, + }], + } + } + catch (error) { + logger.mcp.error('获取推文详情错误:', error) + return { contents: [] } + } + }, + ) + + // 添加用户资料资源 + this.mcpServer.resource( + 'profile', + new ResourceTemplate('twitter://user/{username}', { list: undefined }), + async (uri, { username }) => { + try { + const profile = await this.twitterService.getUserProfile(username) + + return { + contents: [{ + uri: uri.href, + text: `Profile for @${profile.username} (${profile.displayName})\n${profile.bio || ''}`, + }], + } + } + catch (error) { + logger.mcp.error('获取用户资料错误:', error) + return { contents: [] } + } + }, + ) + + // 添加登录工具 + this.mcpServer.tool( + 'login', + { + username: z.string(), + password: z.string(), + }, + async ({ username, password }) => { + try { + const success = await this.twitterService.login({ username, password }) + + return { + content: [{ + type: 'text', + text: success ? '成功登录到 Twitter' : '登录失败，请检查凭据', + }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `登录失败: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // 添加发推工具 + this.mcpServer.tool( + 'post-tweet', + { + content: z.string(), + replyTo: z.string().optional(), + media: z.array(z.string()).optional(), + }, + async ({ content, replyTo, media }) => { + try { + const tweetId = await this.twitterService.postTweet(content, { + inReplyTo: replyTo, + media, + }) + + return { + content: [{ + type: 'text', + text: `成功发布推文: ${tweetId}`, + }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `发推失败: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // 添加点赞工具 + this.mcpServer.tool( + 'like-tweet', + { tweetId: z.string() }, + async ({ tweetId }) => { + try { + const success = await this.twitterService.likeTweet(tweetId) + + return { + content: [{ + type: 'text', + text: success ? '成功点赞' : '点赞失败', + }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `点赞失败: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // 添加转发工具 + this.mcpServer.tool( + 'retweet', + { tweetId: z.string() }, + async ({ tweetId }) => { + try { + const success = await this.twitterService.retweet(tweetId) + + return { + content: [{ + type: 'text', + text: success ? '成功转发' : '转发失败', + }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `转发失败: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // 添加搜索工具 + this.mcpServer.tool( + 'search', + { + query: z.string(), + count: z.number().optional(), + filter: z.enum(['latest', 'photos', 'videos', 'top']).optional(), + }, + async ({ query, count, filter }) => { + try { + const results = await this.twitterService.searchTweets(query, { count, filter }) + + return { + content: [{ + type: 'text', + text: `搜索结果: ${results.length} 条推文`, + }], + resources: results.map(tweet => `twitter://tweet/${tweet.id}`), + } + } + catch (error) { + return { + content: [{ type: 'text', text: `搜索失败: ${(error as Error).message}` }], + isError: true, + } + } + }, + ) + } + + /** + * 设置 H3 路由 + */ + private setupRoutes(): void { + const router = createRouter() + + // 设置 CORS + router.use('*', defineEventHandler((event) => { + event.node.res.setHeader('Access-Control-Allow-Origin', '*') + event.node.res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + event.node.res.setHeader('Access-Control-Allow-Headers', 'Content-Type') + + if (event.node.req.method === 'OPTIONS') { + event.node.res.statusCode = 204 + event.node.res.end() + } + })) + + // SSE 端点 + router.get('/sse', defineEventHandler(async (event) => { + const { req, res } = event.node + + res.setHeader('Content-Type', 'text/event-stream') + res.setHeader('Cache-Control', 'no-cache') + res.setHeader('Connection', 'keep-alive') + + // 创建 SSE 传输 + const transport = new SSEServerTransport('/messages', res) + this.activeTransports.push(transport) + + // 客户端断开连接时清理 + req.on('close', () => { + const index = this.activeTransports.indexOf(transport) + if (index !== -1) { + this.activeTransports.splice(index, 1) + } + }) + + // 连接到 MCP 服务器 + await this.mcpServer.connect(transport) + })) + + // 消息端点，接收客户端请求 + router.post('/messages', defineEventHandler(async (event) => { + if (this.activeTransports.length === 0) { + event.node.res.statusCode = 503 + return { error: 'No active SSE connections' } + } + + try { + // 解析请求体 + const body = await readBody(event) + + // 简单处理 - 发送到最近的传输 + // 注意: 生产环境中应该使用会话ID来路由到正确的传输 + const transport = this.activeTransports[this.activeTransports.length - 1] + + // 手动处理 POST 消息，因为 H3 不是 Express 兼容的 + const response = await transport.handleRawMessage(body) + + return response + } + catch (error) { + event.node.res.statusCode = 500 + return { error: errorToMessage(error) } + } + })) + + // 根路径 - 提供服务信息 + router.get('/', defineEventHandler(() => { + return { + name: 'Twitter MCP Service', + version: '1.0.0', + endpoints: { + sse: '/sse', + messages: '/messages', + }, + } + })) + + // 使用路由 + this.app.use(router) + } + + /** + * 启动 MCP 服务器 + */ + start(): Promise { + return new Promise((resolve) => { + // 创建 Node.js HTTP 服务器 + this.server = createServer(toNodeListener(this.app)) + + this.server.listen(this.port, () => { + logger.mcp.withField('port', this.port).log('MCP 服务器已启动') + resolve() + }) + }) + } + + /** + * 停止 MCP 服务器 + */ + stop(): Promise { + return new Promise((resolve, reject) => { + if (!this.server) { + return resolve() + } + + this.server.close((error) => { + if (error) { + reject(error) + } + else { + logger.mcp.log('MCP 服务器已停止') + resolve() + } + }) + }) + } +} + +// h3 工具函数：从 event 读取请求体 +async function readBody(event: any): Promise { + const buffers = [] + for await (const chunk of event.node.req) { + buffers.push(chunk) + } + const data = Buffer.concat(buffers).toString() + return JSON.parse(data) +} diff --git a/services/twitter-services/src/browser/browserbase.ts b/services/twitter-services/src/browser/browserbase.ts new file mode 100644 index 00000000..c071381b --- /dev/null +++ b/services/twitter-services/src/browser/browserbase.ts @@ -0,0 +1,200 @@ +import { Buffer } from 'node:buffer' + +import { createApiClient } from '../utils/api' +import { logger } from '../utils/logger' + +/** + * BrowserBase API 客户端配置选项 + */ +export interface BrowserBaseClientOptions { + apiKey: string + baseUrl?: string + timeout?: number + retries?: number +} + +/** + * BrowserBase API 客户端 + * 封装 BrowserBase 的 REST API + */ +export class BrowserBaseClient { + private sessionId: string | null = null + private api: ReturnType + + constructor(options: BrowserBaseClientOptions) { + const { apiKey, baseUrl = 'https://api.browserbase.com', timeout = 30000, retries = 1 } = options + + // 创建 API 客户端 + this.api = createApiClient(baseUrl, { + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + timeout, + retry: retries, + }) + } + + /** + * 创建浏览器会话 + */ + async createSession(options: { + headless?: boolean + userAgent?: string + viewport?: { width: number, height: number } + }): Promise { + try { + const data = await this.api('/v1/sessions', { + method: 'POST', + body: options, + }) + + this.sessionId = data.sessionId + logger.browser.withField('sessionId', this.sessionId).log('创建浏览器会话成功') + return this.sessionId || '' + } + catch (error) { + logger.browser.errorWithError('创建浏览器会话失败', error) + throw error + } + } + + /** + * 导航到指定URL + */ + async navigate(url: string): Promise { + this.ensureSessionExists() + + await this.api(`/v1/sessions/${this.sessionId}/url`, { + method: 'POST', + body: { url }, + }) + } + + /** + * 执行JavaScript脚本 + */ + async executeScript(script: string): Promise { + this.ensureSessionExists() + + const data = await this.api(`/v1/sessions/${this.sessionId}/execute`, { + method: 'POST', + body: { script }, + }) + + return data.result + } + + /** + * 获取页面内容 + */ + async getContent(): Promise { + return this.executeScript('document.documentElement.outerHTML') + } + + /** + * 等待元素出现 + */ + async waitForSelector(selector: string, options: { timeout?: number } = {}): Promise { + this.ensureSessionExists() + + await this.api(`/v1/sessions/${this.sessionId}/wait`, { + method: 'POST', + body: { selector, timeout: options.timeout }, + }) + } + + /** + * 点击元素 + */ + async click(selector: string): Promise { + this.ensureSessionExists() + + await this.executeScript(` + const element = document.querySelector('${selector}'); + if (!element) throw new Error('Element not found: ${selector}'); + element.click(); + `) + } + + /** + * 向输入框输入文本 + */ + async type(selector: string, text: string): Promise { + this.ensureSessionExists() + + // 先清空输入框 + await this.executeScript(` + const element = document.querySelector('${selector}'); + if (!element) throw new Error('Element not found: ${selector}'); + element.value = ''; + `) + + // 然后输入文本 + await this.executeScript(` + const element = document.querySelector('${selector}'); + if (!element) throw new Error('Element not found: ${selector}'); + element.value = '${text.replace(/'/g, '\\\'')}'; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + `) + } + + /** + * 获取元素文本内容 + */ + async getText(selector: string): Promise { + this.ensureSessionExists() + + return this.executeScript(` + const element = document.querySelector('${selector}'); + if (!element) throw new Error('Element not found: ${selector}'); + return element.textContent.trim(); + `) + } + + /** + * 获取屏幕截图 + */ + async getScreenshot(): Promise { + this.ensureSessionExists() + + const response = await this.api(`/v1/sessions/${this.sessionId}/screenshot`, { + method: 'GET', + }) + + if (!response.ok) { + throw new Error(`截图失败: ${response.statusText || '未知错误'}`) + } + + const buffer = await response.arrayBuffer() + return Buffer.from(buffer) + } + + /** + * 关闭会话 + */ + async closeSession(): Promise { + if (!this.sessionId) + return + + const response = await this.api(`/v1/sessions/${this.sessionId}`, { + method: 'DELETE', + }) + + if (!response.ok) { + throw new Error(`关闭会话失败: ${response.statusText || '未知错误'}`) + } + + this.sessionId = null + } + + /** + * 确保会话存在 + */ + private ensureSessionExists(): void { + if (!this.sessionId) { + throw new Error('No active session. Call createSession first.') + } + } +} diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts new file mode 100644 index 00000000..343bfb29 --- /dev/null +++ b/services/twitter-services/src/cli.ts @@ -0,0 +1,112 @@ +#!/usr/bin/env node + +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' +import { Command } from 'commander' + +import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { createDefaultConfig } from './config' +import { TwitterService } from './core/twitter-service' +import { TwitterServiceLauncher } from './index' + +// 获取版本 +const packageJsonPath = path.join(__dirname, '..', 'package.json') +const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) + +// 创建程序 +const program = new Command() + +// 设置基本信息 +program + .name('twitter-services') + .description('Twitter 服务 CLI - 访问和管理 Twitter 数据') + .version(packageJson.version) + +// 启动服务命令 +program + .command('start') + .description('启动 Twitter 服务') + .option('-c, --config ', '配置文件路径') + .action(async (options) => { + if (options.config) { + process.env.CONFIG_PATH = options.config + } + + const launcher = new TwitterServiceLauncher() + await launcher.start() + + console.log('服务已启动，按 Ctrl+C 停止') + }) + +// 获取时间线命令 +program + .command('timeline') + .description('获取 Twitter 时间线') + .option('-c, --count ', '要获取的推文数量', '10') + .option('--no-replies', '排除回复') + .option('--no-retweets', '排除转发') + .option('-o, --output ', '输出结果到文件') + .action(async (options) => { + try { + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + // 初始化浏览器 + const browser = new BrowserBaseMCPAdapter(config.browserbase.apiKey) + await browser.initialize(config.browser) + + // 创建服务并登录 + const twitterService = new TwitterService(browser) + + if (!config.twitter.credentials) { + throw new Error('无法获取 Twitter 凭据，请检查配置') + } + + const loggedIn = await twitterService.login(config.twitter.credentials) + + if (!loggedIn) { + throw new Error('登录失败，请检查凭据') + } + + console.log('正在获取时间线...') + + // 获取时间线 + const tweets = await twitterService.getTimeline({ + count: Number.parseInt(options.count), + includeReplies: options.replies, + includeRetweets: options.retweets, + }) + + // 处理结果 + const result = tweets.map(tweet => ({ + id: tweet.id, + text: tweet.text, + author: tweet.author.displayName, + username: tweet.author.username, + timestamp: tweet.timestamp, + likeCount: tweet.likeCount, + retweetCount: tweet.retweetCount, + replyCount: tweet.replyCount, + })) + + // 输出结果 + if (options.output) { + fs.writeFileSync(options.output, JSON.stringify(result, null, 2)) + console.log(`结果已保存到 ${options.output}`) + } + else { + console.log(JSON.stringify(result, null, 2)) + } + + // 关闭浏览器 + await browser.close() + } + catch (error) { + console.error('获取时间线失败:', error.message) + process.exit(1) + } + }) + +// 解析命令行参数 +program.parse() diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts new file mode 100644 index 00000000..e4bf1675 --- /dev/null +++ b/services/twitter-services/src/config/index.ts @@ -0,0 +1,105 @@ +import type { Config } from './types' + +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +import { logger } from '../utils/logger' +import { DEFAULT_CONFIG } from './types' + +/** + * 配置管理器 + * 负责加载、验证和提供配置 + */ +export class ConfigManager { + private config: Config + + /** + * 创建配置管理器 + * @param configPath 配置文件路径 + */ + constructor(configPath?: string) { + this.config = { ...DEFAULT_CONFIG } + + if (configPath) { + this.loadFromFile(configPath) + } + + this.validateConfig() + } + + /** + * 从文件加载配置 + */ + private loadFromFile(filePath: string): void { + try { + const configFile = fs.readFileSync(filePath, 'utf8') + const fileConfig = JSON.parse(configFile) + + // 深度合并配置 + this.config = this.mergeConfigs(this.config, fileConfig) + + logger.config.log(`配置已从 ${filePath} 加载`) + } + catch (error) { + logger.config.errorWithError(`加载配置文件失败: ${(error as Error).message}`, error) + } + } + + /** + * 验证配置有效性 + */ + private validateConfig(): void { + // 验证必要的 API 密钥 + if (!this.config.browserbase.apiKey) { + console.warn('未设置 BrowserBase API 密钥!') + } + + // 验证 Twitter 凭据 + if (!this.config.twitter.credentials?.username || !this.config.twitter.credentials?.password) { + console.warn('未设置 Twitter 凭据!') + } + } + + /** + * 递归合并配置对象 + */ + private mergeConfigs(target: any, source: any): any { + const result = { ...target } + + for (const key in source) { + if (source[key] instanceof Object && key in target) { + result[key] = this.mergeConfigs(target[key], source[key]) + } + else { + result[key] = source[key] + } + } + + return result + } + + /** + * 获取完整配置 + */ + getConfig(): Config { + return this.config + } + + /** + * 更新配置 + */ + updateConfig(newConfig: Partial): void { + this.config = this.mergeConfigs(this.config, newConfig) + this.validateConfig() + } +} + +/** + * 创建默认配置管理器 + */ +export function createDefaultConfig(): ConfigManager { + const configPath = process.env.CONFIG_PATH || path.join(process.cwd(), 'twitter-config.json') + + return new ConfigManager(fs.existsSync(configPath) ? configPath : undefined) +} diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts new file mode 100644 index 00000000..be0033a6 --- /dev/null +++ b/services/twitter-services/src/config/types.ts @@ -0,0 +1,96 @@ +import type { BrowserConfig } from '../types/browser' +import type { SearchOptions, TimelineOptions, TwitterCredentials } from '../types/twitter' + +import process from 'node:process' + +/** + * 完整配置接口 + */ +export interface Config { + // BrowserBase 配置 + browserbase: { + apiKey: string + endpoint?: string + } + + // 浏览器配置 + browser: BrowserConfig + + // Twitter 配置 + twitter: { + credentials?: TwitterCredentials + defaultOptions?: { + timeline?: TimelineOptions + search?: SearchOptions + } + } + + // 适配器配置 + adapters: { + airi?: { + url?: string + token?: string + enabled: boolean + } + mcp?: { + port?: number + enabled: boolean + } + } + + // 系统配置 + system: { + logLevel: 'error' | 'warn' | 'info' | 'verbose' | 'debug' + logFormat?: 'json' | 'pretty' + concurrency: number + } +} + +/** + * 默认配置 + */ +export const DEFAULT_CONFIG: Config = { + browserbase: { + apiKey: process.env.BROWSERBASE_API_KEY || '', + }, + browser: { + headless: true, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + viewport: { + width: 1280, + height: 800, + }, + timeout: 30000, + requestTimeout: 20000, // API 请求超时设置 (20秒) + requestRetries: 2, // 网络错误时重试次数 + }, + twitter: { + credentials: { + username: process.env.TWITTER_USERNAME || '', + password: process.env.TWITTER_PASSWORD || '', + }, + defaultOptions: { + timeline: { + count: 20, + includeReplies: true, + includeRetweets: true, + }, + }, + }, + adapters: { + airi: { + url: process.env.AIRI_URL || 'http://localhost:3000', + token: process.env.AIRI_TOKEN || '', + enabled: process.env.ENABLE_AIRI === 'true', + }, + mcp: { + port: Number(process.env.MCP_PORT || 8080), + enabled: process.env.ENABLE_MCP === 'true' || true, + }, + }, + system: { + logLevel: 'info', + logFormat: process.env.NODE_ENV === 'development' ? 'pretty' : 'json', + concurrency: Number(process.env.CONCURRENCY || 1), + }, +} diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts new file mode 100644 index 00000000..e1c9c378 --- /dev/null +++ b/services/twitter-services/src/core/auth-service.ts @@ -0,0 +1,93 @@ +import type { BrowserAdapter } from '../adapters/browser-adapter' +import type { TwitterCredentials } from '../types/twitter' + +import { logger } from '../utils/logger' +import { SELECTORS } from '../utils/selectors' + +/** + * Twitter 认证服务 + * 处理登录和会话管理 + */ +export class TwitterAuthService { + private browser: BrowserAdapter + private isLoggedIn: boolean = false + + constructor(browser: BrowserAdapter) { + this.browser = browser + } + + /** + * 登录到 Twitter + */ + async login(credentials: TwitterCredentials): Promise { + logger.auth.withField('username', credentials.username.replace(/./g, '*')).log('尝试登录 Twitter') + + try { + // 导航到登录页 + await this.browser.navigate('https://twitter.com/i/flow/login') + + // 等待并输入用户名 + await this.browser.waitForSelector(SELECTORS.LOGIN.USERNAME_INPUT) + await this.browser.type(SELECTORS.LOGIN.USERNAME_INPUT, credentials.username) + await this.browser.click(SELECTORS.LOGIN.NEXT_BUTTON) + + // 等待并输入密码 + await this.browser.waitForSelector(SELECTORS.LOGIN.PASSWORD_INPUT) + await this.browser.type(SELECTORS.LOGIN.PASSWORD_INPUT, credentials.password) + await this.browser.click(SELECTORS.LOGIN.LOGIN_BUTTON) + + // 验证登录是否成功 + logger.auth.log('登录表单已填写完成，提交中...') + const loginSuccess = await this.verifyLogin() + + if (loginSuccess) { + logger.auth.log('登录成功') + this.isLoggedIn = true + } + else { + logger.auth.warn('登录验证失败') + } + + return loginSuccess + } + catch (error) { + logger.auth.errorWithError('登录过程中发生错误', error) + this.isLoggedIn = false + return false + } + } + + /** + * 验证是否已成功登录 + */ + private async verifyLogin(): Promise { + try { + // 等待主页内容加载 + await this.browser.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 10000 }) + return true + } + catch { + return false + } + } + + /** + * 检查当前是否已登录 + */ + async checkLoginStatus(): Promise { + try { + await this.browser.navigate('https://twitter.com/home') + return await this.verifyLogin() + } + catch { + return false + } + } + + /** + * 获取登录状态 + */ + isAuthenticated(): boolean { + return this.isLoggedIn + } +} diff --git a/services/twitter-services/src/core/timeline-service.ts b/services/twitter-services/src/core/timeline-service.ts new file mode 100644 index 00000000..9ecc4bac --- /dev/null +++ b/services/twitter-services/src/core/timeline-service.ts @@ -0,0 +1,66 @@ +import type { BrowserAdapter } from '../adapters/browser-adapter' +import type { TimelineOptions, Tweet } from '../types/twitter' + +import { TweetParser } from '../parsers/tweet-parser' +import { RateLimiter } from '../utils/rate-limiter' +import { SELECTORS } from '../utils/selectors' + +/** + * Twitter 时间线服务 + * 处理获取和解析时间线内容 + */ +export class TwitterTimelineService { + private browser: BrowserAdapter + private rateLimiter: RateLimiter + + constructor(browser: BrowserAdapter) { + this.browser = browser + this.rateLimiter = new RateLimiter(10, 60000) // 每分钟10个请求 + } + + /** + * 获取时间线 + */ + async getTimeline(options: TimelineOptions = {}): Promise { + // 等待频率限制 + await this.rateLimiter.waitUntilReady() + + try { + // 导航到主页 + await this.browser.navigate('https://twitter.com/home') + + // 等待时间线加载 + await this.browser.waitForSelector(SELECTORS.TIMELINE.TWEET) + + // 延迟一下，确保内容加载完成 + await new Promise(resolve => setTimeout(resolve, 2000)) + + // 获取页面HTML内容 + const html = await this.browser.executeScript('document.documentElement.outerHTML') + + // 解析推文 + const tweets = TweetParser.parseTimelineTweets(html) + + // 应用筛选和限制 + let filteredTweets = tweets + + if (options.includeReplies === false) { + filteredTweets = filteredTweets.filter(tweet => !tweet.id.includes('reply')) + } + + if (options.includeRetweets === false) { + filteredTweets = filteredTweets.filter(tweet => !tweet.id.includes('retweet')) + } + + if (options.count) { + filteredTweets = filteredTweets.slice(0, options.count) + } + + return filteredTweets + } + catch (error) { + console.error('Failed to get timeline:', error) + return [] + } + } +} diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts new file mode 100644 index 00000000..449ee269 --- /dev/null +++ b/services/twitter-services/src/core/twitter-service.ts @@ -0,0 +1,114 @@ +import type { BrowserAdapter } from '../adapters/browser-adapter' +import type { + TwitterService as ITwitterService, + PostOptions, + SearchOptions, + TimelineOptions, + Tweet, + TweetDetail, + TwitterCredentials, + UserProfile, +} from '../types/twitter' + +import { TwitterAuthService } from './auth-service' +import { TwitterTimelineService } from './timeline-service' + +/** + * Twitter 服务实现 + * 集成各个服务组件，提供统一的接口 + */ +export class TwitterService implements ITwitterService { + private browser: BrowserAdapter + private authService: TwitterAuthService + private timelineService: TwitterTimelineService + + constructor(browser: BrowserAdapter) { + this.browser = browser + this.authService = new TwitterAuthService(browser) + this.timelineService = new TwitterTimelineService(browser) + } + + /** + * 登录 Twitter + */ + async login(credentials: TwitterCredentials): Promise { + return await this.authService.login(credentials) + } + + /** + * 获取时间线 + */ + async getTimeline(options?: TimelineOptions): Promise { + this.ensureAuthenticated() + return await this.timelineService.getTimeline(options) + } + + /** + * 获取推文详情（MVP暂未实现） + */ + async getTweetDetails(tweetId: string): Promise { + this.ensureAuthenticated() + // MVP阶段，返回一个基本结构 + return { + id: tweetId, + text: '推文详情功能尚未实现', + author: { + username: 'twitter', + displayName: 'Twitter', + }, + timestamp: new Date().toISOString(), + } + } + + /** + * 搜索推文 + */ + async search(_query: string, _options?: SearchOptions): Promise { + throw new Error('搜索功能尚未实现') + } + + /** + * 获取用户资料 + */ + async getUserProfile(_username: string): Promise { + throw new Error('获取用户资料功能尚未实现') + } + + /** + * 关注用户（MVP暂未实现） + */ + async followUser(_username: string): Promise { + this.ensureAuthenticated() + return false + } + + /** + * 点赞推文 + */ + async likeTweet(_tweetId: string): Promise { + throw new Error('点赞功能尚未实现') + } + + /** + * 转发推文 + */ + async retweet(_tweetId: string): Promise { + throw new Error('转发功能尚未实现') + } + + /** + * 发送推文 + */ + async postTweet(_content: string, _options?: PostOptions): Promise { + throw new Error('发送推文功能尚未实现') + } + + /** + * 确保已经登录 + */ + private ensureAuthenticated(): void { + if (!this.authService.isAuthenticated()) { + throw new Error('Not authenticated. Call login() first.') + } + } +} diff --git a/services/twitter-services/src/dev-server.ts b/services/twitter-services/src/dev-server.ts new file mode 100644 index 00000000..6da24c8e --- /dev/null +++ b/services/twitter-services/src/dev-server.ts @@ -0,0 +1,202 @@ +import { Buffer } from 'node:buffer' +import process from 'node:process' +import { McpServer, ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js' +import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js' +import * as dotenv from 'dotenv' +import { createApp, createRouter, defineEventHandler, toNodeListener } from 'h3' +import { listen } from 'listhen' +import { z } from 'zod' + +import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { TwitterService } from './core/twitter-service' +import { errorToMessage } from './utils/error' +import { logger } from './utils/logger' + +// 加载环境变量 +dotenv.config() + +/** + * 开发服务器入口点 + * 使用 listhen 提供开发时的便利功能 + */ +async function startDevServer() { + // 基本检查环境变量 + if (!process.env.BROWSERBASE_API_KEY) { + console.error('错误: 缺少环境变量 BROWSERBASE_API_KEY') + process.exit(1) + } + + const app = createApp() + const router = createRouter() + + // 创建浏览器和 Twitter 服务 + const browser = new BrowserBaseMCPAdapter(process.env.BROWSERBASE_API_KEY) + await browser.initialize({ + headless: true, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + }) + + const twitter = new TwitterService(browser) + + // 可选: 如果有凭据，进行登录 + if (process.env.TWITTER_USERNAME && process.env.TWITTER_PASSWORD) { + const success = await twitter.login({ + username: process.env.TWITTER_USERNAME, + password: process.env.TWITTER_PASSWORD, + }) + + if (success) { + logger.main.log('✅ 已成功登录 Twitter') + } + else { + logger.main.warn('⚠️ Twitter 登录失败') + } + } + + // 创建 MCP 服务器 + const mcpServer = new McpServer({ + name: 'Twitter Service (Dev)', + version: '1.0.0-dev', + }) + + // 配置 MCP 资源 + mcpServer.resource( + 'timeline', + new ResourceTemplate('twitter://timeline/{count}', { list: 'twitter://timeline' }), + async (uri, { count }) => { + try { + const tweets = await twitter.getTimeline({ + count: count ? Number.parseInt(count) : undefined, + }) + + return { + contents: tweets.map(tweet => ({ + uri: `twitter://tweet/${tweet.id}`, + text: `Tweet by @${tweet.author.username} (${tweet.author.displayName}):\n${tweet.text}`, + })), + } + } + catch (error) { + console.error('Error fetching timeline:', error) + return { contents: [] } + } + }, + ) + + // 配置一些基本工具 + mcpServer.tool( + 'post-tweet', + { + content: z.string(), + }, + async ({ content }) => { + try { + const tweetId = await twitter.postTweet(content) + return { + content: [{ type: 'text', text: `成功发布推文: ${tweetId}` }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `发推失败: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // 保存活跃的 SSE 传输 + const activeTransports: SSEServerTransport[] = [] + + // 设置路由 + router.get('/', defineEventHandler(() => { + return { + name: 'Twitter MCP 开发服务', + version: '1.0.0-dev', + status: 'running', + endpoints: { + sse: '/sse', + messages: '/messages', + }, + } + })) + + // SSE 端点 + router.get('/sse', defineEventHandler(async (event) => { + const { req, res } = event.node + + res.setHeader('Content-Type', 'text/event-stream') + res.setHeader('Cache-Control', 'no-cache') + res.setHeader('Connection', 'keep-alive') + + // 创建 SSE 传输 + const transport = new SSEServerTransport('/messages', res) + activeTransports.push(transport) + + // 客户端断开连接时清理 + req.on('close', () => { + const index = activeTransports.indexOf(transport) + if (index !== -1) { + activeTransports.splice(index, 1) + } + }) + + // 连接到 MCP 服务器 + await mcpServer.connect(transport) + })) + + // 消息端点 + router.post('/messages', defineEventHandler(async (event) => { + if (activeTransports.length === 0) { + event.node.res.statusCode = 503 + return { error: 'No active SSE connections' } + } + + try { + // 解析请求体 + const buffers = [] + for await (const chunk of event.node.req) { + buffers.push(chunk) + } + const data = Buffer.concat(buffers).toString() + const body = JSON.parse(data) + + // 使用最近的传输 + const transport = activeTransports[activeTransports.length - 1] + + // 处理消息 + const response = await transport.handleMessage(body) + return response + } + catch (error) { + event.node.res.statusCode = 500 + return { error: errorToMessage(error) } + } + })) + + // 注册路由 + app.use(router) + + // 启动服务器 + const listener = toNodeListener(app) + await listen(listener, { + showURL: true, + port: 8080, + open: true, + }) + + logger.main.log('🚀 Twitter MCP 开发服务器已启动') + + // 处理退出 + process.on('SIGINT', async () => { + logger.main.log('正在关闭服务器...') + await browser.close() + process.exit(0) + }) +} + +// 执行 +startDevServer().catch((error) => { + logger.main.error('启动开发服务器失败:', error) + process.exit(1) +}) diff --git a/services/twitter-services/src/index.ts b/services/twitter-services/src/index.ts new file mode 100644 index 00000000..f9bb5ddb --- /dev/null +++ b/services/twitter-services/src/index.ts @@ -0,0 +1,150 @@ +import process from 'node:process' + +import { AiriAdapter } from './adapters/airi-adapter' +import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { MCPAdapter } from './adapters/mcp-adapter' +import { createDefaultConfig } from './config' +import { TwitterService } from './core/twitter-service' +import { initializeLogger, logger } from './utils/logger' + +/** + * Twitter 服务启动器类 + * 负责初始化和启动服务 + */ +export class TwitterServiceLauncher { + private browser?: BrowserBaseMCPAdapter + private twitterService?: TwitterService + private airiAdapter?: AiriAdapter + private mcpAdapter?: MCPAdapter + + /** + * 启动 Twitter 服务 + */ + async start() { + try { + // 加载配置 + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + // 初始化日志系统 + initializeLogger() + logger.main.log('正在启动 Twitter 服务...') + + // 初始化浏览器 + this.browser = new BrowserBaseMCPAdapter( + config.browserbase.apiKey, + config.browserbase.endpoint, + { + timeout: config.browser.requestTimeout, + retries: config.browser.requestRetries, + }, + ) + + await this.browser.initialize(config.browser) + logger.main.log('浏览器已初始化') + + // 创建 Twitter 服务 + this.twitterService = new TwitterService(this.browser) + + // 尝试登录 + if (config.twitter.credentials) { + const success = await this.twitterService.login(config.twitter.credentials) + if (success) { + logger.main.log('成功登录 Twitter') + } + else { + logger.main.error('Twitter 登录失败!') + } + } + + // 启动适配器 + if (config.adapters.airi?.enabled && this.twitterService) { + this.airiAdapter = new AiriAdapter(this.twitterService, { + url: config.adapters.airi.url, + token: config.adapters.airi.token, + credentials: config.twitter.credentials!, + }) + + await this.airiAdapter.start() + logger.main.log('Airi 适配器已启动') + } + + if (config.adapters.mcp?.enabled && this.twitterService) { + this.mcpAdapter = new MCPAdapter(this.twitterService, config.adapters.mcp.port) + + await this.mcpAdapter.start() + logger.main.log('MCP 适配器已启动') + } + + logger.main.log('Twitter 服务已成功启动!') + + // 设置关闭钩子 + this.setupShutdownHooks() + } + catch (error) { + logger.main.withError(error).error('启动 Twitter 服务失败') + } + } + + /** + * 停止服务 + */ + async stop() { + logger.main.log('正在停止 Twitter 服务...') + + // 停止 MCP 适配器 + if (this.mcpAdapter) { + await this.mcpAdapter.stop() + logger.main.log('MCP 适配器已停止') + } + + // 关闭浏览器 + if (this.browser) { + await this.browser.close() + logger.main.log('浏览器已关闭') + } + + logger.main.log('Twitter 服务已停止') + } + + /** + * 设置关闭钩子 + */ + private setupShutdownHooks() { + // 处理进程退出 + process.on('SIGINT', async () => { + logger.main.log('接收到退出信号...') + await this.stop() + process.exit(0) + }) + + process.on('SIGTERM', async () => { + logger.main.log('接收到终止信号...') + await this.stop() + process.exit(0) + }) + + // 处理未捕获的异常 + process.on('uncaughtException', async (error) => { + logger.main.withError(error).error('未捕获的异常') + await this.stop() + process.exit(1) + }) + } +} + +export { AiriAdapter } from './adapters/airi-adapter' +export * from './adapters/browser-adapter' +export { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +export { MCPAdapter } from './adapters/mcp-adapter' +export * from './core/twitter-service' +export * from './types/browser' + +// 直接启动服务的入口点 +if (require.main === module) { + const launcher = new TwitterServiceLauncher() + launcher.start().catch((error) => { + logger.main.withError(error).error('启动失败') + process.exit(1) + }) +} diff --git a/services/twitter-services/src/parsers/html-parser.ts b/services/twitter-services/src/parsers/html-parser.ts new file mode 100644 index 00000000..bccf43ee --- /dev/null +++ b/services/twitter-services/src/parsers/html-parser.ts @@ -0,0 +1,95 @@ +import type { Element, Root } from 'hast' + +import rehypeParse from 'rehype-parse' +import { unified } from 'unified' +import { visit } from 'unist-util-visit' + +export interface ParseOptions { + fragment?: boolean +} + +/** + * HTML 解析器 + * 使用 rehype 将 HTML 字符串转换为 AST，便于后续处理 + */ +export class HtmlParser { + /** + * 将 HTML 字符串解析为 rehype AST + * @param html HTML 字符串 + * @param options 解析选项 + * @returns hast 语法树 + */ + static parse(html: string, options: ParseOptions = {}): Root { + const processor = unified().use(rehypeParse, { + fragment: options.fragment ?? false, + }) + + const tree = processor.parse(html) + const file = processor.runSync(tree) + + return file as Root + } + + /** + * 根据选择器查找元素 + * @param tree AST 树 + * @param selector 简化版选择器 (tagName, className, id) + * @returns 匹配的元素数组 + */ + static select(tree: Root, selector: string): Element[] { + const elements: Element[] = [] + + visit(tree, 'element', (node) => { + // 简单选择器实现 + if (this.matchesSelector(node, selector)) { + elements.push(node) + } + }) + + return elements + } + + /** + * 简单的选择器匹配逻辑 + */ + private static matchesSelector(node: Element, selector: string): boolean { + // 标签选择器 + if (selector.match(/^[a-z0-9]+$/i)) { + return node.tagName === selector + } + + // 类选择器 + if (selector.startsWith('.')) { + const className = selector.slice(1) + return (node.properties?.className as string[])?.includes(className) ?? false + } + + // ID 选择器 + if (selector.startsWith('#')) { + const id = selector.slice(1) + return node.properties?.id === id + } + + // 数据属性选择器 + if (selector.startsWith('[data-')) { + // Use non-greedy quantifier and more specific character classes to avoid backtracking + const match = selector.match(/\[([^=\]]+)=(['"]?)([^"'\]]+)\2\]/) + if (match) { + const [, attr, , value] = match + return node.properties?.[attr] === value + } + } + + return false + } + + /** + * 访问特定类型的节点 + * @param tree AST 树 + * @param nodeType 节点类型 + * @param visitor 访问器函数 + */ + static visit(tree: Element | Root, nodeType: string, visitor: (node: any) => void): void { + visit(tree, nodeType, visitor) + } +} diff --git a/services/twitter-services/src/parsers/profile-parser.ts b/services/twitter-services/src/parsers/profile-parser.ts new file mode 100644 index 00000000..95e935e0 --- /dev/null +++ b/services/twitter-services/src/parsers/profile-parser.ts @@ -0,0 +1,142 @@ +import type { Element, Root } from 'hast' +import type { UserProfile } from '../types/twitter' + +import { SELECTORS } from '../utils/selectors' +import { HtmlParser } from './html-parser' + +/** + * 用户资料解析器 + * 从 HTML 中提取用户资料信息 + */ +export class ProfileParser { + /** + * 从 HTML 中解析用户资料 + * @param html HTML 字符串 + * @returns 用户资料 + */ + static parseUserProfile(html: string): UserProfile { + const tree = HtmlParser.parse(html) + + // 提取用户名和显示名称 + const displayNameElement = HtmlParser.select(tree, SELECTORS.PROFILE.DISPLAY_NAME)[0] + const displayName = this.extractTextContent(displayNameElement) || 'Unknown User' + + // 从URL或DOM中提取用户名 + const username = this.extractUsername(tree) || 'unknown' + + // 提取用户简介 + const bioElement = HtmlParser.select(tree, SELECTORS.PROFILE.BIO)[0] + const bio = this.extractTextContent(bioElement) + + // 提取用户统计数据 + const stats = this.extractProfileStats(tree) + + // 提取头像和背景图 + const avatarUrl = this.extractAvatarUrl(tree) + const bannerUrl = this.extractBannerUrl(tree) + + return { + username, + displayName, + bio, + avatarUrl, + bannerUrl, + ...stats, + } + } + + /** + * 提取文本内容 + */ + private static extractTextContent(element?: Element): string { + if (!element) + return '' + + let text = '' + HtmlParser.visit(element, 'text', (node) => { + text += node.value + }) + + return text.trim() + } + + /** + * 从页面中提取用户名 + */ + private static extractUsername(_tree: Root): string { + // 可以从URL或特定DOM元素中提取 + return '' + } + + /** + * 提取用户统计数据 + */ + private static extractProfileStats(tree: Root) { + // 提取粉丝数、关注数、推文数等 + const _statsElement = HtmlParser.select(tree, SELECTORS.PROFILE.STATS)[0] + + return { + followersCount: undefined, + followingCount: undefined, + tweetCount: undefined, + isVerified: false, + joinDate: undefined, + } + } + + /** + * 提取头像URL + */ + private static extractAvatarUrl(_tree: Root): string | undefined { + // 提取头像图片URL + return undefined + } + + /** + * 提取背景图URL + */ + private static extractBannerUrl(_tree: Root): string | undefined { + // 提取背景图URL + return undefined + } + + /** + * 从 HTML 提取用户统计信息 + */ + static extractUserStats(_html: string, _tree?: Node): UserStats { + // 解析 HTML 获取统计数据 + const stats: UserStats = { + tweets: 0, + following: 0, + followers: 0, + } + + try { + // 查找统计信息容器 + const _statsElement = document.querySelector('[data-testid="userProfileStats"]') + + // 暂未实现具体解析逻辑 + + return stats + } + catch { + return stats + } + } + + /** + * 从 HTML 提取用户链接 + */ + static extractUserLinks(_html: string, _tree?: Node): UserLink[] { + // 暂未实现 + return [] + } + + /** + * 从 HTML 提取用户加入日期 + */ + static extractJoinDate(_html: string, _tree?: Node): string | null { + // 暂未实现 + return null + } +} diff --git a/services/twitter-services/src/parsers/tweet-parser.ts b/services/twitter-services/src/parsers/tweet-parser.ts new file mode 100644 index 00000000..ac317e46 --- /dev/null +++ b/services/twitter-services/src/parsers/tweet-parser.ts @@ -0,0 +1,104 @@ +import type { Element } from 'hast' +import type { Tweet } from '../types/twitter' + +import { SELECTORS } from '../utils/selectors' +import { HtmlParser } from './html-parser' + +/** + * 推文解析器 + * 从 HTML 中提取推文信息 + */ +export class TweetParser { + /** + * 从 HTML 中解析推文列表 + * @param html HTML 字符串 + * @returns 推文数组 + */ + static parseTimelineTweets(html: string): Tweet[] { + const tree = HtmlParser.parse(html) + const tweetElements = HtmlParser.select(tree, SELECTORS.TIMELINE.TWEET) + + return tweetElements.map(el => this.extractTweetData(el)) + } + + /** + * 从推文元素中提取推文数据 + * @param element 推文元素 + * @returns 推文数据 + */ + static extractTweetData(element: Element): Tweet { + // 获取推文 ID + const id = this.extractTweetId(element) + + // 获取推文文本 + const textElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TEXT)[0] + const text = this.extractTextContent(textElement) + + // 获取作者信息 + const author = this.extractAuthorInfo(element) + + // 获取时间戳 + const timeElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TIME)[0] + const timestamp = timeElement?.properties?.datetime as string || new Date().toISOString() + + // 获取统计数据 + const stats = this.extractTweetStats(element) + + return { + id, + text, + author, + timestamp, + ...stats, + } + } + + /** + * 提取推文ID + */ + private static extractTweetId(element: Element): string { + // 从 data-tweet-id 属性或其他位置提取ID + return element.properties?.['data-tweet-id'] as string + || `tweet-${Math.random().toString(36).substring(2, 15)}` + } + + /** + * 提取文本内容 + */ + private static extractTextContent(element?: Element): string { + if (!element) + return '' + + let text = '' + HtmlParser.visit(element, 'text', (node) => { + text += node.value + }) + + return text.trim() + } + + /** + * 提取作者信息 + */ + private static extractAuthorInfo(_element: Element) { + // 根据选择器提取作者名称、用户名和头像 + // 这部分需要根据Twitter的实际DOM结构调整 + return { + username: '用户名', // 占位，实际实现需要根据DOM结构 + displayName: '显示名称', + avatarUrl: undefined, + } + } + + /** + * 提取推文统计信息 + */ + private static extractTweetStats(_element: Element) { + // 提取点赞数、转发数和评论数 + return { + likeCount: undefined, + retweetCount: undefined, + replyCount: undefined, + } + } +} diff --git a/services/twitter-services/src/types/browser.ts b/services/twitter-services/src/types/browser.ts new file mode 100644 index 00000000..8b27f3e1 --- /dev/null +++ b/services/twitter-services/src/types/browser.ts @@ -0,0 +1,34 @@ +/** + * 浏览器配置接口 + */ +export interface BrowserConfig { + headless?: boolean + userAgent?: string + viewport?: { + width: number + height: number + } + timeout?: number + requestTimeout?: number // API 请求超时设置 + requestRetries?: number // 请求重试次数 + proxy?: string +} + +/** + * 元素句柄接口 + */ +export interface ElementHandle { + getText: () => Promise + getAttribute: (name: string) => Promise + click: () => Promise + type: (text: string) => Promise +} + +/** + * 等待选项接口 + */ +export interface WaitOptions { + timeout?: number + visible?: boolean + hidden?: boolean +} diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts new file mode 100644 index 00000000..b5453838 --- /dev/null +++ b/services/twitter-services/src/types/twitter.ts @@ -0,0 +1,89 @@ +/** + * Twitter 认证凭据 + */ +export interface TwitterCredentials { + username: string + password: string +} + +/** + * 推文接口 + */ +export interface Tweet { + id: string + text: string + author: { + username: string + displayName: string + avatarUrl?: string + } + timestamp: string + likeCount?: number + retweetCount?: number + replyCount?: number + mediaUrls?: string[] +} + +/** + * 推文详情 + */ +export interface TweetDetail extends Tweet { + replies?: Tweet[] + quotedTweet?: Tweet +} + +/** + * 用户资料 + */ +export interface UserProfile { + username: string + displayName: string + bio?: string + avatarUrl?: string + bannerUrl?: string + followersCount?: number + followingCount?: number + tweetCount?: number + isVerified?: boolean + joinDate?: string +} + +/** + * 时间线选项 + */ +export interface TimelineOptions { + count?: number + includeReplies?: boolean + includeRetweets?: boolean +} + +/** + * 搜索选项 + */ +export interface SearchOptions { + count?: number + filter?: 'latest' | 'photos' | 'videos' | 'top' +} + +/** + * 发推选项 + */ +export interface PostOptions { + media?: string[] + inReplyTo?: string +} + +/** + * Twitter 服务接口 + */ +export interface TwitterService { + login: (credentials: TwitterCredentials) => Promise + getTimeline: (options?: TimelineOptions) => Promise + getTweetDetails: (tweetId: string) => Promise + searchTweets: (query: string, options?: SearchOptions) => Promise + getUserProfile: (username: string) => Promise + followUser: (username: string) => Promise + likeTweet: (tweetId: string) => Promise + retweet: (tweetId: string) => Promise + postTweet: (content: string, options?: PostOptions) => Promise +} diff --git a/services/twitter-services/src/utils/api.ts b/services/twitter-services/src/utils/api.ts new file mode 100644 index 00000000..1a7c369c --- /dev/null +++ b/services/twitter-services/src/utils/api.ts @@ -0,0 +1,62 @@ +import { ofetch } from 'ofetch' + +import { logger } from './logger' + +/** + * 创建一个预配置的 ofetch 实例 + * + * @param baseURL - API 的基础 URL + * @param options - 附加选项 + * @returns - 定制的 ofetch 实例 + */ +export function createApiClient(baseURL: string, options: Record = {}) { + const client = ofetch.create({ + baseURL, + retry: 1, + timeout: 30000, // 默认 30 秒超时 + ...options, + + // 请求拦截器 + onRequest({ request, options }) { + const method = options.method || 'GET' + const url = request.toString() + logger.browser.withField('method', method).withField('url', url).debug('API 请求') + }, + + // 请求错误拦截器 + onRequestError({ request, error, options }) { + const method = options.method || 'GET' + const url = request.toString() + logger.browser.withField('method', method).withField('url', url).error('API 请求失败', error) + }, + + // 响应拦截器 + onResponse({ request, response, options }) { + const method = options.method || 'GET' + const url = request.toString() + const status = response.status + + logger.browser + .withField('method', method) + .withField('url', url) + .withField('status', status) + .debug('API 响应') + }, + + // 响应错误拦截器 + onResponseError({ request, response, options }) { + const method = options.method || 'GET' + const url = request.toString() + const status = response.status + + logger.browser + .withField('method', method) + .withField('url', url) + .withField('status', status) + .withField('body', response._data) + .error('API 响应错误') + }, + }) + + return client +} diff --git a/services/twitter-services/src/utils/error.ts b/services/twitter-services/src/utils/error.ts new file mode 100644 index 00000000..e8395793 --- /dev/null +++ b/services/twitter-services/src/utils/error.ts @@ -0,0 +1,74 @@ +/** + * 从任意错误类型中安全地提取错误消息 + * 处理 Error 对象、字符串、对象和其他类型 + * + * @param error - 任意错误对象 + * @param fallbackMessage - 当无法提取消息时的后备消息 + * @returns 格式化的错误消息 + */ +export function errorToMessage(error: unknown, fallbackMessage = '未知错误'): string { + if (error === null || error === undefined) { + return fallbackMessage + } + + // 处理标准 Error 对象 + if (error instanceof Error) { + return error.message + } + + // 处理字符串错误 + if (typeof error === 'string') { + return error + } + + // 处理带有 message 属性的对象 + if (typeof error === 'object') { + // 检查是否有 message 属性 + if ('message' in error && typeof (error as any).message === 'string') { + return (error as any).message + } + + // 尝试将对象转换为字符串 + try { + return JSON.stringify(error) + } + catch { + // 如果无法序列化，返回对象的字符串表示 + return String(error) + } + } + + // 针对其他情况，尝试强制转换为字符串 + return String(error) +} + +/** + * 创建一个带有详细上下文信息的错误 + * + * @param message - 错误消息 + * @param originalError - 原始错误对象（可选） + * @param context - 额外上下文信息（可选） + * @returns 增强的错误对象 + */ +export function createError( + message: string, + originalError?: unknown, + context?: Record, +): Error { + let errorMessage = message + + // 添加原始错误信息 + if (originalError) { + errorMessage += `: ${errorToMessage(originalError)}` + } + + // 创建新的错误对象 + const error = new Error(errorMessage) + + // 添加上下文信息 + if (context) { + Object.assign(error, { context }) + } + + return error +} diff --git a/services/twitter-services/src/utils/logger.ts b/services/twitter-services/src/utils/logger.ts new file mode 100644 index 00000000..01f7fcee --- /dev/null +++ b/services/twitter-services/src/utils/logger.ts @@ -0,0 +1,58 @@ +import { createLogg, Format, LogLevel, setGlobalFormat, setGlobalLogLevel } from '@guiiai/logg' + +import { createDefaultConfig } from '../config' +import { errorToMessage } from './error' + +// 初始化全局日志配置 +export function initializeLogger() { + const config = createDefaultConfig().getConfig() + + // 从环境变量或配置中获取日志级别 + const logLevelString = config.system?.logLevel?.toLowerCase() || 'info' + + // 映射日志级别 + const logLevelMap: Record = { + error: LogLevel.Error, + warn: LogLevel.Warning, + info: LogLevel.Log, + verbose: LogLevel.Verbose, + debug: LogLevel.Debug, + } + + // 设置全局日志级别 + setGlobalLogLevel(logLevelMap[logLevelString] || LogLevel.Log) + + // 根据配置设置格式 + if (config.system?.logFormat === 'pretty') { + setGlobalFormat(Format.Pretty) + } + else { + setGlobalFormat(Format.JSON) + } +} + +// 创建特定上下文的日志记录器 +export function createLogger(context: string) { + const logger = createLogg(context).useGlobalConfig() + + // 添加错误处理方法 + return { + ...logger, + // 使用 errorToMessage 增强错误日志 + errorWithMessage: (message: string, error: unknown) => { + logger.error(`${message}: ${errorToMessage(error)}`) + }, + } +} + +// 创建各种服务的预配置日志记录器 +export const logger = { + auth: createLogger('auth-service'), + timeline: createLogger('timeline-service'), + browser: createLogger('browser-adapter'), + airi: createLogger('airi-adapter'), + mcp: createLogger('mcp-adapter'), + parser: createLogger('parser'), + main: createLogger('twitter-service'), + config: createLogger('config'), +} diff --git a/services/twitter-services/src/utils/rate-limiter.ts b/services/twitter-services/src/utils/rate-limiter.ts new file mode 100644 index 00000000..f08f0808 --- /dev/null +++ b/services/twitter-services/src/utils/rate-limiter.ts @@ -0,0 +1,67 @@ +/** + * 请求频率限制器 + * 控制对 Twitter 的请求频率，避免触发限制 + */ +export class RateLimiter { + private requestHistory: number[] = [] + private maxRequests: number + private timeWindow: number + + /** + * 创建频率限制器 + * @param maxRequests 时间窗口内的最大请求数 + * @param timeWindow 时间窗口大小（毫秒） + */ + constructor(maxRequests: number = 20, timeWindow: number = 60000) { + this.maxRequests = maxRequests + this.timeWindow = timeWindow + } + + /** + * 检查是否可以发送请求 + */ + canRequest(): boolean { + this.cleanOldRequests() + return this.requestHistory.length < this.maxRequests + } + + /** + * 记录一次请求 + */ + recordRequest(): void { + this.requestHistory.push(Date.now()) + } + + /** + * 获取下次可请求的等待时间（毫秒） + * 如果当前可以请求，返回0 + */ + getWaitTime(): number { + if (this.canRequest()) { + return 0 + } + + const oldestRequest = this.requestHistory[0] + return oldestRequest + this.timeWindow - Date.now() + } + + /** + * 清理过期的请求记录 + */ + private cleanOldRequests(): void { + const now = Date.now() + const cutoff = now - this.timeWindow + this.requestHistory = this.requestHistory.filter(time => time >= cutoff) + } + + /** + * 等待直到可以发送请求 + */ + async waitUntilReady(): Promise { + const waitTime = this.getWaitTime() + if (waitTime > 0) { + await new Promise(resolve => setTimeout(resolve, waitTime)) + } + this.recordRequest() + } +} diff --git a/services/twitter-services/src/utils/selectors.ts b/services/twitter-services/src/utils/selectors.ts new file mode 100644 index 00000000..0883ace3 --- /dev/null +++ b/services/twitter-services/src/utils/selectors.ts @@ -0,0 +1,35 @@ +/** + * Twitter 网站 CSS 选择器常量 + * 用于定位页面上的元素 + */ +export const SELECTORS = { + LOGIN: { + USERNAME_INPUT: 'input[autocomplete="username"]', + PASSWORD_INPUT: 'input[type="password"]', + NEXT_BUTTON: '[data-testid="auth-login-button"]', + LOGIN_BUTTON: '[data-testid="LoginForm_Login_Button"]', + }, + HOME: { + TIMELINE: '[data-testid="primaryColumn"]', + }, + TIMELINE: { + TWEET: '[data-testid="tweet"]', + TWEET_TEXT: '[data-testid="tweetText"]', + TWEET_TIME: 'time', + LIKE_BUTTON: '[data-testid="like"]', + RETWEET_BUTTON: '[data-testid="retweet"]', + REPLY_BUTTON: '[data-testid="reply"]', + }, + PROFILE: { + FOLLOW_BUTTON: '[data-testid="followButton"]', + UNFOLLOW_BUTTON: '[data-testid="unfollowButton"]', + DISPLAY_NAME: '[data-testid="UserName"]', + BIO: '[data-testid="UserDescription"]', + STATS: '[data-testid="UserProfileStats"]', + }, + COMPOSE: { + TWEET_INPUT: '[data-testid="tweetTextarea_0"]', + TWEET_BUTTON: '[data-testid="tweetButtonInline"]', + MEDIA_BUTTON: '[data-testid="imageOrGifButton"]', + }, +} diff --git a/services/twitter-services/tsconfig.json b/services/twitter-services/tsconfig.json new file mode 100644 index 00000000..b7208af9 --- /dev/null +++ b/services/twitter-services/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": [ + "ESNext" + ], + "moduleDetection": "auto", + "module": "ESNext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "strict": true, + "strictNullChecks": true, + "noImplicitAny": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": [ + "src/**/*.ts" + ] +} From 04be157fbde56a25a7ccf4239f7beabd5ccc7857 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 00:16:23 +0800 Subject: [PATCH 02/20] fix: package.json --- services/twitter-services/package.json | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index b6d5a210..18b4a790 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -1,25 +1,10 @@ { "name": "twitter-services", "version": "0.1.0", - "description": "Twitter 服务 - 提供结构化的 Twitter 数据访问", - "author": "Your Name", - "license": "MIT", - "repository": { - "type": "git", - "url": "https://github.com/yourusername/twitter-services.git" - }, - "keywords": [ - "twitter", - "api", - "automation", - "scraping", - "web" - ], + "description": "Twitter Services for MCP", + "author": "RainbowBird ", "main": "dist/index.js", "types": "dist/index.d.ts", - "engines": { - "node": ">=14.0.0" - }, "scripts": { "build": "tsc", "start": "tsx dist/index.js", From de070e499199abc927d809f06c4e770a9020bc4e Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 00:41:20 +0800 Subject: [PATCH 03/20] fix: type error --- pnpm-lock.yaml | 89 ++++++++++--------- services/twitter-services/package.json | 9 +- .../src/adapters/airi-adapter.ts | 2 + .../src/adapters/browserbase-adapter.ts | 3 +- .../src/adapters/mcp-adapter.ts | 28 +++--- services/twitter-services/src/cli.ts | 3 +- services/twitter-services/src/dev-server.ts | 14 ++- .../src/parsers/profile-parser.ts | 18 ++-- .../twitter-services/src/types/twitter.ts | 18 ++++ services/twitter-services/src/utils/api.ts | 4 +- 10 files changed, 117 insertions(+), 71 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 69d4a0a7..9a3003f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -465,7 +465,7 @@ importers: version: 0.8.2 '@gcornut/valibot-json-schema': specifier: ^0.42.0 - version: 0.42.0(esbuild@0.19.12)(typescript@5.8.2) + version: 0.42.0(esbuild@0.25.0)(typescript@5.8.2) '@huggingface/transformers': specifier: ^3.3.3 version: 3.3.3 @@ -537,7 +537,7 @@ importers: version: 2.10.3 '@typeschema/valibot': specifier: ^0.14.0 - version: 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) + version: 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) '@unhead/vue': specifier: ^2.0.0-beta.2 version: 2.0.0-beta.2(vue@3.5.13(typescript@5.8.2)) @@ -739,7 +739,7 @@ importers: version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-macros: specifier: ^2.14.5 - version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-markdown: specifier: ^28.3.1 version: 28.3.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) @@ -1317,9 +1317,21 @@ importers: h3: specifier: ^1.11.0 version: 1.15.1 + hast-util-is-element: + specifier: ^3.0.0 + version: 3.0.0 + hast-util-select: + specifier: ^6.0.4 + version: 6.0.4 + hast-util-to-text: + specifier: ^4.0.2 + version: 4.0.2 listhen: specifier: ^1.6.0 version: 1.9.0 + ofetch: + specifier: ^1.3.3 + version: 1.4.1 rehype-parse: specifier: ^9.0.0 version: 9.0.1 @@ -7668,6 +7680,9 @@ packages: hast-util-select@6.0.3: resolution: {integrity: sha512-OVRQlQ1XuuLP8aFVLYmC2atrfWHS5UD3shonxpnyrjcCkwtvmt/+N6kYJdcY4mkMJhxp4kj2EFIxQ9kvkkt/eQ==} + hast-util-select@6.0.4: + resolution: {integrity: sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==} + hast-util-to-estree@3.1.1: resolution: {integrity: sha512-IWtwwmPskfSmma9RpzCappDUitC8t5jhAynHhc1m2+5trOgsrp7txscUSavc5Ic8PATyAjfrCK1wgtxh2cICVQ==} @@ -13550,16 +13565,6 @@ snapshots: '@formkit/auto-animate@0.8.2': {} - '@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2)': - dependencies: - valibot: 0.42.1(typescript@5.8.2) - optionalDependencies: - '@types/json-schema': 7.0.15 - esbuild-runner: 2.2.2(esbuild@0.19.12) - transitivePeerDependencies: - - esbuild - - typescript - '@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2)': dependencies: valibot: 0.42.1(typescript@5.8.2) @@ -15314,20 +15319,11 @@ snapshots: dependencies: '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) optionalDependencies: - '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) + '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) '@typeschema/zod': 0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.3(zod@3.24.2))(zod@3.24.2) transitivePeerDependencies: - '@types/json-schema' - '@typeschema/valibot@0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2))': - dependencies: - '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) - optionalDependencies: - '@gcornut/valibot-json-schema': 0.42.0(esbuild@0.19.12)(typescript@5.8.2) - valibot: 1.0.0-beta.9(typescript@5.8.2) - transitivePeerDependencies: - - '@types/json-schema' - '@typeschema/valibot@0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2))': dependencies: '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) @@ -18176,13 +18172,6 @@ snapshots: transitivePeerDependencies: - supports-color - esbuild-runner@2.2.2(esbuild@0.19.12): - dependencies: - esbuild: 0.19.12 - source-map-support: 0.5.21 - tslib: 2.4.0 - optional: true - esbuild-runner@2.2.2(esbuild@0.25.0): dependencies: esbuild: 0.25.0 @@ -19489,6 +19478,24 @@ snapshots: unist-util-visit: 5.0.0 zwitch: 2.0.4 + hast-util-select@6.0.4: + dependencies: + '@types/hast': 3.0.4 + '@types/unist': 3.0.0 + bcp-47-match: 2.0.3 + comma-separated-tokens: 2.0.3 + css-selector-parser: 3.0.5 + devlop: 1.1.0 + direction: 2.0.1 + hast-util-has-property: 3.0.0 + hast-util-to-string: 3.0.1 + hast-util-whitespace: 3.0.0 + nth-check: 2.1.1 + property-information: 7.0.0 + space-separated-tokens: 2.0.2 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + hast-util-to-estree@3.1.1: dependencies: '@types/estree': 1.0.6 @@ -23956,17 +23963,17 @@ snapshots: '@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.34.9) '@vueuse/core': 12.7.0(typescript@5.8.2) - unplugin-combine@1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: - esbuild: 0.19.12 - rollup: 4.34.9 + esbuild: 0.25.0 + rollup: 2.79.1 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) - unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: esbuild: 0.25.0 - rollup: 2.79.1 + rollup: 4.34.9 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) @@ -24017,7 +24024,7 @@ snapshots: transitivePeerDependencies: - vue - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -24047,9 +24054,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: @@ -24063,7 +24070,7 @@ snapshots: - vue-tsc - webpack - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -24093,9 +24100,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index 18b4a790..ff7862b4 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -1,10 +1,10 @@ { - "name": "twitter-services", + "name": "@proj-airi/twitter-services", + "type": "module", "version": "0.1.0", "description": "Twitter Services for MCP", "author": "RainbowBird ", - "main": "dist/index.js", - "types": "dist/index.d.ts", + "license": "MIT", "scripts": { "build": "tsc", "start": "tsx dist/index.js", @@ -18,6 +18,9 @@ "@types/hast": "^3.0.1", "dotenv": "^16.0.3", "h3": "^1.11.0", + "hast-util-is-element": "^3.0.0", + "hast-util-select": "^6.0.4", + "hast-util-to-text": "^4.0.2", "listhen": "^1.6.0", "ofetch": "^1.3.3", "rehype-parse": "^9.0.0", diff --git a/services/twitter-services/src/adapters/airi-adapter.ts b/services/twitter-services/src/adapters/airi-adapter.ts index 45d1733f..e879e4d3 100644 --- a/services/twitter-services/src/adapters/airi-adapter.ts +++ b/services/twitter-services/src/adapters/airi-adapter.ts @@ -2,6 +2,8 @@ import type { TimelineOptions, TwitterCredentials, TwitterService } from '../typ import { Client } from '@proj-airi/server-sdk' +import { logger } from '../utils/logger' + /** * Airi 适配器 * 将 Twitter 服务适配为 Airi 模块 diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts index 4ebd3a27..2592efff 100644 --- a/services/twitter-services/src/adapters/browserbase-adapter.ts +++ b/services/twitter-services/src/adapters/browserbase-adapter.ts @@ -1,4 +1,5 @@ import type { Buffer } from 'node:buffer' +import type { BrowserBaseClientOptions } from '../browser/browserbase' import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' import type { BrowserAdapter } from './browser-adapter' @@ -60,7 +61,7 @@ export class BrowserBaseMCPAdapter implements BrowserAdapter { headless: config.headless, userAgent: config.userAgent, viewport: config.viewport, - proxyUrl: config.proxy, + // proxyUrl: config.proxy, // TODO: Proxy }) logger.browser.log('浏览器会话已创建', { headless: config.headless }) } diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts index 9c4f65a5..10a7b955 100644 --- a/services/twitter-services/src/adapters/mcp-adapter.ts +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -19,7 +19,7 @@ export class MCPAdapter { private twitterService: TwitterService private mcpServer: McpServer private app: ReturnType - private server: ReturnType + private server: ReturnType | null = null private port: number private activeTransports: SSEServerTransport[] = [] @@ -50,8 +50,14 @@ export class MCPAdapter { // 添加时间线资源 this.mcpServer.resource( 'timeline', - new ResourceTemplate('twitter://timeline/{count}', { list: 'twitter://timeline' }), - async (uri, { count }) => { + new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ + resources: [{ + name: 'timeline', + uri: 'twitter://timeline', + description: '推文时间线', + }], + }) }), + async (_uri: URL, { count }: { count?: string }) => { try { const tweets = await this.twitterService.getTimeline({ count: count ? Number.parseInt(count) : undefined, @@ -65,7 +71,7 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.error('获取时间线错误:', error) + logger.mcp.errorWithError('获取时间线错误:', error) return { contents: [] } } }, @@ -75,9 +81,9 @@ export class MCPAdapter { this.mcpServer.resource( 'tweet', new ResourceTemplate('twitter://tweet/{id}', { list: undefined }), - async (uri, { id }) => { + async (uri: URL, { id }) => { try { - const tweet = await this.twitterService.getTweetDetails(id) + const tweet = await this.twitterService.getTweetDetails(id as string) return { contents: [{ @@ -87,7 +93,7 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.error('获取推文详情错误:', error) + logger.mcp.errorWithError('获取推文详情错误:', error) return { contents: [] } } }, @@ -99,7 +105,7 @@ export class MCPAdapter { new ResourceTemplate('twitter://user/{username}', { list: undefined }), async (uri, { username }) => { try { - const profile = await this.twitterService.getUserProfile(username) + const profile = await this.twitterService.getUserProfile(username as string) return { contents: [{ @@ -109,7 +115,7 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.error('获取用户资料错误:', error) + logger.mcp.errorWithError('获取用户资料错误:', error) return { contents: [] } } }, @@ -243,7 +249,7 @@ export class MCPAdapter { } catch (error) { return { - content: [{ type: 'text', text: `搜索失败: ${(error as Error).message}` }], + content: [{ type: 'text', text: `搜索失败: ${errorToMessage(error)}` }], isError: true, } } @@ -309,7 +315,7 @@ export class MCPAdapter { const transport = this.activeTransports[this.activeTransports.length - 1] // 手动处理 POST 消息，因为 H3 不是 Express 兼容的 - const response = await transport.handleRawMessage(body) + const response = await transport.handleMessage(body) return response } diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts index 343bfb29..973915c3 100644 --- a/services/twitter-services/src/cli.ts +++ b/services/twitter-services/src/cli.ts @@ -9,6 +9,7 @@ import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' import { createDefaultConfig } from './config' import { TwitterService } from './core/twitter-service' import { TwitterServiceLauncher } from './index' +import { errorToMessage } from './utils/error' // 获取版本 const packageJsonPath = path.join(__dirname, '..', 'package.json') @@ -103,7 +104,7 @@ program await browser.close() } catch (error) { - console.error('获取时间线失败:', error.message) + console.error('获取时间线失败:', errorToMessage(error)) process.exit(1) } }) diff --git a/services/twitter-services/src/dev-server.ts b/services/twitter-services/src/dev-server.ts index 6da24c8e..1cc13432 100644 --- a/services/twitter-services/src/dev-server.ts +++ b/services/twitter-services/src/dev-server.ts @@ -62,8 +62,14 @@ async function startDevServer() { // 配置 MCP 资源 mcpServer.resource( 'timeline', - new ResourceTemplate('twitter://timeline/{count}', { list: 'twitter://timeline' }), - async (uri, { count }) => { + new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ + resources: [{ + name: 'twitter-timeline', + uri: 'twitter://timeline', + description: '推文时间线', + }], + }) }), + async (_uri: URL, { count }: { count?: string }) => { try { const tweets = await twitter.getTimeline({ count: count ? Number.parseInt(count) : undefined, @@ -77,7 +83,7 @@ async function startDevServer() { } } catch (error) { - console.error('Error fetching timeline:', error) + logger.mcp.errorWithError('获取时间线错误:', error) return { contents: [] } } }, @@ -89,7 +95,7 @@ async function startDevServer() { { content: z.string(), }, - async ({ content }) => { + async ({ content }: { content: string }) => { try { const tweetId = await twitter.postTweet(content) return { diff --git a/services/twitter-services/src/parsers/profile-parser.ts b/services/twitter-services/src/parsers/profile-parser.ts index 95e935e0..49bb37ac 100644 --- a/services/twitter-services/src/parsers/profile-parser.ts +++ b/services/twitter-services/src/parsers/profile-parser.ts @@ -1,5 +1,7 @@ -import type { Element, Root } from 'hast' -import type { UserProfile } from '../types/twitter' +import type { Element, Node, Root } from 'hast' +import type { UserLink, UserProfile, UserStats } from '../types/twitter' + +import { select } from 'hast-util-select' import { SELECTORS } from '../utils/selectors' import { HtmlParser } from './html-parser' @@ -64,7 +66,7 @@ export class ProfileParser { * 从页面中提取用户名 */ private static extractUsername(_tree: Root): string { - // 可以从URL或特定DOM元素中提取 + // TODO: 可以从URL或特定DOM元素中提取 return '' } @@ -72,7 +74,7 @@ export class ProfileParser { * 提取用户统计数据 */ private static extractProfileStats(tree: Root) { - // 提取粉丝数、关注数、推文数等 + // TODO: 提取粉丝数、关注数、推文数等 const _statsElement = HtmlParser.select(tree, SELECTORS.PROFILE.STATS)[0] return { @@ -113,9 +115,9 @@ export class ProfileParser { try { // 查找统计信息容器 - const _statsElement = document.querySelector('[data-testid="userProfileStats"]') + const _statsElement = _tree ? select('[data-testid="userProfileStats"]', _tree as Root) : null - // 暂未实现具体解析逻辑 + // TODO: 暂未实现具体解析逻辑 return stats } @@ -128,7 +130,7 @@ export class ProfileParser { * 从 HTML 提取用户链接 */ static extractUserLinks(_html: string, _tree?: Node): UserLink[] { - // 暂未实现 + // TODO: 暂未实现 return [] } @@ -136,7 +138,7 @@ export class ProfileParser { * 从 HTML 提取用户加入日期 */ static extractJoinDate(_html: string, _tree?: Node): string | null { - // 暂未实现 + // TODO: 暂未实现 return null } } diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index b5453838..6e33dd26 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -73,6 +73,24 @@ export interface PostOptions { inReplyTo?: string } +/** + * 用户统计信息 + */ +export interface UserStats { + tweets: number + following: number + followers: number +} + +/** + * 用户链接信息 + */ +export interface UserLink { + type: string + url: string + title: string +} + /** * Twitter 服务接口 */ diff --git a/services/twitter-services/src/utils/api.ts b/services/twitter-services/src/utils/api.ts index 1a7c369c..7e70dc0d 100644 --- a/services/twitter-services/src/utils/api.ts +++ b/services/twitter-services/src/utils/api.ts @@ -20,14 +20,14 @@ export function createApiClient(baseURL: string, options: Record = onRequest({ request, options }) { const method = options.method || 'GET' const url = request.toString() - logger.browser.withField('method', method).withField('url', url).debug('API 请求') + logger.browser.withFields({ method, url }).debug('API 请求') }, // 请求错误拦截器 onRequestError({ request, error, options }) { const method = options.method || 'GET' const url = request.toString() - logger.browser.withField('method', method).withField('url', url).error('API 请求失败', error) + logger.browser.withFields({ method, url }).errorWithError('API 请求失败', error) }, // 响应拦截器 From e5d7253ee69bbf3e2965442f8a56618be00aea2e Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 00:52:31 +0800 Subject: [PATCH 04/20] refactor: launcher --- .../src/core/twitter-service.ts | 2 +- services/twitter-services/src/index.ts | 146 +----------------- services/twitter-services/src/launcher.ts | 134 ++++++++++++++++ 3 files changed, 139 insertions(+), 143 deletions(-) create mode 100644 services/twitter-services/src/launcher.ts diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index 449ee269..11df97d9 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -63,7 +63,7 @@ export class TwitterService implements ITwitterService { /** * 搜索推文 */ - async search(_query: string, _options?: SearchOptions): Promise { + async searchTweets(_query: string, _options?: SearchOptions): Promise { throw new Error('搜索功能尚未实现') } diff --git a/services/twitter-services/src/index.ts b/services/twitter-services/src/index.ts index f9bb5ddb..c8de9d24 100644 --- a/services/twitter-services/src/index.ts +++ b/services/twitter-services/src/index.ts @@ -1,150 +1,12 @@ import process from 'node:process' -import { AiriAdapter } from './adapters/airi-adapter' -import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' -import { MCPAdapter } from './adapters/mcp-adapter' -import { createDefaultConfig } from './config' -import { TwitterService } from './core/twitter-service' -import { initializeLogger, logger } from './utils/logger' +import { TwitterServiceLauncher } from './launcher' +import { logger } from './utils/logger'; -/** - * Twitter 服务启动器类 - * 负责初始化和启动服务 - */ -export class TwitterServiceLauncher { - private browser?: BrowserBaseMCPAdapter - private twitterService?: TwitterService - private airiAdapter?: AiriAdapter - private mcpAdapter?: MCPAdapter - - /** - * 启动 Twitter 服务 - */ - async start() { - try { - // 加载配置 - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - // 初始化日志系统 - initializeLogger() - logger.main.log('正在启动 Twitter 服务...') - - // 初始化浏览器 - this.browser = new BrowserBaseMCPAdapter( - config.browserbase.apiKey, - config.browserbase.endpoint, - { - timeout: config.browser.requestTimeout, - retries: config.browser.requestRetries, - }, - ) - - await this.browser.initialize(config.browser) - logger.main.log('浏览器已初始化') - - // 创建 Twitter 服务 - this.twitterService = new TwitterService(this.browser) - - // 尝试登录 - if (config.twitter.credentials) { - const success = await this.twitterService.login(config.twitter.credentials) - if (success) { - logger.main.log('成功登录 Twitter') - } - else { - logger.main.error('Twitter 登录失败!') - } - } - - // 启动适配器 - if (config.adapters.airi?.enabled && this.twitterService) { - this.airiAdapter = new AiriAdapter(this.twitterService, { - url: config.adapters.airi.url, - token: config.adapters.airi.token, - credentials: config.twitter.credentials!, - }) - - await this.airiAdapter.start() - logger.main.log('Airi 适配器已启动') - } - - if (config.adapters.mcp?.enabled && this.twitterService) { - this.mcpAdapter = new MCPAdapter(this.twitterService, config.adapters.mcp.port) - - await this.mcpAdapter.start() - logger.main.log('MCP 适配器已启动') - } - - logger.main.log('Twitter 服务已成功启动!') - - // 设置关闭钩子 - this.setupShutdownHooks() - } - catch (error) { - logger.main.withError(error).error('启动 Twitter 服务失败') - } - } - - /** - * 停止服务 - */ - async stop() { - logger.main.log('正在停止 Twitter 服务...') - - // 停止 MCP 适配器 - if (this.mcpAdapter) { - await this.mcpAdapter.stop() - logger.main.log('MCP 适配器已停止') - } - - // 关闭浏览器 - if (this.browser) { - await this.browser.close() - logger.main.log('浏览器已关闭') - } - - logger.main.log('Twitter 服务已停止') - } - - /** - * 设置关闭钩子 - */ - private setupShutdownHooks() { - // 处理进程退出 - process.on('SIGINT', async () => { - logger.main.log('接收到退出信号...') - await this.stop() - process.exit(0) - }) - - process.on('SIGTERM', async () => { - logger.main.log('接收到终止信号...') - await this.stop() - process.exit(0) - }) - - // 处理未捕获的异常 - process.on('uncaughtException', async (error) => { - logger.main.withError(error).error('未捕获的异常') - await this.stop() - process.exit(1) - }) - } -} - -export { AiriAdapter } from './adapters/airi-adapter' -export * from './adapters/browser-adapter' -export { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' -export { MCPAdapter } from './adapters/mcp-adapter' -export * from './core/twitter-service' -export * from './types/browser' - -// 直接启动服务的入口点 -if (require.main === module) { +(async () => { const launcher = new TwitterServiceLauncher() launcher.start().catch((error) => { logger.main.withError(error).error('启动失败') process.exit(1) }) -} +})() diff --git a/services/twitter-services/src/launcher.ts b/services/twitter-services/src/launcher.ts new file mode 100644 index 00000000..ef1c7ff1 --- /dev/null +++ b/services/twitter-services/src/launcher.ts @@ -0,0 +1,134 @@ +import process from 'node:process' + +import { AiriAdapter } from './adapters/airi-adapter' +import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { MCPAdapter } from './adapters/mcp-adapter' +import { createDefaultConfig } from './config' +import { TwitterService } from './core/twitter-service' +import { initializeLogger, logger } from './utils/logger' + +/** + * Twitter 服务启动器类 + * 负责初始化和启动服务 + */ +export class TwitterServiceLauncher { + private browser?: BrowserBaseMCPAdapter + private twitterService?: TwitterService + private airiAdapter?: AiriAdapter + private mcpAdapter?: MCPAdapter + + /** + * 启动 Twitter 服务 + */ + async start() { + try { + // 加载配置 + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + // 初始化日志系统 + initializeLogger() + logger.main.log('正在启动 Twitter 服务...') + + // 初始化浏览器 + this.browser = new BrowserBaseMCPAdapter( + config.browserbase.apiKey, + config.browserbase.endpoint, + { + timeout: config.browser.requestTimeout, + retries: config.browser.requestRetries, + }, + ) + + await this.browser.initialize(config.browser) + logger.main.log('浏览器已初始化') + + // 创建 Twitter 服务 + this.twitterService = new TwitterService(this.browser) + + // 尝试登录 + if (config.twitter.credentials) { + const success = await this.twitterService.login(config.twitter.credentials) + if (success) { + logger.main.log('成功登录 Twitter') + } + else { + logger.main.error('Twitter 登录失败!') + } + } + + // 启动适配器 + if (config.adapters.airi?.enabled && this.twitterService) { + this.airiAdapter = new AiriAdapter(this.twitterService, { + url: config.adapters.airi.url, + token: config.adapters.airi.token, + credentials: config.twitter.credentials!, + }) + + await this.airiAdapter.start() + logger.main.log('Airi 适配器已启动') + } + + if (config.adapters.mcp?.enabled && this.twitterService) { + this.mcpAdapter = new MCPAdapter(this.twitterService, config.adapters.mcp.port) + + await this.mcpAdapter.start() + logger.main.log('MCP 适配器已启动') + } + + logger.main.log('Twitter 服务已成功启动!') + + // 设置关闭钩子 + this.setupShutdownHooks() + } + catch (error) { + logger.main.withError(error).error('启动 Twitter 服务失败') + } + } + + /** + * 停止服务 + */ + async stop() { + logger.main.log('正在停止 Twitter 服务...') + + // 停止 MCP 适配器 + if (this.mcpAdapter) { + await this.mcpAdapter.stop() + logger.main.log('MCP 适配器已停止') + } + + // 关闭浏览器 + if (this.browser) { + await this.browser.close() + logger.main.log('浏览器已关闭') + } + + logger.main.log('Twitter 服务已停止') + } + + /** + * 设置关闭钩子 + */ + private setupShutdownHooks() { + // 处理进程退出 + process.on('SIGINT', async () => { + logger.main.log('接收到退出信号...') + await this.stop() + process.exit(0) + }) + + process.on('SIGTERM', async () => { + logger.main.log('接收到终止信号...') + await this.stop() + process.exit(0) + }) + + // 处理未捕获的异常 + process.on('uncaughtException', async (error) => { + logger.main.withError(error).error('未捕获的异常') + await this.stop() + process.exit(1) + }) + } +} From 562023f6e5a96ac15998ee5457a8f5040db55bc8 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 01:07:22 +0800 Subject: [PATCH 05/20] refactor: BrowserBase/Stagehand --- pnpm-lock.yaml | 72 ++++- services/twitter-services/package.json | 4 +- .../src/browser/browserbase.ts | 269 +++++++++++------- services/twitter-services/src/config/index.ts | 2 +- services/twitter-services/src/config/types.ts | 19 +- 5 files changed, 248 insertions(+), 118 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9a3003f2..d587a12e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1299,6 +1299,9 @@ importers: services/twitter-services: dependencies: + '@browserbasehq/stagehand': + specifier: ^1.13.1 + version: 1.13.1(@playwright/test@1.50.1)(bufferutil@4.0.9)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.85.0(encoding@0.1.13)(ws@8.18.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.24.2))(utf-8-validate@5.0.10)(zod@3.24.2) '@guiiai/logg': specifier: ^1.0.0 version: 1.0.7 @@ -1332,6 +1335,9 @@ importers: ofetch: specifier: ^1.3.3 version: 1.4.1 + playwright: + specifier: ^1.50.1 + version: 1.50.1 rehype-parse: specifier: ^9.0.0 version: 9.0.1 @@ -1342,7 +1348,7 @@ importers: specifier: ^5.0.0 version: 5.0.0 zod: - specifier: ^3.22.2 + specifier: ^3.24.2 version: 3.24.2 devDependencies: '@types/node': @@ -1433,6 +1439,9 @@ packages: '@antfu/utils@8.1.0': resolution: {integrity: sha512-XPR7Jfwp0FFl/dFYPX8ZjpmU4/1mIXTjnZ1ba48BLMyKOV62/tiRjdsFcPs2hsYcSud4tzk7w3a3LjX8Fu3huA==} + '@anthropic-ai/sdk@0.27.3': + resolution: {integrity: sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==} + '@apideck/better-ajv-errors@0.3.6': resolution: {integrity: sha512-P+ZygBLZtkp0qqOAJJVX4oX/sFo5JR3eBWwwuqHHhK0GIgQOKWrAfiAaWX0aArHkRWHMuggFEgAZNxVPwPZYaA==} engines: {node: '>=10'} @@ -2021,6 +2030,18 @@ packages: resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} + '@browserbasehq/sdk@2.3.0': + resolution: {integrity: sha512-H2nu46C6ydWgHY+7yqaP8qpfRJMJFVGxVIgsuHe1cx9HkfJHqzkuIqaK/k8mU4ZeavQgV5ZrJa0UX6MDGYiT4w==} + + '@browserbasehq/stagehand@1.13.1': + resolution: {integrity: sha512-sty9bDiuuQJDOS+/uBfXpwYQY+mhFyqi6uT5wSOrazagZ5s8tgk3ryCIheB/BGS5iisc6ivAsKe9aC9n5WBTAg==} + peerDependencies: + '@playwright/test': ^1.42.1 + deepmerge: ^4.3.1 + dotenv: ^16.4.5 + openai: ^4.62.1 + zod: ^3.23.8 + '@bundled-es-modules/cookie@2.0.1': resolution: {integrity: sha512-8o+5fRPLNbjbdGRRmJj3h6Hh1AQJf2dk3qQ/5ZFb+PXkRNiSoMGGUKlsgLfrxneb72axVJyIYji64E2+nNfYyw==} @@ -3923,6 +3944,11 @@ packages: resolution: {integrity: sha512-Zwq5OCzuwJC2jwqmpEQt7Ds1DTi6BWSwoGkbb1n9pO3hzb35BoJELx7c0T23iDkBGkh2e7tvOtjF3tr3OaQHDQ==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + '@playwright/test@1.50.1': + resolution: {integrity: sha512-Jii3aBg+CEDpgnuDxEp/h7BimHcUTDlpEtce89xEumlJ5ef2hqepZ+PWp1DDpYC/VO9fmWVI1IlEaoI5fK9FXQ==} + engines: {node: '>=18'} + hasBin: true + '@polka/url@1.0.0-next.24': resolution: {integrity: sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==} @@ -11981,6 +12007,18 @@ snapshots: '@antfu/utils@8.1.0': {} + '@anthropic-ai/sdk@0.27.3(encoding@0.1.13)': + dependencies: + '@types/node': 18.19.76 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0(encoding@0.1.13) + transitivePeerDependencies: + - encoding + '@apideck/better-ajv-errors@0.3.6(ajv@8.12.0)': dependencies: ajv: 8.12.0 @@ -12829,6 +12867,34 @@ snapshots: '@bcoe/v8-coverage@1.0.2': {} + '@browserbasehq/sdk@2.3.0(encoding@0.1.13)': + dependencies: + '@types/node': 18.19.76 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0(encoding@0.1.13) + transitivePeerDependencies: + - encoding + + '@browserbasehq/stagehand@1.13.1(@playwright/test@1.50.1)(bufferutil@4.0.9)(deepmerge@4.3.1)(dotenv@16.4.7)(encoding@0.1.13)(openai@4.85.0(encoding@0.1.13)(ws@8.18.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.24.2))(utf-8-validate@5.0.10)(zod@3.24.2)': + dependencies: + '@anthropic-ai/sdk': 0.27.3(encoding@0.1.13) + '@browserbasehq/sdk': 2.3.0(encoding@0.1.13) + '@playwright/test': 1.50.1 + deepmerge: 4.3.1 + dotenv: 16.4.7 + openai: 4.85.0(encoding@0.1.13)(ws@8.18.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(zod@3.24.2) + ws: 8.18.1(bufferutil@4.0.9)(utf-8-validate@5.0.10) + zod: 3.24.2 + zod-to-json-schema: 3.24.3(zod@3.24.2) + transitivePeerDependencies: + - bufferutil + - encoding + - utf-8-validate + '@bundled-es-modules/cookie@2.0.1': dependencies: cookie: 0.7.2 @@ -14585,6 +14651,10 @@ snapshots: '@pkgr/core@0.1.0': {} + '@playwright/test@1.50.1': + dependencies: + playwright: 1.50.1 + '@polka/url@1.0.0-next.24': {} '@proj-airi/server-sdk@0.1.4': diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index ff7862b4..1b8e2457 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -12,6 +12,7 @@ "dev:mcp": "tsx src/dev-server.ts" }, "dependencies": { + "@browserbasehq/stagehand": "^1.13.1", "@guiiai/logg": "^1.0.0", "@modelcontextprotocol/sdk": "^1.6.1", "@proj-airi/server-sdk": "^0.1.0", @@ -23,10 +24,11 @@ "hast-util-to-text": "^4.0.2", "listhen": "^1.6.0", "ofetch": "^1.3.3", + "playwright": "^1.50.1", "rehype-parse": "^9.0.0", "unified": "^11.0.3", "unist-util-visit": "^5.0.0", - "zod": "^3.22.2" + "zod": "^3.24.2" }, "devDependencies": { "@types/node": "^18.16.3", diff --git a/services/twitter-services/src/browser/browserbase.ts b/services/twitter-services/src/browser/browserbase.ts index c071381b..9400bfd5 100644 --- a/services/twitter-services/src/browser/browserbase.ts +++ b/services/twitter-services/src/browser/browserbase.ts @@ -1,57 +1,83 @@ -import { Buffer } from 'node:buffer' +import type { Buffer } from 'node:buffer' +import type { Browser, Page } from 'playwright' +import type { z } from 'zod' + +import { chromium } from 'playwright' -import { createApiClient } from '../utils/api' import { logger } from '../utils/logger' /** - * BrowserBase API 客户端配置选项 + * Stagehand 客户端配置选项 */ -export interface BrowserBaseClientOptions { +export interface StagehandClientOptions { apiKey: string baseUrl?: string timeout?: number - retries?: number + headless?: boolean + userAgent?: string + viewport?: { width: number, height: number } } /** - * BrowserBase API 客户端 - * 封装 BrowserBase 的 REST API + * Stagehand 客户端 + * 使用 @browserbasehq/stagehand 实现浏览器自动化 */ -export class BrowserBaseClient { - private sessionId: string | null = null - private api: ReturnType - - constructor(options: BrowserBaseClientOptions) { - const { apiKey, baseUrl = 'https://api.browserbase.com', timeout = 30000, retries = 1 } = options - - // 创建 API 客户端 - this.api = createApiClient(baseUrl, { - headers: { - 'Authorization': `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, +export class StagehandClient { + private browser: Browser | null = null + private page: Page | null = null + private apiKey: string + private options: Omit + + constructor(options: StagehandClientOptions) { + const { + apiKey, + baseUrl, + timeout = 30000, + headless = true, + userAgent, + viewport = { width: 1280, height: 800 }, + } = options + + this.apiKey = apiKey + this.options = { + baseUrl, timeout, - retry: retries, - }) + headless, + userAgent, + viewport, + } } /** * 创建浏览器会话 */ - async createSession(options: { + async createSession(options?: { headless?: boolean userAgent?: string viewport?: { width: number, height: number } }): Promise { try { - const data = await this.api('/v1/sessions', { - method: 'POST', - body: options, + // 启动 Playwright 浏览器 + this.browser = await chromium.launch({ + headless: options?.headless ?? this.options.headless, + }) + + // 创建上下文 + const context = await this.browser.newContext({ + userAgent: options?.userAgent ?? this.options.userAgent, + viewport: options?.viewport ?? this.options.viewport, + // 设置任何其他所需的浏览器上下文选项 }) - this.sessionId = data.sessionId - logger.browser.withField('sessionId', this.sessionId).log('创建浏览器会话成功') - return this.sessionId || '' + // 创建页面 + this.page = await context.newPage() + + // 为页面添加 Stagehand 扩展 + await this.setupStagehand() + + const sessionId = `session-${Date.now()}` + logger.browser.withField('sessionId', sessionId).log('创建浏览器会话成功') + return sessionId } catch (error) { logger.browser.errorWithError('创建浏览器会话失败', error) @@ -59,48 +85,119 @@ export class BrowserBaseClient { } } + /** + * 设置 Stagehand 扩展 + * 这将添加 act, extract, observe 方法到 page 对象 + */ + private async setupStagehand(): Promise { + if (!this.page) { + throw new Error('No active page. Call createSession first.') + } + + // 在实际实现中，这里会使用 Stagehand 的 API 设置页面对象 + // 这可能涉及到页面扩展或注入 Stagehand 的功能 + // 示例代码（实际使用需要根据 Stagehand 的文档进行调整）: + // + // import { extendPage } from '@browserbasehq/stagehand' + // await extendPage(this.page, { + // apiKey: this.apiKey, + // // 其他 Stagehand 选项 + // }) + } + /** * 导航到指定URL */ async navigate(url: string): Promise { - this.ensureSessionExists() - - await this.api(`/v1/sessions/${this.sessionId}/url`, { - method: 'POST', - body: { url }, - }) + this.ensurePageExists() + await this.page!.goto(url, { timeout: this.options.timeout }) } /** * 执行JavaScript脚本 */ async executeScript(script: string): Promise { - this.ensureSessionExists() + this.ensurePageExists() + return await this.page!.evaluate(script) as T + } - const data = await this.api(`/v1/sessions/${this.sessionId}/execute`, { - method: 'POST', - body: { script }, - }) + /** + * 使用 Stagehand 的 act API 执行操作 + */ + async act(instruction: string): Promise { + this.ensurePageExists() - return data.result + // 在实际实现中，这将使用 Stagehand 的 act API + // 示例：await this.page!.act(instruction) + + // 临时实现，使用 Playwright 的基本能力模拟 + logger.browser.withField('instruction', instruction).log('执行 act 指令') + + // 这里通过简单的方法来模拟 act 的行为 + // 实际需要使用 Stagehand 的 act API + if (instruction.includes('click')) { + const match = instruction.match(/click on the ['"](.+?)['"]/) + if (match && match[1]) { + await this.page!.getByText(match[1]).first().click() + } + } + } + + /** + * 使用 Stagehand 的 extract API 提取数据 + */ + async extract({ + instruction, + _schema, + }: { + instruction: string + _schema: T + }): Promise> { + this.ensurePageExists() + + // 在实际实现中，这将使用 Stagehand 的 extract API + // 示例：return await this.page!.extract({ instruction, schema }) + + // 临时实现，记录指令并返回一个空对象 + logger.browser.withField('instruction', instruction).log('执行 extract 指令') + + // 这里只是简单地返回一个空对象 + // 实际需要使用 Stagehand 的 extract API + return {} as z.infer + } + + /** + * 使用 Stagehand 的 observe API 观察页面状态 + */ + async observe(instruction: string): Promise { + this.ensurePageExists() + + // 在实际实现中，这将使用 Stagehand 的 observe API + // 示例：return await this.page!.observe(instruction) + + // 临时实现，记录指令并返回空字符串 + logger.browser.withField('instruction', instruction).log('执行 observe 指令') + + // 这里只是简单地返回一个空字符串 + // 实际需要使用 Stagehand 的 observe API + return '' } /** * 获取页面内容 */ async getContent(): Promise { - return this.executeScript('document.documentElement.outerHTML') + this.ensurePageExists() + return await this.page!.content() } /** * 等待元素出现 */ async waitForSelector(selector: string, options: { timeout?: number } = {}): Promise { - this.ensureSessionExists() - - await this.api(`/v1/sessions/${this.sessionId}/wait`, { - method: 'POST', - body: { selector, timeout: options.timeout }, + this.ensurePageExists() + await this.page!.waitForSelector(selector, { + timeout: options.timeout || this.options.timeout, }) } @@ -108,93 +205,59 @@ export class BrowserBaseClient { * 点击元素 */ async click(selector: string): Promise { - this.ensureSessionExists() - - await this.executeScript(` - const element = document.querySelector('${selector}'); - if (!element) throw new Error('Element not found: ${selector}'); - element.click(); - `) + this.ensurePageExists() + await this.page!.click(selector) } /** * 向输入框输入文本 */ async type(selector: string, text: string): Promise { - this.ensureSessionExists() - + this.ensurePageExists() // 先清空输入框 - await this.executeScript(` - const element = document.querySelector('${selector}'); - if (!element) throw new Error('Element not found: ${selector}'); - element.value = ''; - `) - + await this.page!.fill(selector, '') // 然后输入文本 - await this.executeScript(` - const element = document.querySelector('${selector}'); - if (!element) throw new Error('Element not found: ${selector}'); - element.value = '${text.replace(/'/g, '\\\'')}'; - element.dispatchEvent(new Event('input', { bubbles: true })); - element.dispatchEvent(new Event('change', { bubbles: true })); - `) + await this.page!.fill(selector, text) } /** * 获取元素文本内容 */ async getText(selector: string): Promise { - this.ensureSessionExists() - - return this.executeScript(` - const element = document.querySelector('${selector}'); - if (!element) throw new Error('Element not found: ${selector}'); - return element.textContent.trim(); - `) + this.ensurePageExists() + const element = await this.page!.$(selector) + if (!element) { + throw new Error(`Element not found: ${selector}`) + } + return (await element.textContent() || '').trim() } /** * 获取屏幕截图 */ async getScreenshot(): Promise { - this.ensureSessionExists() - - const response = await this.api(`/v1/sessions/${this.sessionId}/screenshot`, { - method: 'GET', - }) - - if (!response.ok) { - throw new Error(`截图失败: ${response.statusText || '未知错误'}`) - } - - const buffer = await response.arrayBuffer() - return Buffer.from(buffer) + this.ensurePageExists() + return await this.page!.screenshot() as Buffer } /** * 关闭会话 */ async closeSession(): Promise { - if (!this.sessionId) - return - - const response = await this.api(`/v1/sessions/${this.sessionId}`, { - method: 'DELETE', - }) - - if (!response.ok) { - throw new Error(`关闭会话失败: ${response.statusText || '未知错误'}`) + if (this.browser) { + await this.browser.close() + this.browser = null + this.page = null + logger.browser.log('浏览器会话已关闭') } - - this.sessionId = null } /** - * 确保会话存在 + * 确保页面存在 */ - private ensureSessionExists(): void { - if (!this.sessionId) { - throw new Error('No active session. Call createSession first.') + private ensurePageExists(): void { + if (!this.page) { + throw new Error('No active page. Call createSession first.') } } } diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts index e4bf1675..7fc89578 100644 --- a/services/twitter-services/src/config/index.ts +++ b/services/twitter-services/src/config/index.ts @@ -51,7 +51,7 @@ export class ConfigManager { */ private validateConfig(): void { // 验证必要的 API 密钥 - if (!this.config.browserbase.apiKey) { + if (!this.config.browser.apiKey) { console.warn('未设置 BrowserBase API 密钥!') } diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index be0033a6..ff722611 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -7,14 +7,11 @@ import process from 'node:process' * 完整配置接口 */ export interface Config { - // BrowserBase 配置 - browserbase: { - apiKey: string - endpoint?: string - } - // 浏览器配置 - browser: BrowserConfig + browser: BrowserConfig & { + apiKey: string // 为 Stagehand 保留 API Key + endpoint?: string // 可选的 Stagehand 服务端点 + } // Twitter 配置 twitter: { @@ -50,10 +47,8 @@ export interface Config { * 默认配置 */ export const DEFAULT_CONFIG: Config = { - browserbase: { - apiKey: process.env.BROWSERBASE_API_KEY || '', - }, browser: { + apiKey: process.env.BROWSERBASE_API_KEY || '', // 将 apiKey 移到 browser 配置中 headless: true, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', viewport: { @@ -61,8 +56,8 @@ export const DEFAULT_CONFIG: Config = { height: 800, }, timeout: 30000, - requestTimeout: 20000, // API 请求超时设置 (20秒) - requestRetries: 2, // 网络错误时重试次数 + requestTimeout: 20000, + requestRetries: 2, }, twitter: { credentials: { From a53333d648acc3adb3f7bab34aade4663529b70c Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 02:10:16 +0800 Subject: [PATCH 06/20] fix: config & startup --- cspell.config.yaml | 1 + pnpm-lock.yaml | 5 +- services/twitter-services/package.json | 11 +-- .../src/adapters/browserbase-adapter.ts | 44 ++++++---- services/twitter-services/src/cli.ts | 2 +- services/twitter-services/src/config/index.ts | 88 ++++++++++++------- services/twitter-services/src/config/types.ts | 76 ++++++++-------- services/twitter-services/src/dev-server.ts | 10 +-- services/twitter-services/src/index.ts | 12 --- services/twitter-services/src/launcher.ts | 27 +++--- services/twitter-services/src/main.ts | 30 +++++++ services/twitter-services/src/utils/logger.ts | 70 +++++++++------ 12 files changed, 223 insertions(+), 153 deletions(-) delete mode 100644 services/twitter-services/src/index.ts create mode 100644 services/twitter-services/src/main.ts diff --git a/cspell.config.yaml b/cspell.config.yaml index f5331861..170864c3 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -15,6 +15,7 @@ words: - baiducloud - bigserial - Bitstream + - browserbasehq - bumpp - catppuccin - changelogithub diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d587a12e..197a2efe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1314,8 +1314,11 @@ importers: '@types/hast': specifier: ^3.0.1 version: 3.0.4 + defu: + specifier: ^6.1.4 + version: 6.1.4 dotenv: - specifier: ^16.0.3 + specifier: ^16.4.7 version: 16.4.7 h3: specifier: ^1.11.0 diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index 1b8e2457..f0665f68 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -6,10 +6,10 @@ "author": "RainbowBird ", "license": "MIT", "scripts": { - "build": "tsc", - "start": "tsx dist/index.js", - "dev": "tsx src/index.ts", - "dev:mcp": "tsx src/dev-server.ts" + "start": "tsx src/main.ts", + "dev": "tsx watch src/main.ts", + "dev:mcp": "tsx src/dev-server.ts", + "postinstall": "playwright install chromium" }, "dependencies": { "@browserbasehq/stagehand": "^1.13.1", @@ -17,7 +17,8 @@ "@modelcontextprotocol/sdk": "^1.6.1", "@proj-airi/server-sdk": "^0.1.0", "@types/hast": "^3.0.1", - "dotenv": "^16.0.3", + "defu": "^6.1.4", + "dotenv": "^16.4.7", "h3": "^1.11.0", "hast-util-is-element": "^3.0.0", "hast-util-select": "^6.0.4", diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts index 2592efff..64bb761e 100644 --- a/services/twitter-services/src/adapters/browserbase-adapter.ts +++ b/services/twitter-services/src/adapters/browserbase-adapter.ts @@ -1,20 +1,20 @@ import type { Buffer } from 'node:buffer' -import type { BrowserBaseClientOptions } from '../browser/browserbase' +import type { StagehandClientOptions } from '../browser/browserbase' import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' import type { BrowserAdapter } from './browser-adapter' -import { BrowserBaseClient } from '../browser/browserbase' +import { StagehandClient } from '../browser/browserbase' import { errorToMessage } from '../utils/error' import { logger } from '../utils/logger' /** - * BrowserBase 元素句柄实现 + * Stagehand 元素句柄实现 */ -class BrowserBaseElementHandle implements ElementHandle { - private client: BrowserBaseClient +class StagehandElementHandle implements ElementHandle { + private client: StagehandClient private selector: string - constructor(client: BrowserBaseClient, selector: string) { + constructor(client: StagehandClient, selector: string) { this.client = client this.selector = selector } @@ -41,14 +41,14 @@ class BrowserBaseElementHandle implements ElementHandle { } /** - * BrowserBase 适配器实现 - * 将 BrowserBase API 适配为通用浏览器接口 + * Stagehand 浏览器适配器实现 + * 将 Stagehand API 适配为通用浏览器接口 */ -export class BrowserBaseMCPAdapter implements BrowserAdapter { - private client: BrowserBaseClient +export class StagehandBrowserAdapter implements BrowserAdapter { + private client: StagehandClient - constructor(apiKey: string, baseUrl?: string, options: Partial = {}) { - this.client = new BrowserBaseClient({ + constructor(apiKey: string, baseUrl?: string, options: Partial = {}) { + this.client = new StagehandClient({ apiKey, baseUrl, ...options, @@ -61,12 +61,13 @@ export class BrowserBaseMCPAdapter implements BrowserAdapter { headless: config.headless, userAgent: config.userAgent, viewport: config.viewport, - // proxyUrl: config.proxy, // TODO: Proxy }) - logger.browser.log('浏览器会话已创建', { headless: config.headless }) + logger.browser.withFields({ + headless: config.headless, + }).log('浏览器会话已创建') } catch (error) { - logger.browser.errorWithError('浏览器初始化失败', error) + logger.browser.withError(error).error('浏览器初始化失败') throw new Error(`无法初始化浏览器: ${errorToMessage(error)}`) } } @@ -101,14 +102,19 @@ export class BrowserBaseMCPAdapter implements BrowserAdapter { // 获取所有匹配元素的选择器 const selectors = await this.executeScript(` Array.from(document.querySelectorAll('${selector}')).map((el, i) => { - const uniqueId = 'browserbase-' + Date.now() + '-' + i; - el.setAttribute('data-browserbase-id', uniqueId); - return '[data-browserbase-id="' + uniqueId + '"]'; + const uniqueId = 'stagehand-' + Date.now() + '-' + i; + el.setAttribute('data-stagehand-id', uniqueId); + return '[data-stagehand-id="' + uniqueId + '"]'; }) `) // 为每个匹配的元素创建一个 ElementHandle - return selectors.map(selector => new BrowserBaseElementHandle(this.client, selector)) + return selectors.map(selector => new StagehandElementHandle(this.client, selector)) + } + + // 新增 Stagehand 相关方法 + async act(instruction: string): Promise { + await this.client.act(instruction) } async getScreenshot(): Promise { diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts index 973915c3..3d15029a 100644 --- a/services/twitter-services/src/cli.ts +++ b/services/twitter-services/src/cli.ts @@ -8,7 +8,7 @@ import { Command } from 'commander' import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' import { createDefaultConfig } from './config' import { TwitterService } from './core/twitter-service' -import { TwitterServiceLauncher } from './index' +import { TwitterServiceLauncher } from './launcher' import { errorToMessage } from './utils/error' // 获取版本 diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts index 7fc89578..3fffa1f9 100644 --- a/services/twitter-services/src/config/index.ts +++ b/services/twitter-services/src/config/index.ts @@ -3,9 +3,39 @@ import type { Config } from './types' import fs from 'node:fs' import path from 'node:path' import process from 'node:process' +import { defu } from 'defu' +import { config as configDotenv } from 'dotenv' import { logger } from '../utils/logger' -import { DEFAULT_CONFIG } from './types' +import { getDefaultConfig } from './types' + +/** + * 加载环境变量文件 + * 按优先级顺序加载 + */ +function loadEnvFiles(): void { + // 加载环境变量文件 + const envFiles = [ + '.env.local', + ] + + // 从当前目录向上查找 .env 文件 + for (const file of envFiles) { + const filePath = path.resolve(process.cwd(), file) + if (fs.existsSync(filePath)) { + const result = configDotenv({ + path: filePath, + override: true, // 允许覆盖已存在的环境变量 + }) + + if (result.parsed) { + logger.config.withFields({ + config: result.parsed, + }).log(`已从 ${file} 加载环境变量`) + } + } + } +} /** * 配置管理器 @@ -19,12 +49,18 @@ export class ConfigManager { * @param configPath 配置文件路径 */ constructor(configPath?: string) { - this.config = { ...DEFAULT_CONFIG } + // 首先加载环境变量 + loadEnvFiles() + + // 设置默认配置 + this.config = getDefaultConfig() + // 然后从配置文件加载（如果指定） if (configPath) { this.loadFromFile(configPath) } + // 验证配置 this.validateConfig() } @@ -36,8 +72,9 @@ export class ConfigManager { const configFile = fs.readFileSync(filePath, 'utf8') const fileConfig = JSON.parse(configFile) - // 深度合并配置 - this.config = this.mergeConfigs(this.config, fileConfig) + // 使用 defu 深度合并配置 + // fileConfig 中的值优先于 this.config 中的值 + this.config = defu(fileConfig, this.config) logger.config.log(`配置已从 ${filePath} 加载`) } @@ -50,33 +87,14 @@ export class ConfigManager { * 验证配置有效性 */ private validateConfig(): void { - // 验证必要的 API 密钥 - if (!this.config.browser.apiKey) { - console.warn('未设置 BrowserBase API 密钥!') - } - // 验证 Twitter 凭据 if (!this.config.twitter.credentials?.username || !this.config.twitter.credentials?.password) { - console.warn('未设置 Twitter 凭据!') - } - } - - /** - * 递归合并配置对象 - */ - private mergeConfigs(target: any, source: any): any { - const result = { ...target } - - for (const key in source) { - if (source[key] instanceof Object && key in target) { - result[key] = this.mergeConfigs(target[key], source[key]) - } - else { - result[key] = source[key] - } + logger.config.warn('未设置 Twitter 凭据!') } - return result + logger.config.withFields({ + config: this.config, + }).log('配置验证完成') } /** @@ -90,16 +108,24 @@ export class ConfigManager { * 更新配置 */ updateConfig(newConfig: Partial): void { - this.config = this.mergeConfigs(this.config, newConfig) + // 使用 defu 合并新配置 + this.config = defu(newConfig, this.config) this.validateConfig() } } +// 单例实例 +let configInstance: ConfigManager | null = null + /** - * 创建默认配置管理器 + * 创建默认配置管理器 (单例) */ export function createDefaultConfig(): ConfigManager { - const configPath = process.env.CONFIG_PATH || path.join(process.cwd(), 'twitter-config.json') + if (configInstance) { + return configInstance + } - return new ConfigManager(fs.existsSync(configPath) ? configPath : undefined) + const configPath = process.env.CONFIG_PATH || path.join(process.cwd(), 'twitter-config.json') + configInstance = new ConfigManager(fs.existsSync(configPath) ? configPath : undefined) + return configInstance } diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index ff722611..bb805d5e 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -46,46 +46,48 @@ export interface Config { /** * 默认配置 */ -export const DEFAULT_CONFIG: Config = { - browser: { - apiKey: process.env.BROWSERBASE_API_KEY || '', // 将 apiKey 移到 browser 配置中 - headless: true, - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - viewport: { - width: 1280, - height: 800, - }, - timeout: 30000, - requestTimeout: 20000, - requestRetries: 2, - }, - twitter: { - credentials: { - username: process.env.TWITTER_USERNAME || '', - password: process.env.TWITTER_PASSWORD || '', +export function getDefaultConfig(): Config { + return { + browser: { + apiKey: process.env.BROWSERBASE_API_KEY || '', // 将 apiKey 移到 browser 配置中 + headless: true, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + viewport: { + width: 1280, + height: 800, + }, + timeout: 30000, + requestTimeout: 20000, + requestRetries: 2, }, - defaultOptions: { - timeline: { - count: 20, - includeReplies: true, - includeRetweets: true, + twitter: { + credentials: { + username: process.env.TWITTER_USERNAME || '', + password: process.env.TWITTER_PASSWORD || '', + }, + defaultOptions: { + timeline: { + count: 20, + includeReplies: true, + includeRetweets: true, + }, }, }, - }, - adapters: { - airi: { - url: process.env.AIRI_URL || 'http://localhost:3000', - token: process.env.AIRI_TOKEN || '', - enabled: process.env.ENABLE_AIRI === 'true', + adapters: { + airi: { + url: process.env.AIRI_URL || 'http://localhost:3000', + token: process.env.AIRI_TOKEN || '', + enabled: process.env.ENABLE_AIRI === 'true', + }, + mcp: { + port: Number(process.env.MCP_PORT || 8080), + enabled: process.env.ENABLE_MCP === 'true' || true, + }, }, - mcp: { - port: Number(process.env.MCP_PORT || 8080), - enabled: process.env.ENABLE_MCP === 'true' || true, + system: { + logLevel: 'debug', + logFormat: 'pretty', + concurrency: Number(process.env.CONCURRENCY || 1), }, - }, - system: { - logLevel: 'info', - logFormat: process.env.NODE_ENV === 'development' ? 'pretty' : 'json', - concurrency: Number(process.env.CONCURRENCY || 1), - }, + } } diff --git a/services/twitter-services/src/dev-server.ts b/services/twitter-services/src/dev-server.ts index 1cc13432..eee03c72 100644 --- a/services/twitter-services/src/dev-server.ts +++ b/services/twitter-services/src/dev-server.ts @@ -7,7 +7,7 @@ import { createApp, createRouter, defineEventHandler, toNodeListener } from 'h3' import { listen } from 'listhen' import { z } from 'zod' -import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' import { TwitterService } from './core/twitter-service' import { errorToMessage } from './utils/error' import { logger } from './utils/logger' @@ -20,17 +20,11 @@ dotenv.config() * 使用 listhen 提供开发时的便利功能 */ async function startDevServer() { - // 基本检查环境变量 - if (!process.env.BROWSERBASE_API_KEY) { - console.error('错误: 缺少环境变量 BROWSERBASE_API_KEY') - process.exit(1) - } - const app = createApp() const router = createRouter() // 创建浏览器和 Twitter 服务 - const browser = new BrowserBaseMCPAdapter(process.env.BROWSERBASE_API_KEY) + const browser = new StagehandBrowserAdapter(process.env.BROWSERBASE_API_KEY || '') await browser.initialize({ headless: true, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', diff --git a/services/twitter-services/src/index.ts b/services/twitter-services/src/index.ts deleted file mode 100644 index c8de9d24..00000000 --- a/services/twitter-services/src/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import process from 'node:process' - -import { TwitterServiceLauncher } from './launcher' -import { logger } from './utils/logger'; - -(async () => { - const launcher = new TwitterServiceLauncher() - launcher.start().catch((error) => { - logger.main.withError(error).error('启动失败') - process.exit(1) - }) -})() diff --git a/services/twitter-services/src/launcher.ts b/services/twitter-services/src/launcher.ts index ef1c7ff1..70f9e6e9 100644 --- a/services/twitter-services/src/launcher.ts +++ b/services/twitter-services/src/launcher.ts @@ -1,18 +1,19 @@ +import type { AiriAdapter } from './adapters/airi-adapter' +import type { StagehandBrowserAdapter } from './adapters/browserbase-adapter' +import type { MCPAdapter } from './adapters/mcp-adapter' + import process from 'node:process' -import { AiriAdapter } from './adapters/airi-adapter' -import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' -import { MCPAdapter } from './adapters/mcp-adapter' import { createDefaultConfig } from './config' import { TwitterService } from './core/twitter-service' -import { initializeLogger, logger } from './utils/logger' +import { logger } from './utils/logger' /** * Twitter 服务启动器类 * 负责初始化和启动服务 */ export class TwitterServiceLauncher { - private browser?: BrowserBaseMCPAdapter + private browser?: StagehandBrowserAdapter private twitterService?: TwitterService private airiAdapter?: AiriAdapter private mcpAdapter?: MCPAdapter @@ -26,17 +27,17 @@ export class TwitterServiceLauncher { const configManager = createDefaultConfig() const config = configManager.getConfig() - // 初始化日志系统 - initializeLogger() logger.main.log('正在启动 Twitter 服务...') // 初始化浏览器 - this.browser = new BrowserBaseMCPAdapter( - config.browserbase.apiKey, - config.browserbase.endpoint, + // 导入处理 + const { StagehandBrowserAdapter } = await import('./adapters/browserbase-adapter') + this.browser = new StagehandBrowserAdapter( + config.browser.apiKey, + config.browser.endpoint, { timeout: config.browser.requestTimeout, - retries: config.browser.requestRetries, + // retries: config.browser.requestRetries, }, ) @@ -59,6 +60,8 @@ export class TwitterServiceLauncher { // 启动适配器 if (config.adapters.airi?.enabled && this.twitterService) { + // 导入处理 + const { AiriAdapter } = await import('./adapters/airi-adapter') this.airiAdapter = new AiriAdapter(this.twitterService, { url: config.adapters.airi.url, token: config.adapters.airi.token, @@ -70,6 +73,8 @@ export class TwitterServiceLauncher { } if (config.adapters.mcp?.enabled && this.twitterService) { + // 导入处理 + const { MCPAdapter } = await import('./adapters/mcp-adapter') this.mcpAdapter = new MCPAdapter(this.twitterService, config.adapters.mcp.port) await this.mcpAdapter.start() diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts new file mode 100644 index 00000000..69cc955c --- /dev/null +++ b/services/twitter-services/src/main.ts @@ -0,0 +1,30 @@ +import process from 'node:process' + +import { TwitterServiceLauncher } from './launcher' +import { initializeLogger, logger } from './utils/logger' + +// 确保初始化只发生一次 +async function bootstrap() { + // 1. 首先初始化日志系统 + initializeLogger() + + // 2. 然后创建并启动服务 + const launcher = new TwitterServiceLauncher() + + try { + await launcher.start() + logger.main.log('Twitter 服务已成功启动') + } + catch (error) { + logger.main.withError(error).error('启动失败') + process.exit(1) + } + + // 设置进程事件处理 + process.on('unhandledRejection', (reason) => { + logger.main.withError(reason).error('未处理的 Promise 拒绝:') + }) +} + +// 启动应用 +bootstrap() diff --git a/services/twitter-services/src/utils/logger.ts b/services/twitter-services/src/utils/logger.ts index 01f7fcee..40708f08 100644 --- a/services/twitter-services/src/utils/logger.ts +++ b/services/twitter-services/src/utils/logger.ts @@ -1,16 +1,23 @@ +import path from 'node:path' import { createLogg, Format, LogLevel, setGlobalFormat, setGlobalLogLevel } from '@guiiai/logg' import { createDefaultConfig } from '../config' -import { errorToMessage } from './error' + +// 记录初始化状态 +let isInitialized = false // 初始化全局日志配置 -export function initializeLogger() { - const config = createDefaultConfig().getConfig() +export function initializeLogger(): void { + if (isInitialized) { + return // 防止多次初始化 + } - // 从环境变量或配置中获取日志级别 - const logLevelString = config.system?.logLevel?.toLowerCase() || 'info' + // 设置全局日志级别 + setGlobalLogLevel(LogLevel.Debug) + setGlobalFormat(Format.Pretty) + + const config = createDefaultConfig().getConfig() - // 映射日志级别 const logLevelMap: Record = { error: LogLevel.Error, warn: LogLevel.Warning, @@ -19,8 +26,7 @@ export function initializeLogger() { debug: LogLevel.Debug, } - // 设置全局日志级别 - setGlobalLogLevel(logLevelMap[logLevelString] || LogLevel.Log) + setGlobalLogLevel(logLevelMap[config.system?.logLevel] || LogLevel.Debug) // 根据配置设置格式 if (config.system?.logFormat === 'pretty') { @@ -29,30 +35,38 @@ export function initializeLogger() { else { setGlobalFormat(Format.JSON) } + + isInitialized = true } -// 创建特定上下文的日志记录器 -export function createLogger(context: string) { - const logger = createLogg(context).useGlobalConfig() - - // 添加错误处理方法 - return { - ...logger, - // 使用 errorToMessage 增强错误日志 - errorWithMessage: (message: string, error: unknown) => { - logger.error(`${message}: ${errorToMessage(error)}`) - }, - } +/** + * Get logger instance with directory name and filename + * @returns logger instance configured with "directoryName/filename" + */ +export function useLogger(name?: string): ReturnType { + if (name) + return createLogg(name).useGlobalConfig() + + const stack = new Error('logger').stack + const caller = stack?.split('\n')[2] + + // Extract directory, filename and line number from stack trace + const match = caller?.match(/(?:([^/]+)\/)?([^/\s]+?)(?:\.[jt]s)?:(\d+)(?::\d+)?\)?$/) + const dirName = match?.[1] || path.basename(path.dirname(__filename)) + const fileName = match?.[2] || path.basename(__filename, '.ts') + const lineNumber = match?.[3] || '?' + + return createLogg(`${dirName}/${fileName}:${lineNumber}`).useGlobalConfig() } // 创建各种服务的预配置日志记录器 export const logger = { - auth: createLogger('auth-service'), - timeline: createLogger('timeline-service'), - browser: createLogger('browser-adapter'), - airi: createLogger('airi-adapter'), - mcp: createLogger('mcp-adapter'), - parser: createLogger('parser'), - main: createLogger('twitter-service'), - config: createLogger('config'), + auth: useLogger('auth-service'), + timeline: useLogger('timeline-service'), + browser: useLogger('browser-adapter'), + airi: useLogger('airi-adapter'), + mcp: useLogger('mcp-adapter'), + parser: useLogger('parser'), + main: useLogger('twitter-service'), + config: useLogger('config'), } From 0a98fb6b24e5619e05b869db7819cc3b70f96981 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 02:26:27 +0800 Subject: [PATCH 07/20] chore: i18n --- .../twitter-services/docs/architecture.md | 197 +++++++++--------- .../src/adapters/airi-adapter.ts | 18 +- .../src/adapters/browserbase-adapter.ts | 18 +- .../src/adapters/mcp-adapter.ts | 98 ++++----- .../src/browser/browserbase.ts | 102 ++++----- services/twitter-services/src/cli.ts | 56 ++--- services/twitter-services/src/config/index.ts | 56 ++--- services/twitter-services/src/config/types.ts | 18 +- .../twitter-services/src/core/auth-service.ts | 32 +-- .../src/core/timeline-service.ts | 22 +- .../src/core/twitter-service.ts | 38 ++-- services/twitter-services/src/dev-server.ts | 64 +++--- services/twitter-services/src/launcher.ts | 77 +++---- services/twitter-services/src/main.ts | 16 +- .../src/parsers/html-parser.ts | 40 ++-- .../src/parsers/profile-parser.ts | 54 ++--- .../src/parsers/tweet-parser.ts | 46 ++-- .../twitter-services/src/types/browser.ts | 10 +- .../twitter-services/src/types/twitter.ts | 20 +- services/twitter-services/src/utils/api.ts | 26 +-- services/twitter-services/src/utils/error.ts | 42 ++-- services/twitter-services/src/utils/logger.ts | 12 +- .../src/utils/rate-limiter.ts | 22 +- .../twitter-services/src/utils/selectors.ts | 4 +- 24 files changed, 547 insertions(+), 541 deletions(-) diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index 8aa2de4c..ccd222f8 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -1,25 +1,26 @@ -# Twitter 服务架构文档 +# Twitter Service Architecture Documentation -## 1. 项目概述 +## 1. Project Overview -Twitter 服务是一个基于 BrowserBase 的 Web 自动化服务，提供结构化的 Twitter 数据访问和交互能力。它采用分层架构设计，支持多种适配器以便与不同的应用集成。 +Twitter Service is a web automation service based on BrowserBase, providing structured access and interaction capabilities with Twitter data. It employs a layered architecture design that supports multiple adapters for integration with different applications. -## 2. 设计目标 +## 2. Design Goals -- **可靠性**：稳定处理 Twitter 页面的变化和限制 -- **可扩展性**：易于添加新功能和支持不同接入方式 -- **性能优化**：智能管理请求频率和浏览器会话 -- **数据结构化**：提供规范、类型化的数据模型 +- **Reliability**: Stable handling of Twitter page changes and limitations +- **Scalability**: Easy to add new features and support different integration methods +- **Performance Optimization**: Intelligent management of request frequency and browser sessions +- **Data Structuring**: Provides standardized, typed data models -## 3. 架构总览 +## 3. Architecture Overview ``` ┌─────────────────────────────────────────────┐ -│ 应用层/消费者层 │ +│ Application/Consumer Layer │ │ │ │ ┌────────────┐ ┌─────────────┐ │ │ │ │ │ │ │ -│ │ Airi Core │ │ 其他 LLM 应用 │ │ +│ │ Airi Core │ │ Other LLM │ │ +│ │ │ │ Applications │ │ │ │ │ │ │ │ │ └──────┬─────┘ └──────┬──────┘ │ └──────────┼─────────────────────┼────────────┘ @@ -57,77 +58,77 @@ Twitter 服务是一个基于 BrowserBase 的 Web 自动化服务，提供结构 └──────────────────────────┘ ``` -## 4. 技术栈与依赖 +## 4. Technology Stack and Dependencies -- **核心库**: TypeScript, Node.js -- **浏览器自动化**: BrowserBase API -- **HTML解析**: unified, rehype-parse, unist-util-visit -- **API服务器**: H3.js, listhen -- **适配器**: Airi Server SDK, MCP SDK -- **日志系统**: @guiiai/logg -- **工具库**: zod（类型验证） +- **Core Library**: TypeScript, Node.js +- **Browser Automation**: BrowserBase API +- **HTML Parsing**: unified, rehype-parse, unist-util-visit +- **API Server**: H3.js, listhen +- **Adapters**: Airi Server SDK, MCP SDK +- **Logging System**: @guiiai/logg +- **Utility Library**: zod (type validation) -## 5. 关键组件 +## 5. Key Components -### 5.1 适配器层 +### 5.1 Adapter Layer -#### 5.1.1 Airi 适配器 +#### 5.1.1 Airi Adapter -提供与 Airi LLM 平台的集成，处理事件驱动的通信。 +Provides integration with the Airi LLM platform, handling event-driven communication. -#### 5.1.2 MCP 适配器 +#### 5.1.2 MCP Adapter -实现 Model Context Protocol 接口，提供基于 HTTP 的通信。现使用官方 MCP SDK 实现，通过 H3.js 提供高性能 HTTP 服务器和 SSE 通信。 +Implements the Model Context Protocol interface, providing communication based on HTTP. Currently using the official MCP SDK implementation, providing high-performance HTTP server and SSE communication through H3.js. -#### 5.1.3 开发服务器 +#### 5.1.3 Development Server -使用 listhen 提供优化的开发体验，包括自动打开浏览器、实时日志和调试工具。 +Using listhen for optimized development experience, including automatic browser opening, real-time logging, and debugging tools. -### 5.2 核心服务层 +### 5.2 Core Service Layer -#### 5.2.1 认证服务 (Auth Service) +#### 5.2.1 Authentication Service (Auth Service) -处理 Twitter 登录和会话维护。 +Handles Twitter login and session maintenance. -#### 5.2.2 时间线服务 (Timeline Service) +#### 5.2.2 Timeline Service (Timeline Service) -获取和处理 Twitter 时间线内容。 +Gets and processes Twitter timeline content. -#### 5.2.3 其他服务 +#### 5.2.3 Other Services -包括搜索服务、互动服务、用户资料服务等（部分未在 MVP 中实现）。 +Includes search service, interaction service, user profile service, etc. (not implemented in MVP) -### 5.3 解析器和工具 +### 5.3 Parsers and Tools -#### 5.3.1 Tweet 解析器 +#### 5.3.1 Tweet Parser -从 HTML 中提取推文结构化数据。 +Extracts structured data from HTML. -#### 5.3.2 频率限制器 +#### 5.3.2 Rate Limiter -控制请求频率，避免触发 Twitter 的限制。 +Controls request frequency to avoid triggering Twitter limits. -## 6. 数据流 +## 6. Data Flow -1. **请求流**：应用层 → 适配器 → 核心服务 → 浏览器适配层 → BrowserBase API → Twitter -2. **响应流**：Twitter → BrowserBase API → 浏览器适配层 → 核心服务 → 数据解析 → 适配器 → 应用层 +1. **Request Flow**: Application Layer → Adapter → Core Service → Browser Adapter Layer → BrowserBase API → Twitter +2. **Response Flow**: Twitter → BrowserBase API → Browser Adapter Layer → Core Service → Data Parsing → Adapter → Application Layer -## 7. 配置系统 +## 7. Configuration System -配置分为以下几个主要部分： +Configuration is divided into several main parts: ```typescript interface Config { - // BrowserBase 配置 + // BrowserBase configuration browserbase: { apiKey: string endpoint?: string } - // 浏览器配置 + // Browser configuration browser: BrowserConfig - // Twitter 配置 + // Twitter configuration twitter: { credentials?: TwitterCredentials defaultOptions?: { @@ -136,7 +137,7 @@ interface Config { } } - // 适配器配置 + // Adapter configuration adapters: { airi?: { url?: string @@ -149,7 +150,7 @@ interface Config { } } - // 系统配置 + // System configuration system: { logLevel: string concurrency: number @@ -157,60 +158,60 @@ interface Config { } ``` -## 8. 开发与测试 +## 8. Development and Testing -### 8.1 开发环境设置 +### 8.1 Development Environment Setup ```bash -# 安装依赖 +# Install dependencies npm install -# 设置环境变量 +# Set environment variables cp .env.example .env -# 编辑 .env 添加 BrowserBase API 密钥和 Twitter 凭据 +# Edit .env to add BrowserBase API key and Twitter credentials -# 开发模式启动 -npm run dev # 标准模式 -npm run dev:mcp # MCP 开发服务器模式 +# Development mode startup +npm run dev # Standard mode +npm run dev:mcp # MCP development server mode ``` -### 8.2 测试策略 +### 8.2 Testing Strategy -- **单元测试**：测试解析器、工具类和业务逻辑 -- **集成测试**：测试服务和适配器的交互 -- **端到端测试**：模拟完整的使用场景 +- **Unit Tests**: Test parsers, utility classes, and business logic +- **Integration Tests**: Test service and adapter interaction +- **End-to-End Tests**: Simulate complete usage scenarios -## 9. 集成示例 +## 9. Integration Example -### 9.1 从其他 Node.js 应用集成 +### 9.1 Integrating from Other Node.js Applications ```typescript import { BrowserBaseMCPAdapter, TwitterService } from 'twitter-services' async function main() { - // 初始化浏览器 + // Initialize browser const browser = new BrowserBaseMCPAdapter('your-api-key') await browser.initialize({ headless: true }) - // 创建 Twitter 服务 + // Create Twitter service const twitter = new TwitterService(browser) - // 登录 + // Login await twitter.login({ username: 'your-username', password: 'your-password' }) - // 获取时间线 + // Get timeline const tweets = await twitter.getTimeline({ count: 10 }) console.log(tweets) - // 释放资源 + // Release resources await browser.close() } ``` -### 9.2 作为 Airi 模块集成 +### 9.2 Integrating as Airi Module ```typescript import { AiriAdapter, BrowserBaseMCPAdapter, TwitterService } from 'twitter-services' @@ -221,7 +222,7 @@ async function startAiriModule() { const twitter = new TwitterService(browser) - // 创建 Airi 适配器 + // Create Airi adapter const airiAdapter = new AiriAdapter(twitter, { url: process.env.AIRI_URL, token: process.env.AIRI_TOKEN, @@ -231,33 +232,33 @@ async function startAiriModule() { } }) - // 启动适配器 + // Start adapter await airiAdapter.start() console.log('Twitter service running as Airi module') } ``` -### 9.3 使用 MCP 进行集成 +### 9.3 Using MCP for Integration ```typescript -// 使用 MCP SDK 与 Twitter 服务交互 +// Use MCP SDK to interact with Twitter service import { McpClient } from '@modelcontextprotocol/sdk/client/mcp.js' import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js' async function connectToTwitterService() { - // 创建 SSE 传输 + // Create SSE transport const transport = new SSEClientTransport('http://localhost:8080/sse', 'http://localhost:8080/messages') - // 创建客户端 + // Create client const client = new McpClient() await client.connect(transport) - // 获取时间线 + // Get timeline const timeline = await client.get('twitter://timeline/10') console.log('Timeline:', timeline.contents) - // 使用工具发送推文 + // Use tool to send tweet const result = await client.useTool('post-tweet', { content: 'Hello from MCP!' }) console.log('Result:', result.content) @@ -265,33 +266,33 @@ async function connectToTwitterService() { } ``` -## 10. 扩展指南 +## 10. Extension Guide -### 10.1 添加新功能 +### 10.1 Adding New Features -例如添加"获取特定用户发布的推文"功能： +For example, adding "Get Tweets from a Specific User" functionality: -1. 在 `src/types/twitter.ts` 中扩展接口 -2. 在 `src/core/twitter-service.ts` 中实现方法 -3. 在适配器中添加对应的处理逻辑 -4. 如果是 MCP 适配器，在 `configureServer()` 中添加相应的资源或工具 +1. Extend the interface in `src/types/twitter.ts` +2. Implement the method in `src/core/twitter-service.ts` +3. Add corresponding handling logic in the adapter +4. If it's an MCP adapter, add appropriate resources or tools in `configureServer()` -### 10.2 支持新的适配器 +### 10.2 Supporting New Adapters -1. 创建新的适配器类 -2. 实现与目标系统的通信逻辑 -3. 在入口文件中添加配置支持 +1. Create a new adapter class +2. Implement communication logic with the target system +3. Add configuration support in the entry file -## 11. 维护建议 +## 11. Maintenance Recommendations -- **自动化测试**：编写单元测试和集成测试 -- **监控与告警**：监控服务状态和 Twitter 的访问限制 -- **选择器更新**：定期验证和更新选择器配置 -- **会话管理**：优化会话管理以提高稳定性 +- **Automated Testing**: Write unit tests and integration tests +- **Monitoring & Alerts**: Monitor service status and Twitter access limitations +- **Selector Updates**: Regularly validate and update selector configurations +- **Session Management**: Optimize session management to improve stability -## 12. 项目路线图 +## 12. Project Roadmap -- MVP 阶段：实现核心功能（认证、浏览时间线） -- 阶段二：完善互动功能（点赞、评论、转发） -- 阶段三：高级功能（搜索、高级过滤、数据分析） -- 阶段四：性能优化和稳定性提升 +- MVP Stage: Implement core functionality (authentication, browsing timeline) +- Stage Two: Enhance interaction features (likes, comments, retweets) +- Stage Three: Advanced features (search, advanced filtering, data analysis) +- Stage Four: Performance optimization and stability improvements diff --git a/services/twitter-services/src/adapters/airi-adapter.ts b/services/twitter-services/src/adapters/airi-adapter.ts index e879e4d3..a169b9da 100644 --- a/services/twitter-services/src/adapters/airi-adapter.ts +++ b/services/twitter-services/src/adapters/airi-adapter.ts @@ -5,8 +5,8 @@ import { Client } from '@proj-airi/server-sdk' import { logger } from '../utils/logger' /** - * Airi 适配器 - * 将 Twitter 服务适配为 Airi 模块 + * Airi Adapter + * Adapts the Twitter service as an Airi module */ export class AiriAdapter { private client: Client @@ -26,7 +26,7 @@ export class AiriAdapter { name: 'twitter-module', token: options.token, possibleEvents: [ - // 定义此模块可以处理的事件类型 + // Define event types this module can handle 'twitter:login', 'twitter:getTimeline', 'twitter:getTweetDetails', @@ -43,10 +43,10 @@ export class AiriAdapter { } /** - * 设置事件处理器 + * Set up event handlers */ private setupEventHandlers(): void { - // 登录处理 + // Login handler this.client.onEvent('twitter:login', async (event) => { try { const credentials = event.data.credentials as TwitterCredentials || this.credentials @@ -64,7 +64,7 @@ export class AiriAdapter { } }) - // 获取时间线处理 + // Timeline handler this.client.onEvent('twitter:getTimeline', async (event) => { try { const options = event.data.options as TimelineOptions || {} @@ -82,14 +82,14 @@ export class AiriAdapter { } }) - // 其他事件处理... + // Other event handlers... } /** - * 启动适配器 + * Start the adapter */ async start(): Promise { - // 可以在这里添加初始化逻辑 + // Initialization logic can be added here logger.airi.log('Airi Twitter adapter started') } } diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts index 64bb761e..e0c60ffd 100644 --- a/services/twitter-services/src/adapters/browserbase-adapter.ts +++ b/services/twitter-services/src/adapters/browserbase-adapter.ts @@ -8,7 +8,7 @@ import { errorToMessage } from '../utils/error' import { logger } from '../utils/logger' /** - * Stagehand 元素句柄实现 + * Stagehand element handle implementation */ class StagehandElementHandle implements ElementHandle { private client: StagehandClient @@ -41,8 +41,8 @@ class StagehandElementHandle implements ElementHandle { } /** - * Stagehand 浏览器适配器实现 - * 将 Stagehand API 适配为通用浏览器接口 + * Stagehand browser adapter implementation + * Adapts the Stagehand API to a common browser interface */ export class StagehandBrowserAdapter implements BrowserAdapter { private client: StagehandClient @@ -64,11 +64,11 @@ export class StagehandBrowserAdapter implements BrowserAdapter { }) logger.browser.withFields({ headless: config.headless, - }).log('浏览器会话已创建') + }).log('Browser session created') } catch (error) { - logger.browser.withError(error).error('浏览器初始化失败') - throw new Error(`无法初始化浏览器: ${errorToMessage(error)}`) + logger.browser.withError(error).error('Failed to initialize browser') + throw new Error(`Unable to initialize browser: ${errorToMessage(error)}`) } } @@ -99,7 +99,7 @@ export class StagehandBrowserAdapter implements BrowserAdapter { } async getElements(selector: string): Promise { - // 获取所有匹配元素的选择器 + // Get all matching element selectors const selectors = await this.executeScript(` Array.from(document.querySelectorAll('${selector}')).map((el, i) => { const uniqueId = 'stagehand-' + Date.now() + '-' + i; @@ -108,11 +108,11 @@ export class StagehandBrowserAdapter implements BrowserAdapter { }) `) - // 为每个匹配的元素创建一个 ElementHandle + // Create an ElementHandle for each match return selectors.map(selector => new StagehandElementHandle(this.client, selector)) } - // 新增 Stagehand 相关方法 + // Add Stagehand specific methods async act(instruction: string): Promise { await this.client.act(instruction) } diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts index 10a7b955..da84dc67 100644 --- a/services/twitter-services/src/adapters/mcp-adapter.ts +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -11,9 +11,9 @@ import { errorToMessage } from '../utils/error' import { logger } from '../utils/logger' /** - * MCP 协议适配器 - * 使用官方 MCP SDK 将 Twitter 服务适配为 MCP 协议服务 - * 基于 H3.js 实现 HTTP 服务器 + * MCP Protocol Adapter + * Adapts the Twitter service to MCP protocol using official MCP SDK + * Implements HTTP server using H3.js */ export class MCPAdapter { private twitterService: TwitterService @@ -27,34 +27,34 @@ export class MCPAdapter { this.twitterService = twitterService this.port = port - // 创建 MCP 服务器 + // Create MCP server this.mcpServer = new McpServer({ name: 'Twitter Service', version: '1.0.0', }) - // 创建 H3 应用 + // Create H3 app this.app = createApp() - // 配置资源和工具 + // Configure resources and tools this.configureServer() - // 设置 H3 路由 + // Set up H3 routes this.setupRoutes() } /** - * 配置 MCP 服务器的资源和工具 + * Configure MCP server resources and tools */ private configureServer(): void { - // 添加时间线资源 + // Add timeline resource this.mcpServer.resource( 'timeline', new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ resources: [{ name: 'timeline', uri: 'twitter://timeline', - description: '推文时间线', + description: 'Tweet timeline', }], }) }), async (_uri: URL, { count }: { count?: string }) => { @@ -71,13 +71,13 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.errorWithError('获取时间线错误:', error) + logger.mcp.errorWithError('Error fetching timeline:', error) return { contents: [] } } }, ) - // 添加推文详情资源 + // Add tweet details resource this.mcpServer.resource( 'tweet', new ResourceTemplate('twitter://tweet/{id}', { list: undefined }), @@ -93,13 +93,13 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.errorWithError('获取推文详情错误:', error) + logger.mcp.errorWithError('Error fetching tweet details:', error) return { contents: [] } } }, ) - // 添加用户资料资源 + // Add user profile resource this.mcpServer.resource( 'profile', new ResourceTemplate('twitter://user/{username}', { list: undefined }), @@ -115,13 +115,13 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.errorWithError('获取用户资料错误:', error) + logger.mcp.errorWithError('Error fetching user profile:', error) return { contents: [] } } }, ) - // 添加登录工具 + // Add login tool this.mcpServer.tool( 'login', { @@ -135,20 +135,20 @@ export class MCPAdapter { return { content: [{ type: 'text', - text: success ? '成功登录到 Twitter' : '登录失败，请检查凭据', + text: success ? 'Successfully logged into Twitter' : 'Login failed, please check credentials', }], } } catch (error) { return { - content: [{ type: 'text', text: `登录失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Login failed: ${errorToMessage(error)}` }], isError: true, } } }, ) - // 添加发推工具 + // Add post tweet tool this.mcpServer.tool( 'post-tweet', { @@ -166,20 +166,20 @@ export class MCPAdapter { return { content: [{ type: 'text', - text: `成功发布推文: ${tweetId}`, + text: `Successfully posted tweet: ${tweetId}`, }], } } catch (error) { return { - content: [{ type: 'text', text: `发推失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Failed to post tweet: ${errorToMessage(error)}` }], isError: true, } } }, ) - // 添加点赞工具 + // Add like tweet tool this.mcpServer.tool( 'like-tweet', { tweetId: z.string() }, @@ -190,20 +190,20 @@ export class MCPAdapter { return { content: [{ type: 'text', - text: success ? '成功点赞' : '点赞失败', + text: success ? 'Successfully liked tweet' : 'Failed to like tweet', }], } } catch (error) { return { - content: [{ type: 'text', text: `点赞失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Failed to like tweet: ${errorToMessage(error)}` }], isError: true, } } }, ) - // 添加转发工具 + // Add retweet tool this.mcpServer.tool( 'retweet', { tweetId: z.string() }, @@ -214,20 +214,20 @@ export class MCPAdapter { return { content: [{ type: 'text', - text: success ? '成功转发' : '转发失败', + text: success ? 'Successfully retweeted' : 'Failed to retweet', }], } } catch (error) { return { - content: [{ type: 'text', text: `转发失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Failed to retweet: ${errorToMessage(error)}` }], isError: true, } } }, ) - // 添加搜索工具 + // Add search tool this.mcpServer.tool( 'search', { @@ -242,14 +242,14 @@ export class MCPAdapter { return { content: [{ type: 'text', - text: `搜索结果: ${results.length} 条推文`, + text: `Search results: ${results.length} tweets`, }], resources: results.map(tweet => `twitter://tweet/${tweet.id}`), } } catch (error) { return { - content: [{ type: 'text', text: `搜索失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Search failed: ${errorToMessage(error)}` }], isError: true, } } @@ -258,12 +258,12 @@ export class MCPAdapter { } /** - * 设置 H3 路由 + * Set up H3 routes */ private setupRoutes(): void { const router = createRouter() - // 设置 CORS + // Set up CORS router.use('*', defineEventHandler((event) => { event.node.res.setHeader('Access-Control-Allow-Origin', '*') event.node.res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') @@ -275,7 +275,7 @@ export class MCPAdapter { } })) - // SSE 端点 + // SSE endpoint router.get('/sse', defineEventHandler(async (event) => { const { req, res } = event.node @@ -283,11 +283,11 @@ export class MCPAdapter { res.setHeader('Cache-Control', 'no-cache') res.setHeader('Connection', 'keep-alive') - // 创建 SSE 传输 + // Create SSE transport const transport = new SSEServerTransport('/messages', res) this.activeTransports.push(transport) - // 客户端断开连接时清理 + // Clean up when client disconnects req.on('close', () => { const index = this.activeTransports.indexOf(transport) if (index !== -1) { @@ -295,11 +295,11 @@ export class MCPAdapter { } }) - // 连接到 MCP 服务器 + // Connect to MCP server await this.mcpServer.connect(transport) })) - // 消息端点，接收客户端请求 + // Messages endpoint - receive client requests router.post('/messages', defineEventHandler(async (event) => { if (this.activeTransports.length === 0) { event.node.res.statusCode = 503 @@ -307,14 +307,14 @@ export class MCPAdapter { } try { - // 解析请求体 + // Parse request body const body = await readBody(event) - // 简单处理 - 发送到最近的传输 - // 注意: 生产环境中应该使用会话ID来路由到正确的传输 + // Simple handling - send to most recent transport + // Note: In production, should use session ID to route to correct transport const transport = this.activeTransports[this.activeTransports.length - 1] - // 手动处理 POST 消息，因为 H3 不是 Express 兼容的 + // Manually handle POST message, as H3 is not Express-compatible const response = await transport.handleMessage(body) return response @@ -325,7 +325,7 @@ export class MCPAdapter { } })) - // 根路径 - 提供服务信息 + // Root path - provide service info router.get('/', defineEventHandler(() => { return { name: 'Twitter MCP Service', @@ -337,27 +337,27 @@ export class MCPAdapter { } })) - // 使用路由 + // Use router this.app.use(router) } /** - * 启动 MCP 服务器 + * Start MCP server */ start(): Promise { return new Promise((resolve) => { - // 创建 Node.js HTTP 服务器 + // Create Node.js HTTP server this.server = createServer(toNodeListener(this.app)) this.server.listen(this.port, () => { - logger.mcp.withField('port', this.port).log('MCP 服务器已启动') + logger.mcp.withField('port', this.port).log('MCP server started') resolve() }) }) } /** - * 停止 MCP 服务器 + * Stop MCP server */ stop(): Promise { return new Promise((resolve, reject) => { @@ -370,7 +370,7 @@ export class MCPAdapter { reject(error) } else { - logger.mcp.log('MCP 服务器已停止') + logger.mcp.log('MCP server stopped') resolve() } }) @@ -378,7 +378,7 @@ export class MCPAdapter { } } -// h3 工具函数：从 event 读取请求体 +// h3 utility function: read body from event async function readBody(event: any): Promise { const buffers = [] for await (const chunk of event.node.req) { diff --git a/services/twitter-services/src/browser/browserbase.ts b/services/twitter-services/src/browser/browserbase.ts index 9400bfd5..adba49f7 100644 --- a/services/twitter-services/src/browser/browserbase.ts +++ b/services/twitter-services/src/browser/browserbase.ts @@ -7,7 +7,7 @@ import { chromium } from 'playwright' import { logger } from '../utils/logger' /** - * Stagehand 客户端配置选项 + * Stagehand client configuration options */ export interface StagehandClientOptions { apiKey: string @@ -19,8 +19,8 @@ export interface StagehandClientOptions { } /** - * Stagehand 客户端 - * 使用 @browserbasehq/stagehand 实现浏览器自动化 + * Stagehand client + * Implements browser automation using @browserbasehq/stagehand */ export class StagehandClient { private browser: Browser | null = null @@ -49,7 +49,7 @@ export class StagehandClient { } /** - * 创建浏览器会话 + * Create browser session */ async createSession(options?: { headless?: boolean @@ -57,56 +57,56 @@ export class StagehandClient { viewport?: { width: number, height: number } }): Promise { try { - // 启动 Playwright 浏览器 + // Launch Playwright browser this.browser = await chromium.launch({ headless: options?.headless ?? this.options.headless, }) - // 创建上下文 + // Create context const context = await this.browser.newContext({ userAgent: options?.userAgent ?? this.options.userAgent, viewport: options?.viewport ?? this.options.viewport, - // 设置任何其他所需的浏览器上下文选项 + // Set any other required browser context options }) - // 创建页面 + // Create page this.page = await context.newPage() - // 为页面添加 Stagehand 扩展 + // Add Stagehand extension to page await this.setupStagehand() const sessionId = `session-${Date.now()}` - logger.browser.withField('sessionId', sessionId).log('创建浏览器会话成功') + logger.browser.withField('sessionId', sessionId).log('Browser session created successfully') return sessionId } catch (error) { - logger.browser.errorWithError('创建浏览器会话失败', error) + logger.browser.errorWithError('Failed to create browser session', error) throw error } } /** - * 设置 Stagehand 扩展 - * 这将添加 act, extract, observe 方法到 page 对象 + * Set up Stagehand extension + * This adds act, extract, observe methods to the page object */ private async setupStagehand(): Promise { if (!this.page) { throw new Error('No active page. Call createSession first.') } - // 在实际实现中，这里会使用 Stagehand 的 API 设置页面对象 - // 这可能涉及到页面扩展或注入 Stagehand 的功能 - // 示例代码（实际使用需要根据 Stagehand 的文档进行调整）: + // In actual implementation, this would use Stagehand's API to set up the page object + // This might involve page extension or injecting Stagehand functionality + // Example code (actual usage would need to be adjusted based on Stagehand's documentation): // // import { extendPage } from '@browserbasehq/stagehand' // await extendPage(this.page, { // apiKey: this.apiKey, - // // 其他 Stagehand 选项 + // // Other Stagehand options // }) } /** - * 导航到指定URL + * Navigate to specified URL */ async navigate(url: string): Promise { this.ensurePageExists() @@ -114,7 +114,7 @@ export class StagehandClient { } /** - * 执行JavaScript脚本 + * Execute JavaScript script */ async executeScript(script: string): Promise { this.ensurePageExists() @@ -122,19 +122,19 @@ export class StagehandClient { } /** - * 使用 Stagehand 的 act API 执行操作 + * Use Stagehand's act API to perform operations */ async act(instruction: string): Promise { this.ensurePageExists() - // 在实际实现中，这将使用 Stagehand 的 act API - // 示例：await this.page!.act(instruction) + // In actual implementation, this would use Stagehand's act API + // Example: await this.page!.act(instruction) - // 临时实现，使用 Playwright 的基本能力模拟 - logger.browser.withField('instruction', instruction).log('执行 act 指令') + // Temporary implementation, simulating act behavior with Playwright basics + logger.browser.withField('instruction', instruction).log('Executing act instruction') - // 这里通过简单的方法来模拟 act 的行为 - // 实际需要使用 Stagehand 的 act API + // Simulate act behavior through simple methods + // Actual implementation would use Stagehand's act API if (instruction.includes('click')) { const match = instruction.match(/click on the ['"](.+?)['"]/) if (match && match[1]) { @@ -144,7 +144,7 @@ export class StagehandClient { } /** - * 使用 Stagehand 的 extract API 提取数据 + * Use Stagehand's extract API to extract data */ async extract({ instruction, @@ -155,36 +155,36 @@ export class StagehandClient { }): Promise> { this.ensurePageExists() - // 在实际实现中，这将使用 Stagehand 的 extract API - // 示例：return await this.page!.extract({ instruction, schema }) + // In actual implementation, this would use Stagehand's extract API + // Example: return await this.page!.extract({ instruction, schema }) - // 临时实现，记录指令并返回一个空对象 - logger.browser.withField('instruction', instruction).log('执行 extract 指令') + // Temporary implementation, log instruction and return empty object + logger.browser.withField('instruction', instruction).log('Executing extract instruction') - // 这里只是简单地返回一个空对象 - // 实际需要使用 Stagehand 的 extract API + // Simply return an empty object + // Actual implementation would use Stagehand's extract API return {} as z.infer } /** - * 使用 Stagehand 的 observe API 观察页面状态 + * Use Stagehand's observe API to observe page state */ async observe(instruction: string): Promise { this.ensurePageExists() - // 在实际实现中，这将使用 Stagehand 的 observe API - // 示例：return await this.page!.observe(instruction) + // In actual implementation, this would use Stagehand's observe API + // Example: return await this.page!.observe(instruction) - // 临时实现，记录指令并返回空字符串 - logger.browser.withField('instruction', instruction).log('执行 observe 指令') + // Temporary implementation, log instruction and return empty string + logger.browser.withField('instruction', instruction).log('Executing observe instruction') - // 这里只是简单地返回一个空字符串 - // 实际需要使用 Stagehand 的 observe API + // Simply return an empty string + // Actual implementation would use Stagehand's observe API return '' } /** - * 获取页面内容 + * Get page content */ async getContent(): Promise { this.ensurePageExists() @@ -192,7 +192,7 @@ export class StagehandClient { } /** - * 等待元素出现 + * Wait for element to appear */ async waitForSelector(selector: string, options: { timeout?: number } = {}): Promise { this.ensurePageExists() @@ -202,7 +202,7 @@ export class StagehandClient { } /** - * 点击元素 + * Click element */ async click(selector: string): Promise { this.ensurePageExists() @@ -210,18 +210,18 @@ export class StagehandClient { } /** - * 向输入框输入文本 + * Type text into input field */ async type(selector: string, text: string): Promise { this.ensurePageExists() - // 先清空输入框 + // Clear input field first await this.page!.fill(selector, '') - // 然后输入文本 + // Then type text await this.page!.fill(selector, text) } /** - * 获取元素文本内容 + * Get element text content */ async getText(selector: string): Promise { this.ensurePageExists() @@ -233,7 +233,7 @@ export class StagehandClient { } /** - * 获取屏幕截图 + * Get screenshot */ async getScreenshot(): Promise { this.ensurePageExists() @@ -241,19 +241,19 @@ export class StagehandClient { } /** - * 关闭会话 + * Close session */ async closeSession(): Promise { if (this.browser) { await this.browser.close() this.browser = null this.page = null - logger.browser.log('浏览器会话已关闭') + logger.browser.log('Browser session closed') } } /** - * 确保页面存在 + * Ensure page exists */ private ensurePageExists(): void { if (!this.page) { diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts index 3d15029a..07e652fd 100644 --- a/services/twitter-services/src/cli.ts +++ b/services/twitter-services/src/cli.ts @@ -5,30 +5,30 @@ import path from 'node:path' import process from 'node:process' import { Command } from 'commander' -import { BrowserBaseMCPAdapter } from './adapters/browserbase-adapter' +import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' import { createDefaultConfig } from './config' import { TwitterService } from './core/twitter-service' import { TwitterServiceLauncher } from './launcher' import { errorToMessage } from './utils/error' -// 获取版本 +// Get version const packageJsonPath = path.join(__dirname, '..', 'package.json') const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) -// 创建程序 +// Create program const program = new Command() -// 设置基本信息 +// Set basic info program .name('twitter-services') - .description('Twitter 服务 CLI - 访问和管理 Twitter 数据') + .description('Twitter Services CLI - Access and manage Twitter data') .version(packageJson.version) -// 启动服务命令 +// Start service command program .command('start') - .description('启动 Twitter 服务') - .option('-c, --config ', '配置文件路径') + .description('Start Twitter service') + .option('-c, --config ', 'Path to config file') .action(async (options) => { if (options.config) { process.env.CONFIG_PATH = options.config @@ -37,49 +37,49 @@ program const launcher = new TwitterServiceLauncher() await launcher.start() - console.log('服务已启动，按 Ctrl+C 停止') + console.log('Service started, press Ctrl+C to stop') }) -// 获取时间线命令 +// Get timeline command program .command('timeline') - .description('获取 Twitter 时间线') - .option('-c, --count ', '要获取的推文数量', '10') - .option('--no-replies', '排除回复') - .option('--no-retweets', '排除转发') - .option('-o, --output ', '输出结果到文件') + .description('Get Twitter timeline') + .option('-c, --count ', 'Number of tweets to fetch', '10') + .option('--no-replies', 'Exclude replies') + .option('--no-retweets', 'Exclude retweets') + .option('-o, --output ', 'Output results to file') .action(async (options) => { try { const configManager = createDefaultConfig() const config = configManager.getConfig() - // 初始化浏览器 - const browser = new BrowserBaseMCPAdapter(config.browserbase.apiKey) + // Initialize browser + const browser = new StagehandBrowserAdapter(config.browser.apiKey) await browser.initialize(config.browser) - // 创建服务并登录 + // Create service and login const twitterService = new TwitterService(browser) if (!config.twitter.credentials) { - throw new Error('无法获取 Twitter 凭据，请检查配置') + throw new Error('Cannot get Twitter credentials, please check configuration') } const loggedIn = await twitterService.login(config.twitter.credentials) if (!loggedIn) { - throw new Error('登录失败，请检查凭据') + throw new Error('Login failed, please check credentials') } - console.log('正在获取时间线...') + console.log('Fetching timeline...') - // 获取时间线 + // Get timeline const tweets = await twitterService.getTimeline({ count: Number.parseInt(options.count), includeReplies: options.replies, includeRetweets: options.retweets, }) - // 处理结果 + // Process results const result = tweets.map(tweet => ({ id: tweet.id, text: tweet.text, @@ -91,23 +91,23 @@ program replyCount: tweet.replyCount, })) - // 输出结果 + // Output results if (options.output) { fs.writeFileSync(options.output, JSON.stringify(result, null, 2)) - console.log(`结果已保存到 ${options.output}`) + console.log(`Results saved to ${options.output}`) } else { console.log(JSON.stringify(result, null, 2)) } - // 关闭浏览器 + // Close browser await browser.close() } catch (error) { - console.error('获取时间线失败:', errorToMessage(error)) + console.error('Failed to get timeline:', errorToMessage(error)) process.exit(1) } }) -// 解析命令行参数 +// Parse command line arguments program.parse() diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts index 3fffa1f9..7cac1512 100644 --- a/services/twitter-services/src/config/index.ts +++ b/services/twitter-services/src/config/index.ts @@ -10,115 +10,115 @@ import { logger } from '../utils/logger' import { getDefaultConfig } from './types' /** - * 加载环境变量文件 - * 按优先级顺序加载 + * Load environment variable files + * Load in order of priority */ function loadEnvFiles(): void { - // 加载环境变量文件 + // Load environment variable files const envFiles = [ '.env.local', ] - // 从当前目录向上查找 .env 文件 + // Look for .env files from current directory upward for (const file of envFiles) { const filePath = path.resolve(process.cwd(), file) if (fs.existsSync(filePath)) { const result = configDotenv({ path: filePath, - override: true, // 允许覆盖已存在的环境变量 + override: true, // Allow overriding existing environment variables }) if (result.parsed) { logger.config.withFields({ config: result.parsed, - }).log(`已从 ${file} 加载环境变量`) + }).log(`Loaded environment variables from ${file}`) } } } } /** - * 配置管理器 - * 负责加载、验证和提供配置 + * Configuration manager + * Responsible for loading, validating and providing configuration */ export class ConfigManager { private config: Config /** - * 创建配置管理器 - * @param configPath 配置文件路径 + * Create configuration manager + * @param configPath Path to configuration file */ constructor(configPath?: string) { - // 首先加载环境变量 + // First load environment variables loadEnvFiles() - // 设置默认配置 + // Set default configuration this.config = getDefaultConfig() - // 然后从配置文件加载（如果指定） + // Then load from configuration file (if specified) if (configPath) { this.loadFromFile(configPath) } - // 验证配置 + // Validate configuration this.validateConfig() } /** - * 从文件加载配置 + * Load configuration from file */ private loadFromFile(filePath: string): void { try { const configFile = fs.readFileSync(filePath, 'utf8') const fileConfig = JSON.parse(configFile) - // 使用 defu 深度合并配置 - // fileConfig 中的值优先于 this.config 中的值 + // Use defu to deeply merge configurations + // Values in fileConfig take precedence over this.config this.config = defu(fileConfig, this.config) - logger.config.log(`配置已从 ${filePath} 加载`) + logger.config.log(`Configuration loaded from ${filePath}`) } catch (error) { - logger.config.errorWithError(`加载配置文件失败: ${(error as Error).message}`, error) + logger.config.errorWithError(`Failed to load configuration file: ${(error as Error).message}`, error) } } /** - * 验证配置有效性 + * Validate configuration validity */ private validateConfig(): void { - // 验证 Twitter 凭据 + // Validate Twitter credentials if (!this.config.twitter.credentials?.username || !this.config.twitter.credentials?.password) { - logger.config.warn('未设置 Twitter 凭据!') + logger.config.warn('Twitter credentials not set!') } logger.config.withFields({ config: this.config, - }).log('配置验证完成') + }).log('Configuration validation complete') } /** - * 获取完整配置 + * Get complete configuration */ getConfig(): Config { return this.config } /** - * 更新配置 + * Update configuration */ updateConfig(newConfig: Partial): void { - // 使用 defu 合并新配置 + // Use defu to merge new configuration this.config = defu(newConfig, this.config) this.validateConfig() } } -// 单例实例 +// Singleton instance let configInstance: ConfigManager | null = null /** - * 创建默认配置管理器 (单例) + * Create default configuration manager (singleton) */ export function createDefaultConfig(): ConfigManager { if (configInstance) { diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index bb805d5e..a6864e21 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -4,16 +4,16 @@ import type { SearchOptions, TimelineOptions, TwitterCredentials } from '../type import process from 'node:process' /** - * 完整配置接口 + * Complete configuration interface */ export interface Config { - // 浏览器配置 + // Browser configuration browser: BrowserConfig & { - apiKey: string // 为 Stagehand 保留 API Key - endpoint?: string // 可选的 Stagehand 服务端点 + apiKey: string // API Key for Stagehand + endpoint?: string // Optional Stagehand service endpoint } - // Twitter 配置 + // Twitter configuration twitter: { credentials?: TwitterCredentials defaultOptions?: { @@ -22,7 +22,7 @@ export interface Config { } } - // 适配器配置 + // Adapter configuration adapters: { airi?: { url?: string @@ -35,7 +35,7 @@ export interface Config { } } - // 系统配置 + // System configuration system: { logLevel: 'error' | 'warn' | 'info' | 'verbose' | 'debug' logFormat?: 'json' | 'pretty' @@ -44,12 +44,12 @@ export interface Config { } /** - * 默认配置 + * Default configuration */ export function getDefaultConfig(): Config { return { browser: { - apiKey: process.env.BROWSERBASE_API_KEY || '', // 将 apiKey 移到 browser 配置中 + apiKey: process.env.BROWSERBASE_API_KEY || '', // Move apiKey to browser config headless: true, userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', viewport: { diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index e1c9c378..8eb80f57 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -5,8 +5,8 @@ import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' /** - * Twitter 认证服务 - * 处理登录和会话管理 + * Twitter Authentication Service + * Handles login and session management */ export class TwitterAuthService { private browser: BrowserAdapter @@ -17,52 +17,52 @@ export class TwitterAuthService { } /** - * 登录到 Twitter + * Login to Twitter */ async login(credentials: TwitterCredentials): Promise { - logger.auth.withField('username', credentials.username.replace(/./g, '*')).log('尝试登录 Twitter') + logger.auth.withField('username', credentials.username.replace(/./g, '*')).log('Attempting to login to Twitter') try { - // 导航到登录页 + // Navigate to login page await this.browser.navigate('https://twitter.com/i/flow/login') - // 等待并输入用户名 + // Wait for and enter username await this.browser.waitForSelector(SELECTORS.LOGIN.USERNAME_INPUT) await this.browser.type(SELECTORS.LOGIN.USERNAME_INPUT, credentials.username) await this.browser.click(SELECTORS.LOGIN.NEXT_BUTTON) - // 等待并输入密码 + // Wait for and enter password await this.browser.waitForSelector(SELECTORS.LOGIN.PASSWORD_INPUT) await this.browser.type(SELECTORS.LOGIN.PASSWORD_INPUT, credentials.password) await this.browser.click(SELECTORS.LOGIN.LOGIN_BUTTON) - // 验证登录是否成功 - logger.auth.log('登录表单已填写完成，提交中...') + // Verify if login was successful + logger.auth.log('Login form submitted, verifying...') const loginSuccess = await this.verifyLogin() if (loginSuccess) { - logger.auth.log('登录成功') + logger.auth.log('Login successful') this.isLoggedIn = true } else { - logger.auth.warn('登录验证失败') + logger.auth.warn('Login verification failed') } return loginSuccess } catch (error) { - logger.auth.errorWithError('登录过程中发生错误', error) + logger.auth.errorWithError('Error during login process', error) this.isLoggedIn = false return false } } /** - * 验证是否已成功登录 + * Verify if login was successful */ private async verifyLogin(): Promise { try { - // 等待主页内容加载 + // Wait for home page content to load await this.browser.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 10000 }) return true } @@ -72,7 +72,7 @@ export class TwitterAuthService { } /** - * 检查当前是否已登录 + * Check current login status */ async checkLoginStatus(): Promise { try { @@ -85,7 +85,7 @@ export class TwitterAuthService { } /** - * 获取登录状态 + * Get login status */ isAuthenticated(): boolean { return this.isLoggedIn diff --git a/services/twitter-services/src/core/timeline-service.ts b/services/twitter-services/src/core/timeline-service.ts index 9ecc4bac..aeb76c37 100644 --- a/services/twitter-services/src/core/timeline-service.ts +++ b/services/twitter-services/src/core/timeline-service.ts @@ -6,8 +6,8 @@ import { RateLimiter } from '../utils/rate-limiter' import { SELECTORS } from '../utils/selectors' /** - * Twitter 时间线服务 - * 处理获取和解析时间线内容 + * Twitter Timeline Service + * Handles fetching and parsing timeline content */ export class TwitterTimelineService { private browser: BrowserAdapter @@ -15,33 +15,33 @@ export class TwitterTimelineService { constructor(browser: BrowserAdapter) { this.browser = browser - this.rateLimiter = new RateLimiter(10, 60000) // 每分钟10个请求 + this.rateLimiter = new RateLimiter(10, 60000) // 10 requests per minute } /** - * 获取时间线 + * Get timeline */ async getTimeline(options: TimelineOptions = {}): Promise { - // 等待频率限制 + // Wait for rate limit await this.rateLimiter.waitUntilReady() try { - // 导航到主页 + // Navigate to home page await this.browser.navigate('https://twitter.com/home') - // 等待时间线加载 + // Wait for timeline to load await this.browser.waitForSelector(SELECTORS.TIMELINE.TWEET) - // 延迟一下，确保内容加载完成 + // Delay a bit to ensure content is fully loaded await new Promise(resolve => setTimeout(resolve, 2000)) - // 获取页面HTML内容 + // Get page HTML content const html = await this.browser.executeScript('document.documentElement.outerHTML') - // 解析推文 + // Parse tweets const tweets = TweetParser.parseTimelineTweets(html) - // 应用筛选和限制 + // Apply filtering and limits let filteredTweets = tweets if (options.includeReplies === false) { diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index 11df97d9..d313f65a 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -14,8 +14,8 @@ import { TwitterAuthService } from './auth-service' import { TwitterTimelineService } from './timeline-service' /** - * Twitter 服务实现 - * 集成各个服务组件，提供统一的接口 + * Twitter service implementation + * Integrates various service components, providing a unified interface */ export class TwitterService implements ITwitterService { private browser: BrowserAdapter @@ -29,14 +29,14 @@ export class TwitterService implements ITwitterService { } /** - * 登录 Twitter + * Log in to Twitter */ async login(credentials: TwitterCredentials): Promise { return await this.authService.login(credentials) } /** - * 获取时间线 + * Get timeline */ async getTimeline(options?: TimelineOptions): Promise { this.ensureAuthenticated() @@ -44,14 +44,14 @@ export class TwitterService implements ITwitterService { } /** - * 获取推文详情（MVP暂未实现） + * Get tweet details (not implemented in MVP) */ async getTweetDetails(tweetId: string): Promise { this.ensureAuthenticated() - // MVP阶段，返回一个基本结构 + // In MVP stage, return a basic structure return { id: tweetId, - text: '推文详情功能尚未实现', + text: 'Tweet details feature not yet implemented', author: { username: 'twitter', displayName: 'Twitter', @@ -61,21 +61,21 @@ export class TwitterService implements ITwitterService { } /** - * 搜索推文 + * Search tweets */ async searchTweets(_query: string, _options?: SearchOptions): Promise { - throw new Error('搜索功能尚未实现') + throw new Error('Search feature not yet implemented') } /** - * 获取用户资料 + * Get user profile */ async getUserProfile(_username: string): Promise { - throw new Error('获取用户资料功能尚未实现') + throw new Error('Get user profile feature not yet implemented') } /** - * 关注用户（MVP暂未实现） + * Follow user (not implemented in MVP) */ async followUser(_username: string): Promise { this.ensureAuthenticated() @@ -83,28 +83,28 @@ export class TwitterService implements ITwitterService { } /** - * 点赞推文 + * Like tweet */ async likeTweet(_tweetId: string): Promise { - throw new Error('点赞功能尚未实现') + throw new Error('Like feature not yet implemented') } /** - * 转发推文 + * Retweet */ async retweet(_tweetId: string): Promise { - throw new Error('转发功能尚未实现') + throw new Error('Retweet feature not yet implemented') } /** - * 发送推文 + * Post tweet */ async postTweet(_content: string, _options?: PostOptions): Promise { - throw new Error('发送推文功能尚未实现') + throw new Error('Post tweet feature not yet implemented') } /** - * 确保已经登录 + * Ensure authenticated */ private ensureAuthenticated(): void { if (!this.authService.isAuthenticated()) { diff --git a/services/twitter-services/src/dev-server.ts b/services/twitter-services/src/dev-server.ts index eee03c72..ec19ec2b 100644 --- a/services/twitter-services/src/dev-server.ts +++ b/services/twitter-services/src/dev-server.ts @@ -12,18 +12,18 @@ import { TwitterService } from './core/twitter-service' import { errorToMessage } from './utils/error' import { logger } from './utils/logger' -// 加载环境变量 +// Load environment variables dotenv.config() /** - * 开发服务器入口点 - * 使用 listhen 提供开发时的便利功能 + * Development server entry point + * Provides convenience features for development */ async function startDevServer() { const app = createApp() const router = createRouter() - // 创建浏览器和 Twitter 服务 + // Create browser and Twitter service const browser = new StagehandBrowserAdapter(process.env.BROWSERBASE_API_KEY || '') await browser.initialize({ headless: true, @@ -32,7 +32,7 @@ async function startDevServer() { const twitter = new TwitterService(browser) - // 可选: 如果有凭据，进行登录 + // Optional: If credentials are available, login if (process.env.TWITTER_USERNAME && process.env.TWITTER_PASSWORD) { const success = await twitter.login({ username: process.env.TWITTER_USERNAME, @@ -40,27 +40,27 @@ async function startDevServer() { }) if (success) { - logger.main.log('✅ 已成功登录 Twitter') + logger.main.log('✅ Successfully logged in Twitter') } else { - logger.main.warn('⚠️ Twitter 登录失败') + logger.main.warn('⚠️ Twitter login failed') } } - // 创建 MCP 服务器 + // Create MCP server const mcpServer = new McpServer({ name: 'Twitter Service (Dev)', version: '1.0.0-dev', }) - // 配置 MCP 资源 + // Configure MCP resources mcpServer.resource( 'timeline', new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ resources: [{ name: 'twitter-timeline', uri: 'twitter://timeline', - description: '推文时间线', + description: 'Twitter timeline', }], }) }), async (_uri: URL, { count }: { count?: string }) => { @@ -77,13 +77,13 @@ async function startDevServer() { } } catch (error) { - logger.mcp.errorWithError('获取时间线错误:', error) + logger.mcp.errorWithError('Get timeline error:', error) return { contents: [] } } }, ) - // 配置一些基本工具 + // Configure some basic tools mcpServer.tool( 'post-tweet', { @@ -93,25 +93,25 @@ async function startDevServer() { try { const tweetId = await twitter.postTweet(content) return { - content: [{ type: 'text', text: `成功发布推文: ${tweetId}` }], + content: [{ type: 'text', text: `Successfully posted tweet: ${tweetId}` }], } } catch (error) { return { - content: [{ type: 'text', text: `发推失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Failed to post tweet: ${errorToMessage(error)}` }], isError: true, } } }, ) - // 保存活跃的 SSE 传输 + // Save active SSE transports const activeTransports: SSEServerTransport[] = [] - // 设置路由 + // Set up routes router.get('/', defineEventHandler(() => { return { - name: 'Twitter MCP 开发服务', + name: 'Twitter MCP Dev Server', version: '1.0.0-dev', status: 'running', endpoints: { @@ -121,7 +121,7 @@ async function startDevServer() { } })) - // SSE 端点 + // SSE endpoint router.get('/sse', defineEventHandler(async (event) => { const { req, res } = event.node @@ -129,11 +129,11 @@ async function startDevServer() { res.setHeader('Cache-Control', 'no-cache') res.setHeader('Connection', 'keep-alive') - // 创建 SSE 传输 + // Create SSE transport const transport = new SSEServerTransport('/messages', res) activeTransports.push(transport) - // 客户端断开连接时清理 + // Clean up when client disconnects req.on('close', () => { const index = activeTransports.indexOf(transport) if (index !== -1) { @@ -141,11 +141,11 @@ async function startDevServer() { } }) - // 连接到 MCP 服务器 + // Connect to MCP server await mcpServer.connect(transport) })) - // 消息端点 + // Messages endpoint router.post('/messages', defineEventHandler(async (event) => { if (activeTransports.length === 0) { event.node.res.statusCode = 503 @@ -153,7 +153,7 @@ async function startDevServer() { } try { - // 解析请求体 + // Parse request body const buffers = [] for await (const chunk of event.node.req) { buffers.push(chunk) @@ -161,10 +161,10 @@ async function startDevServer() { const data = Buffer.concat(buffers).toString() const body = JSON.parse(data) - // 使用最近的传输 + // Use latest transport const transport = activeTransports[activeTransports.length - 1] - // 处理消息 + // Handle message const response = await transport.handleMessage(body) return response } @@ -174,10 +174,10 @@ async function startDevServer() { } })) - // 注册路由 + // Register routes app.use(router) - // 启动服务器 + // Start server const listener = toNodeListener(app) await listen(listener, { showURL: true, @@ -185,18 +185,18 @@ async function startDevServer() { open: true, }) - logger.main.log('🚀 Twitter MCP 开发服务器已启动') + logger.main.log('🚀 Twitter MCP Dev Server started') - // 处理退出 + // Handle exit process.on('SIGINT', async () => { - logger.main.log('正在关闭服务器...') + logger.main.log('Shutting down server...') await browser.close() process.exit(0) }) } -// 执行 +// Execute startDevServer().catch((error) => { - logger.main.error('启动开发服务器失败:', error) + logger.main.error('Failed to start dev server:', error) process.exit(1) }) diff --git a/services/twitter-services/src/launcher.ts b/services/twitter-services/src/launcher.ts index 70f9e6e9..ae9102f9 100644 --- a/services/twitter-services/src/launcher.ts +++ b/services/twitter-services/src/launcher.ts @@ -9,8 +9,8 @@ import { TwitterService } from './core/twitter-service' import { logger } from './utils/logger' /** - * Twitter 服务启动器类 - * 负责初始化和启动服务 + * Twitter service launcher class + * Responsible for initializing and starting services */ export class TwitterServiceLauncher { private browser?: StagehandBrowserAdapter @@ -19,18 +19,18 @@ export class TwitterServiceLauncher { private mcpAdapter?: MCPAdapter /** - * 启动 Twitter 服务 + * Start Twitter service */ async start() { try { - // 加载配置 + // Load configuration const configManager = createDefaultConfig() const config = configManager.getConfig() - logger.main.log('正在启动 Twitter 服务...') + logger.main.log('Starting Twitter service...') - // 初始化浏览器 - // 导入处理 + // Initialize browser + // Import handling const { StagehandBrowserAdapter } = await import('./adapters/browserbase-adapter') this.browser = new StagehandBrowserAdapter( config.browser.apiKey, @@ -42,26 +42,27 @@ export class TwitterServiceLauncher { ) await this.browser.initialize(config.browser) - logger.main.log('浏览器已初始化') + logger.main.log('Browser initialized') - // 创建 Twitter 服务 + // Create Twitter service this.twitterService = new TwitterService(this.browser) - // 尝试登录 + // Try to log in if (config.twitter.credentials) { const success = await this.twitterService.login(config.twitter.credentials) if (success) { - logger.main.log('成功登录 Twitter') + logger.main.log('Successfully logged into Twitter') } else { - logger.main.error('Twitter 登录失败!') + logger.main.error('Twitter login failed!') } } - // 启动适配器 - if (config.adapters.airi?.enabled && this.twitterService) { - // 导入处理 + // Start enabled adapters + if (config.adapters.airi?.enabled) { + logger.main.log('Starting Airi adapter...') const { AiriAdapter } = await import('./adapters/airi-adapter') + this.airiAdapter = new AiriAdapter(this.twitterService, { url: config.adapters.airi.url, token: config.adapters.airi.token, @@ -69,69 +70,73 @@ export class TwitterServiceLauncher { }) await this.airiAdapter.start() - logger.main.log('Airi 适配器已启动') + logger.main.log('Airi adapter started') } - if (config.adapters.mcp?.enabled && this.twitterService) { - // 导入处理 + if (config.adapters.mcp?.enabled) { + logger.main.log('Starting MCP adapter...') const { MCPAdapter } = await import('./adapters/mcp-adapter') - this.mcpAdapter = new MCPAdapter(this.twitterService, config.adapters.mcp.port) + + this.mcpAdapter = new MCPAdapter( + this.twitterService, + config.adapters.mcp.port, + ) await this.mcpAdapter.start() - logger.main.log('MCP 适配器已启动') + logger.main.log('MCP adapter started') } - logger.main.log('Twitter 服务已成功启动!') + logger.main.log('Twitter service successfully started!') - // 设置关闭钩子 + // Set up shutdown hooks this.setupShutdownHooks() } catch (error) { - logger.main.withError(error).error('启动 Twitter 服务失败') + logger.main.withError(error).error('Failed to start Twitter service') } } /** - * 停止服务 + * Stop service */ async stop() { - logger.main.log('正在停止 Twitter 服务...') + logger.main.log('Stopping Twitter service...') - // 停止 MCP 适配器 + // Stop MCP adapter if (this.mcpAdapter) { await this.mcpAdapter.stop() - logger.main.log('MCP 适配器已停止') + logger.main.log('MCP adapter stopped') } - // 关闭浏览器 + // Close browser if (this.browser) { await this.browser.close() - logger.main.log('浏览器已关闭') + logger.main.log('Browser closed') } - logger.main.log('Twitter 服务已停止') + logger.main.log('Twitter service stopped') } /** - * 设置关闭钩子 + * Set up shutdown hooks */ private setupShutdownHooks() { - // 处理进程退出 + // Handle process exit process.on('SIGINT', async () => { - logger.main.log('接收到退出信号...') + logger.main.log('Received exit signal...') await this.stop() process.exit(0) }) process.on('SIGTERM', async () => { - logger.main.log('接收到终止信号...') + logger.main.log('Received termination signal...') await this.stop() process.exit(0) }) - // 处理未捕获的异常 + // Handle uncaught exceptions process.on('uncaughtException', async (error) => { - logger.main.withError(error).error('未捕获的异常') + logger.main.withError(error).error('Uncaught exception') await this.stop() process.exit(1) }) diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index 69cc955c..f758e142 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -3,28 +3,28 @@ import process from 'node:process' import { TwitterServiceLauncher } from './launcher' import { initializeLogger, logger } from './utils/logger' -// 确保初始化只发生一次 +// Ensure initialization only happens once async function bootstrap() { - // 1. 首先初始化日志系统 + // 1. First initialize logging system initializeLogger() - // 2. 然后创建并启动服务 + // 2. Then create and start service const launcher = new TwitterServiceLauncher() try { await launcher.start() - logger.main.log('Twitter 服务已成功启动') + logger.main.log('Twitter service successfully started') } catch (error) { - logger.main.withError(error).error('启动失败') + logger.main.withError(error).error('Startup failed') process.exit(1) } - // 设置进程事件处理 + // Set up process event handling process.on('unhandledRejection', (reason) => { - logger.main.withError(reason).error('未处理的 Promise 拒绝:') + logger.main.withError(reason).error('Unhandled Promise rejection:') }) } -// 启动应用 +// Start application bootstrap() diff --git a/services/twitter-services/src/parsers/html-parser.ts b/services/twitter-services/src/parsers/html-parser.ts index bccf43ee..8d56c0bf 100644 --- a/services/twitter-services/src/parsers/html-parser.ts +++ b/services/twitter-services/src/parsers/html-parser.ts @@ -9,15 +9,15 @@ export interface ParseOptions { } /** - * HTML 解析器 - * 使用 rehype 将 HTML 字符串转换为 AST，便于后续处理 + * HTML Parser + * Uses rehype to convert HTML strings to AST, for further processing */ export class HtmlParser { /** - * 将 HTML 字符串解析为 rehype AST - * @param html HTML 字符串 - * @param options 解析选项 - * @returns hast 语法树 + * Parse HTML string to rehype AST + * @param html HTML string + * @param options Parse options + * @returns hast syntax tree */ static parse(html: string, options: ParseOptions = {}): Root { const processor = unified().use(rehypeParse, { @@ -31,16 +31,16 @@ export class HtmlParser { } /** - * 根据选择器查找元素 - * @param tree AST 树 - * @param selector 简化版选择器 (tagName, className, id) - * @returns 匹配的元素数组 + * Find elements by selector + * @param tree AST tree + * @param selector Simplified selector (tagName, className, id) + * @returns Matching element array */ static select(tree: Root, selector: string): Element[] { const elements: Element[] = [] visit(tree, 'element', (node) => { - // 简单选择器实现 + // Simple selector implementation if (this.matchesSelector(node, selector)) { elements.push(node) } @@ -50,27 +50,27 @@ export class HtmlParser { } /** - * 简单的选择器匹配逻辑 + * Simple selector matching logic */ private static matchesSelector(node: Element, selector: string): boolean { - // 标签选择器 + // Tag selector if (selector.match(/^[a-z0-9]+$/i)) { return node.tagName === selector } - // 类选择器 + // Class selector if (selector.startsWith('.')) { const className = selector.slice(1) return (node.properties?.className as string[])?.includes(className) ?? false } - // ID 选择器 + // ID selector if (selector.startsWith('#')) { const id = selector.slice(1) return node.properties?.id === id } - // 数据属性选择器 + // Data attribute selector if (selector.startsWith('[data-')) { // Use non-greedy quantifier and more specific character classes to avoid backtracking const match = selector.match(/\[([^=\]]+)=(['"]?)([^"'\]]+)\2\]/) @@ -84,10 +84,10 @@ export class HtmlParser { } /** - * 访问特定类型的节点 - * @param tree AST 树 - * @param nodeType 节点类型 - * @param visitor 访问器函数 + * Visit specific node type + * @param tree AST tree + * @param nodeType Node type + * @param visitor Visitor function */ static visit(tree: Element | Root, nodeType: string, visitor: (node: any) => void): void { visit(tree, nodeType, visitor) diff --git a/services/twitter-services/src/parsers/profile-parser.ts b/services/twitter-services/src/parsers/profile-parser.ts index 49bb37ac..15c4e639 100644 --- a/services/twitter-services/src/parsers/profile-parser.ts +++ b/services/twitter-services/src/parsers/profile-parser.ts @@ -7,33 +7,33 @@ import { SELECTORS } from '../utils/selectors' import { HtmlParser } from './html-parser' /** - * 用户资料解析器 - * 从 HTML 中提取用户资料信息 + * Profile Parser + * Extracts user profile information from HTML */ export class ProfileParser { /** - * 从 HTML 中解析用户资料 - * @param html HTML 字符串 - * @returns 用户资料 + * Parse user profile from HTML + * @param html HTML string + * @returns User profile */ static parseUserProfile(html: string): UserProfile { const tree = HtmlParser.parse(html) - // 提取用户名和显示名称 + // Extract username and display name const displayNameElement = HtmlParser.select(tree, SELECTORS.PROFILE.DISPLAY_NAME)[0] const displayName = this.extractTextContent(displayNameElement) || 'Unknown User' - // 从URL或DOM中提取用户名 + // Extract username from URL or DOM const username = this.extractUsername(tree) || 'unknown' - // 提取用户简介 + // Extract user bio const bioElement = HtmlParser.select(tree, SELECTORS.PROFILE.BIO)[0] const bio = this.extractTextContent(bioElement) - // 提取用户统计数据 + // Extract user stats const stats = this.extractProfileStats(tree) - // 提取头像和背景图 + // Extract avatar and banner URL const avatarUrl = this.extractAvatarUrl(tree) const bannerUrl = this.extractBannerUrl(tree) @@ -48,7 +48,7 @@ export class ProfileParser { } /** - * 提取文本内容 + * Extract text content */ private static extractTextContent(element?: Element): string { if (!element) @@ -63,18 +63,18 @@ export class ProfileParser { } /** - * 从页面中提取用户名 + * Extract username from page */ private static extractUsername(_tree: Root): string { - // TODO: 可以从URL或特定DOM元素中提取 + // TODO: Can be extracted from URL or specific DOM element return '' } /** - * 提取用户统计数据 + * Extract user stats */ private static extractProfileStats(tree: Root) { - // TODO: 提取粉丝数、关注数、推文数等 + // TODO: Extract followers, following, tweet count, etc. const _statsElement = HtmlParser.select(tree, SELECTORS.PROFILE.STATS)[0] return { @@ -87,26 +87,26 @@ export class ProfileParser { } /** - * 提取头像URL + * Extract avatar URL */ private static extractAvatarUrl(_tree: Root): string | undefined { - // 提取头像图片URL + // TODO: Extract avatar image URL return undefined } /** - * 提取背景图URL + * Extract banner URL */ private static extractBannerUrl(_tree: Root): string | undefined { - // 提取背景图URL + // TODO: Extract banner image URL return undefined } /** - * 从 HTML 提取用户统计信息 + * Extract user stats */ static extractUserStats(_html: string, _tree?: Node): UserStats { - // 解析 HTML 获取统计数据 + // Parse HTML to get stats const stats: UserStats = { tweets: 0, following: 0, @@ -114,10 +114,10 @@ export class ProfileParser { } try { - // 查找统计信息容器 + // Find stats container const _statsElement = _tree ? select('[data-testid="userProfileStats"]', _tree as Root) : null - // TODO: 暂未实现具体解析逻辑 + // TODO: Not implemented yet return stats } @@ -127,18 +127,18 @@ export class ProfileParser { } /** - * 从 HTML 提取用户链接 + * Extract user links */ static extractUserLinks(_html: string, _tree?: Node): UserLink[] { - // TODO: 暂未实现 + // TODO: Not implemented yet return [] } /** - * 从 HTML 提取用户加入日期 + * Extract user join date */ static extractJoinDate(_html: string, _tree?: Node): string | null { - // TODO: 暂未实现 + // TODO: Not implemented yet return null } } diff --git a/services/twitter-services/src/parsers/tweet-parser.ts b/services/twitter-services/src/parsers/tweet-parser.ts index ac317e46..c14781a5 100644 --- a/services/twitter-services/src/parsers/tweet-parser.ts +++ b/services/twitter-services/src/parsers/tweet-parser.ts @@ -5,14 +5,14 @@ import { SELECTORS } from '../utils/selectors' import { HtmlParser } from './html-parser' /** - * 推文解析器 - * 从 HTML 中提取推文信息 + * Tweet Parser + * Extracts tweet information from HTML */ export class TweetParser { /** - * 从 HTML 中解析推文列表 - * @param html HTML 字符串 - * @returns 推文数组 + * Parse timeline tweets from HTML + * @param html HTML string + * @returns Tweet array */ static parseTimelineTweets(html: string): Tweet[] { const tree = HtmlParser.parse(html) @@ -22,26 +22,26 @@ export class TweetParser { } /** - * 从推文元素中提取推文数据 - * @param element 推文元素 - * @returns 推文数据 + * Extract tweet data from tweet element + * @param element Tweet element + * @returns Tweet data */ static extractTweetData(element: Element): Tweet { - // 获取推文 ID + // Get tweet ID const id = this.extractTweetId(element) - // 获取推文文本 + // Get tweet text const textElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TEXT)[0] const text = this.extractTextContent(textElement) - // 获取作者信息 + // Get author info const author = this.extractAuthorInfo(element) - // 获取时间戳 + // Get timestamp const timeElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TIME)[0] const timestamp = timeElement?.properties?.datetime as string || new Date().toISOString() - // 获取统计数据 + // Get stats const stats = this.extractTweetStats(element) return { @@ -54,16 +54,16 @@ export class TweetParser { } /** - * 提取推文ID + * Extract tweet ID */ private static extractTweetId(element: Element): string { - // 从 data-tweet-id 属性或其他位置提取ID + // Extract ID from data-tweet-id attribute or other location return element.properties?.['data-tweet-id'] as string || `tweet-${Math.random().toString(36).substring(2, 15)}` } /** - * 提取文本内容 + * Extract text content */ private static extractTextContent(element?: Element): string { if (!element) @@ -78,23 +78,23 @@ export class TweetParser { } /** - * 提取作者信息 + * Extract author info */ private static extractAuthorInfo(_element: Element) { - // 根据选择器提取作者名称、用户名和头像 - // 这部分需要根据Twitter的实际DOM结构调整 + // Extract author name, username and avatar + // This part needs to be adjusted based on the actual DOM structure of Twitter return { - username: '用户名', // 占位，实际实现需要根据DOM结构 - displayName: '显示名称', + username: 'username', // Placeholder, actual implementation needs to be based on DOM structure + displayName: 'displayName', avatarUrl: undefined, } } /** - * 提取推文统计信息 + * Extract tweet stats */ private static extractTweetStats(_element: Element) { - // 提取点赞数、转发数和评论数 + // Extract like count, retweet count and reply count return { likeCount: undefined, retweetCount: undefined, diff --git a/services/twitter-services/src/types/browser.ts b/services/twitter-services/src/types/browser.ts index 8b27f3e1..e6b6def0 100644 --- a/services/twitter-services/src/types/browser.ts +++ b/services/twitter-services/src/types/browser.ts @@ -1,5 +1,5 @@ /** - * 浏览器配置接口 + * Browser Config Interface */ export interface BrowserConfig { headless?: boolean @@ -9,13 +9,13 @@ export interface BrowserConfig { height: number } timeout?: number - requestTimeout?: number // API 请求超时设置 - requestRetries?: number // 请求重试次数 + requestTimeout?: number // API request timeout + requestRetries?: number // Request retries proxy?: string } /** - * 元素句柄接口 + * Element Handle Interface */ export interface ElementHandle { getText: () => Promise @@ -25,7 +25,7 @@ export interface ElementHandle { } /** - * 等待选项接口 + * Wait Options Interface */ export interface WaitOptions { timeout?: number diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index 6e33dd26..a24d19f3 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -1,5 +1,5 @@ /** - * Twitter 认证凭据 + * Twitter Credentials */ export interface TwitterCredentials { username: string @@ -7,7 +7,7 @@ export interface TwitterCredentials { } /** - * 推文接口 + * Tweet Interface */ export interface Tweet { id: string @@ -25,7 +25,7 @@ export interface Tweet { } /** - * 推文详情 + * Tweet Detail */ export interface TweetDetail extends Tweet { replies?: Tweet[] @@ -33,7 +33,7 @@ export interface TweetDetail extends Tweet { } /** - * 用户资料 + * User Profile */ export interface UserProfile { username: string @@ -49,7 +49,7 @@ export interface UserProfile { } /** - * 时间线选项 + * Timeline Options */ export interface TimelineOptions { count?: number @@ -58,7 +58,7 @@ export interface TimelineOptions { } /** - * 搜索选项 + * Search Options */ export interface SearchOptions { count?: number @@ -66,7 +66,7 @@ export interface SearchOptions { } /** - * 发推选项 + * Post Options */ export interface PostOptions { media?: string[] @@ -74,7 +74,7 @@ export interface PostOptions { } /** - * 用户统计信息 + * User Stats */ export interface UserStats { tweets: number @@ -83,7 +83,7 @@ export interface UserStats { } /** - * 用户链接信息 + * User Link */ export interface UserLink { type: string @@ -92,7 +92,7 @@ export interface UserLink { } /** - * Twitter 服务接口 + * Twitter Service Interface */ export interface TwitterService { login: (credentials: TwitterCredentials) => Promise diff --git a/services/twitter-services/src/utils/api.ts b/services/twitter-services/src/utils/api.ts index 7e70dc0d..a561fc78 100644 --- a/services/twitter-services/src/utils/api.ts +++ b/services/twitter-services/src/utils/api.ts @@ -3,34 +3,34 @@ import { ofetch } from 'ofetch' import { logger } from './logger' /** - * 创建一个预配置的 ofetch 实例 + * Create a pre-configured ofetch instance * - * @param baseURL - API 的基础 URL - * @param options - 附加选项 - * @returns - 定制的 ofetch 实例 + * @param baseURL - Base URL for the API + * @param options - Additional options + * @returns - Customized ofetch instance */ export function createApiClient(baseURL: string, options: Record = {}) { const client = ofetch.create({ baseURL, retry: 1, - timeout: 30000, // 默认 30 秒超时 + timeout: 30000, // Default 30 second timeout ...options, - // 请求拦截器 + // Request interceptor onRequest({ request, options }) { const method = options.method || 'GET' const url = request.toString() - logger.browser.withFields({ method, url }).debug('API 请求') + logger.browser.withFields({ method, url }).debug('API request') }, - // 请求错误拦截器 + // Request error interceptor onRequestError({ request, error, options }) { const method = options.method || 'GET' const url = request.toString() - logger.browser.withFields({ method, url }).errorWithError('API 请求失败', error) + logger.browser.withFields({ method, url }).errorWithError('API request failed', error) }, - // 响应拦截器 + // Response interceptor onResponse({ request, response, options }) { const method = options.method || 'GET' const url = request.toString() @@ -40,10 +40,10 @@ export function createApiClient(baseURL: string, options: Record = .withField('method', method) .withField('url', url) .withField('status', status) - .debug('API 响应') + .debug('API response') }, - // 响应错误拦截器 + // Response error interceptor onResponseError({ request, response, options }) { const method = options.method || 'GET' const url = request.toString() @@ -54,7 +54,7 @@ export function createApiClient(baseURL: string, options: Record = .withField('url', url) .withField('status', status) .withField('body', response._data) - .error('API 响应错误') + .error('API response error') }, }) diff --git a/services/twitter-services/src/utils/error.ts b/services/twitter-services/src/utils/error.ts index e8395793..15835674 100644 --- a/services/twitter-services/src/utils/error.ts +++ b/services/twitter-services/src/utils/error.ts @@ -1,54 +1,54 @@ /** - * 从任意错误类型中安全地提取错误消息 - * 处理 Error 对象、字符串、对象和其他类型 + * Safely extract error message from any error type + * Handles Error objects, strings, objects, and other types * - * @param error - 任意错误对象 - * @param fallbackMessage - 当无法提取消息时的后备消息 - * @returns 格式化的错误消息 + * @param error - Any error object + * @param fallbackMessage - Fallback message when unable to extract a message + * @returns Formatted error message */ -export function errorToMessage(error: unknown, fallbackMessage = '未知错误'): string { +export function errorToMessage(error: unknown, fallbackMessage = 'Unknown error'): string { if (error === null || error === undefined) { return fallbackMessage } - // 处理标准 Error 对象 + // Handle standard Error objects if (error instanceof Error) { return error.message } - // 处理字符串错误 + // Handle string errors if (typeof error === 'string') { return error } - // 处理带有 message 属性的对象 + // Handle objects with message property if (typeof error === 'object') { - // 检查是否有 message 属性 + // Check if it has a message property if ('message' in error && typeof (error as any).message === 'string') { return (error as any).message } - // 尝试将对象转换为字符串 + // Try to convert object to string try { return JSON.stringify(error) } catch { - // 如果无法序列化，返回对象的字符串表示 + // If serialization fails, return object's string representation return String(error) } } - // 针对其他情况，尝试强制转换为字符串 + // For other cases, try to force convert to string return String(error) } /** - * 创建一个带有详细上下文信息的错误 + * Create an error with detailed context information * - * @param message - 错误消息 - * @param originalError - 原始错误对象（可选） - * @param context - 额外上下文信息（可选） - * @returns 增强的错误对象 + * @param message - Error message + * @param originalError - Original error object (optional) + * @param context - Additional context information (optional) + * @returns Enhanced error object */ export function createError( message: string, @@ -57,15 +57,15 @@ export function createError( ): Error { let errorMessage = message - // 添加原始错误信息 + // Add original error information if (originalError) { errorMessage += `: ${errorToMessage(originalError)}` } - // 创建新的错误对象 + // Create new error object const error = new Error(errorMessage) - // 添加上下文信息 + // Add context information if (context) { Object.assign(error, { context }) } diff --git a/services/twitter-services/src/utils/logger.ts b/services/twitter-services/src/utils/logger.ts index 40708f08..f391ab71 100644 --- a/services/twitter-services/src/utils/logger.ts +++ b/services/twitter-services/src/utils/logger.ts @@ -3,16 +3,16 @@ import { createLogg, Format, LogLevel, setGlobalFormat, setGlobalLogLevel } from import { createDefaultConfig } from '../config' -// 记录初始化状态 +// Track initialization status let isInitialized = false -// 初始化全局日志配置 +// Initialize global logging configuration export function initializeLogger(): void { if (isInitialized) { - return // 防止多次初始化 + return // Prevent multiple initializations } - // 设置全局日志级别 + // Set global log level setGlobalLogLevel(LogLevel.Debug) setGlobalFormat(Format.Pretty) @@ -28,7 +28,7 @@ export function initializeLogger(): void { setGlobalLogLevel(logLevelMap[config.system?.logLevel] || LogLevel.Debug) - // 根据配置设置格式 + // Set format based on configuration if (config.system?.logFormat === 'pretty') { setGlobalFormat(Format.Pretty) } @@ -59,7 +59,7 @@ export function useLogger(name?: string): ReturnType { return createLogg(`${dirName}/${fileName}:${lineNumber}`).useGlobalConfig() } -// 创建各种服务的预配置日志记录器 +// Create pre-configured loggers for various services export const logger = { auth: useLogger('auth-service'), timeline: useLogger('timeline-service'), diff --git a/services/twitter-services/src/utils/rate-limiter.ts b/services/twitter-services/src/utils/rate-limiter.ts index f08f0808..047d2b42 100644 --- a/services/twitter-services/src/utils/rate-limiter.ts +++ b/services/twitter-services/src/utils/rate-limiter.ts @@ -1,6 +1,6 @@ /** - * 请求频率限制器 - * 控制对 Twitter 的请求频率，避免触发限制 + * Request rate limiter + * Controls request frequency to Twitter to avoid triggering limits */ export class RateLimiter { private requestHistory: number[] = [] @@ -8,9 +8,9 @@ export class RateLimiter { private timeWindow: number /** - * 创建频率限制器 - * @param maxRequests 时间窗口内的最大请求数 - * @param timeWindow 时间窗口大小（毫秒） + * Create rate limiter + * @param maxRequests Maximum requests within time window + * @param timeWindow Time window size (milliseconds) */ constructor(maxRequests: number = 20, timeWindow: number = 60000) { this.maxRequests = maxRequests @@ -18,7 +18,7 @@ export class RateLimiter { } /** - * 检查是否可以发送请求 + * Check if request can be sent */ canRequest(): boolean { this.cleanOldRequests() @@ -26,15 +26,15 @@ export class RateLimiter { } /** - * 记录一次请求 + * Record a request */ recordRequest(): void { this.requestHistory.push(Date.now()) } /** - * 获取下次可请求的等待时间（毫秒） - * 如果当前可以请求，返回0 + * Get wait time until next available request (milliseconds) + * Returns 0 if request can be sent now */ getWaitTime(): number { if (this.canRequest()) { @@ -46,7 +46,7 @@ export class RateLimiter { } /** - * 清理过期的请求记录 + * Clean expired request records */ private cleanOldRequests(): void { const now = Date.now() @@ -55,7 +55,7 @@ export class RateLimiter { } /** - * 等待直到可以发送请求 + * Wait until request can be sent */ async waitUntilReady(): Promise { const waitTime = this.getWaitTime() diff --git a/services/twitter-services/src/utils/selectors.ts b/services/twitter-services/src/utils/selectors.ts index 0883ace3..0ac00471 100644 --- a/services/twitter-services/src/utils/selectors.ts +++ b/services/twitter-services/src/utils/selectors.ts @@ -1,6 +1,6 @@ /** - * Twitter 网站 CSS 选择器常量 - * 用于定位页面上的元素 + * Twitter website CSS selector constants + * Used to locate elements on the page */ export const SELECTORS = { LOGIN: { From 95074bb07035f3b038c795dc58a5aeca4a36287d Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 02:27:28 +0800 Subject: [PATCH 08/20] fix: continue i18n --- .../src/adapters/browser-adapter.ts | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/services/twitter-services/src/adapters/browser-adapter.ts b/services/twitter-services/src/adapters/browser-adapter.ts index e43a6622..dd487e5e 100644 --- a/services/twitter-services/src/adapters/browser-adapter.ts +++ b/services/twitter-services/src/adapters/browser-adapter.ts @@ -2,57 +2,57 @@ import type { Buffer } from 'node:buffer' import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' /** - * 浏览器操作的通用接口 - * 定义了与不同浏览器后端交互所需的基本操作 + * Generic browser adapter interface + * Defines the basic operations required for interacting with different browser backends */ export interface BrowserAdapter { /** - * 初始化浏览器会话 + * Initialize browser session */ initialize: (config: BrowserConfig) => Promise /** - * 导航到指定 URL + * Navigate to specified URL */ navigate: (url: string) => Promise /** - * 执行 JavaScript 脚本 + * Execute JavaScript script */ executeScript: (script: string) => Promise /** - * 等待元素出现 + * Wait for element to appear */ waitForSelector: (selector: string, options?: WaitOptions) => Promise /** - * 点击元素 + * Click element */ click: (selector: string) => Promise /** - * 向输入框输入文本 + * Type text into input */ type: (selector: string, text: string) => Promise /** - * 获取元素文本内容 + * Get element text content */ getText: (selector: string) => Promise /** - * 获取多个元素的句柄 + * Get multiple element handles */ getElements: (selector: string) => Promise /** - * 获取屏幕截图 + * Get screenshot */ getScreenshot: () => Promise /** - * 关闭浏览器会话 + * Close browser session */ close: () => Promise } From 4c1040f9b244cd69fe9f2dae399c10b0760146ed Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 03:11:07 +0800 Subject: [PATCH 09/20] feat: cookie login --- services/twitter-services/.env.example | 25 +- .../twitter-services/docs/architecture.md | 35 +- services/twitter-services/package.json | 2 +- .../src/adapters/browser-adapter.ts | 15 + services/twitter-services/src/cli.ts | 112 ++++++ services/twitter-services/src/config/types.ts | 38 +- .../twitter-services/src/core/auth-service.ts | 361 ++++++++++++++++-- .../src/core/twitter-service.ts | 8 + .../twitter-services/src/types/twitter.ts | 5 +- .../twitter-services/src/utils/selectors.ts | 6 +- 10 files changed, 537 insertions(+), 70 deletions(-) diff --git a/services/twitter-services/.env.example b/services/twitter-services/.env.example index 99f77fdf..b393ba2a 100644 --- a/services/twitter-services/.env.example +++ b/services/twitter-services/.env.example @@ -1,11 +1,24 @@ -# BrowserBase 配置 +# BrowserBase Config BROWSERBASE_API_KEY=your_api_key_here -# Twitter 账号配置 +# Browser Config +BROWSER_HEADLESS=false +BROWSER_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 +BROWSER_VIEWPORT_WIDTH=1280 +BROWSER_VIEWPORT_HEIGHT=800 +BROWSER_TIMEOUT=30000 +BROWSER_REQUEST_TIMEOUT=20000 +BROWSER_REQUEST_RETRIES=2 + +# Twitter Account Config TWITTER_USERNAME=your_twitter_username TWITTER_PASSWORD=your_twitter_password +# Cookie can be in two formats: +# 1. JSON format: {"auth_token":"xxx","ct0":"yyy"} +# 2. document.cookie format: auth_token=xxx; ct0=yyy +TWITTER_COOKIES= -# 适配器配置 +# Adapter Config ENABLE_AIRI=false AIRI_URL=http://localhost:3000 AIRI_TOKEN=your_airi_token @@ -13,7 +26,7 @@ AIRI_TOKEN=your_airi_token ENABLE_MCP=true MCP_PORT=8080 -# 系统配置 -LOG_LEVEL=info # 可选: error, warn, info, verbose, debug -LOG_FORMAT=pretty # 可选: json, pretty +# System Config +LOG_LEVEL=info # Optional: error, warn, info, verbose, debug +LOG_FORMAT=pretty # Optional: json, pretty CONCURRENCY=1 diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index ccd222f8..fa471d2b 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -88,7 +88,7 @@ Using listhen for optimized development experience, including automatic browser #### 5.2.1 Authentication Service (Auth Service) -Handles Twitter login and session maintenance. +Handles Twitter session detection and maintenance. Uses a manual login approach where the service opens the Twitter login page and waits for the user to complete the authentication process. After successful login, the service can save cookies for future sessions, allowing cookie-based authentication in subsequent uses. #### 5.2.2 Timeline Service (Timeline Service) @@ -196,15 +196,23 @@ async function main() { // Create Twitter service const twitter = new TwitterService(browser) - // Login - await twitter.login({ - username: 'your-username', - password: 'your-password' - }) + // Initiate manual login and wait for user to complete authentication + const loggedIn = await twitter.login({}) - // Get timeline - const tweets = await twitter.getTimeline({ count: 10 }) - console.log(tweets) + if (loggedIn) { + console.log('Login successful') + + // Export cookies for future use (optional) + const cookies = await twitter.exportCookies() + console.log('Cookies saved for future use:', Object.keys(cookies).length) + + // Get timeline + const tweets = await twitter.getTimeline({ count: 10 }) + console.log(tweets) + } + else { + console.error('Login failed') + } // Release resources await browser.close() @@ -222,14 +230,10 @@ async function startAiriModule() { const twitter = new TwitterService(browser) - // Create Airi adapter + // Create Airi adapter (no credentials needed for manual login) const airiAdapter = new AiriAdapter(twitter, { url: process.env.AIRI_URL, - token: process.env.AIRI_TOKEN, - credentials: { - username: process.env.TWITTER_USERNAME, - password: process.env.TWITTER_PASSWORD - } + token: process.env.AIRI_TOKEN }) // Start adapter @@ -289,6 +293,7 @@ For example, adding "Get Tweets from a Specific User" functionality: - **Monitoring & Alerts**: Monitor service status and Twitter access limitations - **Selector Updates**: Regularly validate and update selector configurations - **Session Management**: Optimize session management to improve stability +- **Cookie Management**: Implement secure storage for saved session cookies ## 12. Project Roadmap diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index f0665f68..a2b80576 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -7,7 +7,7 @@ "license": "MIT", "scripts": { "start": "tsx src/main.ts", - "dev": "tsx watch src/main.ts", + "dev": "tsx src/main.ts", "dev:mcp": "tsx src/dev-server.ts", "postinstall": "playwright install chromium" }, diff --git a/services/twitter-services/src/adapters/browser-adapter.ts b/services/twitter-services/src/adapters/browser-adapter.ts index dd487e5e..330b5287 100644 --- a/services/twitter-services/src/adapters/browser-adapter.ts +++ b/services/twitter-services/src/adapters/browser-adapter.ts @@ -46,6 +46,21 @@ export interface BrowserAdapter { */ getElements: (selector: string) => Promise + /** + * Get all cookies from the browser context + * This includes HTTP_ONLY cookies that can't be accessed via document.cookie + */ + getAllCookies: () => Promise> + /** * Get screenshot */ diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts index 07e652fd..adf3a2aa 100644 --- a/services/twitter-services/src/cli.ts +++ b/services/twitter-services/src/cli.ts @@ -109,5 +109,117 @@ program } }) +// Export cookies command +program + .command('export-cookies') + .description('Login and export cookies for later use') + .option('-o, --output ', 'Output cookies to file', 'twitter-cookies.json') + .option('-f, --format ', 'Cookie format (json or string)', 'json') + .action(async (options) => { + try { + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + // Initialize browser + const browser = new StagehandBrowserAdapter(config.browser.apiKey) + await browser.initialize(config.browser) + + // Create service and login + const twitterService = new TwitterService(browser) + + if (!config.twitter.credentials) { + throw new Error('Cannot get Twitter credentials, please check configuration') + } + + const loggedIn = await twitterService.login(config.twitter.credentials) + + if (!loggedIn) { + throw new Error('Login failed, please check credentials') + } + + console.log('Login successful, exporting cookies...') + + // Export cookies - specify the format based on user option + const cookieFormat = options.format === 'string' ? 'string' : 'object' + const cookies = await twitterService.exportCookies(cookieFormat) + + if (cookieFormat === 'string') { + // Save raw string to file + fs.writeFileSync(options.output, cookies as string) + } + else { + // Save JSON to file + fs.writeFileSync(options.output, JSON.stringify(cookies, null, 2)) + } + + console.log(`Cookies saved to ${options.output} in ${cookieFormat} format`) + + // Close browser + await browser.close() + } + catch (error) { + console.error('Failed to export cookies:', errorToMessage(error)) + process.exit(1) + } + }) + +// Login with cookies file command +program + .command('login-with-cookies') + .description('Login using cookies from a file') + .option('-i, --input ', 'Input cookies file', 'twitter-cookies.json') + .option('-t, --test', 'Test login only, do not perform other actions', false) + .action(async (options) => { + try { + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + // Load cookies from file + if (!fs.existsSync(options.input)) { + throw new Error(`Cookies file not found: ${options.input}`) + } + + console.log(`Loading cookies from ${options.input}`) + const cookies = JSON.parse(fs.readFileSync(options.input, 'utf8')) + + // Initialize browser + const browser = new StagehandBrowserAdapter(config.browser.apiKey) + await browser.initialize(config.browser) + + // Create service and login with cookies + const twitterService = new TwitterService(browser) + const credentials = { + username: config.twitter.credentials?.username || '', + password: config.twitter.credentials?.password || '', + cookies, + } + + console.log('Attempting to login with cookies...') + const loggedIn = await twitterService.login(credentials) + + if (loggedIn) { + console.log('Login successful!') + + if (options.test) { + console.log('Login test passed, exiting.') + } + else { + // Perform additional actions if needed + console.log('Add more actions here when needed') + } + } + else { + throw new Error('Login failed, cookies may be expired') + } + + // Close browser + await browser.close() + } + catch (error) { + console.error('Failed to login with cookies:', errorToMessage(error)) + process.exit(1) + } + }) + // Parse command line arguments program.parse() diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index a6864e21..d5d81869 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -47,23 +47,47 @@ export interface Config { * Default configuration */ export function getDefaultConfig(): Config { + // Parse cookies from environment variable if provided + let cookiesFromEnv: Record | undefined + + if (process.env.TWITTER_COOKIES) { + // Try to parse as JSON first + try { + cookiesFromEnv = JSON.parse(process.env.TWITTER_COOKIES) + } + catch { + // If JSON parsing fails, treat as document.cookie format string + cookiesFromEnv = process.env.TWITTER_COOKIES + .split(';') + .map(v => v.split('=')) + .reduce((acc, v) => { + // Skip empty values + if (v.length < 2) + return acc + acc[decodeURIComponent(v[0].trim())] = decodeURIComponent(v[1].trim()) + return acc + }, {} as Record) + } + } + return { browser: { apiKey: process.env.BROWSERBASE_API_KEY || '', // Move apiKey to browser config - headless: true, - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', + headless: process.env.BROWSER_HEADLESS === 'true', + userAgent: process.env.BROWSER_USER_AGENT || 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', viewport: { - width: 1280, - height: 800, + width: Number.parseInt(process.env.BROWSER_VIEWPORT_WIDTH || '1280'), + height: Number.parseInt(process.env.BROWSER_VIEWPORT_HEIGHT || '800'), }, - timeout: 30000, - requestTimeout: 20000, - requestRetries: 2, + timeout: Number.parseInt(process.env.BROWSER_TIMEOUT || '30000'), + requestTimeout: Number.parseInt(process.env.BROWSER_REQUEST_TIMEOUT || '20000'), + requestRetries: Number.parseInt(process.env.BROWSER_REQUEST_RETRIES || '2'), }, twitter: { credentials: { username: process.env.TWITTER_USERNAME || '', password: process.env.TWITTER_PASSWORD || '', + cookies: cookiesFromEnv, }, defaultOptions: { timeline: { diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index 8eb80f57..f726676f 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -17,54 +17,93 @@ export class TwitterAuthService { } /** - * Login to Twitter + * Login to Twitter - compatibility method for existing code + * Prefers cookie-based login if cookies provided, otherwise redirects to manual login */ - async login(credentials: TwitterCredentials): Promise { - logger.auth.withField('username', credentials.username.replace(/./g, '*')).log('Attempting to login to Twitter') + async login(credentials: TwitterCredentials = {}): Promise { + // If cookies are provided, try to use them for login + if (credentials.cookies && Object.keys(credentials.cookies).length > 0) { + logger.auth.log('Cookies provided, attempting cookie-based login') + const cookieLoginSuccess = await this.loginWithCookies(credentials.cookies) + if (cookieLoginSuccess) { + return true + } + // If cookie login fails, log the issue but continue to manual login + logger.auth.log('Cookie login failed, falling back to manual login') + } - try { - // Navigate to login page - await this.browser.navigate('https://twitter.com/i/flow/login') + // Check for existing session first + logger.auth.log('Checking for existing session before initiating manual login') + const existingSession = await this.checkExistingSession() + if (existingSession) { + return true + } - // Wait for and enter username - await this.browser.waitForSelector(SELECTORS.LOGIN.USERNAME_INPUT) - await this.browser.type(SELECTORS.LOGIN.USERNAME_INPUT, credentials.username) - await this.browser.click(SELECTORS.LOGIN.NEXT_BUTTON) + // Fallback to manual login flow + logger.auth.log('No existing session found, initiating manual login process') + return this.initiateManualLogin() + } - // Wait for and enter password - await this.browser.waitForSelector(SELECTORS.LOGIN.PASSWORD_INPUT) - await this.browser.type(SELECTORS.LOGIN.PASSWORD_INPUT, credentials.password) - await this.browser.click(SELECTORS.LOGIN.LOGIN_BUTTON) + /** + * Verify if login was successful + */ + private async verifyLogin(): Promise { + try { + // Try multiple selectors to determine login status + // First check for timeline which is definitive proof of being logged in + try { + await this.browser.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 15000 }) + return true + } + catch { + // If timeline selector fails, check for other indicators + } - // Verify if login was successful - logger.auth.log('Login form submitted, verifying...') - const loginSuccess = await this.verifyLogin() + // Check for profile button which appears when logged in + try { + const profileSelector = '[data-testid="AppTabBar_Profile_Link"]' + await this.browser.waitForSelector(profileSelector, { timeout: 5000 }) + return true + } + catch { + // Continue to other checks + } - if (loginSuccess) { - logger.auth.log('Login successful') - this.isLoggedIn = true + // Check for login form to confirm NOT logged in + try { + const loginFormSelector = '[data-testid="loginForm"]' + await this.browser.waitForSelector(loginFormSelector, { timeout: 3000 }) + // If login form is visible, we're definitely not logged in + return false } - else { - logger.auth.warn('Login verification failed') + catch { + // Login form not found, could still be logged in or on another page } - return loginSuccess + // If we got here, we couldn't definitively confirm login status + // Check current URL for additional clues + const currentUrl = await this.browser.executeScript('return window.location.href') + if (currentUrl.includes('/home')) { + // On home page but couldn't find timeline - might still be loading + return true + } + + // Default to not logged in if we can't confirm + return false } catch (error) { - logger.auth.errorWithError('Error during login process', error) - this.isLoggedIn = false + logger.auth.errorWithError('Error during login verification', error) return false } } /** - * Verify if login was successful + * Check current login status */ - private async verifyLogin(): Promise { + async checkLoginStatus(): Promise { try { - // Wait for home page content to load - await this.browser.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 10000 }) - return true + await this.browser.navigate('https://twitter.com/home') + return await this.verifyLogin() } catch { return false @@ -72,22 +111,270 @@ export class TwitterAuthService { } /** - * Check current login status + * Get login status */ - async checkLoginStatus(): Promise { + isAuthenticated(): boolean { + return this.isLoggedIn + } + + /** + * Export current session cookies + * Can be used to save and reuse session later + * @param format - The format of the returned cookies ('object' or 'string') + */ + async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { + try { + if (format === 'string') { + // Return raw document.cookie string + const cookieString = await this.browser.executeScript(` + // Wrap in a function to allow return statement + (() => { + return document.cookie; + })(); + `) + return cookieString + } + else { + // Return cookies as object + const cookies = await this.browser.executeScript>(` + // Wrap in a function to allow return statement + (() => { + return document.cookie.split(';') + .map(cookie => cookie.trim().split('=')) + .reduce((acc, v) => { + if (v.length < 2) return acc; + acc[v[0]] = v[1]; + return acc; + }, {}); + })(); + `) + return cookies + } + } + catch (error) { + logger.auth.errorWithError('Error exporting cookies', error) + return format === 'string' ? '' : {} + } + } + + /** + * Login to Twitter using cookies + */ + async loginWithCookies(cookies: Record): Promise { + logger.auth.log(`Attempting to login to Twitter using ${Object.keys(cookies).length} cookies`) + try { + // Navigate to a Twitter page + await this.browser.navigate('https://twitter.com') + + // Set cookies + await this.browser.executeScript(` + const cookies = ${JSON.stringify(cookies)}; + Object.entries(cookies).forEach(([name, value]) => { + document.cookie = \`\${name}=\${value};domain=.twitter.com;path=/\`; + }); + `) + + logger.auth.log(`Set ${Object.keys(cookies).length} cookies`) + + // Refresh page to apply cookies await this.browser.navigate('https://twitter.com/home') - return await this.verifyLogin() + + // Verify if login was successful - try multiple times with longer timeout + logger.auth.log('Cookies set, verifying login status...') + + // Try multiple times with increasing timeouts for verification + // Twitter might be slow to respond or need multiple page refreshes + let loginSuccess = false + const verificationAttempts = 3 + + for (let attempt = 1; attempt <= verificationAttempts; attempt++) { + try { + logger.auth.log(`Verification attempt ${attempt}/${verificationAttempts}`) + loginSuccess = await this.verifyLogin() + + if (loginSuccess) { + break + } + else if (attempt < verificationAttempts) { + // If not successful but not last attempt, refresh page and wait + logger.auth.log('Refreshing page and trying again...') + await this.browser.navigate('https://twitter.com/home') + await new Promise(resolve => setTimeout(resolve, 3000)) + } + } + catch (error) { + logger.auth.withError(error).debug(`Verification attempt ${attempt} failed`) + } + } + + if (loginSuccess) { + logger.auth.log('Login with cookies successful') + this.isLoggedIn = true + + // Try to refresh cookies to ensure they're up to date + try { + const freshCookies = await this.exportCookies('object') + logger.auth.log(`Updated and exported ${typeof freshCookies === 'string' ? freshCookies.length : Object.keys(freshCookies).length} cookies`) + } + catch (error) { + logger.auth.withError(error).debug('Failed to update cookies, but login was successful') + } + } + else { + logger.auth.warn('Login with cookies verification failed, cookies may be expired') + } + + return loginSuccess } - catch { + catch (error) { + logger.auth.errorWithError('Error during cookie login process', error) + this.isLoggedIn = false return false } } /** - * Get login status + * Checks if there's an existing login session and retrieves it + * This should be called before initiateManualLogin */ - isAuthenticated(): boolean { - return this.isLoggedIn + async checkExistingSession(): Promise { + logger.auth.log('Checking for existing Twitter session') + + try { + // Navigate to home page to check session + await this.browser.navigate('https://twitter.com/home') + + // Verify if login is active + const loginSuccess = await this.verifyLogin() + + if (loginSuccess) { + logger.auth.log('Existing session found and valid') + this.isLoggedIn = true + + // Export and save cookies + try { + const cookies = await this.exportCookies('object') + logger.auth.log(`✅ Exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies from existing session`) + } + catch (error) { + logger.auth.errorWithError('Error exporting cookies from existing session', error) + } + } + else { + logger.auth.log('No valid session found') + } + + return loginSuccess + } + catch (error) { + logger.auth.errorWithError('Error checking session status', error) + this.isLoggedIn = false + return false + } + } + + /** + * Initiates the manual login process by navigating to Twitter login page + * and waits for user to complete the login process + */ + async initiateManualLogin(): Promise { + logger.auth.log('Opening Twitter login page for manual login') + + try { + // Store the current URL to detect navigation + const initialUrl = await this.browser.executeScript('return window.location.href') + + // Navigate to login page + await this.browser.navigate('https://twitter.com/i/flow/login') + + // Wait for user to manually log in (detected by timeline presence) + logger.auth.log('==============================================') + logger.auth.log('Please log in to Twitter in the opened browser window') + logger.auth.log('The system will wait for you to complete the login process') + logger.auth.log('Cookies will be automatically saved after login') + logger.auth.log('==============================================') + + // Poll for login success at intervals + let attempts = 0 + const maxAttempts = 60 // 10 minutes (10 seconds * 60) + let lastUrl = initialUrl + + while (attempts < maxAttempts) { + attempts++ + + try { + // Get current URL to detect page changes + const currentUrl = await this.browser.executeScript('return window.location.href') + + // Check if URL has changed significantly - may indicate user interaction + if (currentUrl !== lastUrl && !currentUrl.includes('/flow/login')) { + logger.auth.log(`Detected page change: ${lastUrl} -> ${currentUrl}`) + logger.auth.log('Attempting to navigate to home page and verify login status') + + // URL changed - try navigating to home to verify + await this.browser.navigate('https://twitter.com/home') + + // Check if login was successful + const isLoggedIn = await this.verifyLogin() + if (isLoggedIn) { + logger.auth.log('✅ Login successful! Exporting cookies...') + + // Export cookies for future use + try { + const cookies = await this.exportCookies('object') + logger.auth.log(`✅ Successfully exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies`) + } + catch (error) { + logger.auth.errorWithError('Error exporting cookies', error) + } + + this.isLoggedIn = true + return true + } + + // Update last URL + lastUrl = currentUrl + } + + // Also try direct login verification + const isLoggedIn = await this.verifyLogin() + if (isLoggedIn) { + logger.auth.log('✅ Login successful! Exporting cookies...') + + // Export cookies for future use + try { + const cookies = await this.exportCookies('object') + logger.auth.log(`✅ Successfully exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies`) + } + catch (error) { + logger.auth.errorWithError('Error exporting cookies', error) + } + + this.isLoggedIn = true + return true + } + } + catch (error) { + // Ignore errors during verification, continue polling + logger.auth.debug(`Error during verification: ${error.message}`) + } + + // Wait 10 seconds before checking again + await new Promise(resolve => setTimeout(resolve, 10000)) + + // Only log every 6 attempts (1 minute) to reduce noise + if (attempts % 6 === 0) { + logger.auth.log(`Still waiting for login... (${Math.floor(attempts / 6)} minutes elapsed)`) + } + } + + logger.auth.warn('⚠️ Manual login timeout exceeded') + return false + } + catch (error) { + logger.auth.errorWithError('Error during manual login process', error) + return false + } } } diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index d313f65a..c5f60bd5 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -111,4 +111,12 @@ export class TwitterService implements ITwitterService { throw new Error('Not authenticated. Call login() first.') } } + + /** + * Export current session cookies + * @param format - The format of the returned cookies ('object' or 'string') + */ + async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { + return await this.authService.exportCookies(format) + } } diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index a24d19f3..e9a76f91 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -2,8 +2,9 @@ * Twitter Credentials */ export interface TwitterCredentials { - username: string - password: string + username?: string + password?: string + cookies?: Record } /** diff --git a/services/twitter-services/src/utils/selectors.ts b/services/twitter-services/src/utils/selectors.ts index 0ac00471..b8cac7e5 100644 --- a/services/twitter-services/src/utils/selectors.ts +++ b/services/twitter-services/src/utils/selectors.ts @@ -6,8 +6,10 @@ export const SELECTORS = { LOGIN: { USERNAME_INPUT: 'input[autocomplete="username"]', PASSWORD_INPUT: 'input[type="password"]', - NEXT_BUTTON: '[data-testid="auth-login-button"]', - LOGIN_BUTTON: '[data-testid="LoginForm_Login_Button"]', + NEXT_BUTTON: 'div[role="button"]:has-text("Next")', + LOGIN_BUTTON: 'div[role="button"]:has-text("Log in")', + NEXT_BUTTON_ALT: '[data-testid="login-next-button"]', + LOGIN_BUTTON_ALT: '[data-testid="login-submit-button"]', }, HOME: { TIMELINE: '[data-testid="primaryColumn"]', From 499760ad09545a0c4409d940be3db645d56f3410 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 03:37:28 +0800 Subject: [PATCH 10/20] fix: save cookie to session json --- .gitignore | 2 + .../twitter-services/docs/architecture.md | 39 ++- .../src/adapters/browser-adapter.ts | 15 + .../src/adapters/browserbase-adapter.ts | 26 ++ .../src/browser/browserbase.ts | 51 +++ services/twitter-services/src/config/types.ts | 26 +- .../twitter-services/src/core/auth-service.ts | 305 ++++++++++++++---- services/twitter-services/src/main.ts | 1 - .../src/utils/session-manager.ts | 129 ++++++++ 9 files changed, 495 insertions(+), 99 deletions(-) create mode 100644 services/twitter-services/src/utils/session-manager.ts diff --git a/.gitignore b/.gitignore index 8ac53d4e..dbba6eac 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ coverage/ *.mp3 **/temp/ + +*.session.json diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index fa471d2b..9c53400a 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -88,7 +88,13 @@ Using listhen for optimized development experience, including automatic browser #### 5.2.1 Authentication Service (Auth Service) -Handles Twitter session detection and maintenance. Uses a manual login approach where the service opens the Twitter login page and waits for the user to complete the authentication process. After successful login, the service can save cookies for future sessions, allowing cookie-based authentication in subsequent uses. +Handles Twitter session detection and maintenance. Features a multi-stage authentication approach: + +1. **Session File Loading**: First attempts to load saved sessions from disk using the SessionManager +2. **Existing Session Detection**: Checks if the browser already has a valid Twitter session +3. **Manual Login Process**: If no existing session is found, opens the Twitter login page for user authentication + +After successful login through any method, the service automatically saves the session cookies to file for future use. The SessionManager handles the serialization and persistence of authentication data, reducing the need for repeated manual logins. #### 5.2.2 Timeline Service (Timeline Service) @@ -108,10 +114,21 @@ Extracts structured data from HTML. Controls request frequency to avoid triggering Twitter limits. +#### 5.3.3 Session Manager + +Manages authentication session data, providing methods to: + +- Save session cookies to local files +- Load previous sessions during startup +- Delete invalid or expired sessions +- Validate session age and integrity + ## 6. Data Flow 1. **Request Flow**: Application Layer → Adapter → Core Service → Browser Adapter Layer → BrowserBase API → Twitter 2. **Response Flow**: Twitter → BrowserBase API → Browser Adapter Layer → Core Service → Data Parsing → Adapter → Application Layer +3. **Authentication Flow**: + - Load Session → Check Existing Session → Manual Login → Session Validation → Session Storage ## 7. Configuration System @@ -158,6 +175,8 @@ interface Config { } ``` +The system no longer relies on the `TWITTER_COOKIES` environment variable, as cookies are now managed through the session management system. + ## 8. Development and Testing ### 8.1 Development Environment Setup @@ -168,7 +187,7 @@ npm install # Set environment variables cp .env.example .env -# Edit .env to add BrowserBase API key and Twitter credentials +# Edit .env to add BrowserBase API key and Twitter credentials (optional) # Development mode startup npm run dev # Standard mode @@ -196,15 +215,17 @@ async function main() { // Create Twitter service const twitter = new TwitterService(browser) - // Initiate manual login and wait for user to complete authentication + // Initiate login process - will try: + // 1. Load existing session file + // 2. Check for existing browser session + // 3. Finally fall back to manual login if needed const loggedIn = await twitter.login({}) if (loggedIn) { console.log('Login successful') - // Export cookies for future use (optional) - const cookies = await twitter.exportCookies() - console.log('Cookies saved for future use:', Object.keys(cookies).length) + // Session cookies are automatically saved to file after successful login + // No need to manually export cookies // Get timeline const tweets = await twitter.getTimeline({ count: 10 }) @@ -230,7 +251,7 @@ async function startAiriModule() { const twitter = new TwitterService(browser) - // Create Airi adapter (no credentials needed for manual login) + // Create Airi adapter const airiAdapter = new AiriAdapter(twitter, { url: process.env.AIRI_URL, token: process.env.AIRI_TOKEN @@ -292,8 +313,8 @@ For example, adding "Get Tweets from a Specific User" functionality: - **Automated Testing**: Write unit tests and integration tests - **Monitoring & Alerts**: Monitor service status and Twitter access limitations - **Selector Updates**: Regularly validate and update selector configurations -- **Session Management**: Optimize session management to improve stability -- **Cookie Management**: Implement secure storage for saved session cookies +- **Session Management**: Use the built-in session management system to improve stability and reduce manual login requirements. Consider implementing session rotation and validation. +- **Cookie Management**: The system now automatically manages cookie storage via the SessionManager, but consider adding encrypted storage for production environments. ## 12. Project Roadmap diff --git a/services/twitter-services/src/adapters/browser-adapter.ts b/services/twitter-services/src/adapters/browser-adapter.ts index 330b5287..0c2f2600 100644 --- a/services/twitter-services/src/adapters/browser-adapter.ts +++ b/services/twitter-services/src/adapters/browser-adapter.ts @@ -61,6 +61,21 @@ export interface BrowserAdapter { sameSite?: 'Strict' | 'Lax' | 'None' }>> + /** + * Set cookies in the browser context + * This can set HTTP_ONLY cookies that can't be set via document.cookie + */ + setCookies: (cookies: Array<{ + name: string + value: string + domain?: string + path?: string + expires?: number + httpOnly?: boolean + secure?: boolean + sameSite?: 'Strict' | 'Lax' | 'None' + }>) => Promise + /** * Get screenshot */ diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts index e0c60ffd..d19693c0 100644 --- a/services/twitter-services/src/adapters/browserbase-adapter.ts +++ b/services/twitter-services/src/adapters/browserbase-adapter.ts @@ -121,6 +121,32 @@ export class StagehandBrowserAdapter implements BrowserAdapter { return this.client.getScreenshot() } + async getAllCookies(): Promise> { + return this.client.getAllCookies() + } + + async setCookies(cookies: Array<{ + name: string + value: string + domain?: string + path?: string + expires?: number + httpOnly?: boolean + secure?: boolean + sameSite?: 'Strict' | 'Lax' | 'None' + }>): Promise { + return this.client.setCookies(cookies) + } + async close(): Promise { await this.client.closeSession() } diff --git a/services/twitter-services/src/browser/browserbase.ts b/services/twitter-services/src/browser/browserbase.ts index adba49f7..4d5ca275 100644 --- a/services/twitter-services/src/browser/browserbase.ts +++ b/services/twitter-services/src/browser/browserbase.ts @@ -240,6 +240,57 @@ export class StagehandClient { return await this.page!.screenshot() as Buffer } + /** + * Get all cookies from browser context + * This includes HTTP_ONLY cookies that can't be accessed via document.cookie + */ + async getAllCookies(): Promise> { + this.ensurePageExists() + const context = this.page!.context() + return await context.cookies() + } + + /** + * Set cookies in browser context + * This can set HTTP_ONLY cookies that can't be set via document.cookie + */ + async setCookies(cookies: Array<{ + name: string + value: string + domain?: string + path?: string + expires?: number + httpOnly?: boolean + secure?: boolean + sameSite?: 'Strict' | 'Lax' | 'None' + }>): Promise { + this.ensurePageExists() + const context = this.page!.context() + // Format cookies correctly for Playwright + const formattedCookies = cookies.map((cookie) => { + // Ensure domain is set for Twitter + if (!cookie.domain) { + cookie.domain = '.twitter.com' + } + // Ensure path is set + if (!cookie.path) { + cookie.path = '/' + } + return cookie + }) + + await context.addCookies(formattedCookies) + } + /** * Close session */ diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index d5d81869..69f9a720 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -47,28 +47,8 @@ export interface Config { * Default configuration */ export function getDefaultConfig(): Config { - // Parse cookies from environment variable if provided - let cookiesFromEnv: Record | undefined - - if (process.env.TWITTER_COOKIES) { - // Try to parse as JSON first - try { - cookiesFromEnv = JSON.parse(process.env.TWITTER_COOKIES) - } - catch { - // If JSON parsing fails, treat as document.cookie format string - cookiesFromEnv = process.env.TWITTER_COOKIES - .split(';') - .map(v => v.split('=')) - .reduce((acc, v) => { - // Skip empty values - if (v.length < 2) - return acc - acc[decodeURIComponent(v[0].trim())] = decodeURIComponent(v[1].trim()) - return acc - }, {} as Record) - } - } + // No longer parse cookies from environment variable + // The auth service will load cookies from session file instead return { browser: { @@ -87,7 +67,7 @@ export function getDefaultConfig(): Config { credentials: { username: process.env.TWITTER_USERNAME || '', password: process.env.TWITTER_PASSWORD || '', - cookies: cookiesFromEnv, + // Don't include cookies here, they will be loaded from session file }, defaultOptions: { timeline: { diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index f726676f..d21a8a3f 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -1,8 +1,10 @@ import type { BrowserAdapter } from '../adapters/browser-adapter' import type { TwitterCredentials } from '../types/twitter' +import type { SessionData } from '../utils/session-manager' import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' +import { getSessionManager } from '../utils/session-manager' /** * Twitter Authentication Service @@ -21,27 +23,61 @@ export class TwitterAuthService { * Prefers cookie-based login if cookies provided, otherwise redirects to manual login */ async login(credentials: TwitterCredentials = {}): Promise { - // If cookies are provided, try to use them for login - if (credentials.cookies && Object.keys(credentials.cookies).length > 0) { - logger.auth.log('Cookies provided, attempting cookie-based login') - const cookieLoginSuccess = await this.loginWithCookies(credentials.cookies) - if (cookieLoginSuccess) { + logger.auth.log('Starting Twitter login process') + + try { + // First try to load session from file if no cookies provided + if (!credentials.cookies || Object.keys(credentials.cookies).length === 0) { + logger.auth.log('No cookies provided, attempting to load session from file') + + // Get the session manager and load the session + const sessionManager = getSessionManager() + const sessionData = await sessionManager.loadSession() + + if (sessionData && sessionData.cookies && Object.keys(sessionData.cookies).length > 0) { + logger.auth.log(`Found session file with ${Object.keys(sessionData.cookies).length} cookies, attempting login`) + + // Use the dedicated method for session-based login + const sessionLoginSuccess = await this.loginWithSessionData(sessionData) + if (sessionLoginSuccess) { + logger.auth.log('✅ Successfully logged in using saved session') + return true + } + logger.auth.log('Session login failed, continuing with other login methods') + } + else { + logger.auth.log('No valid session file found') + } + } + + // If cookies are provided directly, try to use them for login + if (credentials.cookies && Object.keys(credentials.cookies).length > 0) { + logger.auth.log('Cookies provided, attempting cookie-based login') + const cookieLoginSuccess = await this.loginWithCookies(credentials.cookies) + if (cookieLoginSuccess) { + return true + } + // If cookie login fails, log the issue but continue to manual login + logger.auth.log('Cookie login failed, falling back to manual login') + } + + // Check for existing session first + logger.auth.log('Checking for existing session before initiating manual login') + const existingSession = await this.checkExistingSession() + if (existingSession) { + logger.auth.log('✅ Successfully logged in using existing browser session') return true } - // If cookie login fails, log the issue but continue to manual login - logger.auth.log('Cookie login failed, falling back to manual login') - } - // Check for existing session first - logger.auth.log('Checking for existing session before initiating manual login') - const existingSession = await this.checkExistingSession() - if (existingSession) { - return true + // Fallback to manual login flow + logger.auth.log('No existing session found, initiating manual login process') + return this.initiateManualLogin() + } + catch (error: unknown) { + logger.auth.withError(error as Error).error('Login process failed') + this.isLoggedIn = false + return false } - - // Fallback to manual login flow - logger.auth.log('No existing session found, initiating manual login process') - return this.initiateManualLogin() } /** @@ -82,7 +118,11 @@ export class TwitterAuthService { // If we got here, we couldn't definitively confirm login status // Check current URL for additional clues - const currentUrl = await this.browser.executeScript('return window.location.href') + const currentUrl = await this.browser.executeScript(` + (() => { + return window.location.href; + })() + `) if (currentUrl.includes('/home')) { // On home page but couldn't find timeline - might still be loading return true @@ -92,7 +132,7 @@ export class TwitterAuthService { return false } catch (error) { - logger.auth.errorWithError('Error during login verification', error) + logger.auth.withError(error as Error).error('Error during login verification') return false } } @@ -124,35 +164,32 @@ export class TwitterAuthService { */ async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { try { + // Use the new getAllCookies method to get all cookies, including HTTP_ONLY ones + const allCookies = await this.browser.getAllCookies() + logger.auth.log(`Retrieved ${allCookies.length} cookies from browser context`) + if (format === 'string') { - // Return raw document.cookie string - const cookieString = await this.browser.executeScript(` - // Wrap in a function to allow return statement - (() => { - return document.cookie; - })(); - `) + // Convert cookie objects to string format + const cookieString = allCookies + .map(cookie => `${cookie.name}=${cookie.value}`) + .join('; ') + + logger.auth.log(`Exported ${allCookies.length} cookies as string`) return cookieString } else { - // Return cookies as object - const cookies = await this.browser.executeScript>(` - // Wrap in a function to allow return statement - (() => { - return document.cookie.split(';') - .map(cookie => cookie.trim().split('=')) - .reduce((acc, v) => { - if (v.length < 2) return acc; - acc[v[0]] = v[1]; - return acc; - }, {}); - })(); - `) - return cookies + // Convert to object format + const cookiesObj = allCookies.reduce>((acc, cookie) => { + acc[cookie.name] = cookie.value + return acc + }, {}) + + logger.auth.log(`Exported ${Object.keys(cookiesObj).length} cookies as object`) + return cookiesObj } } - catch (error) { - logger.auth.errorWithError('Error exporting cookies', error) + catch (error: unknown) { + logger.auth.withError(error as Error).error('Error exporting cookies') return format === 'string' ? '' : {} } } @@ -167,15 +204,18 @@ export class TwitterAuthService { // Navigate to a Twitter page await this.browser.navigate('https://twitter.com') - // Set cookies - await this.browser.executeScript(` - const cookies = ${JSON.stringify(cookies)}; - Object.entries(cookies).forEach(([name, value]) => { - document.cookie = \`\${name}=\${value};domain=.twitter.com;path=/\`; - }); - `) + // Convert cookies object to array format required by setCookies + const cookieArray = Object.entries(cookies).map(([name, value]) => ({ + name, + value, + domain: '.twitter.com', + path: '/', + })) + + // Set cookies using the browser adapter's API that can set HTTP_ONLY cookies + await this.browser.setCookies(cookieArray) - logger.auth.log(`Set ${Object.keys(cookies).length} cookies`) + logger.auth.log(`Set ${cookieArray.length} cookies via browser API`) // Refresh page to apply cookies await this.browser.navigate('https://twitter.com/home') @@ -203,8 +243,8 @@ export class TwitterAuthService { await new Promise(resolve => setTimeout(resolve, 3000)) } } - catch (error) { - logger.auth.withError(error).debug(`Verification attempt ${attempt} failed`) + catch (error: unknown) { + logger.auth.withError(error as Error).debug(`Verification attempt ${attempt} failed`) } } @@ -214,11 +254,11 @@ export class TwitterAuthService { // Try to refresh cookies to ensure they're up to date try { - const freshCookies = await this.exportCookies('object') - logger.auth.log(`Updated and exported ${typeof freshCookies === 'string' ? freshCookies.length : Object.keys(freshCookies).length} cookies`) + await this.saveCurrentSession() + logger.auth.log('✅ Session saved to file') } - catch (error) { - logger.auth.withError(error).debug('Failed to update cookies, but login was successful') + catch (error: unknown) { + logger.auth.withError(error as Error).debug('Failed to save session, but login was successful') } } else { @@ -227,8 +267,8 @@ export class TwitterAuthService { return loginSuccess } - catch (error) { - logger.auth.errorWithError('Error during cookie login process', error) + catch (error: unknown) { + logger.auth.withError(error as Error).error('Error during cookie login process') this.isLoggedIn = false return false } @@ -256,9 +296,13 @@ export class TwitterAuthService { try { const cookies = await this.exportCookies('object') logger.auth.log(`✅ Exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies from existing session`) + + // Save the current session to file + await this.saveCurrentSession() + logger.auth.log('✅ Session saved to file') } catch (error) { - logger.auth.errorWithError('Error exporting cookies from existing session', error) + logger.auth.withError(error as Error).error('Error exporting cookies from existing session') } } else { @@ -268,7 +312,7 @@ export class TwitterAuthService { return loginSuccess } catch (error) { - logger.auth.errorWithError('Error checking session status', error) + logger.auth.withError(error as Error).error('Error checking session status') this.isLoggedIn = false return false } @@ -283,7 +327,11 @@ export class TwitterAuthService { try { // Store the current URL to detect navigation - const initialUrl = await this.browser.executeScript('return window.location.href') + const initialUrl = await this.browser.executeScript(` + (() => { + return window.location.href; + })() + `) // Navigate to login page await this.browser.navigate('https://twitter.com/i/flow/login') @@ -305,7 +353,11 @@ export class TwitterAuthService { try { // Get current URL to detect page changes - const currentUrl = await this.browser.executeScript('return window.location.href') + const currentUrl = await this.browser.executeScript(` + (() => { + return window.location.href; + })() + `) // Check if URL has changed significantly - may indicate user interaction if (currentUrl !== lastUrl && !currentUrl.includes('/flow/login')) { @@ -324,9 +376,13 @@ export class TwitterAuthService { try { const cookies = await this.exportCookies('object') logger.auth.log(`✅ Successfully exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies`) + + // Save the current session to file + await this.saveCurrentSession() + logger.auth.log('✅ Session saved to file') } catch (error) { - logger.auth.errorWithError('Error exporting cookies', error) + logger.auth.withError(error as Error).error('Error exporting cookies') } this.isLoggedIn = true @@ -346,9 +402,13 @@ export class TwitterAuthService { try { const cookies = await this.exportCookies('object') logger.auth.log(`✅ Successfully exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies`) + + // Save the current session to file + await this.saveCurrentSession() + logger.auth.log('✅ Session saved to file') } catch (error) { - logger.auth.errorWithError('Error exporting cookies', error) + logger.auth.withError(error as Error).error('Error exporting cookies') } this.isLoggedIn = true @@ -357,7 +417,7 @@ export class TwitterAuthService { } catch (error) { // Ignore errors during verification, continue polling - logger.auth.debug(`Error during verification: ${error.message}`) + logger.auth.debug(`Error during verification: ${(error as Error).message}`) } // Wait 10 seconds before checking again @@ -373,7 +433,120 @@ export class TwitterAuthService { return false } catch (error) { - logger.auth.errorWithError('Error during manual login process', error) + logger.auth.withError(error as Error).error('Error during manual login process') + return false + } + } + + /** + * Save the current session to a file for future use. + * This includes cookies and authentication details. + */ + async saveCurrentSession(): Promise { + try { + // Export cookies in object format + const cookies = await this.exportCookies('object') + + if (!cookies || (typeof cookies === 'object' && Object.keys(cookies).length === 0)) { + logger.auth.warn('No cookies available to save') + return + } + + // Create a session data object + const sessionData: SessionData = { + cookies: typeof cookies === 'string' ? {} : cookies, + timestamp: new Date().toISOString(), + userAgent: await this.browser.executeScript(` + (() => { + return navigator.userAgent; + })() + `), + } + + // Get the session manager and save the session + const sessionManager = getSessionManager() + await sessionManager.saveSession(sessionData) + + logger.auth.log(`Session saved with ${typeof cookies === 'object' ? Object.keys(cookies).length : 0} cookies`) + } + catch (error: unknown) { + logger.auth.withError(error as Error).error('Failed to save session') + } + } + + /** + * Login with a saved session data object + * @param sessionData The session data with cookies to use for login + */ + private async loginWithSessionData(sessionData: SessionData): Promise { + logger.auth.log(`Attempting to login using session with ${Object.keys(sessionData.cookies).length} cookies`) + + try { + // Navigate to a Twitter page + await this.browser.navigate('https://twitter.com') + + // Convert cookies object to array format required by setCookies + const cookieArray = Object.entries(sessionData.cookies).map(([name, value]) => ({ + name, + value, + domain: '.twitter.com', + path: '/', + })) + + // Set cookies using the browser adapter's API + await this.browser.setCookies(cookieArray) + logger.auth.log(`Set ${cookieArray.length} cookies from session file`) + + // Continue with similar verification logic as loginWithCookies + await this.browser.navigate('https://twitter.com/home') + + // Verify login status - try multiple times + logger.auth.log('Cookies set from session file, verifying login status...') + + let loginSuccess = false + const verificationAttempts = 3 + + for (let attempt = 1; attempt <= verificationAttempts; attempt++) { + try { + logger.auth.log(`Verification attempt ${attempt}/${verificationAttempts}`) + loginSuccess = await this.verifyLogin() + + if (loginSuccess) { + break + } + else if (attempt < verificationAttempts) { + logger.auth.log('Refreshing page and trying again...') + await this.browser.navigate('https://twitter.com/home') + await new Promise(resolve => setTimeout(resolve, 3000)) + } + } + catch (error: unknown) { + logger.auth.withError(error as Error).debug(`Verification attempt ${attempt} failed`) + } + } + + if (loginSuccess) { + logger.auth.log('Login with session data successful') + this.isLoggedIn = true + + // Try to refresh cookies to ensure they're up to date + try { + await this.saveCurrentSession() + logger.auth.log('✅ Session refreshed and saved to file') + } + catch (error: unknown) { + logger.auth.withError(error as Error).debug('Failed to update cookies, but login was successful') + } + } + else { + logger.auth.warn('Login with session data verification failed, cookies may be expired') + } + + return loginSuccess + } + catch (error: unknown) { + logger.auth.withError(error as Error).error('Error during session data login process') + this.isLoggedIn = false return false } } diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index f758e142..b0f048a5 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -13,7 +13,6 @@ async function bootstrap() { try { await launcher.start() - logger.main.log('Twitter service successfully started') } catch (error) { logger.main.withError(error).error('Startup failed') diff --git a/services/twitter-services/src/utils/session-manager.ts b/services/twitter-services/src/utils/session-manager.ts new file mode 100644 index 00000000..2bf549e9 --- /dev/null +++ b/services/twitter-services/src/utils/session-manager.ts @@ -0,0 +1,129 @@ +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +import { logger } from './logger' + +/** + * Session cookie data structure + */ +export interface SessionData { + cookies: Record + timestamp: string + userAgent?: string + username?: string +} + +/** + * Session manager utility + * Responsible for loading and saving session data to/from files + */ +export class SessionManager { + private sessionFilePath: string + + /** + * Create a new session manager instance + * @param sessionFilePath Optional custom path for the session file + */ + constructor(sessionFilePath?: string) { + this.sessionFilePath = sessionFilePath + || path.join(process.cwd(), '.twitter.session.json') + } + + /** + * Save session data to file + * @param data The session data to save + */ + async saveSession(data: SessionData): Promise { + try { + // Create session directory if it doesn't exist + const dir = path.dirname(this.sessionFilePath) + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + + // Write session data to file + fs.writeFileSync( + this.sessionFilePath, + JSON.stringify(data, null, 2), + 'utf8', + ) + + logger.auth.log(`Session saved to ${this.sessionFilePath}`) + return true + } + catch (error: unknown) { + logger.auth.withError(error as Error).error(`Failed to save session to ${this.sessionFilePath}`) + return false + } + } + + /** + * Load session data from file + * @returns The loaded session data or null if file doesn't exist or is invalid + */ + async loadSession(): Promise { + try { + // Check if session file exists + if (!fs.existsSync(this.sessionFilePath)) { + logger.auth.debug(`No session file found at ${this.sessionFilePath}`) + return null + } + + // Read and parse session data + const data = JSON.parse(fs.readFileSync(this.sessionFilePath, 'utf8')) + + // Validate session data + if (!data.cookies || typeof data.cookies !== 'object' || !data.timestamp) { + logger.auth.warn(`Invalid session data in ${this.sessionFilePath}`) + return null + } + + // Check if session is too old (30 days) + const sessionDate = new Date(data.timestamp) + const ageInDays = (Date.now() - sessionDate.getTime()) / (1000 * 60 * 60 * 24) + if (ageInDays > 30) { + logger.auth.warn(`Session is ${Math.floor(ageInDays)} days old and may be expired`) + } + + logger.auth.log(`Loaded session from ${this.sessionFilePath} with ${Object.keys(data.cookies).length} cookies`) + return data + } + catch (error: unknown) { + logger.auth.withError(error as Error).error(`Failed to load session from ${this.sessionFilePath}`) + return null + } + } + + /** + * Delete the session file + * @returns true if deletion was successful, false otherwise + */ + deleteSession(): boolean { + try { + if (fs.existsSync(this.sessionFilePath)) { + fs.unlinkSync(this.sessionFilePath) + logger.auth.log(`Session file deleted: ${this.sessionFilePath}`) + return true + } + return false + } + catch (error: unknown) { + logger.auth.withError(error as Error).error(`Failed to delete session file: ${this.sessionFilePath}`) + return false + } + } +} + +// Create singleton instance +let sessionManagerInstance: SessionManager | null = null + +/** + * Get the session manager instance + */ +export function getSessionManager(sessionFilePath?: string): SessionManager { + if (!sessionManagerInstance) { + sessionManagerInstance = new SessionManager(sessionFilePath) + } + return sessionManagerInstance +} From edbc1c5e72289c6f1541ec362b08fc94cb84caa2 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 13:32:01 +0800 Subject: [PATCH 11/20] docs: architecture --- .../twitter-services/docs/architecture.md | 117 ++++++++++++------ 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index 9c53400a..f0235b3d 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -20,13 +20,13 @@ Twitter Service is a web automation service based on BrowserBase, providing stru │ ┌────────────┐ ┌─────────────┐ │ │ │ │ │ │ │ │ │ Airi Core │ │ Other LLM │ │ -│ │ │ │ Applications │ │ +│ │ │ │ Applications│ │ │ │ │ │ │ │ │ └──────┬─────┘ └──────┬──────┘ │ └──────────┼─────────────────────┼────────────┘ │ │ ┌──────────▼─────────────────────▼────────────┐ -│ 适配器层 │ +│ Adapter Layer │ │ │ │ ┌────────────┐ ┌─────────────┐ │ │ │Airi Adapter│ │ MCP Adapter │ │ @@ -35,37 +35,42 @@ Twitter Service is a web automation service based on BrowserBase, providing stru └──────────┼─────────────────────┼────────────┘ │ │ ┌──────────▼─────────────────────▼────────────┐ -│ 核心服务层 │ +│ Core Services Layer │ │ │ │ ┌──────────────────────────────────┐ │ -│ │ Twitter Services │ │ -│ │ │ │ -│ │ ┌────────┐ ┌────────────┐ │ │ -│ │ │ Auth │ │ Timeline │ │ │ -│ │ │ Service│ │ Service │ │ │ -│ │ └────────┘ └────────────┘ │ │ -│ │ │ │ -│ └──────────────────┬────────────────┘ │ -└──────────────────────┼────────────────────────┘ +│ │ Twitter Services │ │ +│ │ │ │ +│ │ ┌────────┐ ┌────────────┐ │ │ +│ │ │ Auth │ │ Timeline │ │ │ +│ │ │ Service│ │ Service │ │ │ +│ │ └────────┘ └────────────┘ │ │ +│ │ │ │ +│ └──────────────────┬───────────────┘ │ +└──────────────────────┼──────────────────────┘ │ ┌───────────▼────────────┐ - │ 浏览器适配层 │ + │ Browser Adapter Layer │ │ (BrowserAdapter) │ └───────────┬────────────┘ │ ┌───────────▼────────────┐ - │ BrowserBase API │ - └──────────────────────────┘ + │ Stagehand │ + └───────────┬────────────┘ + │ + ┌───────────▼────────────┐ + │ Playwright │ + └────────────────────────┘ ``` ## 4. Technology Stack and Dependencies - **Core Library**: TypeScript, Node.js -- **Browser Automation**: BrowserBase API +- **Browser Automation**: BrowserBase Stagehand, Playwright - **HTML Parsing**: unified, rehype-parse, unist-util-visit - **API Server**: H3.js, listhen - **Adapters**: Airi Server SDK, MCP SDK - **Logging System**: @guiiai/logg +- **Configuration**: defu (deep merging configurations) - **Utility Library**: zod (type validation) ## 5. Key Components @@ -88,13 +93,21 @@ Using listhen for optimized development experience, including automatic browser #### 5.2.1 Authentication Service (Auth Service) -Handles Twitter session detection and maintenance. Features a multi-stage authentication approach: +The Authentication Service has been significantly enhanced to improve reliability and error handling: + +1. **Improved Session Detection**: Enhanced logic for detecting existing browser sessions +2. **Robust Error Handling**: Implemented granular error handling to distinguish between different authentication failure types +3. **Timeout Optimization**: Adjusted timeouts for various operations to enhance stability during network fluctuations +4. **Enhanced Cookie Management**: Improved cookie storage and loading mechanisms to reduce the need for manual login +5. **Session Validation**: Added comprehensive session validation to verify the integrity of saved sessions -1. **Session File Loading**: First attempts to load saved sessions from disk using the SessionManager +The service follows a multi-stage authentication approach: + +1. **Session File Loading**: First attempts to load saved sessions from disk 2. **Existing Session Detection**: Checks if the browser already has a valid Twitter session -3. **Manual Login Process**: If no existing session is found, opens the Twitter login page for user authentication +3. **Manual Login Process**: If necessary, guides through the Twitter login page -After successful login through any method, the service automatically saves the session cookies to file for future use. The SessionManager handles the serialization and persistence of authentication data, reducing the need for repeated manual logins. +After successful authentication through any method, sessions are automatically persisted for future use. #### 5.2.2 Timeline Service (Timeline Service) @@ -123,6 +136,16 @@ Manages authentication session data, providing methods to: - Delete invalid or expired sessions - Validate session age and integrity +### 5.3.4 Browser Adapter Layer + +The service has migrated from direct BrowserBase API usage to Stagehand, an AI-powered web browsing framework built on top of Playwright. Stagehand offers three core APIs that simplify browser automation: + +- **act**: Execute actions on the page through natural language instructions +- **extract**: Retrieve structured data from the page using natural language queries +- **observe**: Analyze the page and suggest possible actions before execution + +Stagehand processes the DOM in chunks to optimize LLM performance and provides fallback vision capabilities for complex page structures. This migration significantly improves code maintainability and automation reliability when interacting with Twitter's interface. + ## 6. Data Flow 1. **Request Flow**: Application Layer → Adapter → Core Service → Browser Adapter Layer → BrowserBase API → Twitter @@ -132,14 +155,21 @@ Manages authentication session data, providing methods to: ## 7. Configuration System -Configuration is divided into several main parts: +The configuration system has been optimized using the `defu` library for deep merging configurations, eliminating redundant initialization. The updated configuration structure includes Stagehand-specific settings: ```typescript interface Config { - // BrowserBase configuration + // BrowserBase/Stagehand configuration browserbase: { apiKey: string + projectId?: string endpoint?: string + stagehand?: { + modelName?: string // e.g., "gpt-4o" or "claude-3-5-sonnet-latest" + modelClientOptions?: { + apiKey: string // OpenAI or Anthropic API key + } + } } // Browser configuration @@ -202,32 +232,34 @@ npm run dev:mcp # MCP development server mode ## 9. Integration Example -### 9.1 Integrating from Other Node.js Applications +### 9.1 Integration Example with Stagehand ```typescript -import { BrowserBaseMCPAdapter, TwitterService } from 'twitter-services' +import { StagehandAdapter, TwitterService } from 'twitter-services' async function main() { - // Initialize browser - const browser = new BrowserBaseMCPAdapter('your-api-key') - await browser.initialize({ headless: true }) + // Initialize Stagehand adapter + const browser = new StagehandAdapter(process.env.BROWSERBASE_API_KEY, process.env.BROWSERBASE_PROJECT_ID) + await browser.initialize({ + headless: true, + stagehand: { + modelName: 'gpt-4o', // Or 'claude-3-5-sonnet-latest' for Anthropic + modelClientOptions: { + apiKey: process.env.OPENAI_API_KEY // Or process.env.ANTHROPIC_API_KEY + } + } + }) // Create Twitter service const twitter = new TwitterService(browser) - // Initiate login process - will try: - // 1. Load existing session file - // 2. Check for existing browser session - // 3. Finally fall back to manual login if needed + // Authenticate - will try multi-stage approach const loggedIn = await twitter.login({}) if (loggedIn) { console.log('Login successful') - // Session cookies are automatically saved to file after successful login - // No need to manually export cookies - - // Get timeline + // Get timeline using natural language capabilities of Stagehand const tweets = await twitter.getTimeline({ count: 10 }) console.log(tweets) } @@ -316,9 +348,16 @@ For example, adding "Get Tweets from a Specific User" functionality: - **Session Management**: Use the built-in session management system to improve stability and reduce manual login requirements. Consider implementing session rotation and validation. - **Cookie Management**: The system now automatically manages cookie storage via the SessionManager, but consider adding encrypted storage for production environments. +### 11.4 Stagehand Maintenance + +- **Model Selection**: Regularly evaluate the performance of different LLM models (GPT-4o, Claude 3.5 Sonnet) for your specific use cases +- **Prompt Engineering**: Refine natural language instructions to improve reliability and performance +- **Vision Capabilities**: Consider enabling vision capabilities for complex DOM structures by setting `useVision: true` in appropriate operations +- **DOM Chunking**: Monitor and optimize chunk sizes based on the complexity of the Twitter interface + ## 12. Project Roadmap -- MVP Stage: Implement core functionality (authentication, browsing timeline) -- Stage Two: Enhance interaction features (likes, comments, retweets) -- Stage Three: Advanced features (search, advanced filtering, data analysis) -- Stage Four: Performance optimization and stability improvements +- MVP Stage: Core functionality with Stagehand integration (authentication, browsing timeline) +- Stage Two: Enhanced interaction features utilizing Stagehand's natural language capabilities +- Stage Three: Advanced search and filtering features with optimized LLM prompts +- Stage Four: Performance optimization and multi-model support From 17743fa62783a895ef216bc4af3d478499710b80 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 15:20:47 +0800 Subject: [PATCH 12/20] chore: remove cli & use mcp adapter for mcp-server --- .../twitter-services/docs/architecture.md | 4 +- services/twitter-services/package.json | 4 +- services/twitter-services/src/cli.ts | 225 ------------------ services/twitter-services/src/dev-server.ts | 202 ---------------- services/twitter-services/src/mcp-server.ts | 60 +++++ 5 files changed, 64 insertions(+), 431 deletions(-) delete mode 100644 services/twitter-services/src/cli.ts delete mode 100644 services/twitter-services/src/dev-server.ts create mode 100644 services/twitter-services/src/mcp-server.ts diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index f0235b3d..cfe47376 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -300,7 +300,7 @@ async function startAiriModule() { ```typescript // Use MCP SDK to interact with Twitter service -import { McpClient } from '@modelcontextprotocol/sdk/client/mcp.js' +import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js' async function connectToTwitterService() { @@ -308,7 +308,7 @@ async function connectToTwitterService() { const transport = new SSEClientTransport('http://localhost:8080/sse', 'http://localhost:8080/messages') // Create client - const client = new McpClient() + const client = new Client() await client.connect(transport) // Get timeline diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index a2b80576..874a2000 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -6,9 +6,9 @@ "author": "RainbowBird ", "license": "MIT", "scripts": { - "start": "tsx src/main.ts", "dev": "tsx src/main.ts", - "dev:mcp": "tsx src/dev-server.ts", + "dev:mcp": "tsx src/mcp-server.ts", + "mcp:ui": "pnpx @modelcontextprotocol/inspector", "postinstall": "playwright install chromium" }, "dependencies": { diff --git a/services/twitter-services/src/cli.ts b/services/twitter-services/src/cli.ts deleted file mode 100644 index adf3a2aa..00000000 --- a/services/twitter-services/src/cli.ts +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env node - -import fs from 'node:fs' -import path from 'node:path' -import process from 'node:process' -import { Command } from 'commander' - -import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' -import { createDefaultConfig } from './config' -import { TwitterService } from './core/twitter-service' -import { TwitterServiceLauncher } from './launcher' -import { errorToMessage } from './utils/error' - -// Get version -const packageJsonPath = path.join(__dirname, '..', 'package.json') -const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) - -// Create program -const program = new Command() - -// Set basic info -program - .name('twitter-services') - .description('Twitter Services CLI - Access and manage Twitter data') - .version(packageJson.version) - -// Start service command -program - .command('start') - .description('Start Twitter service') - .option('-c, --config ', 'Path to config file') - .action(async (options) => { - if (options.config) { - process.env.CONFIG_PATH = options.config - } - - const launcher = new TwitterServiceLauncher() - await launcher.start() - - console.log('Service started, press Ctrl+C to stop') - }) - -// Get timeline command -program - .command('timeline') - .description('Get Twitter timeline') - .option('-c, --count ', 'Number of tweets to fetch', '10') - .option('--no-replies', 'Exclude replies') - .option('--no-retweets', 'Exclude retweets') - .option('-o, --output ', 'Output results to file') - .action(async (options) => { - try { - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - // Initialize browser - const browser = new StagehandBrowserAdapter(config.browser.apiKey) - await browser.initialize(config.browser) - - // Create service and login - const twitterService = new TwitterService(browser) - - if (!config.twitter.credentials) { - throw new Error('Cannot get Twitter credentials, please check configuration') - } - - const loggedIn = await twitterService.login(config.twitter.credentials) - - if (!loggedIn) { - throw new Error('Login failed, please check credentials') - } - - console.log('Fetching timeline...') - - // Get timeline - const tweets = await twitterService.getTimeline({ - count: Number.parseInt(options.count), - includeReplies: options.replies, - includeRetweets: options.retweets, - }) - - // Process results - const result = tweets.map(tweet => ({ - id: tweet.id, - text: tweet.text, - author: tweet.author.displayName, - username: tweet.author.username, - timestamp: tweet.timestamp, - likeCount: tweet.likeCount, - retweetCount: tweet.retweetCount, - replyCount: tweet.replyCount, - })) - - // Output results - if (options.output) { - fs.writeFileSync(options.output, JSON.stringify(result, null, 2)) - console.log(`Results saved to ${options.output}`) - } - else { - console.log(JSON.stringify(result, null, 2)) - } - - // Close browser - await browser.close() - } - catch (error) { - console.error('Failed to get timeline:', errorToMessage(error)) - process.exit(1) - } - }) - -// Export cookies command -program - .command('export-cookies') - .description('Login and export cookies for later use') - .option('-o, --output ', 'Output cookies to file', 'twitter-cookies.json') - .option('-f, --format ', 'Cookie format (json or string)', 'json') - .action(async (options) => { - try { - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - // Initialize browser - const browser = new StagehandBrowserAdapter(config.browser.apiKey) - await browser.initialize(config.browser) - - // Create service and login - const twitterService = new TwitterService(browser) - - if (!config.twitter.credentials) { - throw new Error('Cannot get Twitter credentials, please check configuration') - } - - const loggedIn = await twitterService.login(config.twitter.credentials) - - if (!loggedIn) { - throw new Error('Login failed, please check credentials') - } - - console.log('Login successful, exporting cookies...') - - // Export cookies - specify the format based on user option - const cookieFormat = options.format === 'string' ? 'string' : 'object' - const cookies = await twitterService.exportCookies(cookieFormat) - - if (cookieFormat === 'string') { - // Save raw string to file - fs.writeFileSync(options.output, cookies as string) - } - else { - // Save JSON to file - fs.writeFileSync(options.output, JSON.stringify(cookies, null, 2)) - } - - console.log(`Cookies saved to ${options.output} in ${cookieFormat} format`) - - // Close browser - await browser.close() - } - catch (error) { - console.error('Failed to export cookies:', errorToMessage(error)) - process.exit(1) - } - }) - -// Login with cookies file command -program - .command('login-with-cookies') - .description('Login using cookies from a file') - .option('-i, --input ', 'Input cookies file', 'twitter-cookies.json') - .option('-t, --test', 'Test login only, do not perform other actions', false) - .action(async (options) => { - try { - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - // Load cookies from file - if (!fs.existsSync(options.input)) { - throw new Error(`Cookies file not found: ${options.input}`) - } - - console.log(`Loading cookies from ${options.input}`) - const cookies = JSON.parse(fs.readFileSync(options.input, 'utf8')) - - // Initialize browser - const browser = new StagehandBrowserAdapter(config.browser.apiKey) - await browser.initialize(config.browser) - - // Create service and login with cookies - const twitterService = new TwitterService(browser) - const credentials = { - username: config.twitter.credentials?.username || '', - password: config.twitter.credentials?.password || '', - cookies, - } - - console.log('Attempting to login with cookies...') - const loggedIn = await twitterService.login(credentials) - - if (loggedIn) { - console.log('Login successful!') - - if (options.test) { - console.log('Login test passed, exiting.') - } - else { - // Perform additional actions if needed - console.log('Add more actions here when needed') - } - } - else { - throw new Error('Login failed, cookies may be expired') - } - - // Close browser - await browser.close() - } - catch (error) { - console.error('Failed to login with cookies:', errorToMessage(error)) - process.exit(1) - } - }) - -// Parse command line arguments -program.parse() diff --git a/services/twitter-services/src/dev-server.ts b/services/twitter-services/src/dev-server.ts deleted file mode 100644 index ec19ec2b..00000000 --- a/services/twitter-services/src/dev-server.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { Buffer } from 'node:buffer' -import process from 'node:process' -import { McpServer, ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js' -import { SSEServerTransport } from '@modelcontextprotocol/sdk/server/sse.js' -import * as dotenv from 'dotenv' -import { createApp, createRouter, defineEventHandler, toNodeListener } from 'h3' -import { listen } from 'listhen' -import { z } from 'zod' - -import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' -import { TwitterService } from './core/twitter-service' -import { errorToMessage } from './utils/error' -import { logger } from './utils/logger' - -// Load environment variables -dotenv.config() - -/** - * Development server entry point - * Provides convenience features for development - */ -async function startDevServer() { - const app = createApp() - const router = createRouter() - - // Create browser and Twitter service - const browser = new StagehandBrowserAdapter(process.env.BROWSERBASE_API_KEY || '') - await browser.initialize({ - headless: true, - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - }) - - const twitter = new TwitterService(browser) - - // Optional: If credentials are available, login - if (process.env.TWITTER_USERNAME && process.env.TWITTER_PASSWORD) { - const success = await twitter.login({ - username: process.env.TWITTER_USERNAME, - password: process.env.TWITTER_PASSWORD, - }) - - if (success) { - logger.main.log('✅ Successfully logged in Twitter') - } - else { - logger.main.warn('⚠️ Twitter login failed') - } - } - - // Create MCP server - const mcpServer = new McpServer({ - name: 'Twitter Service (Dev)', - version: '1.0.0-dev', - }) - - // Configure MCP resources - mcpServer.resource( - 'timeline', - new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ - resources: [{ - name: 'twitter-timeline', - uri: 'twitter://timeline', - description: 'Twitter timeline', - }], - }) }), - async (_uri: URL, { count }: { count?: string }) => { - try { - const tweets = await twitter.getTimeline({ - count: count ? Number.parseInt(count) : undefined, - }) - - return { - contents: tweets.map(tweet => ({ - uri: `twitter://tweet/${tweet.id}`, - text: `Tweet by @${tweet.author.username} (${tweet.author.displayName}):\n${tweet.text}`, - })), - } - } - catch (error) { - logger.mcp.errorWithError('Get timeline error:', error) - return { contents: [] } - } - }, - ) - - // Configure some basic tools - mcpServer.tool( - 'post-tweet', - { - content: z.string(), - }, - async ({ content }: { content: string }) => { - try { - const tweetId = await twitter.postTweet(content) - return { - content: [{ type: 'text', text: `Successfully posted tweet: ${tweetId}` }], - } - } - catch (error) { - return { - content: [{ type: 'text', text: `Failed to post tweet: ${errorToMessage(error)}` }], - isError: true, - } - } - }, - ) - - // Save active SSE transports - const activeTransports: SSEServerTransport[] = [] - - // Set up routes - router.get('/', defineEventHandler(() => { - return { - name: 'Twitter MCP Dev Server', - version: '1.0.0-dev', - status: 'running', - endpoints: { - sse: '/sse', - messages: '/messages', - }, - } - })) - - // SSE endpoint - router.get('/sse', defineEventHandler(async (event) => { - const { req, res } = event.node - - res.setHeader('Content-Type', 'text/event-stream') - res.setHeader('Cache-Control', 'no-cache') - res.setHeader('Connection', 'keep-alive') - - // Create SSE transport - const transport = new SSEServerTransport('/messages', res) - activeTransports.push(transport) - - // Clean up when client disconnects - req.on('close', () => { - const index = activeTransports.indexOf(transport) - if (index !== -1) { - activeTransports.splice(index, 1) - } - }) - - // Connect to MCP server - await mcpServer.connect(transport) - })) - - // Messages endpoint - router.post('/messages', defineEventHandler(async (event) => { - if (activeTransports.length === 0) { - event.node.res.statusCode = 503 - return { error: 'No active SSE connections' } - } - - try { - // Parse request body - const buffers = [] - for await (const chunk of event.node.req) { - buffers.push(chunk) - } - const data = Buffer.concat(buffers).toString() - const body = JSON.parse(data) - - // Use latest transport - const transport = activeTransports[activeTransports.length - 1] - - // Handle message - const response = await transport.handleMessage(body) - return response - } - catch (error) { - event.node.res.statusCode = 500 - return { error: errorToMessage(error) } - } - })) - - // Register routes - app.use(router) - - // Start server - const listener = toNodeListener(app) - await listen(listener, { - showURL: true, - port: 8080, - open: true, - }) - - logger.main.log('🚀 Twitter MCP Dev Server started') - - // Handle exit - process.on('SIGINT', async () => { - logger.main.log('Shutting down server...') - await browser.close() - process.exit(0) - }) -} - -// Execute -startDevServer().catch((error) => { - logger.main.error('Failed to start dev server:', error) - process.exit(1) -}) diff --git a/services/twitter-services/src/mcp-server.ts b/services/twitter-services/src/mcp-server.ts new file mode 100644 index 00000000..d6b06cf9 --- /dev/null +++ b/services/twitter-services/src/mcp-server.ts @@ -0,0 +1,60 @@ +import process from 'node:process' +import * as dotenv from 'dotenv' + +import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' +import { MCPAdapter } from './adapters/mcp-adapter' +import { TwitterService } from './core/twitter-service' +import { logger } from './utils/logger' + +// Load environment variables +dotenv.config() + +/** + * Development server entry point + * Provides convenience features for development + */ +async function startDevServer() { + // Create browser and Twitter service + const browser = new StagehandBrowserAdapter(process.env.BROWSERBASE_API_KEY || '') + await browser.initialize({ + headless: true, + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + }) + + const twitter = new TwitterService(browser) + + // Optional: If credentials are available, login + if (process.env.TWITTER_USERNAME && process.env.TWITTER_PASSWORD) { + const success = await twitter.login({ + username: process.env.TWITTER_USERNAME, + password: process.env.TWITTER_PASSWORD, + }) + + if (success) { + logger.main.log('✅ Successfully logged in Twitter') + } + else { + logger.main.warn('⚠️ Twitter login failed') + } + } + + // Create and start MCP adapter + const mcpAdapter = new MCPAdapter(twitter, 8080) + await mcpAdapter.start() + + logger.main.log('🚀 Twitter MCP Dev Server started') + + // Handle exit + process.on('SIGINT', async () => { + logger.main.log('Shutting down server...') + await mcpAdapter.stop() + await browser.close() + process.exit(0) + }) +} + +// Execute +startDevServer().catch((error) => { + logger.main.error('Failed to start dev server:', error) + process.exit(1) +}) From 446b830c54023768e4d1ea10729ffa1f2df58079 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 16:29:50 +0800 Subject: [PATCH 13/20] refactor: remove browser adapter, use playwright directly --- services/twitter-services/package.json | 1 - .../src/adapters/browser-adapter.ts | 88 ---- .../src/adapters/browserbase-adapter.ts | 153 ------- .../src/browser/browserbase.ts | 314 ------------- .../twitter-services/src/core/auth-service.ts | 428 ++++++++++-------- .../src/core/timeline-service.ts | 39 +- .../src/core/twitter-service.ts | 14 +- services/twitter-services/src/launcher.ts | 144 ------ services/twitter-services/src/main.ts | 159 ++++++- services/twitter-services/src/mcp-server.ts | 60 --- .../twitter-services/src/types/browser.ts | 34 -- services/twitter-services/src/utils/api.ts | 62 --- .../src/utils/session-manager.ts | 129 ------ 13 files changed, 415 insertions(+), 1210 deletions(-) delete mode 100644 services/twitter-services/src/adapters/browser-adapter.ts delete mode 100644 services/twitter-services/src/adapters/browserbase-adapter.ts delete mode 100644 services/twitter-services/src/browser/browserbase.ts delete mode 100644 services/twitter-services/src/launcher.ts delete mode 100644 services/twitter-services/src/mcp-server.ts delete mode 100644 services/twitter-services/src/types/browser.ts delete mode 100644 services/twitter-services/src/utils/api.ts delete mode 100644 services/twitter-services/src/utils/session-manager.ts diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index 874a2000..61e4377c 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -7,7 +7,6 @@ "license": "MIT", "scripts": { "dev": "tsx src/main.ts", - "dev:mcp": "tsx src/mcp-server.ts", "mcp:ui": "pnpx @modelcontextprotocol/inspector", "postinstall": "playwright install chromium" }, diff --git a/services/twitter-services/src/adapters/browser-adapter.ts b/services/twitter-services/src/adapters/browser-adapter.ts deleted file mode 100644 index 0c2f2600..00000000 --- a/services/twitter-services/src/adapters/browser-adapter.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { Buffer } from 'node:buffer' -import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' - -/** - * Generic browser adapter interface - * Defines the basic operations required for interacting with different browser backends - */ -export interface BrowserAdapter { - /** - * Initialize browser session - */ - initialize: (config: BrowserConfig) => Promise - - /** - * Navigate to specified URL - */ - navigate: (url: string) => Promise - - /** - * Execute JavaScript script - */ - executeScript: (script: string) => Promise - - /** - * Wait for element to appear - */ - waitForSelector: (selector: string, options?: WaitOptions) => Promise - - /** - * Click element - */ - click: (selector: string) => Promise - - /** - * Type text into input - */ - type: (selector: string, text: string) => Promise - - /** - * Get element text content - */ - getText: (selector: string) => Promise - - /** - * Get multiple element handles - */ - getElements: (selector: string) => Promise - - /** - * Get all cookies from the browser context - * This includes HTTP_ONLY cookies that can't be accessed via document.cookie - */ - getAllCookies: () => Promise> - - /** - * Set cookies in the browser context - * This can set HTTP_ONLY cookies that can't be set via document.cookie - */ - setCookies: (cookies: Array<{ - name: string - value: string - domain?: string - path?: string - expires?: number - httpOnly?: boolean - secure?: boolean - sameSite?: 'Strict' | 'Lax' | 'None' - }>) => Promise - - /** - * Get screenshot - */ - getScreenshot: () => Promise - - /** - * Close browser session - */ - close: () => Promise -} diff --git a/services/twitter-services/src/adapters/browserbase-adapter.ts b/services/twitter-services/src/adapters/browserbase-adapter.ts deleted file mode 100644 index d19693c0..00000000 --- a/services/twitter-services/src/adapters/browserbase-adapter.ts +++ /dev/null @@ -1,153 +0,0 @@ -import type { Buffer } from 'node:buffer' -import type { StagehandClientOptions } from '../browser/browserbase' -import type { BrowserConfig, ElementHandle, WaitOptions } from '../types/browser' -import type { BrowserAdapter } from './browser-adapter' - -import { StagehandClient } from '../browser/browserbase' -import { errorToMessage } from '../utils/error' -import { logger } from '../utils/logger' - -/** - * Stagehand element handle implementation - */ -class StagehandElementHandle implements ElementHandle { - private client: StagehandClient - private selector: string - - constructor(client: StagehandClient, selector: string) { - this.client = client - this.selector = selector - } - - async getText(): Promise { - return this.client.executeScript(` - document.querySelector('${this.selector}').textContent.trim() - `) - } - - async getAttribute(name: string): Promise { - return this.client.executeScript(` - document.querySelector('${this.selector}').getAttribute('${name}') - `) - } - - async click(): Promise { - await this.client.click(this.selector) - } - - async type(text: string): Promise { - await this.client.type(this.selector, text) - } -} - -/** - * Stagehand browser adapter implementation - * Adapts the Stagehand API to a common browser interface - */ -export class StagehandBrowserAdapter implements BrowserAdapter { - private client: StagehandClient - - constructor(apiKey: string, baseUrl?: string, options: Partial = {}) { - this.client = new StagehandClient({ - apiKey, - baseUrl, - ...options, - }) - } - - async initialize(config: BrowserConfig): Promise { - try { - await this.client.createSession({ - headless: config.headless, - userAgent: config.userAgent, - viewport: config.viewport, - }) - logger.browser.withFields({ - headless: config.headless, - }).log('Browser session created') - } - catch (error) { - logger.browser.withError(error).error('Failed to initialize browser') - throw new Error(`Unable to initialize browser: ${errorToMessage(error)}`) - } - } - - async navigate(url: string): Promise { - await this.client.navigate(url) - } - - async executeScript(script: string): Promise { - return this.client.executeScript(script) - } - - async waitForSelector(selector: string, options?: WaitOptions): Promise { - await this.client.waitForSelector(selector, { - timeout: options?.timeout, - }) - } - - async click(selector: string): Promise { - await this.client.click(selector) - } - - async type(selector: string, text: string): Promise { - await this.client.type(selector, text) - } - - async getText(selector: string): Promise { - return this.client.getText(selector) - } - - async getElements(selector: string): Promise { - // Get all matching element selectors - const selectors = await this.executeScript(` - Array.from(document.querySelectorAll('${selector}')).map((el, i) => { - const uniqueId = 'stagehand-' + Date.now() + '-' + i; - el.setAttribute('data-stagehand-id', uniqueId); - return '[data-stagehand-id="' + uniqueId + '"]'; - }) - `) - - // Create an ElementHandle for each match - return selectors.map(selector => new StagehandElementHandle(this.client, selector)) - } - - // Add Stagehand specific methods - async act(instruction: string): Promise { - await this.client.act(instruction) - } - - async getScreenshot(): Promise { - return this.client.getScreenshot() - } - - async getAllCookies(): Promise> { - return this.client.getAllCookies() - } - - async setCookies(cookies: Array<{ - name: string - value: string - domain?: string - path?: string - expires?: number - httpOnly?: boolean - secure?: boolean - sameSite?: 'Strict' | 'Lax' | 'None' - }>): Promise { - return this.client.setCookies(cookies) - } - - async close(): Promise { - await this.client.closeSession() - } -} diff --git a/services/twitter-services/src/browser/browserbase.ts b/services/twitter-services/src/browser/browserbase.ts deleted file mode 100644 index 4d5ca275..00000000 --- a/services/twitter-services/src/browser/browserbase.ts +++ /dev/null @@ -1,314 +0,0 @@ -import type { Buffer } from 'node:buffer' -import type { Browser, Page } from 'playwright' -import type { z } from 'zod' - -import { chromium } from 'playwright' - -import { logger } from '../utils/logger' - -/** - * Stagehand client configuration options - */ -export interface StagehandClientOptions { - apiKey: string - baseUrl?: string - timeout?: number - headless?: boolean - userAgent?: string - viewport?: { width: number, height: number } -} - -/** - * Stagehand client - * Implements browser automation using @browserbasehq/stagehand - */ -export class StagehandClient { - private browser: Browser | null = null - private page: Page | null = null - private apiKey: string - private options: Omit - - constructor(options: StagehandClientOptions) { - const { - apiKey, - baseUrl, - timeout = 30000, - headless = true, - userAgent, - viewport = { width: 1280, height: 800 }, - } = options - - this.apiKey = apiKey - this.options = { - baseUrl, - timeout, - headless, - userAgent, - viewport, - } - } - - /** - * Create browser session - */ - async createSession(options?: { - headless?: boolean - userAgent?: string - viewport?: { width: number, height: number } - }): Promise { - try { - // Launch Playwright browser - this.browser = await chromium.launch({ - headless: options?.headless ?? this.options.headless, - }) - - // Create context - const context = await this.browser.newContext({ - userAgent: options?.userAgent ?? this.options.userAgent, - viewport: options?.viewport ?? this.options.viewport, - // Set any other required browser context options - }) - - // Create page - this.page = await context.newPage() - - // Add Stagehand extension to page - await this.setupStagehand() - - const sessionId = `session-${Date.now()}` - logger.browser.withField('sessionId', sessionId).log('Browser session created successfully') - return sessionId - } - catch (error) { - logger.browser.errorWithError('Failed to create browser session', error) - throw error - } - } - - /** - * Set up Stagehand extension - * This adds act, extract, observe methods to the page object - */ - private async setupStagehand(): Promise { - if (!this.page) { - throw new Error('No active page. Call createSession first.') - } - - // In actual implementation, this would use Stagehand's API to set up the page object - // This might involve page extension or injecting Stagehand functionality - // Example code (actual usage would need to be adjusted based on Stagehand's documentation): - // - // import { extendPage } from '@browserbasehq/stagehand' - // await extendPage(this.page, { - // apiKey: this.apiKey, - // // Other Stagehand options - // }) - } - - /** - * Navigate to specified URL - */ - async navigate(url: string): Promise { - this.ensurePageExists() - await this.page!.goto(url, { timeout: this.options.timeout }) - } - - /** - * Execute JavaScript script - */ - async executeScript(script: string): Promise { - this.ensurePageExists() - return await this.page!.evaluate(script) as T - } - - /** - * Use Stagehand's act API to perform operations - */ - async act(instruction: string): Promise { - this.ensurePageExists() - - // In actual implementation, this would use Stagehand's act API - // Example: await this.page!.act(instruction) - - // Temporary implementation, simulating act behavior with Playwright basics - logger.browser.withField('instruction', instruction).log('Executing act instruction') - - // Simulate act behavior through simple methods - // Actual implementation would use Stagehand's act API - if (instruction.includes('click')) { - const match = instruction.match(/click on the ['"](.+?)['"]/) - if (match && match[1]) { - await this.page!.getByText(match[1]).first().click() - } - } - } - - /** - * Use Stagehand's extract API to extract data - */ - async extract({ - instruction, - _schema, - }: { - instruction: string - _schema: T - }): Promise> { - this.ensurePageExists() - - // In actual implementation, this would use Stagehand's extract API - // Example: return await this.page!.extract({ instruction, schema }) - - // Temporary implementation, log instruction and return empty object - logger.browser.withField('instruction', instruction).log('Executing extract instruction') - - // Simply return an empty object - // Actual implementation would use Stagehand's extract API - return {} as z.infer - } - - /** - * Use Stagehand's observe API to observe page state - */ - async observe(instruction: string): Promise { - this.ensurePageExists() - - // In actual implementation, this would use Stagehand's observe API - // Example: return await this.page!.observe(instruction) - - // Temporary implementation, log instruction and return empty string - logger.browser.withField('instruction', instruction).log('Executing observe instruction') - - // Simply return an empty string - // Actual implementation would use Stagehand's observe API - return '' - } - - /** - * Get page content - */ - async getContent(): Promise { - this.ensurePageExists() - return await this.page!.content() - } - - /** - * Wait for element to appear - */ - async waitForSelector(selector: string, options: { timeout?: number } = {}): Promise { - this.ensurePageExists() - await this.page!.waitForSelector(selector, { - timeout: options.timeout || this.options.timeout, - }) - } - - /** - * Click element - */ - async click(selector: string): Promise { - this.ensurePageExists() - await this.page!.click(selector) - } - - /** - * Type text into input field - */ - async type(selector: string, text: string): Promise { - this.ensurePageExists() - // Clear input field first - await this.page!.fill(selector, '') - // Then type text - await this.page!.fill(selector, text) - } - - /** - * Get element text content - */ - async getText(selector: string): Promise { - this.ensurePageExists() - const element = await this.page!.$(selector) - if (!element) { - throw new Error(`Element not found: ${selector}`) - } - return (await element.textContent() || '').trim() - } - - /** - * Get screenshot - */ - async getScreenshot(): Promise { - this.ensurePageExists() - return await this.page!.screenshot() as Buffer - } - - /** - * Get all cookies from browser context - * This includes HTTP_ONLY cookies that can't be accessed via document.cookie - */ - async getAllCookies(): Promise> { - this.ensurePageExists() - const context = this.page!.context() - return await context.cookies() - } - - /** - * Set cookies in browser context - * This can set HTTP_ONLY cookies that can't be set via document.cookie - */ - async setCookies(cookies: Array<{ - name: string - value: string - domain?: string - path?: string - expires?: number - httpOnly?: boolean - secure?: boolean - sameSite?: 'Strict' | 'Lax' | 'None' - }>): Promise { - this.ensurePageExists() - const context = this.page!.context() - // Format cookies correctly for Playwright - const formattedCookies = cookies.map((cookie) => { - // Ensure domain is set for Twitter - if (!cookie.domain) { - cookie.domain = '.twitter.com' - } - // Ensure path is set - if (!cookie.path) { - cookie.path = '/' - } - return cookie - }) - - await context.addCookies(formattedCookies) - } - - /** - * Close session - */ - async closeSession(): Promise { - if (this.browser) { - await this.browser.close() - this.browser = null - this.page = null - logger.browser.log('Browser session closed') - } - } - - /** - * Ensure page exists - */ - private ensurePageExists(): void { - if (!this.page) { - throw new Error('No active page. Call createSession first.') - } - } -} diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index d21a8a3f..58d24620 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -1,21 +1,99 @@ -import type { BrowserAdapter } from '../adapters/browser-adapter' +import type { BrowserContext, Cookie, Page } from 'playwright' import type { TwitterCredentials } from '../types/twitter' -import type { SessionData } from '../utils/session-manager' + +import fs from 'node:fs/promises' +import path from 'node:path' +import process from 'node:process' import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' -import { getSessionManager } from '../utils/session-manager' + +/** + * Playwright storage state type definition + */ +interface StorageState { + cookies: Cookie[] + origins: { + origin: string + localStorage: { + name: string + value: string + }[] + }[] + path?: string +} + +/** + * Simple session manager for storing and retrieving browser session data + */ +class SessionManager { + private sessionPath: string + + constructor() { + this.sessionPath = path.join(process.cwd(), 'data', 'twitter-session.json') + } + + /** + * Load storage state from disk + */ + async loadStorageState(): Promise { + try { + // Ensure directory exists + const dir = path.dirname(this.sessionPath) + await fs.mkdir(dir, { recursive: true }) + + // Check if file exists + try { + await fs.access(this.sessionPath) + } + catch { + // File doesn't exist + return null + } + + // Read file + const data = await fs.readFile(this.sessionPath, 'utf-8') + return JSON.parse(data) as StorageState + } + catch (error) { + logger.auth.withError(error as Error).warn('Failed to load session data') + return null + } + } + + /** + * Save storage state to disk + */ + async saveStorageState(state: StorageState): Promise { + try { + // Ensure directory exists + const dir = path.dirname(this.sessionPath) + await fs.mkdir(dir, { recursive: true }) + + // Write to file + await fs.writeFile(this.sessionPath, JSON.stringify(state, null, 2)) + } + catch (error) { + logger.auth.withError(error as Error).warn('Failed to save session data') + } + } +} + +// Singleton instance +const sessionManager = new SessionManager() /** * Twitter Authentication Service * Handles login and session management */ export class TwitterAuthService { - private browser: BrowserAdapter + private page: Page + private context: BrowserContext private isLoggedIn: boolean = false - constructor(browser: BrowserAdapter) { - this.browser = browser + constructor(page: Page, context: BrowserContext) { + this.page = page + this.context = context } /** @@ -26,52 +104,30 @@ export class TwitterAuthService { logger.auth.log('Starting Twitter login process') try { - // First try to load session from file if no cookies provided - if (!credentials.cookies || Object.keys(credentials.cookies).length === 0) { - logger.auth.log('No cookies provided, attempting to load session from file') - - // Get the session manager and load the session - const sessionManager = getSessionManager() - const sessionData = await sessionManager.loadSession() - - if (sessionData && sessionData.cookies && Object.keys(sessionData.cookies).length > 0) { - logger.auth.log(`Found session file with ${Object.keys(sessionData.cookies).length} cookies, attempting login`) - - // Use the dedicated method for session-based login - const sessionLoginSuccess = await this.loginWithSessionData(sessionData) - if (sessionLoginSuccess) { - logger.auth.log('✅ Successfully logged in using saved session') - return true - } - logger.auth.log('Session login failed, continuing with other login methods') - } - else { - logger.auth.log('No valid session file found') - } - } - - // If cookies are provided directly, try to use them for login + // Check if cookies are provided if (credentials.cookies && Object.keys(credentials.cookies).length > 0) { - logger.auth.log('Cookies provided, attempting cookie-based login') - const cookieLoginSuccess = await this.loginWithCookies(credentials.cookies) - if (cookieLoginSuccess) { - return true - } - // If cookie login fails, log the issue but continue to manual login - logger.auth.log('Cookie login failed, falling back to manual login') + logger.auth.log(`Attempting to login with ${Object.keys(credentials.cookies).length} provided cookies`) + return await this.loginWithCookies(credentials.cookies) } - // Check for existing session first - logger.auth.log('Checking for existing session before initiating manual login') - const existingSession = await this.checkExistingSession() - if (existingSession) { - logger.auth.log('✅ Successfully logged in using existing browser session') + // Try to login with existing session first + logger.auth.log('No cookies provided, attempting to load session from file') + const sessionSuccess = await this.checkExistingSession() + + if (sessionSuccess) { + logger.auth.log('Successfully logged in with session file') return true } - // Fallback to manual login flow - logger.auth.log('No existing session found, initiating manual login process') - return this.initiateManualLogin() + // If credentials are provided, try username/password login + if (credentials.username && credentials.password) { + logger.auth.log('Session login failed, attempting username/password login') + return await this.initiateManualLogin(credentials.username, credentials.password) + } + + // No credentials and no session, fail + logger.auth.warn('No cookies, no valid session, and no credentials provided') + return false } catch (error: unknown) { logger.auth.withError(error as Error).error('Login process failed') @@ -88,7 +144,7 @@ export class TwitterAuthService { // Try multiple selectors to determine login status // First check for timeline which is definitive proof of being logged in try { - await this.browser.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 15000 }) + await this.page.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 15000 }) return true } catch { @@ -98,7 +154,7 @@ export class TwitterAuthService { // Check for profile button which appears when logged in try { const profileSelector = '[data-testid="AppTabBar_Profile_Link"]' - await this.browser.waitForSelector(profileSelector, { timeout: 5000 }) + await this.page.waitForSelector(profileSelector, { timeout: 5000 }) return true } catch { @@ -108,7 +164,7 @@ export class TwitterAuthService { // Check for login form to confirm NOT logged in try { const loginFormSelector = '[data-testid="loginForm"]' - await this.browser.waitForSelector(loginFormSelector, { timeout: 3000 }) + await this.page.waitForSelector(loginFormSelector, { timeout: 3000 }) // If login form is visible, we're definitely not logged in return false } @@ -118,7 +174,7 @@ export class TwitterAuthService { // If we got here, we couldn't definitively confirm login status // Check current URL for additional clues - const currentUrl = await this.browser.executeScript(` + const currentUrl = await this.page.evaluate(` (() => { return window.location.href; })() @@ -142,7 +198,7 @@ export class TwitterAuthService { */ async checkLoginStatus(): Promise { try { - await this.browser.navigate('https://twitter.com/home') + await this.page.goto('https://twitter.com/home') return await this.verifyLogin() } catch { @@ -158,15 +214,13 @@ export class TwitterAuthService { } /** - * Export current session cookies - * Can be used to save and reuse session later - * @param format - The format of the returned cookies ('object' or 'string') + * Export current cookies + * @param format Export format, either 'object' or 'string' */ async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { try { - // Use the new getAllCookies method to get all cookies, including HTTP_ONLY ones - const allCookies = await this.browser.getAllCookies() - logger.auth.log(`Retrieved ${allCookies.length} cookies from browser context`) + // Get all cookies from browser + const allCookies = await this.context.cookies() if (format === 'string') { // Convert cookie objects to string format @@ -188,9 +242,12 @@ export class TwitterAuthService { return cookiesObj } } - catch (error: unknown) { - logger.auth.withError(error as Error).error('Error exporting cookies') - return format === 'string' ? '' : {} + catch (error) { + logger.auth.withError(error as Error).error('Failed to export cookies') + if (format === 'string') { + return '' + } + return {} } } @@ -202,7 +259,7 @@ export class TwitterAuthService { try { // Navigate to a Twitter page - await this.browser.navigate('https://twitter.com') + await this.page.goto('https://twitter.com') // Convert cookies object to array format required by setCookies const cookieArray = Object.entries(cookies).map(([name, value]) => ({ @@ -213,12 +270,12 @@ export class TwitterAuthService { })) // Set cookies using the browser adapter's API that can set HTTP_ONLY cookies - await this.browser.setCookies(cookieArray) + await this.context.addCookies(cookieArray) logger.auth.log(`Set ${cookieArray.length} cookies via browser API`) // Refresh page to apply cookies - await this.browser.navigate('https://twitter.com/home') + await this.page.goto('https://twitter.com/home') // Verify if login was successful - try multiple times with longer timeout logger.auth.log('Cookies set, verifying login status...') @@ -239,7 +296,7 @@ export class TwitterAuthService { else if (attempt < verificationAttempts) { // If not successful but not last attempt, refresh page and wait logger.auth.log('Refreshing page and trying again...') - await this.browser.navigate('https://twitter.com/home') + await this.page.goto('https://twitter.com/home') await new Promise(resolve => setTimeout(resolve, 3000)) } } @@ -275,85 +332,94 @@ export class TwitterAuthService { } /** - * Checks if there's an existing login session and retrieves it - * This should be called before initiateManualLogin + * Attempt to login with an existing session if available */ async checkExistingSession(): Promise { - logger.auth.log('Checking for existing Twitter session') - try { - // Navigate to home page to check session - await this.browser.navigate('https://twitter.com/home') + // Get the session data + const sessionData = await sessionManager.loadStorageState() - // Verify if login is active - const loginSuccess = await this.verifyLogin() - - if (loginSuccess) { - logger.auth.log('Existing session found and valid') - this.isLoggedIn = true - - // Export and save cookies - try { - const cookies = await this.exportCookies('object') - logger.auth.log(`✅ Exported ${typeof cookies === 'string' ? cookies.length : Object.keys(cookies).length} cookies from existing session`) - - // Save the current session to file - await this.saveCurrentSession() - logger.auth.log('✅ Session saved to file') - } - catch (error) { - logger.auth.withError(error as Error).error('Error exporting cookies from existing session') - } - } - else { - logger.auth.log('No valid session found') + if (!sessionData || !sessionData.cookies || sessionData.cookies.length === 0) { + logger.auth.log('No valid session data found') + return false } - return loginSuccess + logger.auth.log(`Found session file with ${sessionData.cookies.length} cookies, attempting login`) + + // Login with the session data + return await this.loginWithSessionData(sessionData) } catch (error) { - logger.auth.withError(error as Error).error('Error checking session status') - this.isLoggedIn = false + logger.auth.withError(error as Error).warn('Error checking existing session') return false } } /** - * Initiates the manual login process by navigating to Twitter login page - * and waits for user to complete the login process + * Initiate manual login process with username and password + * @param username Twitter username or email + * @param password Twitter password */ - async initiateManualLogin(): Promise { - logger.auth.log('Opening Twitter login page for manual login') + async initiateManualLogin(username?: string, password?: string): Promise { + logger.auth.log('Initiating manual login process') try { - // Store the current URL to detect navigation - const initialUrl = await this.browser.executeScript(` - (() => { - return window.location.href; - })() - `) - // Navigate to login page - await this.browser.navigate('https://twitter.com/i/flow/login') + await this.page.goto('https://twitter.com/login') + + // Wait for login form to appear and enter credentials + try { + // Wait for username input + await this.page.waitForSelector(SELECTORS.LOGIN.USERNAME_INPUT, { timeout: 10000 }) + + // Use provided credentials if available, otherwise fall back to env vars + const loginUsername = username || process.env.TWITTER_USERNAME + const loginPassword = password || process.env.TWITTER_PASSWORD - // Wait for user to manually log in (detected by timeline presence) - logger.auth.log('==============================================') - logger.auth.log('Please log in to Twitter in the opened browser window') - logger.auth.log('The system will wait for you to complete the login process') - logger.auth.log('Cookies will be automatically saved after login') - logger.auth.log('==============================================') + if (!loginUsername || !loginPassword) { + logger.auth.warn('Missing Twitter credentials, manual login cannot proceed') + return false + } + + // Enter username + await this.page.fill(SELECTORS.LOGIN.USERNAME_INPUT, loginUsername) + logger.auth.debug('Username entered') + + // Click next button + await this.page.click(SELECTORS.LOGIN.NEXT_BUTTON) + logger.auth.debug('Next button clicked') + + // Wait for password input + await this.page.waitForSelector(SELECTORS.LOGIN.PASSWORD_INPUT, { timeout: 10000 }) + + // Enter password + await this.page.fill(SELECTORS.LOGIN.PASSWORD_INPUT, loginPassword) + logger.auth.debug('Password entered') + + // Click login button + await this.page.click(SELECTORS.LOGIN.LOGIN_BUTTON) + logger.auth.debug('Login button clicked') + } + catch (error) { + logger.auth.withError(error as Error).error('Error during manual login process') + return false + } - // Poll for login success at intervals + // Wait for login success at intervals let attempts = 0 const maxAttempts = 60 // 10 minutes (10 seconds * 60) - let lastUrl = initialUrl + let lastUrl = await this.page.evaluate(` + (() => { + return window.location.href; + })() + `) while (attempts < maxAttempts) { attempts++ try { // Get current URL to detect page changes - const currentUrl = await this.browser.executeScript(` + const currentUrl = await this.page.evaluate(` (() => { return window.location.href; })() @@ -365,7 +431,7 @@ export class TwitterAuthService { logger.auth.log('Attempting to navigate to home page and verify login status') // URL changed - try navigating to home to verify - await this.browser.navigate('https://twitter.com/home') + await this.page.goto('https://twitter.com/home') // Check if login was successful const isLoggedIn = await this.verifyLogin() @@ -439,114 +505,92 @@ export class TwitterAuthService { } /** - * Save the current session to a file for future use. - * This includes cookies and authentication details. + * Save the current session to a file */ async saveCurrentSession(): Promise { try { - // Export cookies in object format + // Get the cookies from the browser const cookies = await this.exportCookies('object') - if (!cookies || (typeof cookies === 'object' && Object.keys(cookies).length === 0)) { - logger.auth.warn('No cookies available to save') - return - } + // Format the cookies for the session manager + // We need to convert Record to the Cookie[] format + const cookieArray = Object.entries(cookies).map(([name, value]) => ({ + name, + value, // This will be a string now + domain: '.twitter.com', + path: '/', + expires: -1, // Session cookie + })) - // Create a session data object - const sessionData: SessionData = { - cookies: typeof cookies === 'string' ? {} : cookies, - timestamp: new Date().toISOString(), - userAgent: await this.browser.executeScript(` - (() => { - return navigator.userAgent; - })() - `), + // Get the storage state from the browser + const storageState = await this.context.storageState() + + // Create a new session data object with proper type cast + const sessionData: StorageState = { + cookies: cookieArray.map(cookie => ({ + ...cookie, + sameSite: 'Lax' as const, + secure: true, + httpOnly: true, + })), + origins: storageState.origins, + // Don't include path property here as it's not needed for saving } - // Get the session manager and save the session - const sessionManager = getSessionManager() - await sessionManager.saveSession(sessionData) + // Save the session + await sessionManager.saveStorageState(sessionData) logger.auth.log(`Session saved with ${typeof cookies === 'object' ? Object.keys(cookies).length : 0} cookies`) } - catch (error: unknown) { - logger.auth.withError(error as Error).error('Failed to save session') + catch (error) { + logger.auth.withError(error as Error).warn('Failed to save session') } } /** - * Login with a saved session data object - * @param sessionData The session data with cookies to use for login + * Login with stored session data */ - private async loginWithSessionData(sessionData: SessionData): Promise { - logger.auth.log(`Attempting to login using session with ${Object.keys(sessionData.cookies).length} cookies`) - + private async loginWithSessionData(sessionData: StorageState): Promise { try { - // Navigate to a Twitter page - await this.browser.navigate('https://twitter.com') - - // Convert cookies object to array format required by setCookies - const cookieArray = Object.entries(sessionData.cookies).map(([name, value]) => ({ - name, - value, - domain: '.twitter.com', - path: '/', + // Extract cookies from session data + const { cookies } = sessionData + + // Create array of cookie objects for the browser + const cookieArray = cookies.map(cookie => ({ + name: cookie.name, + value: cookie.value, + domain: cookie.domain, + path: cookie.path, })) // Set cookies using the browser adapter's API - await this.browser.setCookies(cookieArray) + await this.context.addCookies(cookieArray) logger.auth.log(`Set ${cookieArray.length} cookies from session file`) - // Continue with similar verification logic as loginWithCookies - await this.browser.navigate('https://twitter.com/home') - - // Verify login status - try multiple times - logger.auth.log('Cookies set from session file, verifying login status...') - - let loginSuccess = false - const verificationAttempts = 3 + // Set localStorage if available + if (sessionData.origins && sessionData.origins.length > 0) { + await this.context.storageState(sessionData) + logger.auth.log(`Set localStorage for ${sessionData.origins.length} origins`) + } - for (let attempt = 1; attempt <= verificationAttempts; attempt++) { - try { - logger.auth.log(`Verification attempt ${attempt}/${verificationAttempts}`) - loginSuccess = await this.verifyLogin() + // Navigate to home to verify login + await this.page.goto('https://twitter.com/home') - if (loginSuccess) { - break - } - else if (attempt < verificationAttempts) { - logger.auth.log('Refreshing page and trying again...') - await this.browser.navigate('https://twitter.com/home') - await new Promise(resolve => setTimeout(resolve, 3000)) - } - } - catch (error: unknown) { - logger.auth.withError(error as Error).debug(`Verification attempt ${attempt} failed`) - } - } + // Verify if login was successful + const loginSuccess = await this.verifyLogin() if (loginSuccess) { - logger.auth.log('Login with session data successful') this.isLoggedIn = true - - // Try to refresh cookies to ensure they're up to date - try { - await this.saveCurrentSession() - logger.auth.log('✅ Session refreshed and saved to file') - } - catch (error: unknown) { - logger.auth.withError(error as Error).debug('Failed to update cookies, but login was successful') - } + logger.auth.log('✅ Successfully logged in with session data') } else { - logger.auth.warn('Login with session data verification failed, cookies may be expired') + logger.auth.warn('⚠️ Session data login failed verification') } return loginSuccess } - catch (error: unknown) { - logger.auth.withError(error as Error).error('Error during session data login process') - this.isLoggedIn = false + catch (error) { + logger.auth.withError(error as Error).error('Failed to login with session data') return false } } diff --git a/services/twitter-services/src/core/timeline-service.ts b/services/twitter-services/src/core/timeline-service.ts index aeb76c37..0e0fb17b 100644 --- a/services/twitter-services/src/core/timeline-service.ts +++ b/services/twitter-services/src/core/timeline-service.ts @@ -1,8 +1,8 @@ -import type { BrowserAdapter } from '../adapters/browser-adapter' +import type { Page } from 'playwright' import type { TimelineOptions, Tweet } from '../types/twitter' import { TweetParser } from '../parsers/tweet-parser' -import { RateLimiter } from '../utils/rate-limiter' +import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' /** @@ -10,48 +10,41 @@ import { SELECTORS } from '../utils/selectors' * Handles fetching and parsing timeline content */ export class TwitterTimelineService { - private browser: BrowserAdapter - private rateLimiter: RateLimiter + private page: Page - constructor(browser: BrowserAdapter) { - this.browser = browser - this.rateLimiter = new RateLimiter(10, 60000) // 10 requests per minute + constructor(page: Page) { + this.page = page } /** * Get timeline */ async getTimeline(options: TimelineOptions = {}): Promise { - // Wait for rate limit - await this.rateLimiter.waitUntilReady() - try { // Navigate to home page - await this.browser.navigate('https://twitter.com/home') + await this.page.goto('https://twitter.com/home') // Wait for timeline to load - await this.browser.waitForSelector(SELECTORS.TIMELINE.TWEET) - - // Delay a bit to ensure content is fully loaded - await new Promise(resolve => setTimeout(resolve, 2000)) + await this.page.waitForSelector(SELECTORS.TIMELINE.TWEET, { timeout: 10000 }) - // Get page HTML content - const html = await this.browser.executeScript('document.documentElement.outerHTML') - - // Parse tweets + // Get page HTML and parse all tweets + const html = await this.page.content() const tweets = TweetParser.parseTimelineTweets(html) - // Apply filtering and limits + logger.main.log(`Found ${tweets.length} tweets in timeline`) + + // Apply filters let filteredTweets = tweets if (options.includeReplies === false) { - filteredTweets = filteredTweets.filter(tweet => !tweet.id.includes('reply')) + filteredTweets = filteredTweets.filter(tweet => !tweet.text.startsWith('@')) } if (options.includeRetweets === false) { - filteredTweets = filteredTweets.filter(tweet => !tweet.id.includes('retweet')) + filteredTweets = filteredTweets.filter(tweet => !tweet.text.startsWith('RT @')) } + // Apply count limit if specified if (options.count) { filteredTweets = filteredTweets.slice(0, options.count) } @@ -59,7 +52,7 @@ export class TwitterTimelineService { return filteredTweets } catch (error) { - console.error('Failed to get timeline:', error) + logger.main.withError(error as Error).error('Failed to get timeline') return [] } } diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index c5f60bd5..1d2b4850 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -1,4 +1,3 @@ -import type { BrowserAdapter } from '../adapters/browser-adapter' import type { TwitterService as ITwitterService, PostOptions, @@ -9,23 +8,20 @@ import type { TwitterCredentials, UserProfile, } from '../types/twitter' - -import { TwitterAuthService } from './auth-service' -import { TwitterTimelineService } from './timeline-service' +import type { TwitterAuthService } from './auth-service' +import type { TwitterTimelineService } from './timeline-service' /** * Twitter service implementation * Integrates various service components, providing a unified interface */ export class TwitterService implements ITwitterService { - private browser: BrowserAdapter private authService: TwitterAuthService private timelineService: TwitterTimelineService - constructor(browser: BrowserAdapter) { - this.browser = browser - this.authService = new TwitterAuthService(browser) - this.timelineService = new TwitterTimelineService(browser) + constructor(authService: TwitterAuthService, timelineService: TwitterTimelineService) { + this.authService = authService + this.timelineService = timelineService } /** diff --git a/services/twitter-services/src/launcher.ts b/services/twitter-services/src/launcher.ts deleted file mode 100644 index ae9102f9..00000000 --- a/services/twitter-services/src/launcher.ts +++ /dev/null @@ -1,144 +0,0 @@ -import type { AiriAdapter } from './adapters/airi-adapter' -import type { StagehandBrowserAdapter } from './adapters/browserbase-adapter' -import type { MCPAdapter } from './adapters/mcp-adapter' - -import process from 'node:process' - -import { createDefaultConfig } from './config' -import { TwitterService } from './core/twitter-service' -import { logger } from './utils/logger' - -/** - * Twitter service launcher class - * Responsible for initializing and starting services - */ -export class TwitterServiceLauncher { - private browser?: StagehandBrowserAdapter - private twitterService?: TwitterService - private airiAdapter?: AiriAdapter - private mcpAdapter?: MCPAdapter - - /** - * Start Twitter service - */ - async start() { - try { - // Load configuration - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - logger.main.log('Starting Twitter service...') - - // Initialize browser - // Import handling - const { StagehandBrowserAdapter } = await import('./adapters/browserbase-adapter') - this.browser = new StagehandBrowserAdapter( - config.browser.apiKey, - config.browser.endpoint, - { - timeout: config.browser.requestTimeout, - // retries: config.browser.requestRetries, - }, - ) - - await this.browser.initialize(config.browser) - logger.main.log('Browser initialized') - - // Create Twitter service - this.twitterService = new TwitterService(this.browser) - - // Try to log in - if (config.twitter.credentials) { - const success = await this.twitterService.login(config.twitter.credentials) - if (success) { - logger.main.log('Successfully logged into Twitter') - } - else { - logger.main.error('Twitter login failed!') - } - } - - // Start enabled adapters - if (config.adapters.airi?.enabled) { - logger.main.log('Starting Airi adapter...') - const { AiriAdapter } = await import('./adapters/airi-adapter') - - this.airiAdapter = new AiriAdapter(this.twitterService, { - url: config.adapters.airi.url, - token: config.adapters.airi.token, - credentials: config.twitter.credentials!, - }) - - await this.airiAdapter.start() - logger.main.log('Airi adapter started') - } - - if (config.adapters.mcp?.enabled) { - logger.main.log('Starting MCP adapter...') - const { MCPAdapter } = await import('./adapters/mcp-adapter') - - this.mcpAdapter = new MCPAdapter( - this.twitterService, - config.adapters.mcp.port, - ) - - await this.mcpAdapter.start() - logger.main.log('MCP adapter started') - } - - logger.main.log('Twitter service successfully started!') - - // Set up shutdown hooks - this.setupShutdownHooks() - } - catch (error) { - logger.main.withError(error).error('Failed to start Twitter service') - } - } - - /** - * Stop service - */ - async stop() { - logger.main.log('Stopping Twitter service...') - - // Stop MCP adapter - if (this.mcpAdapter) { - await this.mcpAdapter.stop() - logger.main.log('MCP adapter stopped') - } - - // Close browser - if (this.browser) { - await this.browser.close() - logger.main.log('Browser closed') - } - - logger.main.log('Twitter service stopped') - } - - /** - * Set up shutdown hooks - */ - private setupShutdownHooks() { - // Handle process exit - process.on('SIGINT', async () => { - logger.main.log('Received exit signal...') - await this.stop() - process.exit(0) - }) - - process.on('SIGTERM', async () => { - logger.main.log('Received termination signal...') - await this.stop() - process.exit(0) - }) - - // Handle uncaught exceptions - process.on('uncaughtException', async (error) => { - logger.main.withError(error).error('Uncaught exception') - await this.stop() - process.exit(1) - }) - } -} diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index b0f048a5..c3c033f6 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -1,8 +1,165 @@ +import type { Browser, BrowserContext } from 'playwright' +import type { AiriAdapter } from './adapters/airi-adapter' +import type { MCPAdapter } from './adapters/mcp-adapter' + import process from 'node:process' +import { chromium } from 'playwright' -import { TwitterServiceLauncher } from './launcher' +import { createDefaultConfig } from './config' +import { TwitterAuthService } from './core/auth-service' +import { TwitterTimelineService } from './core/timeline-service' +import { TwitterService } from './core/twitter-service' import { initializeLogger, logger } from './utils/logger' +/** + * Twitter service launcher class + * Responsible for initializing and starting services + */ +export class TwitterServiceLauncher { + private browser?: Browser + private context?: BrowserContext + private twitterService?: TwitterService + private airiAdapter?: AiriAdapter + private mcpAdapter?: MCPAdapter + + /** + * Start Twitter service + */ + async start() { + try { + // Load configuration + const configManager = createDefaultConfig() + const config = configManager.getConfig() + + logger.main.log('Starting Twitter service...') + + // Initialize Playwright browser + this.browser = await chromium.launch({ + headless: config.browser.headless, + }) + + // Create a browser context + this.context = await this.browser.newContext({ + userAgent: config.browser.userAgent, + viewport: config.browser.viewport, + bypassCSP: true, + }) + + // Set default timeout for navigation and actions + this.context.setDefaultTimeout(config.browser.timeout || 30000) + + // Create a page + const page = await this.context.newPage() + + logger.main.log('Browser initialized') + + // Create service instances + const authService = new TwitterAuthService(page, this.context) + const timelineService = new TwitterTimelineService(page) + + // Create Twitter service with direct service dependencies + this.twitterService = new TwitterService(authService, timelineService) + + // Try to log in + if (config.twitter.credentials) { + const success = await this.twitterService.login(config.twitter.credentials) + if (success) { + logger.main.log('Successfully logged into Twitter') + } + else { + logger.main.error('Twitter login failed!') + } + } + + // Start enabled adapters + if (config.adapters.airi?.enabled) { + logger.main.log('Starting Airi adapter...') + const { AiriAdapter } = await import('./adapters/airi-adapter') + + this.airiAdapter = new AiriAdapter(this.twitterService, { + url: config.adapters.airi.url, + token: config.adapters.airi.token, + credentials: config.twitter.credentials!, + }) + + await this.airiAdapter.start() + logger.main.log('Airi adapter started') + } + + if (config.adapters.mcp?.enabled) { + logger.main.log('Starting MCP adapter...') + const { MCPAdapter } = await import('./adapters/mcp-adapter') + + this.mcpAdapter = new MCPAdapter( + this.twitterService, + config.adapters.mcp.port, + ) + + await this.mcpAdapter.start() + logger.main.log('MCP adapter started') + } + + logger.main.log('Twitter service successfully started!') + + // Set up shutdown hooks + this.setupShutdownHooks() + } + catch (error) { + logger.main.withError(error).error('Failed to start Twitter service') + } + } + + /** + * Stop service + */ + async stop() { + logger.main.log('Stopping Twitter service...') + + // Stop MCP adapter + if (this.mcpAdapter) { + await this.mcpAdapter.stop() + logger.main.log('MCP adapter stopped') + } + + // Close browser + if (this.context) { + await this.context.close() + } + + if (this.browser) { + await this.browser.close() + logger.main.log('Browser closed') + } + + logger.main.log('Twitter service stopped') + } + + /** + * Set up shutdown hooks + */ + private setupShutdownHooks() { + // Handle process exit + process.on('SIGINT', async () => { + logger.main.log('Received exit signal...') + await this.stop() + process.exit(0) + }) + + process.on('SIGTERM', async () => { + logger.main.log('Received termination signal...') + await this.stop() + process.exit(0) + }) + + // Handle uncaught exceptions + process.on('uncaughtException', async (error) => { + logger.main.withError(error).error('Uncaught exception') + await this.stop() + process.exit(1) + }) + } +} + // Ensure initialization only happens once async function bootstrap() { // 1. First initialize logging system diff --git a/services/twitter-services/src/mcp-server.ts b/services/twitter-services/src/mcp-server.ts deleted file mode 100644 index d6b06cf9..00000000 --- a/services/twitter-services/src/mcp-server.ts +++ /dev/null @@ -1,60 +0,0 @@ -import process from 'node:process' -import * as dotenv from 'dotenv' - -import { StagehandBrowserAdapter } from './adapters/browserbase-adapter' -import { MCPAdapter } from './adapters/mcp-adapter' -import { TwitterService } from './core/twitter-service' -import { logger } from './utils/logger' - -// Load environment variables -dotenv.config() - -/** - * Development server entry point - * Provides convenience features for development - */ -async function startDevServer() { - // Create browser and Twitter service - const browser = new StagehandBrowserAdapter(process.env.BROWSERBASE_API_KEY || '') - await browser.initialize({ - headless: true, - userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - }) - - const twitter = new TwitterService(browser) - - // Optional: If credentials are available, login - if (process.env.TWITTER_USERNAME && process.env.TWITTER_PASSWORD) { - const success = await twitter.login({ - username: process.env.TWITTER_USERNAME, - password: process.env.TWITTER_PASSWORD, - }) - - if (success) { - logger.main.log('✅ Successfully logged in Twitter') - } - else { - logger.main.warn('⚠️ Twitter login failed') - } - } - - // Create and start MCP adapter - const mcpAdapter = new MCPAdapter(twitter, 8080) - await mcpAdapter.start() - - logger.main.log('🚀 Twitter MCP Dev Server started') - - // Handle exit - process.on('SIGINT', async () => { - logger.main.log('Shutting down server...') - await mcpAdapter.stop() - await browser.close() - process.exit(0) - }) -} - -// Execute -startDevServer().catch((error) => { - logger.main.error('Failed to start dev server:', error) - process.exit(1) -}) diff --git a/services/twitter-services/src/types/browser.ts b/services/twitter-services/src/types/browser.ts deleted file mode 100644 index e6b6def0..00000000 --- a/services/twitter-services/src/types/browser.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Browser Config Interface - */ -export interface BrowserConfig { - headless?: boolean - userAgent?: string - viewport?: { - width: number - height: number - } - timeout?: number - requestTimeout?: number // API request timeout - requestRetries?: number // Request retries - proxy?: string -} - -/** - * Element Handle Interface - */ -export interface ElementHandle { - getText: () => Promise - getAttribute: (name: string) => Promise - click: () => Promise - type: (text: string) => Promise -} - -/** - * Wait Options Interface - */ -export interface WaitOptions { - timeout?: number - visible?: boolean - hidden?: boolean -} diff --git a/services/twitter-services/src/utils/api.ts b/services/twitter-services/src/utils/api.ts deleted file mode 100644 index a561fc78..00000000 --- a/services/twitter-services/src/utils/api.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { ofetch } from 'ofetch' - -import { logger } from './logger' - -/** - * Create a pre-configured ofetch instance - * - * @param baseURL - Base URL for the API - * @param options - Additional options - * @returns - Customized ofetch instance - */ -export function createApiClient(baseURL: string, options: Record = {}) { - const client = ofetch.create({ - baseURL, - retry: 1, - timeout: 30000, // Default 30 second timeout - ...options, - - // Request interceptor - onRequest({ request, options }) { - const method = options.method || 'GET' - const url = request.toString() - logger.browser.withFields({ method, url }).debug('API request') - }, - - // Request error interceptor - onRequestError({ request, error, options }) { - const method = options.method || 'GET' - const url = request.toString() - logger.browser.withFields({ method, url }).errorWithError('API request failed', error) - }, - - // Response interceptor - onResponse({ request, response, options }) { - const method = options.method || 'GET' - const url = request.toString() - const status = response.status - - logger.browser - .withField('method', method) - .withField('url', url) - .withField('status', status) - .debug('API response') - }, - - // Response error interceptor - onResponseError({ request, response, options }) { - const method = options.method || 'GET' - const url = request.toString() - const status = response.status - - logger.browser - .withField('method', method) - .withField('url', url) - .withField('status', status) - .withField('body', response._data) - .error('API response error') - }, - }) - - return client -} diff --git a/services/twitter-services/src/utils/session-manager.ts b/services/twitter-services/src/utils/session-manager.ts deleted file mode 100644 index 2bf549e9..00000000 --- a/services/twitter-services/src/utils/session-manager.ts +++ /dev/null @@ -1,129 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import process from 'node:process' - -import { logger } from './logger' - -/** - * Session cookie data structure - */ -export interface SessionData { - cookies: Record - timestamp: string - userAgent?: string - username?: string -} - -/** - * Session manager utility - * Responsible for loading and saving session data to/from files - */ -export class SessionManager { - private sessionFilePath: string - - /** - * Create a new session manager instance - * @param sessionFilePath Optional custom path for the session file - */ - constructor(sessionFilePath?: string) { - this.sessionFilePath = sessionFilePath - || path.join(process.cwd(), '.twitter.session.json') - } - - /** - * Save session data to file - * @param data The session data to save - */ - async saveSession(data: SessionData): Promise { - try { - // Create session directory if it doesn't exist - const dir = path.dirname(this.sessionFilePath) - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }) - } - - // Write session data to file - fs.writeFileSync( - this.sessionFilePath, - JSON.stringify(data, null, 2), - 'utf8', - ) - - logger.auth.log(`Session saved to ${this.sessionFilePath}`) - return true - } - catch (error: unknown) { - logger.auth.withError(error as Error).error(`Failed to save session to ${this.sessionFilePath}`) - return false - } - } - - /** - * Load session data from file - * @returns The loaded session data or null if file doesn't exist or is invalid - */ - async loadSession(): Promise { - try { - // Check if session file exists - if (!fs.existsSync(this.sessionFilePath)) { - logger.auth.debug(`No session file found at ${this.sessionFilePath}`) - return null - } - - // Read and parse session data - const data = JSON.parse(fs.readFileSync(this.sessionFilePath, 'utf8')) - - // Validate session data - if (!data.cookies || typeof data.cookies !== 'object' || !data.timestamp) { - logger.auth.warn(`Invalid session data in ${this.sessionFilePath}`) - return null - } - - // Check if session is too old (30 days) - const sessionDate = new Date(data.timestamp) - const ageInDays = (Date.now() - sessionDate.getTime()) / (1000 * 60 * 60 * 24) - if (ageInDays > 30) { - logger.auth.warn(`Session is ${Math.floor(ageInDays)} days old and may be expired`) - } - - logger.auth.log(`Loaded session from ${this.sessionFilePath} with ${Object.keys(data.cookies).length} cookies`) - return data - } - catch (error: unknown) { - logger.auth.withError(error as Error).error(`Failed to load session from ${this.sessionFilePath}`) - return null - } - } - - /** - * Delete the session file - * @returns true if deletion was successful, false otherwise - */ - deleteSession(): boolean { - try { - if (fs.existsSync(this.sessionFilePath)) { - fs.unlinkSync(this.sessionFilePath) - logger.auth.log(`Session file deleted: ${this.sessionFilePath}`) - return true - } - return false - } - catch (error: unknown) { - logger.auth.withError(error as Error).error(`Failed to delete session file: ${this.sessionFilePath}`) - return false - } - } -} - -// Create singleton instance -let sessionManagerInstance: SessionManager | null = null - -/** - * Get the session manager instance - */ -export function getSessionManager(sessionFilePath?: string): SessionManager { - if (!sessionManagerInstance) { - sessionManagerInstance = new SessionManager(sessionFilePath) - } - return sessionManagerInstance -} From 84571641ffacfac1db82f259749006149a98e97c Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 16:37:08 +0800 Subject: [PATCH 14/20] refactor: remove launcher services, replace twitter.com to x.com --- services/twitter-services/src/config/index.ts | 2 +- .../twitter-services/src/core/auth-service.ts | 18 +- .../src/core/timeline-service.ts | 2 +- services/twitter-services/src/main.ts | 273 +++++++++--------- services/twitter-services/src/utils/logger.ts | 6 +- 5 files changed, 144 insertions(+), 157 deletions(-) diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts index 7cac1512..a4d05a9b 100644 --- a/services/twitter-services/src/config/index.ts +++ b/services/twitter-services/src/config/index.ts @@ -120,7 +120,7 @@ let configInstance: ConfigManager | null = null /** * Create default configuration manager (singleton) */ -export function createDefaultConfig(): ConfigManager { +export function useConfigManager(): ConfigManager { if (configInstance) { return configInstance } diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index 58d24620..7c92dcb2 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -198,7 +198,7 @@ export class TwitterAuthService { */ async checkLoginStatus(): Promise { try { - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') return await this.verifyLogin() } catch { @@ -259,13 +259,13 @@ export class TwitterAuthService { try { // Navigate to a Twitter page - await this.page.goto('https://twitter.com') + await this.page.goto('https://x.com') // Convert cookies object to array format required by setCookies const cookieArray = Object.entries(cookies).map(([name, value]) => ({ name, value, - domain: '.twitter.com', + domain: '.x.com', path: '/', })) @@ -275,7 +275,7 @@ export class TwitterAuthService { logger.auth.log(`Set ${cookieArray.length} cookies via browser API`) // Refresh page to apply cookies - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') // Verify if login was successful - try multiple times with longer timeout logger.auth.log('Cookies set, verifying login status...') @@ -296,7 +296,7 @@ export class TwitterAuthService { else if (attempt < verificationAttempts) { // If not successful but not last attempt, refresh page and wait logger.auth.log('Refreshing page and trying again...') - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') await new Promise(resolve => setTimeout(resolve, 3000)) } } @@ -365,7 +365,7 @@ export class TwitterAuthService { try { // Navigate to login page - await this.page.goto('https://twitter.com/login') + await this.page.goto('https://x.com/login') // Wait for login form to appear and enter credentials try { @@ -431,7 +431,7 @@ export class TwitterAuthService { logger.auth.log('Attempting to navigate to home page and verify login status') // URL changed - try navigating to home to verify - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') // Check if login was successful const isLoggedIn = await this.verifyLogin() @@ -517,7 +517,7 @@ export class TwitterAuthService { const cookieArray = Object.entries(cookies).map(([name, value]) => ({ name, value, // This will be a string now - domain: '.twitter.com', + domain: '.x.com', path: '/', expires: -1, // Session cookie })) @@ -574,7 +574,7 @@ export class TwitterAuthService { } // Navigate to home to verify login - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') // Verify if login was successful const loginSuccess = await this.verifyLogin() diff --git a/services/twitter-services/src/core/timeline-service.ts b/services/twitter-services/src/core/timeline-service.ts index 0e0fb17b..04d71a1b 100644 --- a/services/twitter-services/src/core/timeline-service.ts +++ b/services/twitter-services/src/core/timeline-service.ts @@ -22,7 +22,7 @@ export class TwitterTimelineService { async getTimeline(options: TimelineOptions = {}): Promise { try { // Navigate to home page - await this.page.goto('https://twitter.com/home') + await this.page.goto('https://x.com/home') // Wait for timeline to load await this.page.waitForSelector(SELECTORS.TIMELINE.TWEET, { timeout: 10000 }) diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index c3c033f6..9e811c07 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -5,182 +5,169 @@ import type { MCPAdapter } from './adapters/mcp-adapter' import process from 'node:process' import { chromium } from 'playwright' -import { createDefaultConfig } from './config' +import { useConfigManager } from './config' import { TwitterAuthService } from './core/auth-service' import { TwitterTimelineService } from './core/timeline-service' import { TwitterService } from './core/twitter-service' -import { initializeLogger, logger } from './utils/logger' +import { initLogger, logger } from './utils/logger' /** - * Twitter service launcher class - * Responsible for initializing and starting services + * Initialize browser and create page */ -export class TwitterServiceLauncher { - private browser?: Browser - private context?: BrowserContext - private twitterService?: TwitterService - private airiAdapter?: AiriAdapter - private mcpAdapter?: MCPAdapter - - /** - * Start Twitter service - */ - async start() { - try { - // Load configuration - const configManager = createDefaultConfig() - const config = configManager.getConfig() - - logger.main.log('Starting Twitter service...') - - // Initialize Playwright browser - this.browser = await chromium.launch({ - headless: config.browser.headless, - }) - - // Create a browser context - this.context = await this.browser.newContext({ - userAgent: config.browser.userAgent, - viewport: config.browser.viewport, - bypassCSP: true, - }) - - // Set default timeout for navigation and actions - this.context.setDefaultTimeout(config.browser.timeout || 30000) - - // Create a page - const page = await this.context.newPage() - - logger.main.log('Browser initialized') - - // Create service instances - const authService = new TwitterAuthService(page, this.context) - const timelineService = new TwitterTimelineService(page) - - // Create Twitter service with direct service dependencies - this.twitterService = new TwitterService(authService, timelineService) - - // Try to log in - if (config.twitter.credentials) { - const success = await this.twitterService.login(config.twitter.credentials) - if (success) { - logger.main.log('Successfully logged into Twitter') - } - else { - logger.main.error('Twitter login failed!') - } - } - - // Start enabled adapters - if (config.adapters.airi?.enabled) { - logger.main.log('Starting Airi adapter...') - const { AiriAdapter } = await import('./adapters/airi-adapter') - - this.airiAdapter = new AiriAdapter(this.twitterService, { - url: config.adapters.airi.url, - token: config.adapters.airi.token, - credentials: config.twitter.credentials!, - }) - - await this.airiAdapter.start() - logger.main.log('Airi adapter started') - } - - if (config.adapters.mcp?.enabled) { - logger.main.log('Starting MCP adapter...') - const { MCPAdapter } = await import('./adapters/mcp-adapter') - - this.mcpAdapter = new MCPAdapter( - this.twitterService, - config.adapters.mcp.port, - ) - - await this.mcpAdapter.start() - logger.main.log('MCP adapter started') - } - - logger.main.log('Twitter service successfully started!') - - // Set up shutdown hooks - this.setupShutdownHooks() +async function initBrowser(config: any): Promise<{ browser: Browser, context: BrowserContext, page: any }> { + const browser = await chromium.launch({ + headless: config.browser.headless, + }) + + const context = await browser.newContext({ + userAgent: config.browser.userAgent, + viewport: config.browser.viewport, + bypassCSP: true, + }) + + context.setDefaultTimeout(config.browser.timeout || 30000) + const page = await context.newPage() + + logger.main.log('Browser initialized') + return { browser, context, page } +} + +/** + * Initialize Twitter service and login + */ +async function initTwitterService(page: any, context: BrowserContext, config: any): Promise { + const authService = new TwitterAuthService(page, context) + const timelineService = new TwitterTimelineService(page) + const twitterService = new TwitterService(authService, timelineService) + + if (config.twitter.credentials) { + const success = await twitterService.login(config.twitter.credentials) + if (success) { + logger.main.log('Successfully logged into Twitter') } - catch (error) { - logger.main.withError(error).error('Failed to start Twitter service') + else { + logger.main.error('Twitter login failed!') } } - /** - * Stop service - */ - async stop() { - logger.main.log('Stopping Twitter service...') + return twitterService +} - // Stop MCP adapter - if (this.mcpAdapter) { - await this.mcpAdapter.stop() - logger.main.log('MCP adapter stopped') - } +/** + * Initialize adapters + */ +async function initAdapters(twitterService: TwitterService, config: any): Promise<{ airi?: AiriAdapter, mcp?: MCPAdapter }> { + const adapters: { airi?: AiriAdapter, mcp?: MCPAdapter } = {} - // Close browser - if (this.context) { - await this.context.close() - } + if (config.adapters.airi?.enabled) { + logger.main.log('Starting Airi adapter...') + const { AiriAdapter } = await import('./adapters/airi-adapter') - if (this.browser) { - await this.browser.close() - logger.main.log('Browser closed') - } + adapters.airi = new AiriAdapter(twitterService, { + url: config.adapters.airi.url, + token: config.adapters.airi.token, + credentials: config.twitter.credentials!, + }) - logger.main.log('Twitter service stopped') + await adapters.airi.start() + logger.main.log('Airi adapter started') } - /** - * Set up shutdown hooks - */ - private setupShutdownHooks() { - // Handle process exit - process.on('SIGINT', async () => { - logger.main.log('Received exit signal...') - await this.stop() - process.exit(0) - }) + if (config.adapters.mcp?.enabled) { + logger.main.log('Starting MCP adapter...') + const { MCPAdapter } = await import('./adapters/mcp-adapter') - process.on('SIGTERM', async () => { - logger.main.log('Received termination signal...') - await this.stop() - process.exit(0) - }) + adapters.mcp = new MCPAdapter( + twitterService, + config.adapters.mcp.port, + ) - // Handle uncaught exceptions - process.on('uncaughtException', async (error) => { - logger.main.withError(error).error('Uncaught exception') - await this.stop() - process.exit(1) - }) + await adapters.mcp.start() + logger.main.log('MCP adapter started') } + + return adapters } -// Ensure initialization only happens once -async function bootstrap() { - // 1. First initialize logging system - initializeLogger() +/** + * Clean up resources + */ +async function cleanup( + adapters: { airi?: AiriAdapter, mcp?: MCPAdapter }, + context?: BrowserContext, + browser?: Browser, +) { + logger.main.log('Stopping Twitter service...') + + if (adapters.mcp) { + await adapters.mcp.stop() + logger.main.log('MCP adapter stopped') + } + + if (context) { + await context.close() + } + + if (browser) { + await browser.close() + logger.main.log('Browser closed') + } + + logger.main.log('Twitter service stopped') +} + +/** + * Set up process shutdown hooks + */ +function setupShutdownHooks( + adapters: { airi?: AiriAdapter, mcp?: MCPAdapter }, + context?: BrowserContext, + browser?: Browser, +) { + const handleShutdown = async (signal: string) => { + logger.main.log(`Received ${signal} signal...`) + await cleanup(adapters, context, browser) + process.exit(0) + } - // 2. Then create and start service - const launcher = new TwitterServiceLauncher() + process.on('SIGINT', () => handleShutdown('exit')) + process.on('SIGTERM', () => handleShutdown('termination')) + + process.on('uncaughtException', async (error) => { + logger.main.withError(error).error('Uncaught exception') + await cleanup(adapters, context, browser) + process.exit(1) + }) +} + +// Start application +async function bootstrap() { + // Initialize logging system + initLogger() try { - await launcher.start() + const config = useConfigManager().getConfig() + logger.main.log('Starting Twitter service...') + + // Initialize core components + const { browser, context, page } = await initBrowser(config) + const twitterService = await initTwitterService(page, context, config) + const adapters = await initAdapters(twitterService, config) + + // Set up shutdown hooks + setupShutdownHooks(adapters, context, browser) + + logger.main.log('Twitter service successfully started!') } catch (error) { logger.main.withError(error).error('Startup failed') process.exit(1) } - // Set up process event handling + // Handle unhandled rejections process.on('unhandledRejection', (reason) => { logger.main.withError(reason).error('Unhandled Promise rejection:') }) } -// Start application bootstrap() diff --git a/services/twitter-services/src/utils/logger.ts b/services/twitter-services/src/utils/logger.ts index f391ab71..987ef56a 100644 --- a/services/twitter-services/src/utils/logger.ts +++ b/services/twitter-services/src/utils/logger.ts @@ -1,13 +1,13 @@ import path from 'node:path' import { createLogg, Format, LogLevel, setGlobalFormat, setGlobalLogLevel } from '@guiiai/logg' -import { createDefaultConfig } from '../config' +import { useConfigManager } from '../config' // Track initialization status let isInitialized = false // Initialize global logging configuration -export function initializeLogger(): void { +export function initLogger(): void { if (isInitialized) { return // Prevent multiple initializations } @@ -16,7 +16,7 @@ export function initializeLogger(): void { setGlobalLogLevel(LogLevel.Debug) setGlobalFormat(Format.Pretty) - const config = createDefaultConfig().getConfig() + const config = useConfigManager().getConfig() const logLevelMap: Record = { error: LogLevel.Error, From 1438d095ab2553e99b9dd50499ca8edd1fc5ad87 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 16:53:06 +0800 Subject: [PATCH 15/20] feat: load session to auto login --- .gitignore | 2 +- .../src/adapters/mcp-adapter.ts | 41 ++++++-- services/twitter-services/src/config/index.ts | 18 ---- services/twitter-services/src/config/types.ts | 8 +- .../twitter-services/src/core/auth-service.ts | 96 +++++++++---------- .../src/core/twitter-service.ts | 65 ++++++++++++- services/twitter-services/src/main.ts | 35 +++++-- .../twitter-services/src/types/twitter.ts | 13 +-- 8 files changed, 172 insertions(+), 106 deletions(-) diff --git a/.gitignore b/.gitignore index dbba6eac..5e715f34 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,4 @@ coverage/ **/temp/ -*.session.json +twitter-session.json diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts index da84dc67..1b6fb48c 100644 --- a/services/twitter-services/src/adapters/mcp-adapter.ts +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -124,24 +124,23 @@ export class MCPAdapter { // Add login tool this.mcpServer.tool( 'login', - { - username: z.string(), - password: z.string(), - }, - async ({ username, password }) => { + {}, + async () => { try { - const success = await this.twitterService.login({ username, password }) + const success = await this.twitterService.login() return { content: [{ type: 'text', - text: success ? 'Successfully logged into Twitter' : 'Login failed, please check credentials', + text: success + ? '成功从会话文件加载登录状态！如果您是手动登录，系统已设置自动监控来保存您的会话。' + : '没有找到有效的会话文件。请在浏览器中手动登录，系统会自动保存您的会话。', }], } } catch (error) { return { - content: [{ type: 'text', text: `Login failed: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `检查登录状态失败: ${errorToMessage(error)}` }], isError: true, } } @@ -227,6 +226,32 @@ export class MCPAdapter { }, ) + // Add save session tool + this.mcpServer.tool( + 'save-session', + {}, + async () => { + try { + const success = await this.twitterService.saveSession() + + return { + content: [{ + type: 'text', + text: success + ? 'Successfully saved browser session to file. This session will be loaded automatically next time.' + : 'Failed to save browser session', + }], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `Failed to save session: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + // Add search tool this.mcpServer.tool( 'search', diff --git a/services/twitter-services/src/config/index.ts b/services/twitter-services/src/config/index.ts index a4d05a9b..c48e4132 100644 --- a/services/twitter-services/src/config/index.ts +++ b/services/twitter-services/src/config/index.ts @@ -59,9 +59,6 @@ export class ConfigManager { if (configPath) { this.loadFromFile(configPath) } - - // Validate configuration - this.validateConfig() } /** @@ -83,20 +80,6 @@ export class ConfigManager { } } - /** - * Validate configuration validity - */ - private validateConfig(): void { - // Validate Twitter credentials - if (!this.config.twitter.credentials?.username || !this.config.twitter.credentials?.password) { - logger.config.warn('Twitter credentials not set!') - } - - logger.config.withFields({ - config: this.config, - }).log('Configuration validation complete') - } - /** * Get complete configuration */ @@ -110,7 +93,6 @@ export class ConfigManager { updateConfig(newConfig: Partial): void { // Use defu to merge new configuration this.config = defu(newConfig, this.config) - this.validateConfig() } } diff --git a/services/twitter-services/src/config/types.ts b/services/twitter-services/src/config/types.ts index 69f9a720..5c631318 100644 --- a/services/twitter-services/src/config/types.ts +++ b/services/twitter-services/src/config/types.ts @@ -1,5 +1,5 @@ import type { BrowserConfig } from '../types/browser' -import type { SearchOptions, TimelineOptions, TwitterCredentials } from '../types/twitter' +import type { SearchOptions, TimelineOptions } from '../types/twitter' import process from 'node:process' @@ -15,7 +15,6 @@ export interface Config { // Twitter configuration twitter: { - credentials?: TwitterCredentials defaultOptions?: { timeline?: TimelineOptions search?: SearchOptions @@ -64,11 +63,6 @@ export function getDefaultConfig(): Config { requestRetries: Number.parseInt(process.env.BROWSER_REQUEST_RETRIES || '2'), }, twitter: { - credentials: { - username: process.env.TWITTER_USERNAME || '', - password: process.env.TWITTER_PASSWORD || '', - // Don't include cookies here, they will be loaded from session file - }, defaultOptions: { timeline: { count: 20, diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index 7c92dcb2..8b2663fd 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -1,5 +1,4 @@ import type { BrowserContext, Cookie, Page } from 'playwright' -import type { TwitterCredentials } from '../types/twitter' import fs from 'node:fs/promises' import path from 'node:path' @@ -97,21 +96,15 @@ export class TwitterAuthService { } /** - * Login to Twitter - compatibility method for existing code - * Prefers cookie-based login if cookies provided, otherwise redirects to manual login + * Login to Twitter - simplified method that only tries to use session file + * Users are expected to manually login and save the session */ - async login(credentials: TwitterCredentials = {}): Promise { + async login(): Promise { logger.auth.log('Starting Twitter login process') try { - // Check if cookies are provided - if (credentials.cookies && Object.keys(credentials.cookies).length > 0) { - logger.auth.log(`Attempting to login with ${Object.keys(credentials.cookies).length} provided cookies`) - return await this.loginWithCookies(credentials.cookies) - } - // Try to login with existing session first - logger.auth.log('No cookies provided, attempting to load session from file') + logger.auth.log('Attempting to load session from file') const sessionSuccess = await this.checkExistingSession() if (sessionSuccess) { @@ -119,14 +112,8 @@ export class TwitterAuthService { return true } - // If credentials are provided, try username/password login - if (credentials.username && credentials.password) { - logger.auth.log('Session login failed, attempting username/password login') - return await this.initiateManualLogin(credentials.username, credentials.password) - } - - // No credentials and no session, fail - logger.auth.warn('No cookies, no valid session, and no credentials provided') + // Log session failure but don't attempt automatic login + logger.auth.log('No valid session found, manual login is required') return false } catch (error: unknown) { @@ -145,6 +132,17 @@ export class TwitterAuthService { // First check for timeline which is definitive proof of being logged in try { await this.page.waitForSelector(SELECTORS.HOME.TIMELINE, { timeout: 15000 }) + + // Login verification successful - automatically save session + this.isLoggedIn = true + try { + await this.saveCurrentSession() + logger.auth.log('✅ Auto-saved session after successful login verification') + } + catch (error) { + logger.auth.withError(error as Error).warn('Failed to auto-save session') + } + return true } catch { @@ -155,6 +153,17 @@ export class TwitterAuthService { try { const profileSelector = '[data-testid="AppTabBar_Profile_Link"]' await this.page.waitForSelector(profileSelector, { timeout: 5000 }) + + // Profile link found - automatically save session + this.isLoggedIn = true + try { + await this.saveCurrentSession() + logger.auth.log('✅ Auto-saved session after finding profile link') + } + catch (error) { + logger.auth.withError(error as Error).warn('Failed to auto-save session') + } + return true } catch { @@ -199,7 +208,21 @@ export class TwitterAuthService { async checkLoginStatus(): Promise { try { await this.page.goto('https://x.com/home') - return await this.verifyLogin() + const isLoggedIn = await this.verifyLogin() + + // If already logged in, update state and automatically save session + if (isLoggedIn && !this.isLoggedIn) { + this.isLoggedIn = true + try { + await this.saveCurrentSession() + logger.auth.log('✅ Auto-saved session during status check') + } + catch (error) { + logger.auth.withError(error as Error).warn('Failed to auto-save session during status check') + } + } + + return isLoggedIn } catch { return false @@ -509,38 +532,13 @@ export class TwitterAuthService { */ async saveCurrentSession(): Promise { try { - // Get the cookies from the browser - const cookies = await this.exportCookies('object') - - // Format the cookies for the session manager - // We need to convert Record to the Cookie[] format - const cookieArray = Object.entries(cookies).map(([name, value]) => ({ - name, - value, // This will be a string now - domain: '.x.com', - path: '/', - expires: -1, // Session cookie - })) - - // Get the storage state from the browser + // Get the storage state directly from context const storageState = await this.context.storageState() - // Create a new session data object with proper type cast - const sessionData: StorageState = { - cookies: cookieArray.map(cookie => ({ - ...cookie, - sameSite: 'Lax' as const, - secure: true, - httpOnly: true, - })), - origins: storageState.origins, - // Don't include path property here as it's not needed for saving - } - - // Save the session - await sessionManager.saveStorageState(sessionData) + // Save the session using the session manager + await sessionManager.saveStorageState(storageState) - logger.auth.log(`Session saved with ${typeof cookies === 'object' ? Object.keys(cookies).length : 0} cookies`) + logger.auth.log('✅ Session saved to file using browserContext.storageState()') } catch (error) { logger.auth.withError(error as Error).warn('Failed to save session') diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index 1d2b4850..e8ec8d4d 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -5,12 +5,13 @@ import type { TimelineOptions, Tweet, TweetDetail, - TwitterCredentials, UserProfile, } from '../types/twitter' import type { TwitterAuthService } from './auth-service' import type { TwitterTimelineService } from './timeline-service' +import process from 'node:process' + /** * Twitter service implementation * Integrates various service components, providing a unified interface @@ -27,8 +28,8 @@ export class TwitterService implements ITwitterService { /** * Log in to Twitter */ - async login(credentials: TwitterCredentials): Promise { - return await this.authService.login(credentials) + async login(): Promise { + return await this.authService.login() } /** @@ -99,6 +100,20 @@ export class TwitterService implements ITwitterService { throw new Error('Post tweet feature not yet implemented') } + /** + * Save current browser session to file + * This allows users to manually save their session after logging in + */ + async saveSession(): Promise { + try { + await this.authService.saveCurrentSession() + return true + } + catch { + return false + } + } + /** * Ensure authenticated */ @@ -115,4 +130,48 @@ export class TwitterService implements ITwitterService { async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { return await this.authService.exportCookies(format) } + + /** + * Start automatic session monitoring + * Checks login status at regular intervals and saves the session if login is detected + * @param interval Interval in milliseconds, defaults to 30 seconds + */ + startSessionMonitor(interval: number = 30000): void { + // Check immediately in case we're already logged in + this.checkAndSaveSession() + + // Set interval for regular checks + const timer = setInterval(() => { + this.checkAndSaveSession() + }, interval) + + // Clean up timer on process exit + process.on('exit', () => { + clearInterval(timer) + }) + + process.on('SIGINT', () => { + clearInterval(timer) + }) + + process.on('SIGTERM', () => { + clearInterval(timer) + }) + } + + /** + * Check login status and save session if logged in + * @private + */ + private async checkAndSaveSession(): Promise { + try { + const isLoggedIn = this.authService.isAuthenticated() || await this.authService.checkLoginStatus() + if (isLoggedIn) { + await this.saveSession() + } + } + catch { + // Silently handle errors - don't disrupt the application flow + } + } } diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index 9e811c07..3376a22c 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -1,6 +1,7 @@ -import type { Browser, BrowserContext } from 'playwright' +import type { Browser, BrowserContext, Page } from 'playwright' import type { AiriAdapter } from './adapters/airi-adapter' import type { MCPAdapter } from './adapters/mcp-adapter' +import type { Config } from './config/types' import process from 'node:process' import { chromium } from 'playwright' @@ -14,7 +15,7 @@ import { initLogger, logger } from './utils/logger' /** * Initialize browser and create page */ -async function initBrowser(config: any): Promise<{ browser: Browser, context: BrowserContext, page: any }> { +async function initBrowser(config: Config): Promise<{ browser: Browser, context: BrowserContext, page: Page }> { const browser = await chromium.launch({ headless: config.browser.headless, }) @@ -28,6 +29,9 @@ async function initBrowser(config: any): Promise<{ browser: Browser, context: Br context.setDefaultTimeout(config.browser.timeout || 30000) const page = await context.newPage() + // Navigate to Twitter login page by default + await page.goto('https://twitter.com/login') + logger.main.log('Browser initialized') return { browser, context, page } } @@ -35,20 +39,31 @@ async function initBrowser(config: any): Promise<{ browser: Browser, context: Br /** * Initialize Twitter service and login */ -async function initTwitterService(page: any, context: BrowserContext, config: any): Promise { +async function initTwitterService(page: Page, context: BrowserContext, _config: Config): Promise { const authService = new TwitterAuthService(page, context) const timelineService = new TwitterTimelineService(page) const twitterService = new TwitterService(authService, timelineService) - if (config.twitter.credentials) { - const success = await twitterService.login(config.twitter.credentials) - if (success) { - logger.main.log('Successfully logged into Twitter') + // Check if we have a saved session + try { + const sessionSuccess = await authService.checkExistingSession() + if (sessionSuccess) { + logger.main.log('Successfully loaded existing Twitter session') } else { - logger.main.error('Twitter login failed!') + // Instead of automatic login, navigate to login page + logger.main.log('No valid session found, navigating to login page for manual login') + await page.goto('https://twitter.com/login') } } + catch (error) { + logger.main.withError(error as Error).warn('Error checking session, navigating to login page') + await page.goto('https://twitter.com/login') + } + + // Start session monitoring to automatically save session when user logs in + twitterService.startSessionMonitor() + logger.main.log('Started automatic session monitoring') return twitterService } @@ -56,7 +71,7 @@ async function initTwitterService(page: any, context: BrowserContext, config: an /** * Initialize adapters */ -async function initAdapters(twitterService: TwitterService, config: any): Promise<{ airi?: AiriAdapter, mcp?: MCPAdapter }> { +async function initAdapters(twitterService: TwitterService, config: Config): Promise<{ airi?: AiriAdapter, mcp?: MCPAdapter }> { const adapters: { airi?: AiriAdapter, mcp?: MCPAdapter } = {} if (config.adapters.airi?.enabled) { @@ -66,7 +81,7 @@ async function initAdapters(twitterService: TwitterService, config: any): Promis adapters.airi = new AiriAdapter(twitterService, { url: config.adapters.airi.url, token: config.adapters.airi.token, - credentials: config.twitter.credentials!, + credentials: {}, }) await adapters.airi.start() diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index e9a76f91..73359ebe 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -1,12 +1,3 @@ -/** - * Twitter Credentials - */ -export interface TwitterCredentials { - username?: string - password?: string - cookies?: Record -} - /** * Tweet Interface */ @@ -96,7 +87,7 @@ export interface UserLink { * Twitter Service Interface */ export interface TwitterService { - login: (credentials: TwitterCredentials) => Promise + login: () => Promise getTimeline: (options?: TimelineOptions) => Promise getTweetDetails: (tweetId: string) => Promise searchTweets: (query: string, options?: SearchOptions) => Promise @@ -105,4 +96,6 @@ export interface TwitterService { likeTweet: (tweetId: string) => Promise retweet: (tweetId: string) => Promise postTweet: (content: string, options?: PostOptions) => Promise + saveSession: () => Promise + startSessionMonitor: (interval?: number) => void } From f0ddc1f08abf5444f5b90931f6d8b12cb519ffb9 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 17:01:34 +0800 Subject: [PATCH 16/20] docs: update architecture md --- .../twitter-services/docs/architecture.md | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index cfe47376..4be68980 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -85,6 +85,21 @@ Provides integration with the Airi LLM platform, handling event-driven communica Implements the Model Context Protocol interface, providing communication based on HTTP. Currently using the official MCP SDK implementation, providing high-performance HTTP server and SSE communication through H3.js. +The MCP adapter exposes several tools and resources: + +- **Timeline Resource**: Access tweets from the user's timeline +- **Tweet Details Resource**: Get detailed information about a specific tweet +- **User Profile Resource**: Retrieve user profile information + +Additionally, it provides tools for interaction: + +- **Login Tool**: Simplified authentication tool that provides clear feedback on session status. It attempts to load existing sessions, and clearly communicates whether a session was loaded successfully or if manual login is required. The tool no longer requires username/password parameters, as it relies on the enhanced session management system. +- **Post Tweet Tool**: Create and publish new tweets +- **Like Tweet Tool**: Like a tweet by its ID +- **Retweet Tool**: Retweet a tweet by its ID + +The adapter uses internationalized messages (Chinese/English) to provide clear feedback to users about login status and session management. + #### 5.1.3 Development Server Using listhen for optimized development experience, including automatic browser opening, real-time logging, and debugging tools. @@ -100,6 +115,7 @@ The Authentication Service has been significantly enhanced to improve reliabilit 3. **Timeout Optimization**: Adjusted timeouts for various operations to enhance stability during network fluctuations 4. **Enhanced Cookie Management**: Improved cookie storage and loading mechanisms to reduce the need for manual login 5. **Session Validation**: Added comprehensive session validation to verify the integrity of saved sessions +6. **Simplified API**: Removed the need for explicit username/password in the login method, relying instead on session files and browser session detection The service follows a multi-stage authentication approach: @@ -107,7 +123,7 @@ The service follows a multi-stage authentication approach: 2. **Existing Session Detection**: Checks if the browser already has a valid Twitter session 3. **Manual Login Process**: If necessary, guides through the Twitter login page -After successful authentication through any method, sessions are automatically persisted for future use. +After successful authentication through any method, sessions are automatically persisted for future use. The system provides clear feedback to users about the current login state and automatically monitors and saves sessions when changes are detected. #### 5.2.2 Timeline Service (Timeline Service) @@ -152,6 +168,7 @@ Stagehand processes the DOM in chunks to optimize LLM performance and provides f 2. **Response Flow**: Twitter → BrowserBase API → Browser Adapter Layer → Core Service → Data Parsing → Adapter → Application Layer 3. **Authentication Flow**: - Load Session → Check Existing Session → Manual Login → Session Validation → Session Storage + - Clear feedback is provided at each step of the authentication process ## 7. Configuration System @@ -254,7 +271,7 @@ async function main() { const twitter = new TwitterService(browser) // Authenticate - will try multi-stage approach - const loggedIn = await twitter.login({}) + const loggedIn = await twitter.login() if (loggedIn) { console.log('Login successful') @@ -315,6 +332,10 @@ async function connectToTwitterService() { const timeline = await client.get('twitter://timeline/10') console.log('Timeline:', timeline.contents) + // Use simplified login tool without parameters + const loginResult = await client.useTool('login', {}) + console.log('Login result:', loginResult.content[0].text) + // Use tool to send tweet const result = await client.useTool('post-tweet', { content: 'Hello from MCP!' }) console.log('Result:', result.content) @@ -347,6 +368,7 @@ For example, adding "Get Tweets from a Specific User" functionality: - **Selector Updates**: Regularly validate and update selector configurations - **Session Management**: Use the built-in session management system to improve stability and reduce manual login requirements. Consider implementing session rotation and validation. - **Cookie Management**: The system now automatically manages cookie storage via the SessionManager, but consider adding encrypted storage for production environments. +- **User Feedback**: Maintain clear, internationalized feedback messages for authentication status to improve user experience. ### 11.4 Stagehand Maintenance From 471ce13ba2be15f58b80ca2e71ea210e59f8e7bf Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 17:13:36 +0800 Subject: [PATCH 17/20] chore: more debug logs --- .../twitter-services/docs/architecture.md | 11 + .../src/adapters/mcp-adapter.ts | 223 +++++++++++++++--- .../twitter-services/src/core/auth-service.ts | 23 +- .../src/core/twitter-service.ts | 16 ++ .../twitter-services/src/types/twitter.ts | 1 + 5 files changed, 242 insertions(+), 32 deletions(-) diff --git a/services/twitter-services/docs/architecture.md b/services/twitter-services/docs/architecture.md index 4be68980..778c6687 100644 --- a/services/twitter-services/docs/architecture.md +++ b/services/twitter-services/docs/architecture.md @@ -97,6 +97,8 @@ Additionally, it provides tools for interaction: - **Post Tweet Tool**: Create and publish new tweets - **Like Tweet Tool**: Like a tweet by its ID - **Retweet Tool**: Retweet a tweet by its ID +- **Refresh Timeline Tool**: Refresh the timeline with the latest tweets, with options to control the count and whether to include replies and retweets. +- **Get My Profile Tool**: Get information about a user's profile. It can extract the username from the current URL or accept a specific username as a parameter. The adapter uses internationalized messages (Chinese/English) to provide clear feedback to users about login status and session management. @@ -336,6 +338,15 @@ async function connectToTwitterService() { const loginResult = await client.useTool('login', {}) console.log('Login result:', loginResult.content[0].text) + // Use refresh timeline tool to get latest tweets + const refreshResult = await client.useTool('refresh-timeline', { count: 15, includeReplies: false }) + console.log('Refresh result:', refreshResult.content[0].text) + console.log('New tweets:', refreshResult.resources) + + // Get user profile information + const profileResult = await client.useTool('get-my-profile', { username: 'twitter' }) + console.log('Profile info:', profileResult.content[0].text) + // Use tool to send tweet const result = await client.useTool('post-tweet', { content: 'Hello from MCP!' }) console.log('Result:', result.content) diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts index 1b6fb48c..c670da6c 100644 --- a/services/twitter-services/src/adapters/mcp-adapter.ts +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -22,6 +22,7 @@ export class MCPAdapter { private server: ReturnType | null = null private port: number private activeTransports: SSEServerTransport[] = [] + private extraResourceInfo: string[] = [] constructor(twitterService: TwitterService, port: number = 8080) { this.twitterService = twitterService @@ -47,22 +48,33 @@ export class MCPAdapter { * Configure MCP server resources and tools */ private configureServer(): void { - // Add timeline resource + logger.mcp.debug('Configuring MCP server resources and tools...') + + // Add timeline resource with improved registration this.mcpServer.resource( 'timeline', - new ResourceTemplate('twitter://timeline/{count}', { list: async () => ({ - resources: [{ - name: 'timeline', - uri: 'twitter://timeline', - description: 'Tweet timeline', - }], - }) }), - async (_uri: URL, { count }: { count?: string }) => { + new ResourceTemplate('twitter://timeline/{count}', { + list: async () => { + logger.mcp.debug('Listing available timeline resources') + return { + resources: [{ + name: 'timeline', + uri: 'twitter://timeline/10', // Default number of tweets + description: 'Tweet timeline', + }], + } + }, + }), + async (uri: URL, { count }: { count?: string }) => { try { + logger.mcp.withField('uri', uri.toString()).withField('count', count || 'default').debug('Getting timeline') + const tweets = await this.twitterService.getTimeline({ count: count ? Number.parseInt(count) : undefined, }) + logger.mcp.withField('tweetCount', tweets.length).debug('Successfully retrieved timeline tweets') + return { contents: tweets.map(tweet => ({ uri: `twitter://tweet/${tweet.id}`, @@ -71,7 +83,7 @@ export class MCPAdapter { } } catch (error) { - logger.mcp.errorWithError('Error fetching timeline:', error) + logger.mcp.errorWithError('Failed to get timeline:', error) return { contents: [] } } }, @@ -133,14 +145,14 @@ export class MCPAdapter { content: [{ type: 'text', text: success - ? '成功从会话文件加载登录状态！如果您是手动登录，系统已设置自动监控来保存您的会话。' - : '没有找到有效的会话文件。请在浏览器中手动登录，系统会自动保存您的会话。', + ? 'Successfully loaded login state from session file! If you logged in manually, auto-monitoring is set up to save your session.' + : 'No valid session file found. Please log in manually in the browser, the system will automatically save your session.', }], } } catch (error) { return { - content: [{ type: 'text', text: `检查登录状态失败: ${errorToMessage(error)}` }], + content: [{ type: 'text', text: `Failed to check login status: ${errorToMessage(error)}` }], isError: true, } } @@ -280,6 +292,114 @@ export class MCPAdapter { } }, ) + + // Add refresh timeline tool + this.mcpServer.tool( + 'refresh-timeline', + { + count: z.number().optional(), + includeReplies: z.boolean().optional(), + includeRetweets: z.boolean().optional(), + }, + async ({ count, includeReplies, includeRetweets }) => { + try { + const tweets = await this.twitterService.getTimeline({ + count, + includeReplies, + includeRetweets, + }) + + return { + content: [{ + type: 'text', + text: `Successfully refreshed timeline, retrieved ${tweets.length} tweets`, + }], + resources: tweets.map(tweet => `twitter://tweet/${tweet.id}`), + } + } + catch (error) { + return { + content: [{ type: 'text', text: `Failed to refresh timeline: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + + // Add get my profile tool + this.mcpServer.tool( + 'get-my-profile', + { + username: z.string().optional(), + }, + async ({ username }) => { + try { + let profileUsername = username + + // If no username provided, try to get from current URL + if (!profileUsername) { + const currentUrl = await this.twitterService.getCurrentUrl() + profileUsername = this.extractUsernameFromUrl(currentUrl) + } + + // If we still don't have a username, return an error + if (!profileUsername) { + return { + content: [{ + type: 'text', + text: `Failed to get profile: Please provide a username or navigate to a profile page`, + }], + isError: true, + } + } + + const profile = await this.twitterService.getUserProfile(profileUsername) + + return { + content: [{ + type: 'text', + text: `Profile Information:\n` + + `Username: @${profile.username}\n` + + `Display Name: ${profile.displayName}\n` + + `Bio: ${profile.bio || 'Not set'}\n` + + `Followers: ${profile.followersCount || 'N/A'}\n` + + `Following: ${profile.followingCount || 'N/A'}\n` + + `Tweets: ${profile.tweetCount || 'N/A'}\n` + + `Joined: ${profile.joinDate || 'N/A'}`, + }], + resources: [`twitter://user/${profile.username}`], + } + } + catch (error) { + return { + content: [{ type: 'text', text: `Failed to get profile: ${errorToMessage(error)}` }], + isError: true, + } + } + }, + ) + } + + /** + * Extract username from Twitter URL + * @param url Twitter URL + * @returns Username or undefined if not a profile URL + */ + private extractUsernameFromUrl(url: string): string | undefined { + try { + const parsedUrl = new URL(url) + if (parsedUrl.hostname === 'twitter.com' || parsedUrl.hostname === 'x.com') { + const pathParts = parsedUrl.pathname.split('/').filter(Boolean) + if (pathParts.length > 0 && !['search', 'explore', 'home', 'notifications', 'messages'].includes(pathParts[0])) { + return pathParts[0] + } + } + return undefined + } + catch (e) { + logger.mcp.errorWithError('Error extracting username from URL:', e) + return undefined + } } /** @@ -327,6 +447,7 @@ export class MCPAdapter { // Messages endpoint - receive client requests router.post('/messages', defineEventHandler(async (event) => { if (this.activeTransports.length === 0) { + logger.mcp.warn('Received message request but no active SSE connections') event.node.res.statusCode = 503 return { error: 'No active SSE connections' } } @@ -334,6 +455,7 @@ export class MCPAdapter { try { // Parse request body const body = await readBody(event) + logger.mcp.debug(`Received MCP request: ${JSON.stringify(body)}`) // Simple handling - send to most recent transport // Note: In production, should use session ID to route to correct transport @@ -342,9 +464,13 @@ export class MCPAdapter { // Manually handle POST message, as H3 is not Express-compatible const response = await transport.handleMessage(body) + // Log response for debugging + logger.mcp.debug(`MCP response: ${JSON.stringify(response)}`) + return response } catch (error) { + logger.mcp.errorWithError('Error handling MCP message:', error) event.node.res.statusCode = 500 return { error: errorToMessage(error) } } @@ -370,14 +496,46 @@ export class MCPAdapter { * Start MCP server */ start(): Promise { - return new Promise((resolve) => { - // Create Node.js HTTP server - this.server = createServer(toNodeListener(this.app)) - - this.server.listen(this.port, () => { - logger.mcp.withField('port', this.port).log('MCP server started') + return new Promise((resolve, reject) => { + if (this.server !== null) { + logger.mcp.warn('MCP server is already running') resolve() - }) + return + } + + try { + // Create Node.js HTTP server + this.server = createServer(toNodeListener(this.app)) + + // Add error event handlers + this.server.on('error', (error) => { + logger.mcp.errorWithError('MCP server error:', error) + reject(error) + }) + + // Log available resources for debugging + logger.mcp.debug('Registered MCP resources:') + logger.mcp.debug('- twitter://timeline/{count}: Get tweet timeline') + logger.mcp.debug('- twitter://tweet/{id}: Get single tweet details') + logger.mcp.debug('- twitter://user/{username}: Get user profile information') + if (this.extraResourceInfo?.length) { + this.extraResourceInfo.forEach((info) => { + logger.mcp.debug(`- ${info}`) + }) + } + + this.server.listen(this.port, () => { + const serverAddress = `http://localhost:${this.port}` + logger.mcp.log(`MCP server started at: ${serverAddress}`) + logger.mcp.log(`SSE endpoint: ${serverAddress}/sse`) + logger.mcp.log(`Messages endpoint: ${serverAddress}/messages`) + resolve() + }) + } + catch (error) { + logger.mcp.errorWithError('Error starting MCP server:', error) + reject(error) + } }) } @@ -385,20 +543,25 @@ export class MCPAdapter { * Stop MCP server */ stop(): Promise { - return new Promise((resolve, reject) => { - if (!this.server) { - return resolve() + return new Promise((resolve) => { + if (this.server === null) { + logger.mcp.warn('MCP server is not running') + resolve() + return } - this.server.close((error) => { - if (error) { - reject(error) - } - else { + try { + this.server.close(() => { + this.server = null logger.mcp.log('MCP server stopped') resolve() - } - }) + }) + } + catch (error) { + logger.mcp.errorWithError('Error stopping MCP server:', error) + this.server = null + resolve() + } }) } } diff --git a/services/twitter-services/src/core/auth-service.ts b/services/twitter-services/src/core/auth-service.ts index 8b2663fd..f1c103dc 100644 --- a/services/twitter-services/src/core/auth-service.ts +++ b/services/twitter-services/src/core/auth-service.ts @@ -237,8 +237,27 @@ export class TwitterAuthService { } /** - * Export current cookies - * @param format Export format, either 'object' or 'string' + * Get current page URL + * @returns Current URL of the Twitter page + */ + async getCurrentUrl(): Promise { + try { + const currentUrl = await this.page.evaluate(` + (() => { + return window.location.href; + })() + `) + return currentUrl + } + catch (error) { + logger.auth.withError(error as Error).error('Error getting current URL') + throw new Error('Failed to get current URL') + } + } + + /** + * Export cookies from the browser context + * @param format - The format of the returned cookies ('object' or 'string') */ async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { try { diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index e8ec8d4d..40c80a78 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -159,6 +159,22 @@ export class TwitterService implements ITwitterService { }) } + /** + * Get current page URL + * @returns Current URL of the Twitter page + */ + async getCurrentUrl(): Promise { + try { + // We need to access the page from one of our services + // AuthService has direct access to the page object + const currentUrl = await this.authService.getCurrentUrl() + return currentUrl + } + catch (error) { + throw new Error(`Failed to get current URL: ${error}`) + } + } + /** * Check login status and save session if logged in * @private diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index 73359ebe..082e8d31 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -98,4 +98,5 @@ export interface TwitterService { postTweet: (content: string, options?: PostOptions) => Promise saveSession: () => Promise startSessionMonitor: (interval?: number) => void + getCurrentUrl: () => Promise } From 6fbd13377dd460082d2e1d74fe956d7984f71af1 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 18:48:30 +0800 Subject: [PATCH 18/20] feat: tweet parser without hast and rehype --- pnpm-lock.yaml | 98 +++--- services/twitter-services/package.json | 7 - .../src/adapters/airi-adapter.ts | 92 +---- .../src/adapters/mcp-adapter.ts | 4 +- .../src/core/timeline-service.ts | 71 +++- .../src/core/twitter-service.ts | 153 +++++---- services/twitter-services/src/main.ts | 28 +- .../src/parsers/html-parser.ts | 95 ----- .../src/parsers/profile-parser.ts | 325 +++++++++++++----- .../src/parsers/tweet-parser.ts | 286 +++++++++++---- .../twitter-services/src/types/twitter.ts | 19 +- .../twitter-services/src/utils/selectors.ts | 48 ++- 12 files changed, 720 insertions(+), 506 deletions(-) delete mode 100644 services/twitter-services/src/parsers/html-parser.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 197a2efe..260d10f8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -465,7 +465,7 @@ importers: version: 0.8.2 '@gcornut/valibot-json-schema': specifier: ^0.42.0 - version: 0.42.0(esbuild@0.25.0)(typescript@5.8.2) + version: 0.42.0(esbuild@0.19.12)(typescript@5.8.2) '@huggingface/transformers': specifier: ^3.3.3 version: 3.3.3 @@ -537,7 +537,7 @@ importers: version: 2.10.3 '@typeschema/valibot': specifier: ^0.14.0 - version: 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) + version: 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) '@unhead/vue': specifier: ^2.0.0-beta.2 version: 2.0.0-beta.2(vue@3.5.13(typescript@5.8.2)) @@ -739,7 +739,7 @@ importers: version: 28.4.1(@babel/parser@7.26.7)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.34.9))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-macros: specifier: ^2.14.5 - version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + version: 2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin-vue-markdown: specifier: ^28.3.1 version: 28.3.1(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) @@ -1311,9 +1311,6 @@ importers: '@proj-airi/server-sdk': specifier: ^0.1.0 version: 0.1.4 - '@types/hast': - specifier: ^3.0.1 - version: 3.0.4 defu: specifier: ^6.1.4 version: 6.1.4 @@ -1323,15 +1320,6 @@ importers: h3: specifier: ^1.11.0 version: 1.15.1 - hast-util-is-element: - specifier: ^3.0.0 - version: 3.0.0 - hast-util-select: - specifier: ^6.0.4 - version: 6.0.4 - hast-util-to-text: - specifier: ^4.0.2 - version: 4.0.2 listhen: specifier: ^1.6.0 version: 1.9.0 @@ -1341,15 +1329,6 @@ importers: playwright: specifier: ^1.50.1 version: 1.50.1 - rehype-parse: - specifier: ^9.0.0 - version: 9.0.1 - unified: - specifier: ^11.0.3 - version: 11.0.5 - unist-util-visit: - specifier: ^5.0.0 - version: 5.0.0 zod: specifier: ^3.24.2 version: 3.24.2 @@ -7709,9 +7688,6 @@ packages: hast-util-select@6.0.3: resolution: {integrity: sha512-OVRQlQ1XuuLP8aFVLYmC2atrfWHS5UD3shonxpnyrjcCkwtvmt/+N6kYJdcY4mkMJhxp4kj2EFIxQ9kvkkt/eQ==} - hast-util-select@6.0.4: - resolution: {integrity: sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==} - hast-util-to-estree@3.1.1: resolution: {integrity: sha512-IWtwwmPskfSmma9RpzCappDUitC8t5jhAynHhc1m2+5trOgsrp7txscUSavc5Ic8PATyAjfrCK1wgtxh2cICVQ==} @@ -13634,6 +13610,16 @@ snapshots: '@formkit/auto-animate@0.8.2': {} + '@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2)': + dependencies: + valibot: 0.42.1(typescript@5.8.2) + optionalDependencies: + '@types/json-schema': 7.0.15 + esbuild-runner: 2.2.2(esbuild@0.19.12) + transitivePeerDependencies: + - esbuild + - typescript + '@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2)': dependencies: valibot: 0.42.1(typescript@5.8.2) @@ -15392,11 +15378,20 @@ snapshots: dependencies: '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) optionalDependencies: - '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) + '@typeschema/valibot': 0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2)) '@typeschema/zod': 0.14.0(@types/json-schema@7.0.15)(zod-to-json-schema@3.24.3(zod@3.24.2))(zod@3.24.2) transitivePeerDependencies: - '@types/json-schema' + '@typeschema/valibot@0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.19.12)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2))': + dependencies: + '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) + optionalDependencies: + '@gcornut/valibot-json-schema': 0.42.0(esbuild@0.19.12)(typescript@5.8.2) + valibot: 1.0.0-beta.9(typescript@5.8.2) + transitivePeerDependencies: + - '@types/json-schema' + '@typeschema/valibot@0.14.0(@gcornut/valibot-json-schema@0.42.0(esbuild@0.25.0)(typescript@5.8.2))(@types/json-schema@7.0.15)(valibot@1.0.0-beta.9(typescript@5.8.2))': dependencies: '@typeschema/core': 0.14.0(@types/json-schema@7.0.15) @@ -18245,6 +18240,13 @@ snapshots: transitivePeerDependencies: - supports-color + esbuild-runner@2.2.2(esbuild@0.19.12): + dependencies: + esbuild: 0.19.12 + source-map-support: 0.5.21 + tslib: 2.4.0 + optional: true + esbuild-runner@2.2.2(esbuild@0.25.0): dependencies: esbuild: 0.25.0 @@ -19551,24 +19553,6 @@ snapshots: unist-util-visit: 5.0.0 zwitch: 2.0.4 - hast-util-select@6.0.4: - dependencies: - '@types/hast': 3.0.4 - '@types/unist': 3.0.0 - bcp-47-match: 2.0.3 - comma-separated-tokens: 2.0.3 - css-selector-parser: 3.0.5 - devlop: 1.1.0 - direction: 2.0.1 - hast-util-has-property: 3.0.0 - hast-util-to-string: 3.0.1 - hast-util-whitespace: 3.0.0 - nth-check: 2.1.1 - property-information: 7.0.0 - space-separated-tokens: 2.0.2 - unist-util-visit: 5.0.0 - zwitch: 2.0.4 - hast-util-to-estree@3.1.1: dependencies: '@types/estree': 1.0.6 @@ -24036,17 +24020,17 @@ snapshots: '@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.34.9) '@vueuse/core': 12.7.0(typescript@5.8.2) - unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: - esbuild: 0.25.0 - rollup: 2.79.1 + esbuild: 0.19.12 + rollup: 4.34.9 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) - unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): + unplugin-combine@1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)): optionalDependencies: esbuild: 0.25.0 - rollup: 4.34.9 + rollup: 2.79.1 unplugin: 1.16.1 vite: 6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0) @@ -24097,7 +24081,7 @@ snapshots: transitivePeerDependencies: - vue - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.19.12)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -24127,9 +24111,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.19.12)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: @@ -24143,7 +24127,7 @@ snapshots: - vue-tsc - webpack - unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@4.34.9)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): + unplugin-vue-macros@2.14.5(@vueuse/core@12.7.0(typescript@5.8.2))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.2)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)): dependencies: '@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.2)) '@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.2)) @@ -24173,9 +24157,9 @@ snapshots: '@vue-macros/short-bind': 1.1.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-emits': 1.6.5(vue@3.5.13(typescript@5.8.2)) '@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.2)) - '@vue-macros/volar': 0.30.15(rollup@4.34.9)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) + '@vue-macros/volar': 0.30.15(rollup@2.79.1)(typescript@5.8.2)(vue-tsc@2.2.6(typescript@5.8.2))(vue@3.5.13(typescript@5.8.2)) unplugin: 1.16.1 - unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@4.34.9)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) + unplugin-combine: 1.2.0(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.0(@types/node@22.13.8)(jiti@2.4.2)(less@4.2.1)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)) unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.2)) vue: 3.5.13(typescript@5.8.2) transitivePeerDependencies: diff --git a/services/twitter-services/package.json b/services/twitter-services/package.json index 61e4377c..00892d42 100644 --- a/services/twitter-services/package.json +++ b/services/twitter-services/package.json @@ -15,19 +15,12 @@ "@guiiai/logg": "^1.0.0", "@modelcontextprotocol/sdk": "^1.6.1", "@proj-airi/server-sdk": "^0.1.0", - "@types/hast": "^3.0.1", "defu": "^6.1.4", "dotenv": "^16.4.7", "h3": "^1.11.0", - "hast-util-is-element": "^3.0.0", - "hast-util-select": "^6.0.4", - "hast-util-to-text": "^4.0.2", "listhen": "^1.6.0", "ofetch": "^1.3.3", "playwright": "^1.50.1", - "rehype-parse": "^9.0.0", - "unified": "^11.0.3", - "unist-util-visit": "^5.0.0", "zod": "^3.24.2" }, "devDependencies": { diff --git a/services/twitter-services/src/adapters/airi-adapter.ts b/services/twitter-services/src/adapters/airi-adapter.ts index a169b9da..c4099e97 100644 --- a/services/twitter-services/src/adapters/airi-adapter.ts +++ b/services/twitter-services/src/adapters/airi-adapter.ts @@ -1,95 +1,5 @@ -import type { TimelineOptions, TwitterCredentials, TwitterService } from '../types/twitter' - -import { Client } from '@proj-airi/server-sdk' - -import { logger } from '../utils/logger' - /** * Airi Adapter * Adapts the Twitter service as an Airi module */ -export class AiriAdapter { - private client: Client - private twitterService: TwitterService - private credentials: TwitterCredentials - - constructor(twitterService: TwitterService, options: { - url?: string - token?: string - credentials: TwitterCredentials - }) { - this.twitterService = twitterService - this.credentials = options.credentials - - this.client = new Client({ - url: options.url, - name: 'twitter-module', - token: options.token, - possibleEvents: [ - // Define event types this module can handle - 'twitter:login', - 'twitter:getTimeline', - 'twitter:getTweetDetails', - 'twitter:searchTweets', - 'twitter:getUserProfile', - 'twitter:followUser', - 'twitter:likeTweet', - 'twitter:retweet', - 'twitter:postTweet', - ], - }) - - this.setupEventHandlers() - } - - /** - * Set up event handlers - */ - private setupEventHandlers(): void { - // Login handler - this.client.onEvent('twitter:login', async (event) => { - try { - const credentials = event.data.credentials as TwitterCredentials || this.credentials - const success = await this.twitterService.login(credentials) - this.client.send({ - type: 'twitter:loginResult', - data: { success }, - }) - } - catch (error) { - this.client.send({ - type: 'twitter:error', - data: { error: error.message, operation: 'login' }, - }) - } - }) - - // Timeline handler - this.client.onEvent('twitter:getTimeline', async (event) => { - try { - const options = event.data.options as TimelineOptions || {} - const tweets = await this.twitterService.getTimeline(options) - this.client.send({ - type: 'twitter:timelineResult', - data: { tweets }, - }) - } - catch (error) { - this.client.send({ - type: 'twitter:error', - data: { error: error.message, operation: 'getTimeline' }, - }) - } - }) - - // Other event handlers... - } - - /** - * Start the adapter - */ - async start(): Promise { - // Initialization logic can be added here - logger.airi.log('Airi Twitter adapter started') - } -} +export class AiriAdapter {} diff --git a/services/twitter-services/src/adapters/mcp-adapter.ts b/services/twitter-services/src/adapters/mcp-adapter.ts index c670da6c..c1fb4e9a 100644 --- a/services/twitter-services/src/adapters/mcp-adapter.ts +++ b/services/twitter-services/src/adapters/mcp-adapter.ts @@ -1,4 +1,4 @@ -import type { TwitterService } from '../types/twitter' +import type { TwitterService } from '../core/twitter-service' import { Buffer } from 'node:buffer' import { createServer } from 'node:http' @@ -388,7 +388,7 @@ export class MCPAdapter { private extractUsernameFromUrl(url: string): string | undefined { try { const parsedUrl = new URL(url) - if (parsedUrl.hostname === 'twitter.com' || parsedUrl.hostname === 'x.com') { + if (parsedUrl.hostname === 'x.com') { const pathParts = parsedUrl.pathname.split('/').filter(Boolean) if (pathParts.length > 0 && !['search', 'explore', 'home', 'notifications', 'messages'].includes(pathParts[0])) { return pathParts[0] diff --git a/services/twitter-services/src/core/timeline-service.ts b/services/twitter-services/src/core/timeline-service.ts index 04d71a1b..5b4c6a59 100644 --- a/services/twitter-services/src/core/timeline-service.ts +++ b/services/twitter-services/src/core/timeline-service.ts @@ -17,21 +17,29 @@ export class TwitterTimelineService { } /** - * Get timeline + * Fetches the Twitter timeline + * @param options Configuration options for timeline fetching + * @returns Promise resolving to an array of tweets */ async getTimeline(options: TimelineOptions = {}): Promise { try { + logger.timeline.withFields({ options }).log('Fetching timeline') + // Navigate to home page await this.page.goto('https://x.com/home') // Wait for timeline to load await this.page.waitForSelector(SELECTORS.TIMELINE.TWEET, { timeout: 10000 }) - // Get page HTML and parse all tweets - const html = await this.page.content() - const tweets = TweetParser.parseTimelineTweets(html) + // Optional: scroll to load more tweets if needed + if (options.count && options.count > 5) { + await this.scrollToLoadMoreTweets(Math.min(options.count, 20)) + } + + // Parse all tweets directly from the DOM using Playwright + const tweets = await TweetParser.parseTimelineTweets(this.page) - logger.main.log(`Found ${tweets.length} tweets in timeline`) + logger.timeline.log(`Found ${tweets.length} tweets in timeline`) // Apply filters let filteredTweets = tweets @@ -52,8 +60,59 @@ export class TwitterTimelineService { return filteredTweets } catch (error) { - logger.main.withError(error as Error).error('Failed to get timeline') + logger.timeline.error('Failed to get timeline:', (error as Error).message) return [] } } + + /** + * Scrolls down the timeline to load more tweets + * @param targetCount Approximate number of tweets to load + */ + private async scrollToLoadMoreTweets(targetCount: number): Promise { + try { + // Initial tweet count + let previousTweetCount = 0 + let currentTweetCount = await this.countVisibleTweets() + let scrollAttempts = 0 + const maxScrollAttempts = 10 + + logger.timeline.log(`Initial tweet count: ${currentTweetCount}, target: ${targetCount}`) + + // Scroll until we have enough tweets or reach maximum scroll attempts + while (currentTweetCount < targetCount && scrollAttempts < maxScrollAttempts) { + // Scroll down using Playwright's mouse wheel simulation + await this.page.mouse.wheel(0, 800) + + // Wait for new content to load + await this.page.waitForTimeout(1000) + + // Check if we have new tweets + previousTweetCount = currentTweetCount + currentTweetCount = await this.countVisibleTweets() + + // If no new tweets were loaded, we might have reached the end + if (currentTweetCount === previousTweetCount) { + scrollAttempts++ + } + else { + scrollAttempts = 0 // Reset counter if we're still loading tweets + } + + logger.timeline.debug(`Scrolled for more tweets: ${currentTweetCount}/${targetCount}`) + } + } + catch (error) { + logger.timeline.error('Error while scrolling for more tweets:', (error as Error).message) + } + } + + /** + * Counts the number of visible tweets on the page + * @returns Promise resolving to the count of visible tweets + */ + private async countVisibleTweets(): Promise { + const tweetElements = await this.page.$$(SELECTORS.TIMELINE.TWEET) + return tweetElements.length + } } diff --git a/services/twitter-services/src/core/twitter-service.ts b/services/twitter-services/src/core/twitter-service.ts index 40c80a78..d7f91c0c 100644 --- a/services/twitter-services/src/core/twitter-service.ts +++ b/services/twitter-services/src/core/twitter-service.ts @@ -1,24 +1,13 @@ -import type { - TwitterService as ITwitterService, - PostOptions, - SearchOptions, - TimelineOptions, - Tweet, - TweetDetail, - UserProfile, -} from '../types/twitter' +import type { PostOptions, SearchOptions, TimelineOptions, Tweet, TweetDetail, UserProfile } from '../types/twitter' import type { TwitterAuthService } from './auth-service' import type { TwitterTimelineService } from './timeline-service' -import process from 'node:process' +import { logger } from '../utils/logger' -/** - * Twitter service implementation - * Integrates various service components, providing a unified interface - */ -export class TwitterService implements ITwitterService { +export class TwitterService { private authService: TwitterAuthService private timelineService: TwitterTimelineService + private sessionMonitorInterval: NodeJS.Timeout | null = null constructor(authService: TwitterAuthService, timelineService: TwitterTimelineService) { this.authService = authService @@ -26,10 +15,31 @@ export class TwitterService implements ITwitterService { } /** - * Log in to Twitter + * Login to Twitter + * Attempts to restore session from saved cookies first + * If that fails, will need manual login in the browser */ async login(): Promise { - return await this.authService.login() + try { + // Try to restore session from cookies + const success = await this.authService.login() + + if (success) { + logger.main.log('Successfully restored Twitter session from cookies') + return true + } + + logger.main.log('No saved session found, waiting for manual login') + + // Set up session monitoring to detect when user has manually logged in + this.startSessionMonitor() + + return false + } + catch (error) { + logger.main.error('Error during login:', (error as Error).message) + return false + } } /** @@ -37,15 +47,15 @@ export class TwitterService implements ITwitterService { */ async getTimeline(options?: TimelineOptions): Promise { this.ensureAuthenticated() - return await this.timelineService.getTimeline(options) + return this.timelineService.getTimeline(options) } /** - * Get tweet details (not implemented in MVP) + * Get tweet details */ async getTweetDetails(tweetId: string): Promise { this.ensureAuthenticated() - // In MVP stage, return a basic structure + // This is a stub implementation return { id: tweetId, text: 'Tweet details feature not yet implemented', @@ -94,100 +104,119 @@ export class TwitterService implements ITwitterService { } /** - * Post tweet + * Post a tweet */ async postTweet(_content: string, _options?: PostOptions): Promise { throw new Error('Post tweet feature not yet implemented') } /** - * Save current browser session to file - * This allows users to manually save their session after logging in + * Manually trigger a session save + * Typically this is handled by the session monitor */ async saveSession(): Promise { try { + if (!this.authService.isAuthenticated()) { + logger.main.warn('Cannot save session when not authenticated') + return false + } + await this.authService.saveCurrentSession() + logger.main.log('Successfully saved Twitter session') return true } - catch { + catch (error) { + logger.main.error('Error saving session:', (error as Error).message) return false } } /** - * Ensure authenticated + * Ensure the user is authenticated before performing operations + * @private */ private ensureAuthenticated(): void { if (!this.authService.isAuthenticated()) { - throw new Error('Not authenticated. Call login() first.') + throw new Error('You must be logged in to perform this action. Please call login() first.') } } /** - * Export current session cookies - * @param format - The format of the returned cookies ('object' or 'string') + * Export the current session cookies + * @param format The format to export cookies in */ async exportCookies(format: 'object' | 'string' = 'object'): Promise | string> { - return await this.authService.exportCookies(format) + this.ensureAuthenticated() + return this.authService.exportCookies(format) } /** - * Start automatic session monitoring - * Checks login status at regular intervals and saves the session if login is detected - * @param interval Interval in milliseconds, defaults to 30 seconds + * Start monitoring for session changes + * This will periodically check if the user is logged in + * and save the session if they are + * @param interval Time in ms between checks */ startSessionMonitor(interval: number = 30000): void { - // Check immediately in case we're already logged in - this.checkAndSaveSession() - - // Set interval for regular checks - const timer = setInterval(() => { - this.checkAndSaveSession() - }, interval) - - // Clean up timer on process exit - process.on('exit', () => { - clearInterval(timer) - }) + // Clear any existing monitor + if (this.sessionMonitorInterval) { + clearInterval(this.sessionMonitorInterval) + } - process.on('SIGINT', () => { - clearInterval(timer) - }) + logger.main.log(`Starting Twitter session monitor with ${interval}ms interval`) - process.on('SIGTERM', () => { - clearInterval(timer) - }) + this.sessionMonitorInterval = setInterval(async () => { + try { + await this.checkAndSaveSession() + } + catch (error) { + logger.main.error('Error in session monitor:', (error as Error).message) + } + }, interval) } /** - * Get current page URL - * @returns Current URL of the Twitter page + * Get the current page URL + * Useful for debugging and checking the current state */ async getCurrentUrl(): Promise { try { - // We need to access the page from one of our services - // AuthService has direct access to the page object - const currentUrl = await this.authService.getCurrentUrl() - return currentUrl + return await this.authService.getCurrentUrl() } catch (error) { - throw new Error(`Failed to get current URL: ${error}`) + logger.main.error('Error getting current URL:', (error as Error).message) + return 'unknown' } } /** - * Check login status and save session if logged in + * Stop the session monitor + */ + stopSessionMonitor(): void { + if (this.sessionMonitorInterval) { + clearInterval(this.sessionMonitorInterval) + this.sessionMonitorInterval = null + logger.main.log('Stopped Twitter session monitor') + } + } + + /** + * Check and save the session if logged in * @private */ private async checkAndSaveSession(): Promise { try { - const isLoggedIn = this.authService.isAuthenticated() || await this.authService.checkLoginStatus() - if (isLoggedIn) { + const isLoggedIn = await this.authService.checkLoginStatus() + + if (isLoggedIn && !this.authService.isAuthenticated()) { + logger.main.log('User has logged in manually, saving session') await this.saveSession() } + else if (!isLoggedIn && this.authService.isAuthenticated()) { + logger.main.warn('User appears to be logged out, updating state') + } } - catch { - // Silently handle errors - don't disrupt the application flow + catch (error) { + logger.main.error('Error checking session status:', (error as Error).message) } } } diff --git a/services/twitter-services/src/main.ts b/services/twitter-services/src/main.ts index 3376a22c..1f338d75 100644 --- a/services/twitter-services/src/main.ts +++ b/services/twitter-services/src/main.ts @@ -30,7 +30,7 @@ async function initBrowser(config: Config): Promise<{ browser: Browser, context: const page = await context.newPage() // Navigate to Twitter login page by default - await page.goto('https://twitter.com/login') + await page.goto('https://x.com/login') logger.main.log('Browser initialized') return { browser, context, page } @@ -53,12 +53,12 @@ async function initTwitterService(page: Page, context: BrowserContext, _config: else { // Instead of automatic login, navigate to login page logger.main.log('No valid session found, navigating to login page for manual login') - await page.goto('https://twitter.com/login') + await page.goto('https://x.com/login') } } catch (error) { logger.main.withError(error as Error).warn('Error checking session, navigating to login page') - await page.goto('https://twitter.com/login') + await page.goto('https://x.com/login') } // Start session monitoring to automatically save session when user logs in @@ -74,19 +74,19 @@ async function initTwitterService(page: Page, context: BrowserContext, _config: async function initAdapters(twitterService: TwitterService, config: Config): Promise<{ airi?: AiriAdapter, mcp?: MCPAdapter }> { const adapters: { airi?: AiriAdapter, mcp?: MCPAdapter } = {} - if (config.adapters.airi?.enabled) { - logger.main.log('Starting Airi adapter...') - const { AiriAdapter } = await import('./adapters/airi-adapter') + // if (config.adapters.airi?.enabled) { + // logger.main.log('Starting Airi adapter...') + // const { AiriAdapter } = await import('./adapters/airi-adapter') - adapters.airi = new AiriAdapter(twitterService, { - url: config.adapters.airi.url, - token: config.adapters.airi.token, - credentials: {}, - }) + // adapters.airi = new AiriAdapter(twitterService, { + // url: config.adapters.airi.url, + // token: config.adapters.airi.token, + // credentials: {}, + // }) - await adapters.airi.start() - logger.main.log('Airi adapter started') - } + // await adapters.airi.start() + // logger.main.log('Airi adapter started') + // } if (config.adapters.mcp?.enabled) { logger.main.log('Starting MCP adapter...') diff --git a/services/twitter-services/src/parsers/html-parser.ts b/services/twitter-services/src/parsers/html-parser.ts deleted file mode 100644 index 8d56c0bf..00000000 --- a/services/twitter-services/src/parsers/html-parser.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { Element, Root } from 'hast' - -import rehypeParse from 'rehype-parse' -import { unified } from 'unified' -import { visit } from 'unist-util-visit' - -export interface ParseOptions { - fragment?: boolean -} - -/** - * HTML Parser - * Uses rehype to convert HTML strings to AST, for further processing - */ -export class HtmlParser { - /** - * Parse HTML string to rehype AST - * @param html HTML string - * @param options Parse options - * @returns hast syntax tree - */ - static parse(html: string, options: ParseOptions = {}): Root { - const processor = unified().use(rehypeParse, { - fragment: options.fragment ?? false, - }) - - const tree = processor.parse(html) - const file = processor.runSync(tree) - - return file as Root - } - - /** - * Find elements by selector - * @param tree AST tree - * @param selector Simplified selector (tagName, className, id) - * @returns Matching element array - */ - static select(tree: Root, selector: string): Element[] { - const elements: Element[] = [] - - visit(tree, 'element', (node) => { - // Simple selector implementation - if (this.matchesSelector(node, selector)) { - elements.push(node) - } - }) - - return elements - } - - /** - * Simple selector matching logic - */ - private static matchesSelector(node: Element, selector: string): boolean { - // Tag selector - if (selector.match(/^[a-z0-9]+$/i)) { - return node.tagName === selector - } - - // Class selector - if (selector.startsWith('.')) { - const className = selector.slice(1) - return (node.properties?.className as string[])?.includes(className) ?? false - } - - // ID selector - if (selector.startsWith('#')) { - const id = selector.slice(1) - return node.properties?.id === id - } - - // Data attribute selector - if (selector.startsWith('[data-')) { - // Use non-greedy quantifier and more specific character classes to avoid backtracking - const match = selector.match(/\[([^=\]]+)=(['"]?)([^"'\]]+)\2\]/) - if (match) { - const [, attr, , value] = match - return node.properties?.[attr] === value - } - } - - return false - } - - /** - * Visit specific node type - * @param tree AST tree - * @param nodeType Node type - * @param visitor Visitor function - */ - static visit(tree: Element | Root, nodeType: string, visitor: (node: any) => void): void { - visit(tree, nodeType, visitor) - } -} diff --git a/services/twitter-services/src/parsers/profile-parser.ts b/services/twitter-services/src/parsers/profile-parser.ts index 15c4e639..d29e0577 100644 --- a/services/twitter-services/src/parsers/profile-parser.ts +++ b/services/twitter-services/src/parsers/profile-parser.ts @@ -1,144 +1,293 @@ -import type { Element, Node, Root } from 'hast' +import type { Page } from 'playwright' import type { UserLink, UserProfile, UserStats } from '../types/twitter' -import { select } from 'hast-util-select' - +import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' -import { HtmlParser } from './html-parser' /** * Profile Parser - * Extracts user profile information from HTML + * Extracts user profile information directly from the page DOM using Playwright */ export class ProfileParser { /** - * Parse user profile from HTML - * @param html HTML string - * @returns User profile + * Parse user profile from a Twitter profile page + * @param page Playwright page instance + * @returns Promise resolving to UserProfile object */ - static parseUserProfile(html: string): UserProfile { - const tree = HtmlParser.parse(html) + static async parseUserProfile(page: Page): Promise { + try { + // Extract basic profile info + const displayNameElement = await page.$(SELECTORS.PROFILE.DISPLAY_NAME) + const displayName = await displayNameElement?.textContent() || 'Unknown User' - // Extract username and display name - const displayNameElement = HtmlParser.select(tree, SELECTORS.PROFILE.DISPLAY_NAME)[0] - const displayName = this.extractTextContent(displayNameElement) || 'Unknown User' + // Get username from URL or profile elements + let username = '' + const url = page.url() + const urlUsername = this.extractUsernameFromUrl(url) - // Extract username from URL or DOM - const username = this.extractUsername(tree) || 'unknown' + if (urlUsername) { + username = urlUsername + } + else { + // Try to find username in the DOM + const usernameElement = await page.$('[data-testid="UserName"] span:has-text("@")') + const usernameText = await usernameElement?.textContent() + username = usernameText?.replace('@', '') || 'unknown' + } - // Extract user bio - const bioElement = HtmlParser.select(tree, SELECTORS.PROFILE.BIO)[0] - const bio = this.extractTextContent(bioElement) + // Get bio + const bioElement = await page.$(SELECTORS.PROFILE.BIO) + const bio = await bioElement?.textContent() - // Extract user stats - const stats = this.extractProfileStats(tree) + // Get profile images + const avatarUrl = await this.extractAvatarUrl(page) + const bannerUrl = await this.extractBannerUrl(page) - // Extract avatar and banner URL - const avatarUrl = this.extractAvatarUrl(tree) - const bannerUrl = this.extractBannerUrl(tree) + // Get statistics + const stats = await this.extractUserStats(page) - return { - username, - displayName, - bio, - avatarUrl, - bannerUrl, - ...stats, - } - } + // Get join date + const joinDate = await this.extractJoinDate(page) - /** - * Extract text content - */ - private static extractTextContent(element?: Element): string { - if (!element) - return '' + // Get user links + // const _links = await this.extractUserLinks(page) + + const profile: UserProfile = { + username, + displayName, + } + + // Add optional fields if they exist + if (bio) + profile.bio = bio + if (avatarUrl) + profile.avatarUrl = avatarUrl + if (bannerUrl) + profile.bannerUrl = bannerUrl + if (stats.followers) + profile.followersCount = stats.followers + if (stats.following) + profile.followingCount = stats.following + if (stats.tweets) + profile.tweetCount = stats.tweets + if (joinDate) + profile.joinDate = joinDate - let text = '' - HtmlParser.visit(element, 'text', (node) => { - text += node.value - }) + // Check for verification badge + const isVerified = await page.$('[data-testid="icon-verified"]') !== null + if (isVerified) + profile.isVerified = true - return text.trim() + return profile + } + catch (error) { + logger.parser.error('Error parsing user profile:', (error as Error).message) + + // Return minimal profile to avoid breaking + return { + username: 'unknown', + displayName: 'Unknown User', + } + } } /** - * Extract username from page + * Extract username from Twitter profile URL + * @param url Twitter profile URL + * @returns Username or null if not found */ - private static extractUsername(_tree: Root): string { - // TODO: Can be extracted from URL or specific DOM element - return '' + private static extractUsernameFromUrl(url: string): string | null { + try { + const match = url.match(/twitter\.com\/([^/]+)/) + if (match && match[1] && !['home', 'explore', 'notifications', 'messages'].includes(match[1])) { + return match[1] + } + return null + } + catch { + return null + } } /** - * Extract user stats + * Extract user statistics (followers, following, tweets) + * @param page Playwright page instance + * @returns Promise resolving to UserStats object */ - private static extractProfileStats(tree: Root) { - // TODO: Extract followers, following, tweet count, etc. - const _statsElement = HtmlParser.select(tree, SELECTORS.PROFILE.STATS)[0] + private static async extractUserStats(page: Page): Promise { + const stats: UserStats = { + followers: 0, + following: 0, + tweets: 0, + } + + try { + // Get stats container + const statsContainer = await page.$(SELECTORS.PROFILE.STATS) + if (!statsContainer) + return stats - return { - followersCount: undefined, - followingCount: undefined, - tweetCount: undefined, - isVerified: false, - joinDate: undefined, + // Get all stat items + const statItems = await statsContainer.$$('a') + + for (const statItem of statItems) { + const text = await statItem.textContent() || '' + + if (text.includes('Following')) { + const countText = text.replace(/Following.*/, '').trim() + stats.following = this.parseStatNumber(countText) + } + else if (text.includes('Followers')) { + const countText = text.replace(/Followers.*/, '').trim() + stats.followers = this.parseStatNumber(countText) + } + else if (text.includes('posts') || text.includes('Posts')) { + const countText = text.replace(/posts|Posts.*/, '').trim() + stats.tweets = this.parseStatNumber(countText) + } + } + + return stats + } + catch (error) { + logger.parser.error('Error extracting user stats:', (error as Error).message) + return stats } } /** - * Extract avatar URL + * Extract profile avatar URL + * @param page Playwright page instance + * @returns Promise resolving to avatar URL or undefined */ - private static extractAvatarUrl(_tree: Root): string | undefined { - // TODO: Extract avatar image URL - return undefined + private static async extractAvatarUrl(page: Page): Promise { + try { + const avatarElement = await page.$('img[src*="profile_images"]') + const src = await avatarElement?.getAttribute('src') + return src || undefined + } + catch (error) { + logger.parser.error('Error extracting avatar URL:', (error as Error).message) + return undefined + } } /** - * Extract banner URL + * Extract profile banner URL + * @param page Playwright page instance + * @returns Promise resolving to banner URL or undefined */ - private static extractBannerUrl(_tree: Root): string | undefined { - // TODO: Extract banner image URL - return undefined + private static async extractBannerUrl(page: Page): Promise { + try { + const bannerElement = await page.$('img[src*="profile_banners"]') + const src = await bannerElement?.getAttribute('src') + return src || undefined + } + catch (error) { + logger.parser.error('Error extracting banner URL:', (error as Error).message) + return undefined + } } /** - * Extract user stats + * Extract join date from profile + * @param page Playwright page instance + * @returns Promise resolving to join date string or undefined */ - static extractUserStats(_html: string, _tree?: Node): UserStats { - // Parse HTML to get stats - const stats: UserStats = { - tweets: 0, - following: 0, - followers: 0, - } - + private static async extractJoinDate(page: Page): Promise { try { - // Find stats container - const _statsElement = _tree ? select('[data-testid="userProfileStats"]', _tree as Root) : null + // Try to find join date text that usually appears as "Joined Month Year" + const joinedText = await page.$('span:has-text("Joined")') + if (!joinedText) + return undefined - // TODO: Not implemented yet + const fullText = await joinedText.textContent() + if (fullText && fullText.includes('Joined')) { + // Extract just the date part + const datePart = fullText.replace('Joined', '').trim() + return datePart || undefined + } - return stats + return undefined } - catch { - return stats + catch (error) { + logger.parser.error('Error extracting join date:', (error as Error).message) + return undefined } } /** - * Extract user links + * Extract user links (website, location) + * @param page Playwright page instance + * @returns Promise resolving to array of user links */ - static extractUserLinks(_html: string, _tree?: Node): UserLink[] { - // TODO: Not implemented yet - return [] + private static async extractUserLinks(page: Page): Promise { + const links: UserLink[] = [] + + try { + // Find all link elements in profile + const linkElements = await page.$$('a[href^="https"]:not([href*="twitter.com"])') + + for (const linkElement of linkElements) { + // Extract href and title + const href = await linkElement.getAttribute('href') + const title = await linkElement.textContent() + + if (href && title) { + links.push({ + type: 'url', + url: href, + title, + }) + } + } + + // Try to find location + const locationElement = await page.$('span:has-text("Location")') + if (locationElement) { + const locationText = await locationElement.textContent() + if (locationText) { + links.push({ + type: 'location', + url: '', + title: locationText.replace('Location', '').trim(), + }) + } + } + + return links + } + catch (error) { + logger.parser.error('Error extracting user links:', (error as Error).message) + return links + } } /** - * Extract user join date + * Parse stat number with K, M suffixes + * @param text Number text (e.g., "10.5K") + * @returns Parsed number */ - static extractJoinDate(_html: string, _tree?: Node): string | null { - // TODO: Not implemented yet - return null + private static parseStatNumber(text: string): number { + try { + text = text.trim() + + if (!text) + return 0 + + if (text.includes('K')) { + return Math.round(Number.parseFloat(text.replace('K', '')) * 1000) + } + else if (text.includes('M')) { + return Math.round(Number.parseFloat(text.replace('M', '')) * 1000000) + } + + // Handle other formats like 1,234 + const normalized = text.replace(/,/g, '') + return Number.parseInt(normalized, 10) || 0 + } + catch { + return 0 + } } } diff --git a/services/twitter-services/src/parsers/tweet-parser.ts b/services/twitter-services/src/parsers/tweet-parser.ts index c14781a5..4f535548 100644 --- a/services/twitter-services/src/parsers/tweet-parser.ts +++ b/services/twitter-services/src/parsers/tweet-parser.ts @@ -1,104 +1,260 @@ -import type { Element } from 'hast' +import type { ElementHandle, Page } from 'playwright' import type { Tweet } from '../types/twitter' +import { logger } from '../utils/logger' import { SELECTORS } from '../utils/selectors' -import { HtmlParser } from './html-parser' /** * Tweet Parser - * Extracts tweet information from HTML + * Extracts tweet information directly from the page DOM using Playwright */ export class TweetParser { /** - * Parse timeline tweets from HTML - * @param html HTML string - * @returns Tweet array + * Parse timeline tweets directly from the page + * @param page Playwright page instance + * @returns Promise resolving to Tweet array */ - static parseTimelineTweets(html: string): Tweet[] { - const tree = HtmlParser.parse(html) - const tweetElements = HtmlParser.select(tree, SELECTORS.TIMELINE.TWEET) + static async parseTimelineTweets(page: Page): Promise { + try { + const tweetElements = await page.$$(SELECTORS.TIMELINE.TWEET) + logger.parser.log(`Found ${tweetElements.length} tweet elements`) - return tweetElements.map(el => this.extractTweetData(el)) + const tweets: Tweet[] = [] + + for (const tweetElement of tweetElements) { + const tweet = await this.extractTweetData(page, tweetElement) + if (tweet) { + tweets.push(tweet) + } + } + + return tweets + } + catch (error) { + logger.parser.error('Error parsing timeline tweets:', (error as Error).message) + return [] + } } /** * Extract tweet data from tweet element - * @param element Tweet element - * @returns Tweet data + * @param page Playwright page instance + * @param tweetElement Tweet element handle + * @returns Promise resolving to Tweet object */ - static extractTweetData(element: Element): Tweet { - // Get tweet ID - const id = this.extractTweetId(element) - - // Get tweet text - const textElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TEXT)[0] - const text = this.extractTextContent(textElement) - - // Get author info - const author = this.extractAuthorInfo(element) - - // Get timestamp - const timeElement = HtmlParser.select(element, SELECTORS.TIMELINE.TWEET_TIME)[0] - const timestamp = timeElement?.properties?.datetime as string || new Date().toISOString() - - // Get stats - const stats = this.extractTweetStats(element) - - return { - id, - text, - author, - timestamp, - ...stats, + static async extractTweetData(page: Page, tweetElement: ElementHandle): Promise { + try { + // Extract tweet ID + const id = await this.extractTweetId(tweetElement) + + // Extract tweet text + const textElement = await tweetElement.$(SELECTORS.TIMELINE.TWEET_TEXT) + const text = textElement ? await textElement.textContent() : '' + + // Extract author info + const author = await this.extractAuthorInfo(tweetElement) + + // Extract timestamp + const timeElement = await tweetElement.$('time') + const timestamp = timeElement ? await timeElement.getAttribute('datetime') : new Date().toISOString() + + // Extract engagement stats + const stats = await this.extractTweetStats(tweetElement) + + // Extract media URLs + const mediaUrls = await this.extractMediaUrls(tweetElement) + + const tweet: Tweet = { + id, + text: text || '', + author, + timestamp, + ...stats, + } + + if (mediaUrls.length > 0) { + tweet.mediaUrls = mediaUrls + } + + return tweet + } + catch (error) { + logger.parser.error('Error extracting tweet data:', (error as Error).message) + return null } } /** - * Extract tweet ID + * Extract tweet ID from tweet element + * @param tweetElement Tweet element handle + * @returns Promise resolving to tweet ID */ - private static extractTweetId(element: Element): string { - // Extract ID from data-tweet-id attribute or other location - return element.properties?.['data-tweet-id'] as string - || `tweet-${Math.random().toString(36).substring(2, 15)}` + private static async extractTweetId(tweetElement: ElementHandle): Promise { + try { + // Try to get ID from status link + const statusLink = await tweetElement.$('a[href*="/status/"]') + if (statusLink) { + const href = await statusLink.getAttribute('href') + if (href) { + const match = href.match(/\/status\/(\d+)/) + if (match && match[1]) { + return match[1] + } + } + } + + // Fallback to a random ID + return `tweet-${Date.now()}-${Math.floor(Math.random() * 1000)}` + } + catch (error) { + logger.parser.error('Error extracting tweet ID:', (error as Error).message) + return `tweet-${Date.now()}` + } } /** - * Extract text content + * Extract author info from tweet element + * @param tweetElement Tweet element handle + * @returns Promise resolving to author object */ - private static extractTextContent(element?: Element): string { - if (!element) - return '' + private static async extractAuthorInfo(tweetElement: ElementHandle): Promise { + try { + // Find author element + const authorElement = await tweetElement.$('[data-testid="User-Name"]') + if (!authorElement) { + return { + username: 'unknown', + displayName: 'Unknown User', + } + } + + // Get display name + const displayNameElement = await authorElement.$('span:first-child') + const displayName = displayNameElement ? await displayNameElement.textContent() : 'Unknown User' - let text = '' - HtmlParser.visit(element, 'text', (node) => { - text += node.value - }) + // Get username + const usernameElement = await authorElement.$('span a[href^="/"]') + let username = usernameElement ? await usernameElement.textContent() : 'unknown' + username = username.replace('@', '') - return text.trim() + // Get avatar URL + const avatarElement = await tweetElement.$('img[src*="/profile_images/"]') + const avatarUrl = avatarElement ? await avatarElement.getAttribute('src') : undefined + + return { + username, + displayName, + ...(avatarUrl && { avatarUrl }), + } + } + catch (error) { + logger.parser.error('Error extracting author info:', (error as Error).message) + return { + username: 'unknown', + displayName: 'Unknown User', + } + } + } + + /** + * Extract tweet stats (likes, retweets, replies) + * @param tweetElement Tweet element handle + * @returns Promise resolving to stats object + */ + private static async extractTweetStats(tweetElement: ElementHandle): Promise<{ + likeCount?: number + retweetCount?: number + replyCount?: number + }> { + const stats: { + likeCount?: number + retweetCount?: number + replyCount?: number + } = {} + + try { + // Extract like count + const likeElement = await tweetElement.$(SELECTORS.TIMELINE.LIKE_BUTTON) + if (likeElement) { + const likeCountElement = await likeElement.$('span span') + const likeCountText = likeCountElement ? await likeCountElement.textContent() : null + stats.likeCount = this.parseCount(likeCountText) + } + + // Extract retweet count + const retweetElement = await tweetElement.$(SELECTORS.TIMELINE.RETWEET_BUTTON) + if (retweetElement) { + const retweetCountElement = await retweetElement.$('span span') + const retweetCountText = retweetCountElement ? await retweetCountElement.textContent() : null + stats.retweetCount = this.parseCount(retweetCountText) + } + + // Extract reply count + const replyElement = await tweetElement.$(SELECTORS.TIMELINE.REPLY_BUTTON) + if (replyElement) { + const replyCountElement = await replyElement.$('span span') + const replyCountText = replyCountElement ? await replyCountElement.textContent() : null + stats.replyCount = this.parseCount(replyCountText) + } + + return stats + } + catch (error) { + logger.parser.error('Error extracting tweet stats:', (error as Error).message) + return stats + } } /** - * Extract author info + * Extract media URLs from tweet element + * @param tweetElement Tweet element handle + * @returns Promise resolving to array of media URLs */ - private static extractAuthorInfo(_element: Element) { - // Extract author name, username and avatar - // This part needs to be adjusted based on the actual DOM structure of Twitter - return { - username: 'username', // Placeholder, actual implementation needs to be based on DOM structure - displayName: 'displayName', - avatarUrl: undefined, + private static async extractMediaUrls(tweetElement: ElementHandle): Promise { + try { + const mediaElements = await tweetElement.$$('img[src*="pbs.twimg.com/media/"]') + + const mediaUrls: string[] = [] + for (const mediaElement of mediaElements) { + const src = await mediaElement.getAttribute('src') + if (src) { + mediaUrls.push(src) + } + } + + return mediaUrls + } + catch (error) { + logger.parser.error('Error extracting media URLs:', (error as Error).message) + return [] } } /** - * Extract tweet stats + * Parse count text (handles K, M suffixes) + * @param countText Count text from tweet + * @returns Parsed number or undefined */ - private static extractTweetStats(_element: Element) { - // Extract like count, retweet count and reply count - return { - likeCount: undefined, - retweetCount: undefined, - replyCount: undefined, + private static parseCount(countText: string | null): number | undefined { + if (!countText) + return undefined + + try { + countText = countText.trim() + if (!countText) + return undefined + + if (countText.includes('K')) { + return Math.round(Number.parseFloat(countText.replace('K', '')) * 1000) + } + else if (countText.includes('M')) { + return Math.round(Number.parseFloat(countText.replace('M', '')) * 1000000) + } + + return Number.parseInt(countText, 10) || undefined + } + catch { + return undefined } } } diff --git a/services/twitter-services/src/types/twitter.ts b/services/twitter-services/src/types/twitter.ts index 082e8d31..e446e383 100644 --- a/services/twitter-services/src/types/twitter.ts +++ b/services/twitter-services/src/types/twitter.ts @@ -47,6 +47,7 @@ export interface TimelineOptions { count?: number includeReplies?: boolean includeRetweets?: boolean + limit?: number } /** @@ -82,21 +83,3 @@ export interface UserLink { url: string title: string } - -/** - * Twitter Service Interface - */ -export interface TwitterService { - login: () => Promise - getTimeline: (options?: TimelineOptions) => Promise - getTweetDetails: (tweetId: string) => Promise - searchTweets: (query: string, options?: SearchOptions) => Promise - getUserProfile: (username: string) => Promise - followUser: (username: string) => Promise - likeTweet: (tweetId: string) => Promise - retweet: (tweetId: string) => Promise - postTweet: (content: string, options?: PostOptions) => Promise - saveSession: () => Promise - startSessionMonitor: (interval?: number) => void - getCurrentUrl: () => Promise -} diff --git a/services/twitter-services/src/utils/selectors.ts b/services/twitter-services/src/utils/selectors.ts index b8cac7e5..01fdb945 100644 --- a/services/twitter-services/src/utils/selectors.ts +++ b/services/twitter-services/src/utils/selectors.ts @@ -13,6 +13,8 @@ export const SELECTORS = { }, HOME: { TIMELINE: '[data-testid="primaryColumn"]', + TRENDING: '[data-testid="sidebarColumn"]', + TWEET_COMPOSER: '[data-testid="tweetButtonInline"]', }, TIMELINE: { TWEET: '[data-testid="tweet"]', @@ -21,17 +23,61 @@ export const SELECTORS = { LIKE_BUTTON: '[data-testid="like"]', RETWEET_BUTTON: '[data-testid="retweet"]', REPLY_BUTTON: '[data-testid="reply"]', + AUTHOR_LINK: '[data-testid="User-Name"] a', + AUTHOR_NAME: '[data-testid="User-Name"] a div span', + AUTHOR_USERNAME: '[data-testid="User-Name"] a div div span', + TWEET_STATS: '[data-testid="tweet"] [role="group"]', + REPLY_COUNT: '[data-testid="reply"] span span', + RETWEET_COUNT: '[data-testid="retweet"] span span', + LIKE_COUNT: '[data-testid="like"] span span', + VIEW_COUNT: '[data-testid="tweet"] [data-testid="app-text-transition-container"] span span', + MEDIA_CONTAINER: '[data-testid="tweetPhoto"]', + VIDEO_CONTAINER: '[data-testid="videoPlayer"]', + TWEET_LINK: 'a[aria-label*="posted"]', + TWEET_ID_CONTAINER: 'a[href*="/status/"]', }, PROFILE: { FOLLOW_BUTTON: '[data-testid="followButton"]', UNFOLLOW_BUTTON: '[data-testid="unfollowButton"]', - DISPLAY_NAME: '[data-testid="UserName"]', + DISPLAY_NAME: '[data-testid="UserName"] div span', + USERNAME: '[data-testid="UserName"] div span:has-text("@")', BIO: '[data-testid="UserDescription"]', STATS: '[data-testid="UserProfileStats"]', + FOLLOWING_STAT: '[href$="/following"]', + FOLLOWERS_STAT: '[href$="/followers"]', + AVATAR: '[data-testid="UserAvatar-Container"] img', + BANNER: '[data-testid="UserProfileHeader_Items"] img', + JOIN_DATE: '[data-testid="UserProfileHeader_Items"] span:has-text("Joined")', + LOCATION: '[data-testid="UserProfileHeader_Items"] span:has-text("Location")', + WEBSITE: '[data-testid="UserProfileHeader_Items"] a[href^="https"]', + VERIFIED_BADGE: '[data-testid="icon-verified"]', }, COMPOSE: { TWEET_INPUT: '[data-testid="tweetTextarea_0"]', TWEET_BUTTON: '[data-testid="tweetButtonInline"]', MEDIA_BUTTON: '[data-testid="imageOrGifButton"]', + POLL_BUTTON: '[data-testid="createPollButton"]', + EMOJI_BUTTON: '[data-testid="emojiButton"]', + SCHEDULE_BUTTON: '[data-testid="scheduleButton"]', + }, + SEARCH: { + INPUT: '[data-testid="SearchBox_Search_Input"]', + RESULT_TAB: '[role="tablist"] [role="presentation"]', + PEOPLE_TAB: '[role="tab"]:has-text("People")', + LATEST_TAB: '[role="tab"]:has-text("Latest")', + TOP_TAB: '[role="tab"]:has-text("Top")', + SEARCH_FILTERS: '[data-testid="searchFiltersButton"]', + }, + NOTIFICATIONS: { + ITEM: '[data-testid="cellInnerDiv"]', + MENTION: '[data-testid="notification"]', + FOLLOW: '[data-testid="notification"]', + LIKE: '[data-testid="notification"]', + RETWEET: '[data-testid="notification"]', + }, + MESSAGES: { + CONVERSATION: '[data-testid="conversation"]', + MESSAGE_INPUT: '[data-testid="dmComposerTextInput"]', + SEND_BUTTON: '[data-testid="dmComposerSendButton"]', }, } From d0582f22557304bb25be71cb3e49f16ea585460a Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 18:57:12 +0800 Subject: [PATCH 19/20] fix: type error --- services/twitter-services/src/parsers/tweet-parser.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/twitter-services/src/parsers/tweet-parser.ts b/services/twitter-services/src/parsers/tweet-parser.ts index 4f535548..1fb7d0b8 100644 --- a/services/twitter-services/src/parsers/tweet-parser.ts +++ b/services/twitter-services/src/parsers/tweet-parser.ts @@ -68,7 +68,7 @@ export class TweetParser { id, text: text || '', author, - timestamp, + timestamp: timestamp || new Date().toISOString(), ...stats, } @@ -130,12 +130,12 @@ export class TweetParser { // Get display name const displayNameElement = await authorElement.$('span:first-child') - const displayName = displayNameElement ? await displayNameElement.textContent() : 'Unknown User' + const displayName = displayNameElement ? await displayNameElement.textContent() || 'Unknown User' : 'Unknown User' // Get username const usernameElement = await authorElement.$('span a[href^="/"]') let username = usernameElement ? await usernameElement.textContent() : 'unknown' - username = username.replace('@', '') + username = username?.replace('@', '') || 'unknown' // Get avatar URL const avatarElement = await tweetElement.$('img[src*="/profile_images/"]') From b7430b3cd68e36cb312d913854f87927328aead3 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Mar 2025 18:58:33 +0800 Subject: [PATCH 20/20] fix: env example --- services/twitter-services/.env.example | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/services/twitter-services/.env.example b/services/twitter-services/.env.example index b393ba2a..9d652ead 100644 --- a/services/twitter-services/.env.example +++ b/services/twitter-services/.env.example @@ -1,6 +1,3 @@ -# BrowserBase Config -BROWSERBASE_API_KEY=your_api_key_here - # Browser Config BROWSER_HEADLESS=false BROWSER_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 @@ -10,14 +7,6 @@ BROWSER_TIMEOUT=30000 BROWSER_REQUEST_TIMEOUT=20000 BROWSER_REQUEST_RETRIES=2 -# Twitter Account Config -TWITTER_USERNAME=your_twitter_username -TWITTER_PASSWORD=your_twitter_password -# Cookie can be in two formats: -# 1. JSON format: {"auth_token":"xxx","ct0":"yyy"} -# 2. document.cookie format: auth_token=xxx; ct0=yyy -TWITTER_COOKIES= - # Adapter Config ENABLE_AIRI=false AIRI_URL=http://localhost:3000