Skip to content

Commit 7820624

Browse files
odilitimeodilitime
authored
Merge pull request #8 from Manifest-Holdings/twitter_proxy_url
Twitter proxy url
2 parents b6680af + 4333ed6 commit 7820624

File tree

7 files changed

+708
-601
lines changed

7 files changed

+708
-601
lines changed

‎agent/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -621,6 +621,7 @@ export async function initializeClients(
621621
password: getSecret(character, "TWITTER_PASSWORD"),
622622
email: getSecret(character, "TWITTER_EMAIL"),
623623
twitter2faSecret: getSecret(character, "TWITTER_2FA_SECRET"),
624+
proxyUrl: getSecret(character, "TWITTER_PROXY_URL"),
624625
});
625626
if (isValidKey.success) {
626627
const twitterClient = await TwitterClientInterface.start(runtime);

‎packages/client-twitter/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"dependencies": {
2222
"@elizaos/core": "workspace:*",
2323
"agent-twitter-client": "0.0.18",
24+
"undici": "7.3.0",
2425
"glob": "11.0.0",
2526
"zod": "3.23.8",
2627
"discord.js": "14.16.3"

‎packages/client-twitter/src/base.ts

+79-28
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
} from "agent-twitter-client";
1919
import { EventEmitter } from "events";
2020
import { TwitterConfig } from "./environment.ts";
21+
import { fetch, ProxyAgent, setGlobalDispatcher } from "undici";
2122

2223
export function extractAnswer(text: string): string {
2324
const startIndex = text.indexOf("Answer: ") + 8;
@@ -60,8 +61,8 @@ class RequestQueue {
6061
while (this.queue.length > 0) {
6162
const request = this.queue.shift()!;
6263
try {
63-
await request().catch(e => {
64-
console.error('client.twitter.base - request err', e)
64+
await request().catch((e) => {
65+
console.error("client.twitter.base - request err", e);
6566
});
6667
} catch (error) {
6768
console.error("Error processing request:", error);
@@ -85,35 +86,80 @@ class RequestQueue {
8586
}
8687
}
8788

88-
let lastStart = Date.now()
89+
let lastStart = Date.now();
8990

90-
function doLogin(username, cb) {
91-
const ts = Date.now()
92-
const since = ts - lastStart
93-
elizaLogger.log('last twitter scrapper created', since, 'ms ago')
94-
const delay = 5 * 1000
91+
function doLogin(username, cb, proxyUrl?: string) {
92+
const ts = Date.now();
93+
const since = ts - lastStart;
94+
elizaLogger.log("last twitter scrapper created", since, "ms ago");
95+
const delay = 5 * 1000;
9596
if (since > delay) {
96-
const twitterClient = new Scraper();
97+
let agent;
98+
99+
// Add proxy configuration if TWITTER_PROXY_URL exists
100+
if (proxyUrl) {
101+
elizaLogger.log("Using proxy", proxyUrl);
102+
const url = new URL(proxyUrl);
103+
const username = url.username;
104+
const password = url.password;
105+
106+
// Strip auth from URL if present
107+
url.username = "";
108+
url.password = "";
109+
110+
const agentOptions: any = {
111+
uri: url.toString(),
112+
requestTls: {
113+
rejectUnauthorized: false,
114+
},
115+
};
116+
117+
// Add Basic auth if credentials exist
118+
if (username && password) {
119+
agentOptions.token = `Basic ${Buffer.from(
120+
`${username}:${password}`
121+
).toString("base64")}`;
122+
}
123+
124+
agent = new ProxyAgent(agentOptions);
125+
setGlobalDispatcher(agent);
126+
}
127+
128+
const twitterClient = new Scraper({
129+
fetch: fetch,
130+
transform: {
131+
request: (input: any, init: any) => {
132+
if (agent) {
133+
return [input, { ...init, dispatcher: agent }];
134+
}
135+
return [input, init];
136+
},
137+
},
138+
});
139+
97140
ClientBase._twitterClients[username] = twitterClient;
98-
lastStart = ts
99-
cb(twitterClient)
141+
lastStart = ts;
142+
cb(twitterClient);
100143
} else {
101-
elizaLogger.log('Delaying twitter scrapper creation for', username)
144+
elizaLogger.log("Delaying twitter scrapper creation for", username);
102145
setTimeout(() => {
103-
doLogin(username, cb)
104-
}, delay)
146+
doLogin(username, cb);
147+
}, delay);
105148
}
106149
}
107150

108-
export function getScrapper(username:string):twitterClient {
109-
return new Promise(resolve => {
151+
export function getScraper(
152+
username: string,
153+
proxyUrl?: string
154+
): Promise<Scraper> {
155+
return new Promise((resolve) => {
110156
if (ClientBase._twitterClients[username]) {
111157
const twitterClient = ClientBase._twitterClients[username];
112-
resolve(twitterClient)
158+
resolve(twitterClient);
113159
} else {
114-
doLogin(username, resolve)
160+
doLogin(username, resolve, proxyUrl);
115161
}
116-
})
162+
});
117163
}
118164

119165
export class ClientBase extends EventEmitter {
@@ -174,10 +220,13 @@ export class ClientBase extends EventEmitter {
174220
super();
175221
this.runtime = runtime;
176222
this.twitterConfig = twitterConfig;
223+
224+
// Store proxy URL statically so it's available to doLogin
225+
const proxyUrl = twitterConfig.TWITTER_PROXY_URL;
177226
const username = twitterConfig.TWITTER_USERNAME;
178-
getScrapper(username).then(tc => {
179-
this.twitterClient = tc;
180-
})
227+
getScraper(username, proxyUrl).then((tc) => {
228+
this.twitterClient = tc;
229+
});
181230

182231
this.directions =
183232
"- " +
@@ -193,7 +242,7 @@ export class ClientBase extends EventEmitter {
193242
let retries = this.twitterConfig.TWITTER_RETRY_LIMIT;
194243
const twitter2faSecret = this.twitterConfig.TWITTER_2FA_SECRET;
195244
// if twitter says its bad, trust twitter
196-
retries = 1 // mee.fun, lets no hammer this, it should work or not
245+
retries = 1; // mee.fun, lets no hammer this, it should work or not
197246

198247
if (!username) {
199248
throw new Error("Twitter username not configured");
@@ -221,7 +270,7 @@ export class ClientBase extends EventEmitter {
221270
twitter2faSecret
222271
);
223272
if (await this.twitterClient.isLoggedIn()) {
224-
lastStart = Date.now()
273+
lastStart = Date.now();
225274
// fresh login, store new cookies
226275
elizaLogger.info("Successfully logged in.");
227276
elizaLogger.info("Caching cookies");
@@ -233,7 +282,9 @@ export class ClientBase extends EventEmitter {
233282
}
234283
}
235284
} catch (error) {
236-
elizaLogger.error(`${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`);
285+
elizaLogger.error(
286+
`${this.runtime.character.name}(${this.runtime.agentId}): Login attempt failed: ${error.message} for twitter @${username}`
287+
);
237288
}
238289

239290
retries--;
@@ -255,7 +306,7 @@ export class ClientBase extends EventEmitter {
255306
this.profile = await this.fetchProfile(username);
256307

257308
if (!this.profile) {
258-
elizaLogger.error('cl-tw::init - profile did not load')
309+
elizaLogger.error("cl-tw::init - profile did not load");
259310
return false;
260311
}
261312
elizaLogger.log("Twitter user ID:", this.profile.id);
@@ -724,8 +775,8 @@ export class ClientBase extends EventEmitter {
724775
if (latestCheckedTweetId) {
725776
this.lastCheckedTweetId = BigInt(latestCheckedTweetId);
726777
}
727-
} catch(e) {
728-
elizaLogger.error('cl-tw::loadLatestCheckedTweetId - err', e)
778+
} catch (e) {
779+
elizaLogger.error("cl-tw::loadLatestCheckedTweetId - err", e);
729780
}
730781
}
731782

‎packages/client-twitter/src/environment.ts

+9-1
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ export const twitterEnvSchema = z.object({
7676
ACTION_TIMELINE_TYPE: z
7777
.nativeEnum(ActionTimelineType)
7878
.default(ActionTimelineType.ForYou),
79+
TWITTER_PROXY_URL: z.string().optional(),
7980
});
8081

8182
export type TwitterConfig = z.infer<typeof twitterEnvSchema>;
@@ -224,6 +225,11 @@ export async function validateTwitterConfig(
224225
ACTION_TIMELINE_TYPE:
225226
runtime.getSetting("ACTION_TIMELINE_TYPE") ||
226227
process.env.ACTION_TIMELINE_TYPE,
228+
229+
TWITTER_PROXY_URL:
230+
(runtime.getSetting("TWITTER_PROXY_URL") ||
231+
process.env.TWITTER_PROXY_URL) ??
232+
"",
227233
};
228234

229235
return twitterEnvSchema.parse(twitterConfig);
@@ -232,7 +238,9 @@ export async function validateTwitterConfig(
232238
const errorMessages = error.errors
233239
.map((err) => `${err.path.join(".")}: ${err.message}`)
234240
.join("\n");
235-
elizaLogger.error(`Twitter configuration validation failed:\n${errorMessages}`)
241+
elizaLogger.error(
242+
`Twitter configuration validation failed:\n${errorMessages}`
243+
);
236244
/*
237245
throw new Error(
238246
`X/Twitter configuration validation failed:\n${errorMessages}`

‎packages/client-twitter/src/index.ts

+9-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { Client, elizaLogger, IAgentRuntime } from "@elizaos/core";
2-
import { ClientBase, getScrapper } from "./base.ts";
2+
import { ClientBase, getScraper } from "./base.ts";
33
import { validateTwitterConfig, TwitterConfig } from "./environment.ts";
44
import { TwitterInteractionClient } from "./interactions.ts";
55
import { TwitterPostClient } from "./post.ts";
@@ -65,13 +65,15 @@ export const TwitterClientInterface: Client = {
6565
let twitterConfig: TwitterConfig;
6666
try {
6767
twitterConfig = await validateTwitterConfig(runtime);
68-
} catch (e) {
68+
} catch (error) {
6969
elizaLogger.error(
7070
"TwitterConfig validation failed for",
7171
runtime.getSetting("TWITTER_USERNAME") ||
7272
process.env.TWITTER_USERNAME,
7373
"email",
74-
runtime.getSetting("TWITTER_EMAIL") || process.env.TWITTER_EMAIL
74+
runtime.getSetting("TWITTER_EMAIL") ||
75+
process.env.TWITTER_EMAIL,
76+
error
7577
);
7678
return;
7779
}
@@ -115,7 +117,10 @@ export const TwitterClientInterface: Client = {
115117
},
116118
validate: async (secrets) => {
117119
try {
118-
const twClient = await getScrapper(secrets.username);
120+
const twClient = await getScraper(
121+
secrets.username,
122+
secrets.proxyUrl
123+
);
119124
// try logging in
120125
console.log("trying to log in");
121126
await twClient.login(

‎packages/client-twitter/src/post.ts

+41-25
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ export class TwitterPostClient {
130130
elizaLogger.log(
131131
`- Search Enabled: ${this.client.twitterConfig.TWITTER_SEARCH_ENABLE ? "enabled" : "disabled"}`
132132
);
133+
elizaLogger.log(
134+
`- Proxy URL: ${this.client.twitterConfig.TWITTER_PROXY_URL}`
135+
);
133136

134137
const targetUsers = this.client.twitterConfig.TWITTER_TARGET_USERS;
135138
if (targetUsers) {
@@ -251,7 +254,11 @@ export class TwitterPostClient {
251254
const actionInterval = this.client.twitterConfig.ACTION_INTERVAL; // Defaults to 5 minutes
252255

253256
while (!this.stopProcessingActions) {
254-
elizaLogger.log('ACTION_INTERVAL', actionInterval, this.client.twitterConfig.TWITTER_USERNAME)
257+
elizaLogger.log(
258+
"ACTION_INTERVAL",
259+
actionInterval,
260+
this.client.twitterConfig.TWITTER_USERNAME
261+
);
255262
try {
256263
const results = await this.processTweetActions();
257264
if (results) {
@@ -414,27 +421,30 @@ export class TwitterPostClient {
414421
const body = await standardTweetResult.json();
415422
if (!body?.data?.create_tweet?.tweet_results?.result) {
416423
elizaLogger.error("Error sending tweet; Bad response:", body);
417-
if (body?.errors?.[0].message === 'Authorization: Denied by access control: Missing TwitterUserNotSuspended') {
418-
elizaLogger.error("Account suspended");
419-
//this.client is base
420-
//this.runtime needs a stop client
421-
// this is
422-
const manager = this.runtime.clients.twitter
423-
// stop post/search/interaction
424-
if (manager) {
425-
if (manager.client.twitterClient) {
426-
await manager.post.stop();
427-
await manager.interaction.stop();
428-
if (manager.search) {
429-
await manager.search.stop();
430-
}
431-
} else {
432-
// it's still starting up
433-
}
434-
} // already stoped
435-
436-
// mark it offline
437-
delete runtime.clients.twitter;
424+
if (
425+
body?.errors?.[0].message ===
426+
"Authorization: Denied by access control: Missing TwitterUserNotSuspended"
427+
) {
428+
elizaLogger.error("Account suspended");
429+
//this.client is base
430+
//this.runtime needs a stop client
431+
// this is
432+
const manager = this.runtime.clients.twitter;
433+
// stop post/search/interaction
434+
if (manager) {
435+
if (manager.client.twitterClient) {
436+
await manager.post.stop();
437+
await manager.interaction.stop();
438+
if (manager.search) {
439+
await manager.search.stop();
440+
}
441+
} else {
442+
// it's still starting up
443+
}
444+
} // already stoped
445+
446+
// mark it offline
447+
delete runtime.clients.twitter;
438448
}
439449
return;
440450
}
@@ -531,7 +541,7 @@ export class TwitterPostClient {
531541
modelClass: ModelClass.SMALL,
532542
});
533543

534-
const newTweetContent = response
544+
const newTweetContent = response
535545
.replace(/```json\s*/g, "") // Remove ```json
536546
.replace(/```\s*/g, "") // Remove any remaining ```
537547
.replace(/(\r\n|\n|\r)/g, "") // Remove line break
@@ -618,10 +628,16 @@ export class TwitterPostClient {
618628
);
619629
}
620630
} catch (error) {
621-
elizaLogger.error("generateNewTweet Error sending tweet:", error);
631+
elizaLogger.error(
632+
"generateNewTweet Error sending tweet:",
633+
error
634+
);
622635
}
623636
} catch (error) {
624-
elizaLogger.error("generateNewTweet Error generating new tweet:", error);
637+
elizaLogger.error(
638+
"generateNewTweet Error generating new tweet:",
639+
error
640+
);
625641
}
626642
}
627643

0 commit comments

Comments
 (0)