Skip to content

Commit 44232d4

Browse files
azep-ninjaazep-ninja
committed
multi-agent rag/autonomous community
1 parent 40d8b9b commit 44232d4

File tree

15 files changed

+1728
-274
lines changed

15 files changed

+1728
-274
lines changed

‎packages/adapter-sqlite/src/index.ts

+211-7
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
export * from "./sqliteTables.ts";
22
export * from "./sqlite_vec.ts";
33

4-
import { DatabaseAdapter, IDatabaseCacheAdapter } from "@ai16z/eliza";
4+
import { DatabaseAdapter, elizaLogger, IDatabaseCacheAdapter } from "@ai16z/eliza";
55
import {
66
Account,
77
Actor,
@@ -11,6 +11,7 @@ import {
1111
type Memory,
1212
type Relationship,
1313
type UUID,
14+
RAGKnowledgeItem
1415
} from "@ai16z/eliza";
1516
import { Database } from "better-sqlite3";
1617
import { v4 } from "uuid";
@@ -248,8 +249,8 @@ export class SqliteDatabaseAdapter
248249

249250
let sql = `
250251
SELECT *, vec_distance_L2(embedding, ?) AS similarity
251-
FROM memories
252-
WHERE type = ?
252+
FROM memories
253+
WHERE type = ?
253254
AND roomId = ?`;
254255

255256
if (params.unique) {
@@ -340,24 +341,24 @@ export class SqliteDatabaseAdapter
340341
// First get content text and calculate Levenshtein distance
341342
const sql = `
342343
WITH content_text AS (
343-
SELECT
344+
SELECT
344345
embedding,
345346
json_extract(
346347
json(content),
347348
'$.' || ? || '.' || ?
348349
) as content_text
349-
FROM memories
350+
FROM memories
350351
WHERE type = ?
351352
AND json_extract(
352353
json(content),
353354
'$.' || ? || '.' || ?
354355
) IS NOT NULL
355356
)
356-
SELECT
357+
SELECT
357358
embedding,
358359
length(?) + length(content_text) - (
359360
length(?) + length(content_text) - (
360-
length(replace(lower(?), lower(content_text), '')) +
361+
length(replace(lower(?), lower(content_text), '')) +
361362
length(replace(lower(content_text), lower(?), ''))
362363
) / 2
363364
) as levenshtein_score
@@ -707,4 +708,207 @@ export class SqliteDatabaseAdapter
707708
return false;
708709
}
709710
}
711+
712+
async getKnowledge(params: {
713+
id?: UUID;
714+
agentId: UUID;
715+
limit?: number;
716+
query?: string;
717+
}): Promise<RAGKnowledgeItem[]> {
718+
let sql = `SELECT * FROM knowledge WHERE (agentId = ? OR isShared = 1)`;
719+
const queryParams: any[] = [params.agentId];
720+
721+
if (params.id) {
722+
sql += ` AND id = ?`;
723+
queryParams.push(params.id);
724+
}
725+
726+
if (params.limit) {
727+
sql += ` LIMIT ?`;
728+
queryParams.push(params.limit);
729+
}
730+
731+
interface KnowledgeRow {
732+
id: UUID;
733+
agentId: UUID;
734+
content: string;
735+
embedding: Buffer | null;
736+
createdAt: string | number;
737+
}
738+
739+
const rows = this.db.prepare(sql).all(...queryParams) as KnowledgeRow[];
740+
741+
return rows.map(row => ({
742+
id: row.id,
743+
agentId: row.agentId,
744+
content: JSON.parse(row.content),
745+
embedding: row.embedding ? Array.from(new Float32Array(row.embedding)) : undefined,
746+
createdAt: typeof row.createdAt === 'string' ? Date.parse(row.createdAt) : row.createdAt
747+
}));
748+
}
749+
750+
async searchKnowledge(params: {
751+
agentId: UUID;
752+
embedding: Float32Array;
753+
match_threshold: number;
754+
match_count: number;
755+
searchText?: string;
756+
}): Promise<RAGKnowledgeItem[]> {
757+
const cacheKey = `embedding_${params.agentId}_${params.searchText}`;
758+
const cachedResult = await this.getCache({
759+
key: cacheKey,
760+
agentId: params.agentId
761+
});
762+
763+
if (cachedResult) {
764+
return JSON.parse(cachedResult);
765+
}
766+
767+
interface KnowledgeSearchRow {
768+
id: UUID;
769+
agentId: UUID;
770+
content: string;
771+
embedding: Buffer | null;
772+
createdAt: string | number;
773+
vector_score: number;
774+
keyword_score: number;
775+
combined_score: number;
776+
}
777+
778+
const sql = `
779+
WITH vector_scores AS (
780+
SELECT id,
781+
1 / (1 + vec_distance_L2(embedding, ?)) as vector_score
782+
FROM knowledge
783+
WHERE (agentId IS NULL AND isShared = 1) OR agentId = ?
784+
AND embedding IS NOT NULL
785+
),
786+
keyword_matches AS (
787+
SELECT id,
788+
CASE
789+
WHEN lower(json_extract(content, '$.text')) LIKE ? THEN 3.0
790+
ELSE 1.0
791+
END *
792+
CASE
793+
WHEN json_extract(content, '$.metadata.isChunk') = 1 THEN 1.5
794+
WHEN json_extract(content, '$.metadata.isMain') = 1 THEN 1.2
795+
ELSE 1.0
796+
END as keyword_score
797+
FROM knowledge
798+
WHERE (agentId IS NULL AND isShared = 1) OR agentId = ?
799+
)
800+
SELECT k.*,
801+
v.vector_score,
802+
kw.keyword_score,
803+
(v.vector_score * kw.keyword_score) as combined_score
804+
FROM knowledge k
805+
JOIN vector_scores v ON k.id = v.id
806+
LEFT JOIN keyword_matches kw ON k.id = kw.id
807+
WHERE (k.agentId IS NULL AND k.isShared = 1) OR k.agentId = ?
808+
AND (
809+
v.vector_score >= ? -- Using match_threshold parameter
810+
OR (kw.keyword_score > 1.0 AND v.vector_score >= 0.3)
811+
)
812+
ORDER BY combined_score DESC
813+
LIMIT ?
814+
`;
815+
816+
const searchParams = [
817+
params.embedding,
818+
params.agentId,
819+
`%${params.searchText?.toLowerCase() || ''}%`,
820+
params.agentId,
821+
params.agentId,
822+
params.match_threshold,
823+
params.match_count
824+
];
825+
826+
try {
827+
const rows = this.db.prepare(sql).all(...searchParams) as KnowledgeSearchRow[];
828+
const results = rows.map(row => ({
829+
id: row.id,
830+
agentId: row.agentId,
831+
content: JSON.parse(row.content),
832+
embedding: row.embedding ? Array.from(new Float32Array(row.embedding)) : undefined,
833+
createdAt: typeof row.createdAt === 'string' ? Date.parse(row.createdAt) : row.createdAt,
834+
similarity: row.combined_score
835+
}));
836+
837+
await this.setCache({
838+
key: cacheKey,
839+
agentId: params.agentId,
840+
value: JSON.stringify(results)
841+
});
842+
843+
return results;
844+
} catch (error) {
845+
elizaLogger.error('Error in searchKnowledge:', error);
846+
throw error;
847+
}
848+
849+
}
850+
851+
async createKnowledge(knowledge: RAGKnowledgeItem): Promise<void> {
852+
try {
853+
this.db.transaction(() => {
854+
const sql = `
855+
INSERT INTO knowledge (
856+
id, agentId, content, embedding, createdAt,
857+
isMain, originalId, chunkIndex, isShared
858+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
859+
`;
860+
861+
const embeddingArray = knowledge.embedding ?
862+
new Float32Array(knowledge.embedding) : null;
863+
864+
const metadata = knowledge.content.metadata || {};
865+
const isShared = metadata.isShared ? 1 : 0;
866+
867+
this.db.prepare(sql).run(
868+
knowledge.id,
869+
metadata.isShared ? null : knowledge.agentId,
870+
JSON.stringify(knowledge.content),
871+
embeddingArray,
872+
knowledge.createdAt || Date.now(),
873+
metadata.isMain ? 1 : 0,
874+
metadata.originalId || null,
875+
metadata.chunkIndex || null,
876+
isShared
877+
);
878+
879+
})();
880+
} catch (error: any) {
881+
const isShared = knowledge.content.metadata?.isShared;
882+
const isPrimaryKeyError = error?.code === 'SQLITE_CONSTRAINT_PRIMARYKEY';
883+
884+
if (isShared && isPrimaryKeyError) {
885+
elizaLogger.info(`Shared knowledge ${knowledge.id} already exists, skipping`);
886+
return;
887+
} else if (!isShared && !error.message?.includes('SQLITE_CONSTRAINT_PRIMARYKEY')) {
888+
elizaLogger.error(`Error creating knowledge ${knowledge.id}:`, {
889+
error,
890+
embeddingLength: knowledge.embedding?.length,
891+
content: knowledge.content
892+
});
893+
throw error;
894+
}
895+
896+
elizaLogger.debug(`Knowledge ${knowledge.id} already exists, skipping`);
897+
}
898+
}
899+
900+
async removeKnowledge(id: UUID): Promise<void> {
901+
const sql = `DELETE FROM knowledge WHERE id = ?`;
902+
this.db.prepare(sql).run(id);
903+
}
904+
905+
async clearKnowledge(agentId: UUID, shared?: boolean): Promise<void> {
906+
const sql = shared ? `DELETE FROM knowledge WHERE (agentId = ? OR isShared = 1)` : `DELETE FROM knowledge WHERE agentId = ?`;
907+
try {
908+
this.db.prepare(sql).run(agentId);
909+
} catch (error) {
910+
elizaLogger.error(`Error clearing knowledge for agent ${agentId}:`, error);
911+
throw error;
912+
}
913+
}
710914
}

‎packages/adapter-sqlite/src/sqliteTables.ts

+27-1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,22 @@ CREATE TABLE IF NOT EXISTS "cache" (
9292
PRIMARY KEY ("key", "agentId")
9393
);
9494
95+
-- Table: knowledge
96+
CREATE TABLE IF NOT EXISTS "knowledge" (
97+
"id" TEXT PRIMARY KEY,
98+
"agentId" TEXT,
99+
"content" TEXT NOT NULL CHECK(json_valid("content")),
100+
"embedding" BLOB,
101+
"createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
102+
"isMain" INTEGER DEFAULT 0,
103+
"originalId" TEXT,
104+
"chunkIndex" INTEGER,
105+
"isShared" INTEGER DEFAULT 0,
106+
FOREIGN KEY ("agentId") REFERENCES "accounts"("id"),
107+
FOREIGN KEY ("originalId") REFERENCES "knowledge"("id"),
108+
CHECK((isShared = 1 AND agentId IS NULL) OR (isShared = 0 AND agentId IS NOT NULL))
109+
);
110+
95111
-- Index: relationships_id_key
96112
CREATE UNIQUE INDEX IF NOT EXISTS "relationships_id_key" ON "relationships" ("id");
97113
@@ -101,4 +117,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS "memories_id_key" ON "memories" ("id");
101117
-- Index: participants_id_key
102118
CREATE UNIQUE INDEX IF NOT EXISTS "participants_id_key" ON "participants" ("id");
103119
104-
COMMIT;`;
120+
-- Index: knowledge
121+
CREATE INDEX IF NOT EXISTS "knowledge_agent_key" ON "knowledge" ("agentId");
122+
CREATE INDEX IF NOT EXISTS "knowledge_agent_main_key" ON "knowledge" ("agentId", "isMain");
123+
CREATE INDEX IF NOT EXISTS "knowledge_original_key" ON "knowledge" ("originalId");
124+
CREATE INDEX IF NOT EXISTS "knowledge_content_key" ON "knowledge"
125+
((json_extract(content, '$.text')))
126+
WHERE json_extract(content, '$.text') IS NOT NULL;
127+
CREATE INDEX IF NOT EXISTS "knowledge_created_key" ON "knowledge" ("agentId", "createdAt");
128+
CREATE INDEX IF NOT EXISTS "knowledge_shared_key" ON "knowledge" ("isShared");
129+
130+
COMMIT;`;

‎packages/adapter-sqlite/src/sqlite_vec.ts

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export function loadVecExtensions(db: Database): void {
77
try {
88
// Load sqlite-vec extensions
99
sqliteVec.load(db);
10+
1011
elizaLogger.log("sqlite-vec extensions loaded successfully.");
1112
} catch (error) {
1213
elizaLogger.error("Failed to load sqlite-vec extensions:", error);

0 commit comments

Comments
 (0)