From 729c12e6c768e991bd4a4aa782837784d64554cc Mon Sep 17 00:00:00 2001 From: Codetrauma Date: Tue, 18 Mar 2025 15:25:39 -0700 Subject: [PATCH 1/6] feat: retrieve only latest headers and cookies --- src/entries/Background/db.ts | 120 +++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 35 deletions(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index cddab85..19fadbc 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -4,6 +4,7 @@ import { RequestHistory, RequestProgress } from './rpc'; import mutex from './mutex'; import { minimatch } from 'minimatch'; const charwise = require('charwise'); +import { safeParseJSON } from '../../utils/misc'; export const db = new Level('./ext-db', { valueEncoding: 'json', @@ -335,9 +336,29 @@ export async function setConnection(origin: string) { return true; } -export async function setCookies(host: string, name: string, value: string) { +export async function setCookies(host: string, name: string, value: string): Promise { return mutex.runExclusive(async () => { - await cookiesDb.sublevel(host).put(name, value); + const timestamp = Date.now(); + const cookieStore = cookiesDb.sublevel(host); + const newEntry = { value, timestamp }; + + let existingEntry: { value: string; timestamp: number } | null = null; + try { + const rawEntry = await cookieStore.get(name); + existingEntry = safeParseJSON(rawEntry); + } catch {} + + if (existingEntry) { + const { value: storedValue, timestamp: storedTimestamp } = existingEntry; + if (storedValue === value && storedTimestamp === timestamp) { + return true; + } + if (storedTimestamp >= timestamp) { + return true; + } + } + + await cookieStore.put(name, JSON.stringify(newEntry)); return true; }); } @@ -358,26 +379,31 @@ export async function getCookies(link: string, name: string) { } } -export async function getCookiesByHost(link: string) { - const ret: { [key: string]: string } = {}; - const links: { [k: string]: boolean } = {}; - const url = urlify(link); +export async function getCookiesByHost(cookieLink: string): Promise> { + const latestCookies: Record = {}; - for await (const sublevel of cookiesDb.keys({ keyEncoding: 'utf8' })) { - const l = sublevel.split('!')[1]; - links[l] = true; + for await (const [key, value] of cookiesDb.sublevel(cookieLink).iterator()) { + const parsed = parseCookieValue(value); + if (!latestCookies[key] || parsed.timestamp > latestCookies[key].timestamp) { + latestCookies[key] = parsed; + } } - const cookieLink = url - ? Object.keys(links).filter((l) => minimatch(l, link))[0] - : Object.keys(links).filter((l) => urlify(l)?.host === link)[0]; - - if (!cookieLink) return ret; + return Object.fromEntries( + Object.entries(latestCookies).map(([key, { value }]) => [key, value]) + ); +} - for await (const [key, value] of cookiesDb.sublevel(cookieLink).iterator()) { - ret[key] = value; +function parseCookieValue(rawValue: string): { value: any; timestamp: number } { + try { + const parsed = safeParseJSON(rawValue); + if (parsed && typeof parsed === 'object' && 'value' in parsed && 'timestamp' in parsed) { + return parsed; + } + return { value: parsed, timestamp: 0 }; + } catch { + return { value: rawValue, timestamp: 0 }; } - return ret; } export async function deleteConnection(origin: string) { @@ -388,6 +414,7 @@ export async function deleteConnection(origin: string) { }); } + export async function getConnection(origin: string) { try { const existing = await connectionDb.get(origin); @@ -397,14 +424,34 @@ export async function getConnection(origin: string) { } } -export async function setHeaders(link: string, name: string, value?: string) { +export async function setHeaders(link: string, name: string, value?: string): Promise { if (!value) return null; + return mutex.runExclusive(async () => { - await headersDb.sublevel(link).put(name, value); - return true; + const sublevel = headersDb.sublevel(link); + const timestamp = Date.now(); + const newEntry = { value, timestamp }; + + try { + const existingRaw = await sublevel.get(name).catch(() => null); + if (existingRaw) { + const existing = JSON.parse(existingRaw) as { value: string; timestamp: number }; + if (existing.value === value && existing.timestamp === timestamp) { + return true; + } + } + await Promise.all([ + sublevel.put(name, JSON.stringify(newEntry)), + sublevel.put("__timestamp__", timestamp.toString()), + ]); + return true; + } catch (error) { + return true; + } }); } + export async function clearHeaders(host: string) { return mutex.runExclusive(async () => { await headersDb.sublevel(host).clear(); @@ -420,26 +467,29 @@ export async function getHeaders(host: string, name: string) { return null; } } -export async function getHeadersByHost(link: string) { - const ret: { [key: string]: string } = {}; - const url = urlify(link); - const links: { [k: string]: boolean } = {}; - for await (const sublevel of headersDb.keys({ keyEncoding: 'utf8' })) { - const l = sublevel.split('!')[1]; - links[l] = true; - } +export async function getHeadersByHost(link: string): Promise> { + const headers: Record = {}; + const sublevel = headersDb.sublevel(link); - const headerLink = url - ? Object.keys(links).filter((l) => minimatch(l, link))[0] - : Object.keys(links).filter((l) => urlify(l)?.host === link)[0]; + try { + for await (const [key, value] of sublevel.iterator({ keyEncoding: "utf8", valueEncoding: "utf8" })) { + if (key === "__timestamp__") continue; + const parsed = parseHeaderValue(value); + headers[key] = parsed.value; + } + } catch (error) {} - if (!headerLink) return ret; + return headers; +} - for await (const [key, value] of headersDb.sublevel(headerLink).iterator()) { - ret[key] = value; +function parseHeaderValue(rawValue: string): { value: string; timestamp: number } { + try { + const parsed = safeParseJSON(rawValue) as { value: string; timestamp: number }; + return { value: parsed.value || "", timestamp: parsed.timestamp }; + } catch { + return { value: rawValue, timestamp: 0 }; } - return ret; } export async function setLocalStorage( From 7361b761455f2881a7f1ab3ec8890dc88bcdcc15 Mon Sep 17 00:00:00 2001 From: Codetrauma Date: Wed, 19 Mar 2025 11:47:46 -0700 Subject: [PATCH 2/6] fix: linting --- src/entries/Background/db.ts | 62 ++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index 19fadbc..c5326ce 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -336,7 +336,11 @@ export async function setConnection(origin: string) { return true; } -export async function setCookies(host: string, name: string, value: string): Promise { +export async function setCookies( + host: string, + name: string, + value: string, +): Promise { return mutex.runExclusive(async () => { const timestamp = Date.now(); const cookieStore = cookiesDb.sublevel(host); @@ -379,25 +383,35 @@ export async function getCookies(link: string, name: string) { } } -export async function getCookiesByHost(cookieLink: string): Promise> { +export async function getCookiesByHost( + cookieLink: string, +): Promise> { const latestCookies: Record = {}; for await (const [key, value] of cookiesDb.sublevel(cookieLink).iterator()) { const parsed = parseCookieValue(value); - if (!latestCookies[key] || parsed.timestamp > latestCookies[key].timestamp) { + if ( + !latestCookies[key] || + parsed.timestamp > latestCookies[key].timestamp + ) { latestCookies[key] = parsed; } } return Object.fromEntries( - Object.entries(latestCookies).map(([key, { value }]) => [key, value]) + Object.entries(latestCookies).map(([key, { value }]) => [key, value]), ); } function parseCookieValue(rawValue: string): { value: any; timestamp: number } { try { const parsed = safeParseJSON(rawValue); - if (parsed && typeof parsed === 'object' && 'value' in parsed && 'timestamp' in parsed) { + if ( + parsed && + typeof parsed === 'object' && + 'value' in parsed && + 'timestamp' in parsed + ) { return parsed; } return { value: parsed, timestamp: 0 }; @@ -414,7 +428,6 @@ export async function deleteConnection(origin: string) { }); } - export async function getConnection(origin: string) { try { const existing = await connectionDb.get(origin); @@ -424,7 +437,11 @@ export async function getConnection(origin: string) { } } -export async function setHeaders(link: string, name: string, value?: string): Promise { +export async function setHeaders( + link: string, + name: string, + value?: string, +): Promise { if (!value) return null; return mutex.runExclusive(async () => { @@ -435,14 +452,17 @@ export async function setHeaders(link: string, name: string, value?: string): Pr try { const existingRaw = await sublevel.get(name).catch(() => null); if (existingRaw) { - const existing = JSON.parse(existingRaw) as { value: string; timestamp: number }; + const existing = JSON.parse(existingRaw) as { + value: string; + timestamp: number; + }; if (existing.value === value && existing.timestamp === timestamp) { return true; } } await Promise.all([ sublevel.put(name, JSON.stringify(newEntry)), - sublevel.put("__timestamp__", timestamp.toString()), + sublevel.put('__timestamp__', timestamp.toString()), ]); return true; } catch (error) { @@ -451,7 +471,6 @@ export async function setHeaders(link: string, name: string, value?: string): Pr }); } - export async function clearHeaders(host: string) { return mutex.runExclusive(async () => { await headersDb.sublevel(host).clear(); @@ -468,13 +487,18 @@ export async function getHeaders(host: string, name: string) { } } -export async function getHeadersByHost(link: string): Promise> { +export async function getHeadersByHost( + link: string, +): Promise> { const headers: Record = {}; const sublevel = headersDb.sublevel(link); try { - for await (const [key, value] of sublevel.iterator({ keyEncoding: "utf8", valueEncoding: "utf8" })) { - if (key === "__timestamp__") continue; + for await (const [key, value] of sublevel.iterator({ + keyEncoding: 'utf8', + valueEncoding: 'utf8', + })) { + if (key === '__timestamp__') continue; const parsed = parseHeaderValue(value); headers[key] = parsed.value; } @@ -483,10 +507,16 @@ export async function getHeadersByHost(link: string): Promise Date: Wed, 19 Mar 2025 11:50:18 -0700 Subject: [PATCH 3/6] chore: cleanup --- src/entries/Background/db.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index c5326ce..b1df734 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -374,15 +374,6 @@ export async function clearCookies(host: string) { }); } -export async function getCookies(link: string, name: string) { - try { - const existing = await cookiesDb.sublevel(link).get(name); - return existing; - } catch (e) { - return null; - } -} - export async function getCookiesByHost( cookieLink: string, ): Promise> { @@ -478,15 +469,6 @@ export async function clearHeaders(host: string) { }); } -export async function getHeaders(host: string, name: string) { - try { - const existing = await headersDb.sublevel(host).get(name); - return existing; - } catch (e) { - return null; - } -} - export async function getHeadersByHost( link: string, ): Promise> { From 5134f58650eb4a3ddde021fa056f6b3ec43e3183 Mon Sep 17 00:00:00 2001 From: Codetrauma Date: Wed, 26 Mar 2025 19:55:55 -0700 Subject: [PATCH 4/6] fix: getHeadersByHost properly retrieves latest headers now --- src/entries/Background/db.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index b1df734..9a17ac1 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -472,21 +472,29 @@ export async function clearHeaders(host: string) { export async function getHeadersByHost( link: string, ): Promise> { - const headers: Record = {}; - const sublevel = headersDb.sublevel(link); + const latestHeaders: Record = {}; try { - for await (const [key, value] of sublevel.iterator({ + for await (const [key, value] of headersDb.sublevel(link).iterator({ keyEncoding: 'utf8', valueEncoding: 'utf8', })) { if (key === '__timestamp__') continue; + const parsed = parseHeaderValue(value); - headers[key] = parsed.value; + if ( + !latestHeaders[key] || + parsed.timestamp > latestHeaders[key].timestamp + ) { + latestHeaders[key] = parsed; + } } - } catch (error) {} - - return headers; + } catch (error) { + console.error(`Error retrieving headers for ${link}:`, error); + } + return Object.fromEntries( + Object.entries(latestHeaders).map(([key, { value }]) => [key, value]), + ); } function parseHeaderValue(rawValue: string): { From baa2bbce4fa00aab91378017c8c329280fb3cf6c Mon Sep 17 00:00:00 2001 From: Codetrauma Date: Mon, 31 Mar 2025 12:21:48 -0700 Subject: [PATCH 5/6] chore: lint --- src/entries/Background/db.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index 9a17ac1..52fda1d 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -472,7 +472,8 @@ export async function clearHeaders(host: string) { export async function getHeadersByHost( link: string, ): Promise> { - const latestHeaders: Record = {}; + const latestHeaders: Record = + {}; try { for await (const [key, value] of headersDb.sublevel(link).iterator({ From ce30c9c5de8759e228e115c952333443264d0f4b Mon Sep 17 00:00:00 2001 From: Codetrauma Date: Tue, 8 Apr 2025 15:10:57 -0700 Subject: [PATCH 6/6] feat: added minimatching for url/host and simplified functionality for setting and retrieving cookies/headers --- src/entries/Background/db.ts | 212 +++++++++++++---------------------- 1 file changed, 79 insertions(+), 133 deletions(-) diff --git a/src/entries/Background/db.ts b/src/entries/Background/db.ts index 52fda1d..c8171f8 100644 --- a/src/entries/Background/db.ts +++ b/src/entries/Background/db.ts @@ -24,10 +24,10 @@ const pluginMetadataDb = db.sublevel('pluginMetadata', { const connectionDb = db.sublevel('connections', { valueEncoding: 'json', }); -const cookiesDb = db.sublevel('cookies', { +export const cookiesDb = db.sublevel('cookies', { valueEncoding: 'json', }); -const headersDb = db.sublevel('headers', { +export const headersDb = db.sublevel('headers', { valueEncoding: 'json', }); const localStorageDb = db.sublevel('sessionStorage', { @@ -336,79 +336,40 @@ export async function setConnection(origin: string) { return true; } -export async function setCookies( - host: string, +async function setValue( + db: typeof cookiesDb | typeof headersDb, + key: string, name: string, value: string, ): Promise { return mutex.runExclusive(async () => { + const sublevel = db.sublevel(key); + const timestampSublevel = sublevel.sublevel('timestamp'); const timestamp = Date.now(); - const cookieStore = cookiesDb.sublevel(host); - const newEntry = { value, timestamp }; - - let existingEntry: { value: string; timestamp: number } | null = null; - try { - const rawEntry = await cookieStore.get(name); - existingEntry = safeParseJSON(rawEntry); - } catch {} - - if (existingEntry) { - const { value: storedValue, timestamp: storedTimestamp } = existingEntry; - if (storedValue === value && storedTimestamp === timestamp) { - return true; - } - if (storedTimestamp >= timestamp) { - return true; - } - } - await cookieStore.put(name, JSON.stringify(newEntry)); + await Promise.all([ + sublevel.put(name, value), + timestampSublevel.put(name, timestamp.toString()), + ]); return true; }); } -export async function clearCookies(host: string) { - return mutex.runExclusive(async () => { - await cookiesDb.sublevel(host).clear(); - return true; - }); -} - -export async function getCookiesByHost( - cookieLink: string, -): Promise> { - const latestCookies: Record = {}; - - for await (const [key, value] of cookiesDb.sublevel(cookieLink).iterator()) { - const parsed = parseCookieValue(value); - if ( - !latestCookies[key] || - parsed.timestamp > latestCookies[key].timestamp - ) { - latestCookies[key] = parsed; - } - } - - return Object.fromEntries( - Object.entries(latestCookies).map(([key, { value }]) => [key, value]), - ); +export async function setCookies( + host: string, + name: string, + value: string, +): Promise { + return setValue(cookiesDb, host, name, value); } -function parseCookieValue(rawValue: string): { value: any; timestamp: number } { - try { - const parsed = safeParseJSON(rawValue); - if ( - parsed && - typeof parsed === 'object' && - 'value' in parsed && - 'timestamp' in parsed - ) { - return parsed; - } - return { value: parsed, timestamp: 0 }; - } catch { - return { value: rawValue, timestamp: 0 }; - } +export async function setHeaders( + link: string, + name: string, + value?: string, +): Promise { + if (!value) return null; + return setValue(headersDb, link, name, value); } export async function deleteConnection(origin: string) { @@ -428,40 +389,6 @@ export async function getConnection(origin: string) { } } -export async function setHeaders( - link: string, - name: string, - value?: string, -): Promise { - if (!value) return null; - - return mutex.runExclusive(async () => { - const sublevel = headersDb.sublevel(link); - const timestamp = Date.now(); - const newEntry = { value, timestamp }; - - try { - const existingRaw = await sublevel.get(name).catch(() => null); - if (existingRaw) { - const existing = JSON.parse(existingRaw) as { - value: string; - timestamp: number; - }; - if (existing.value === value && existing.timestamp === timestamp) { - return true; - } - } - await Promise.all([ - sublevel.put(name, JSON.stringify(newEntry)), - sublevel.put('__timestamp__', timestamp.toString()), - ]); - return true; - } catch (error) { - return true; - } - }); -} - export async function clearHeaders(host: string) { return mutex.runExclusive(async () => { await headersDb.sublevel(host).clear(); @@ -469,48 +396,67 @@ export async function clearHeaders(host: string) { }); } -export async function getHeadersByHost( +function parseValue(rawValue: string): string { + try { + const parsed = safeParseJSON(rawValue); + return parsed && typeof parsed === 'object' && 'value' in parsed + ? parsed.value + : rawValue; + } catch { + return rawValue; + } +} + +async function getValuesByHost( + db: typeof cookiesDb | typeof headersDb, link: string, -): Promise> { - const latestHeaders: Record = - {}; + type: 'cookie' | 'header', +): Promise<{ [key: string]: string }> { + const ret: { [key: string]: { value: string; timestamp: number } } = {}; + const links: { [k: string]: boolean } = {}; + const url = urlify(link); + + for await (const sublevel of db.keys({ keyEncoding: 'utf8' })) { + const l = sublevel.split('!')[1] || sublevel; + links[l] = true; + } - try { - for await (const [key, value] of headersDb.sublevel(link).iterator({ - keyEncoding: 'utf8', - valueEncoding: 'utf8', - })) { - if (key === '__timestamp__') continue; - - const parsed = parseHeaderValue(value); - if ( - !latestHeaders[key] || - parsed.timestamp > latestHeaders[key].timestamp - ) { - latestHeaders[key] = parsed; - } + const matchedLink = url + ? Object.keys(links).filter((l) => l === link || minimatch(l, link))[0] + : Object.keys(links).filter((l) => urlify(l)?.host === link)[0]; + + if (!matchedLink) return {}; + + const sublevel = db.sublevel(matchedLink); + const timestampSublevel = sublevel.sublevel('timestamp'); + + for await (const [key, rawValue] of sublevel.iterator({ + valueEncoding: 'utf8', + })) { + if (key === 'timestamp' || key.startsWith('!timestamp!')) continue; + + const timestamp = + parseInt(await timestampSublevel.get(key).catch(() => '0'), 10) || 0; + const value = parseValue(rawValue); + + if (!ret[key] || timestamp > ret[key].timestamp) { + ret[key] = { value, timestamp }; } - } catch (error) { - console.error(`Error retrieving headers for ${link}:`, error); } - return Object.fromEntries( - Object.entries(latestHeaders).map(([key, { value }]) => [key, value]), - ); + + return Object.fromEntries(Object.entries(ret).map(([k, v]) => [k, v.value])); } -function parseHeaderValue(rawValue: string): { - value: string; - timestamp: number; -} { - try { - const parsed = safeParseJSON(rawValue) as { - value: string; - timestamp: number; - }; - return { value: parsed.value || '', timestamp: parsed.timestamp }; - } catch { - return { value: rawValue, timestamp: 0 }; - } +export async function getCookiesByHost( + link: string, +): Promise<{ [key: string]: string }> { + return getValuesByHost(cookiesDb, link, 'cookie'); +} + +export async function getHeadersByHost( + link: string, +): Promise<{ [key: string]: string }> { + return getValuesByHost(headersDb, link, 'header'); } export async function setLocalStorage(