add lru cache

Author: nemanjam

config/server.ts

@@ -23,10 +23,13 @@ export const SERVER_CONFIG = {
   plausibleServerUrl: process.env.PLAUSIBLE_SERVER_URL, // build time, Docker
   // paths
   databaseFilePath: join(projectRootFolder, './data/database/', databaseFileName),
+  // unused
   cacheHttpFilePath: join(projectRootFolder, './data/cache/', 'cache-http.json'),
   cacheDatabaseFilePath: join(projectRootFolder, './data/cache/', 'cache-database.json'),
   logFilePath: join(projectRootFolder, './data/logs/', 'app.html'),
   // cache
+  cacheDatabaseLruItems: 100,
+  cacheHttpLruItems: 10,
   cacheHttpTtlMinutes: 5, // one old-many call
   // parser
   oldMonthsCount: 12, // one year

docs/working-notes/notes4.md

@@ -11,10 +11,13 @@ case insensitive company name consensys
 click up api
 
 https://play.clickhouse.com/play?user=play#U0VMRUNUIG1heCh0aW1lKSBGUk9NIGhhY2tlcm5ld3NfaGlzdG9yeQ==
-lru cache
+        lru cache
 lighthouse in ssr section readme
 need to restart container after cron new month, maybe cache or database volume 
 jeste do cache, nema sa cache disabled
+cache invalidate fails?
 mozda samo portainer container recreate fails
 ----
-x new ads since yesterday, both debugging and ui, compare last month updatedAt
+N new ads since yesterday, both debugging and ui, compare last month updatedAt // dobar feature
+deleteLastNMonths for test updating
+remove plausible proxy for country, open issue and ask

libs/keyv.ts

@@ -1,12 +1,20 @@
 import fs from 'fs';
 
-import Keyv from 'keyv';
+import Keyv, { KeyvStoreAdapter } from 'keyv';
 import KeyvFile from 'keyv-file';
 
 import { Singleton } from '@/utils/singleton';
 import { SERVER_CONFIG } from '@/config/server';
 
-const { cacheHttpFilePath, cacheDatabaseFilePath, cacheDatabaseDisabled } = SERVER_CONFIG;
+const { KeyvLruManagedTtl } = require('keyv-lru');
+
+const {
+  cacheHttpFilePath,
+  cacheDatabaseFilePath,
+  cacheDatabaseLruItems,
+  cacheHttpLruItems,
+  cacheDatabaseDisabled,
+} = SERVER_CONFIG;
 
 // deletes cache for testing
 try {
@@ -15,11 +23,14 @@ try {
 } catch (error) {}
 
 class CacheDatabaseInstance {
+  private static storeAdapter: KeyvStoreAdapter | null = null;
+
   private static createCacheDatabase(): Keyv {
-    // ttl: undefined - infinite duration
-    return new Keyv({
-      store: new KeyvFile({ filename: cacheDatabaseFilePath }),
-    });
+    return new Keyv({ store: CacheDatabaseInstance.storeAdapter });
+  }
+
+  public static setAdapter(adapter: KeyvStoreAdapter): void {
+    CacheDatabaseInstance.storeAdapter = adapter;
   }
 
   public static getCacheDatabase(): Keyv {
@@ -29,12 +40,23 @@ class CacheDatabaseInstance {
   }
 }
 
+// ttl: undefined - infinite duration
+const _databaseFileAdapter = new KeyvFile({ filename: cacheDatabaseFilePath });
+
+const databaseLruAdapter = new KeyvLruManagedTtl({ max: cacheDatabaseLruItems });
+CacheDatabaseInstance.setAdapter(databaseLruAdapter);
+
 export const getCacheDatabase = CacheDatabaseInstance.getCacheDatabase;
+
 class CacheHttpInstance {
+  private static storeAdapter: KeyvStoreAdapter | null = null;
+
   private static createCacheHttp(): Keyv {
-    return new Keyv({
-      store: new KeyvFile({ filename: cacheHttpFilePath }),
-    });
+    return new Keyv({ store: CacheHttpInstance.storeAdapter });
+  }
+
+  public static setAdapter(adapter: KeyvStoreAdapter): void {
+    CacheHttpInstance.storeAdapter = adapter;
   }
 
   public static getCacheHttp(): Keyv {
@@ -44,6 +66,11 @@ class CacheHttpInstance {
   }
 }
 
+const _httpFileAdapter = new KeyvFile({ filename: cacheHttpFilePath });
+
+const httpLruAdapter = new KeyvLruManagedTtl({ max: cacheHttpLruItems });
+CacheHttpInstance.setAdapter(httpLruAdapter);
+
 export const getCacheHttp = CacheHttpInstance.getCacheHttp;
 
 export const cacheDatabaseWrapper = async <T, A extends any[]>(

package.json

@@ -46,6 +46,7 @@
     "jsdom": "^25.0.1",
     "keyv": "^5.1.2",
     "keyv-file": "^5.0.3",
+    "keyv-lru": "^3.0.4",
     "lucide-react": "^0.456.0",
     "next": "^15.0.0",
     "next-plausible": "^3.12.4",

yarn.lock

@@ -3042,6 +3042,13 @@ keyv-file@^5.0.3:
     fs-extra "^4.0.1"
     tslib "^1.14.1"
 
+keyv-lru@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/keyv-lru/-/keyv-lru-3.0.4.tgz#95ec97077a18d9a6ceb16a48b6530c26c5ce9099"
+  integrity sha512-4XALKaBM/3nZr0o0Hskq+SPS2tBkfZXYQRDl0SqywIimZf0wrrMEgSfoOM7eNuKXnICIOHs3exPucGKDYAp/8g==
+  dependencies:
+    tiny-lru "^1.6.1"
+
 keyv@^4.5.3:
   version "4.5.4"
   resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93"
@@ -4279,6 +4286,11 @@ tiny-invariant@^1.3.1:
   resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127"
   integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==
 
+tiny-lru@^1.6.1:
+  version "1.6.4"
+  resolved "https://registry.yarnpkg.com/tiny-lru/-/tiny-lru-1.6.4.tgz#ee5226a29a7b734ef8aee47673dd5079bf987441"
+  integrity sha512-Et+J3Css66XPSLWjLF9wmgbECsGiExlEL+jxsFerTQF6N6dpxswDTPAfIrAbQKO5c1uhgq2xvo5zMk1W+kBDNA==
+
 tldts-core@^6.1.58:
   version "6.1.58"
   resolved "https://registry.yarnpkg.com/tldts-core/-/tldts-core-6.1.58.tgz#f0b5c1fcb2e214f558c7cb380fb1e6f4b2459d8b"