Skip to content

Commit 8acf0c9

Browse files
nemanjamnemanjam
authored
Merge pull request #2 from nemanjam/fix/stale-cache-issue
Keyv stale cache
2 parents dbc9b8b + a42f8fb commit 8acf0c9

File tree

18 files changed

+191
-39
lines changed

18 files changed

+191
-39
lines changed

‎app/[[...month]]/page.tsx

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import Heading from '@/components/heading';
66
import NewOldCompaniesSection from '@/components/new-old-companies-section';
77

88
import { getNewOldCompaniesForMonthCached } from '@/modules/database/select/company';
9+
import { clearCacheIfDatabaseUpdated } from '@/modules/database/select/is-updated';
910
import { getNewOldCompaniesCountForAllMonthsCached } from '@/modules/database/select/line-chart';
1011
import { getAllMonths } from '@/modules/database/select/month';
1112
import { getStatisticsCached } from '@/modules/database/select/statistics';
@@ -19,6 +20,8 @@ export interface Props extends MonthQueryParam {}
1920
const { title } = METADATA;
2021

2122
const IndexPage: FC<Props> = async ({ params }) => {
23+
await clearCacheIfDatabaseUpdated();
24+
2225
const statistics = await getStatisticsCached();
2326
const lineChartMultipleData = await getNewOldCompaniesCountForAllMonthsCached();
2427

‎app/month/[[...month]]/page.tsx

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import CompaniesCommentsTable from '@/components/companies-comments-table';
66
import Heading from '@/components/heading';
77

88
import { getNewOldCompaniesForMonthCached } from '@/modules/database/select/company';
9+
import { clearCacheIfDatabaseUpdated } from '@/modules/database/select/is-updated';
910
import { getAllMonths } from '@/modules/database/select/month';
1011
import { getBarChartSimpleData } from '@/modules/transform/bar-chart';
1112
import { getCompanyTableData } from '@/modules/transform/table';
@@ -23,6 +24,8 @@ const CurrentMonthPage: FC<Props> = async ({ params }) => {
2324

2425
if (!isValidMonthNameWithDb(selectedMonth)) return notFound();
2526

27+
await clearCacheIfDatabaseUpdated();
28+
2629
const newOldCompanies = await getNewOldCompaniesForMonthCached(selectedMonth);
2730
const companyTableData = getCompanyTableData(newOldCompanies.allCompanies);
2831
const barChartSimpleData = getBarChartSimpleData(newOldCompanies.allCompanies);

‎config/server.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ const projectRootFolder = process.cwd();
77

88
const isProd = process.env.NODE_ENV === 'production';
99

10-
const dbSuffix = isProd ? 'prod' : 'dev';
10+
// const dbSuffix = isProd ? 'prod' : 'dev';
11+
const dbSuffix = 'dev';
1112
const databaseFileName = `hn-new-jobs-database-${dbSuffix}.sqlite3`;
1213

1314
export const SERVER_CONFIG = {

‎constants/cache.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ export const CACHE_KEYS_DATABASE = {
22
getNewOldCompaniesCountForAllMonthsCacheKey: 'getNewOldCompaniesCountForAllMonthsCacheKey',
33
getStatisticsCacheKey: 'getStatisticsCacheKey',
44
getNewOldCompaniesForMonthCacheKey: 'getNewOldCompaniesForMonthCacheKey',
5-
};
5+
getUpdatedAtCacheKey: 'getUpdatedAtCacheKey',
6+
} as const;

‎constants/scripts.ts

+1
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ export const SCRIPTS = {
33
parseOld: 'old',
44
parseOldMany: 'old-many',
55
trimOld: 'trim-old',
6+
trimNew: 'trim-new',
67
} as const;

‎data/database/hn-new-jobs-database-dev.sqlite3

28 KB
Binary file not shown.

‎docs/working-notes/notes4.md

+19
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,22 @@ mozda samo portainer container recreate fails
2121
N new ads since yesterday, both debugging and ui, compare last month updatedAt // dobar feature
2222
deleteLastNMonths to debug updating
2323
remove plausible proxy for country, open issue and ask
24+
-----------------
25+
first newer month, check db too, not latest
26+
await getCacheDatabase().clear(); simply FAILS
27+
after clear() doesnt work
28+
----
29+
cacheDatabaseWrapper() must accept keyv as arg, pass getCacheDatabase() every time
30+
doesnt mutate
31+
always has data from memory, get() has memory
32+
const cachedResult = await getCacheDatabase().get<T>(key);
33+
console.log('key', key, 'cachedResult', cachedResult);
34+
if (cachedResult) return cachedResult;
35+
// set() Stores a Promise
36+
const dbResult = func(...args);
37+
await getCacheDatabase().set(key, dbResult);
38+
NIJE ISTI PROCESS u scheduler script, cli script, i u next.js page
39+
cache.clear() zove u drugoj aplikaciji - procesu iako je isti import // glavna POENTA
40+
event?
41+
get number of months from db, check in next.js page and cache.clear() // to
42+
compare cached number of months and db number of months

‎libs/datetime.ts

+4-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ import { format as formatTz, toZonedTime } from 'date-fns-tz';
1111

1212
import { SERVER_CONFIG } from '@/config/server';
1313

14+
export type DateUnion = Date | string | number;
15+
1416
const { appTimeZone, appDateTimeFormat } = SERVER_CONFIG;
1517

1618
export const DATETIME = {
@@ -36,10 +38,10 @@ export const convertMonthNameToDate = (monthString: string): Date =>
3638
* Format to 'dd/MM/yyyy HH:mm:ss' to Belgrade time zone.
3739
* @example 05/11/2024 14:30:01
3840
*/
39-
export const humanFormat = (date: Date): string =>
41+
export const humanFormat = (date: DateUnion): string =>
4042
formatTz(date, appDateTimeFormat, { timeZone: appTimeZone });
4143

42-
export const getAppTime = (dateTime: Date): Date => toZonedTime(dateTime, appTimeZone);
44+
export const getAppTime = (dateTime: DateUnion): Date => toZonedTime(dateTime, appTimeZone);
4345

4446
export const getAppNow = (): Date => getAppTime(new Date());
4547

‎libs/keyv.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import fs from 'fs';
1+
// import fs from 'fs';
22

33
import Keyv, { KeyvStoreAdapter } from 'keyv';
44
import KeyvFile from 'keyv-file';
@@ -22,6 +22,8 @@ try {
2222
// fs.unlinkSync(cacheDatabaseFilePath);
2323
} catch (error) {}
2424

25+
/*-------------------------------- database cache ------------------------------*/
26+
2527
class CacheDatabaseInstance {
2628
private static storeAdapter: KeyvStoreAdapter | null = null;
2729

@@ -48,6 +50,8 @@ CacheDatabaseInstance.setAdapter(databaseLruAdapter);
4850

4951
export const getCacheDatabase = CacheDatabaseInstance.getCacheDatabase;
5052

53+
/*-------------------------------- http cache ------------------------------*/
54+
5155
class CacheHttpInstance {
5256
private static storeAdapter: KeyvStoreAdapter | null = null;
5357

‎modules/database/delete.ts

+16
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { getDb } from '@/modules/database/schema';
2+
import { getAllMonths } from '@/modules/database/select/month';
23
import { isValidMonthName } from '@/utils/validation';
34
import { ALGOLIA } from '@/constants/algolia';
45

@@ -20,3 +21,18 @@ export const deleteMonthsAndCompaniesOlderThanMonth = (
2021

2122
return changes;
2223
};
24+
25+
/**
26+
* Newer than (excluding) monthName. Delete last month by default.
27+
* For debugging cache invalidation on new month.
28+
*/
29+
export const deleteMonthsAndCompaniesNewerThanMonth = (
30+
monthName = getAllMonths()[1].name
31+
): number => {
32+
if (!isValidMonthName(monthName))
33+
throw new Error(`Invalid format, monthName: ${monthName}. Expected "YYYY-MM".`);
34+
35+
const changes = getDb().prepare(`DELETE FROM month WHERE name > ?`).run(monthName).changes;
36+
37+
return changes;
38+
};

‎modules/database/select/is-updated.ts

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import { cacheDatabaseWrapper, getCacheDatabase } from '@/libs/keyv';
2+
import logger from '@/libs/winston';
3+
import { CACHE_KEYS_DATABASE } from '@/constants/cache';
4+
import { getAllMonths } from './month';
5+
6+
import { DbMonth } from '@/types/database';
7+
8+
const { getUpdatedAtCacheKey } = CACHE_KEYS_DATABASE;
9+
10+
export const getUpdatedAt = (): string[] => getAllMonths().map((month) => month.updatedAt);
11+
12+
export const getUpdatedAtCached = () => cacheDatabaseWrapper(getUpdatedAtCacheKey, getUpdatedAt);
13+
14+
export const findUpdatedMonth = (
15+
allMonths: DbMonth[],
16+
updatedAtArray: string[]
17+
): DbMonth | undefined => {
18+
const updatedAtSet = new Set(updatedAtArray);
19+
20+
const dbMonth = allMonths.find((month) => !updatedAtSet.has(month.updatedAt));
21+
if (dbMonth) return dbMonth;
22+
23+
const allMonthsUpdatedAtSet = new Set(allMonths.map((month) => month.updatedAt));
24+
const missingUpdatedAt = updatedAtArray.find(
25+
(updatedAt) => !allMonthsUpdatedAtSet.has(updatedAt)
26+
);
27+
if (missingUpdatedAt) {
28+
return {
29+
name: 'missing-in-db',
30+
threadId: 'missing-in-db',
31+
createdAtOriginal: new Date().toISOString(),
32+
createdAt: new Date().toISOString(),
33+
updatedAt: new Date(missingUpdatedAt).toISOString(),
34+
};
35+
}
36+
37+
return undefined;
38+
};
39+
40+
export const getUpdatedMonth = async (): Promise<DbMonth | undefined> => {
41+
const allMonths = getAllMonths();
42+
const updatedAtArrayCached = await getUpdatedAtCached();
43+
44+
const updatedMonth = findUpdatedMonth(allMonths, updatedAtArrayCached);
45+
46+
return updatedMonth;
47+
};
48+
49+
/** This must run on every request, to detect change. */
50+
51+
export const clearCacheIfDatabaseUpdated = async (): Promise<DbMonth | undefined> => {
52+
const updatedMonth = await getUpdatedMonth();
53+
54+
if (updatedMonth) {
55+
logger.info('Database changed, clearing cache, updatedMonth:', updatedMonth);
56+
await getCacheDatabase().clear();
57+
58+
// populate cache again
59+
await getUpdatedAtCached();
60+
}
61+
62+
return updatedMonth;
63+
};

‎modules/database/select/line-chart.ts

+5-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { CACHE_KEYS_DATABASE } from '@/constants/cache';
44

55
import { LineChartMultipleData } from '@/types/charts';
66

7-
const { getNewOldCompaniesCountForAllMonthsCacheKey: lineChartCacheKey } = CACHE_KEYS_DATABASE;
7+
const { getNewOldCompaniesCountForAllMonthsCacheKey } = CACHE_KEYS_DATABASE;
88

99
export const getNewOldCompaniesCountForAllMonths = (): LineChartMultipleData[] => {
1010
const query = `
@@ -87,4 +87,7 @@ export const getNewOldCompaniesCountForAllMonths = (): LineChartMultipleData[] =
8787
};
8888

8989
export const getNewOldCompaniesCountForAllMonthsCached = () =>
90-
cacheDatabaseWrapper(lineChartCacheKey, getNewOldCompaniesCountForAllMonths);
90+
cacheDatabaseWrapper(
91+
getNewOldCompaniesCountForAllMonthsCacheKey,
92+
getNewOldCompaniesCountForAllMonths
93+
);

‎modules/database/select/utils.ts

+1-8
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,6 @@ export const withCommentsQuery = (
5555
`;
5656

5757
export const convertCompanyRowType = (row: CompanyWithCommentsAsStrings): CompanyWithComments => ({
58-
company: {
59-
name: row.name,
60-
commentId: row.commentId,
61-
monthName: row.monthName,
62-
createdAtOriginal: new Date(row.createdAtOriginal),
63-
createdAt: new Date(row.createdAt),
64-
updatedAt: new Date(row.updatedAt),
65-
},
58+
company: row,
6659
comments: JSON.parse(row.comments) as DbCompany[],
6760
});

‎modules/parser/calls.ts

+5-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import { parseNewMonth, parseNOldMonths, parseOldMonth } from '@/modules/parser/parse';
22
import { getAppNow } from '@/libs/datetime';
3-
import { getCacheDatabase } from '@/libs/keyv';
43
import { SERVER_CONFIG } from '@/config/server';
54

65
import { ParserResponse } from '@/types/api';
@@ -15,21 +14,23 @@ const { oldMonthsCount } = SERVER_CONFIG;
1514
* app/api/parser/[script]/route.ts
1615
* modules/parser/main.ts
1716
*
18-
* ! invalidate all database keys here, for scheduler, api, cli
17+
* ! invalidate all database keys here, for scheduler, api, cli // WRONG
1918
*
2019
* These can throw.
2120
*/
2221

2322
export const callParseNewMonth = async (): Promise<ParserResponse> => {
2423
const parserResult: ParserResult = await parseNewMonth();
2524

25+
// ! here this WONT work, because this function is called in another process, scheduler, cli
26+
// ! must detect database change or message queue
27+
// await getCacheDatabase().clear();
28+
2629
const parserResponse: ParserResponse = {
2730
parseMessage: `Parsing new month successful, now: ${getAppNow()}.`,
2831
parserResults: [parserResult],
2932
};
3033

31-
await getCacheDatabase().clear();
32-
3334
return parserResponse;
3435
};
3536

@@ -41,8 +42,6 @@ export const callParseOldMonth = async (): Promise<ParserResponse> => {
4142
parserResults: [parserResult],
4243
};
4344

44-
await getCacheDatabase().clear();
45-
4645
return parserResponse;
4746
};
4847

@@ -54,7 +53,5 @@ export const callParseNOldMonths = async (): Promise<ParserResponse> => {
5453
parseMessage: `Parsing ${parserResults.length} old months successful, now: ${getAppNow()}.`,
5554
};
5655

57-
await getCacheDatabase().clear();
58-
5956
return parserResponse;
6057
};

‎modules/parser/main.ts

+17-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { callParseNewMonth, callParseNOldMonths, callParseOldMonth } from '@/modules/parser/calls';
2-
import { deleteMonthsAndCompaniesOlderThanMonth } from '@/modules/database/delete';
2+
import {
3+
deleteMonthsAndCompaniesNewerThanMonth,
4+
deleteMonthsAndCompaniesOlderThanMonth,
5+
} from '@/modules/database/delete';
36
import logger from '@/libs/winston';
47
import { SCRIPTS } from '@/constants/scripts';
58
import { SERVER_CONFIG } from '@/config/server';
@@ -15,7 +18,7 @@ const main = async (script: ScriptType) => {
1518
switch (script) {
1619
case SCRIPTS.parseNew: {
1720
const parserResponse: ParserResponse = await callParseNewMonth();
18-
logger.info('main.ts parseNew script, parserResponse:', parserResponse);
21+
logger.info('main.ts parseNew script, parserResponse:', { parserResponse });
1922
break;
2023
}
2124
case SCRIPTS.parseOld: {
@@ -35,7 +38,18 @@ const main = async (script: ScriptType) => {
3538
const statisticsAfter = getStatistics();
3639

3740
const context = { rowsCount, statisticsBefore, statisticsAfter };
38-
logger.info('main.ts script, deleted rows:', context);
41+
logger.info('main.ts script, deleted old rows:', context);
42+
43+
break;
44+
}
45+
// for debugging cache.clear()
46+
case SCRIPTS.trimNew: {
47+
const statisticsBefore = getStatistics();
48+
const rowsCount = deleteMonthsAndCompaniesNewerThanMonth();
49+
const statisticsAfter = getStatistics();
50+
51+
const context = { rowsCount, statisticsBefore, statisticsAfter };
52+
logger.info('main.ts script, deleted new rows:', context);
3953
break;
4054
}
4155
}

‎modules/parser/months.ts

+32-4
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,43 @@
11
import { getAllMonths } from '@/modules/parser/algolia/threads';
2-
import { getFirstMonth } from '@/modules/database/select/month';
2+
import { getFirstMonth, getLastMonth } from '@/modules/database/select/month';
33

4-
/** Always update latest month. */
4+
/**
5+
* Parse first month newer than in the database, not only the latest.
6+
* Always update the latest month.
7+
*/
58

69
export const getNewMonthName = async (): Promise<string> => {
10+
const lastMonth = getLastMonth();
11+
712
const parsedMonths = await getAllMonths();
813
if (!(parsedMonths.length > 0))
914
throw new Error(`Invalid parsedMonths length: ${parsedMonths.length}`);
1015

11-
// overwrite
12-
return parsedMonths[0];
16+
let newMonthName: string;
17+
18+
// handle empty db
19+
if (!lastMonth) {
20+
// gets last thread on empty db
21+
newMonthName = parsedMonths[0];
22+
return newMonthName;
23+
}
24+
25+
const index = parsedMonths.indexOf(lastMonth.name);
26+
// index not found or out of bounds
27+
if (!(index !== -1 && index < parsedMonths.length - 1))
28+
throw new Error(
29+
`IndexOf lastMonth.name: ${lastMonth.name} from database not found in parsedMonths.`
30+
);
31+
32+
// redo last month
33+
if (index === 0) {
34+
newMonthName = parsedMonths[0];
35+
return newMonthName;
36+
}
37+
38+
// one month newer
39+
newMonthName = parsedMonths[index - 1];
40+
return newMonthName;
1341
};
1442

1543
export const getOldMonthName = async (): Promise<string> => {

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
"hn:dev:old": "tsx modules/parser/main.ts old",
1414
"hn:dev:old-many": "tsx modules/parser/main.ts old-many",
1515
"hn:dev:trim-old": "tsx modules/parser/main.ts trim-old",
16+
"hn:dev:trim-new": "tsx modules/parser/main.ts trim-new",
1617
"lint": "next lint",
1718
"lint:fix": "next lint --fix",
1819
"preview": "next build && next start",

0 commit comments

Comments
 (0)