From 89047efa10cd2c2b4bd837441dbbde6b3b84f1d3 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 16:11:37 -0800 Subject: [PATCH 01/11] Make runQuery generic to make it easy to type DB query results. --- packages/loot-core/src/server/budget/base.ts | 4 +- .../loot-core/src/server/budget/statements.ts | 4 +- .../src/server/budget/template-notes.test.ts | 5 +- packages/loot-core/src/server/db/index.ts | 111 ++++-- packages/loot-core/src/server/db/mappings.ts | 14 +- packages/loot-core/src/server/db/types.d.ts | 9 - .../loot-core/src/server/db/types/index.ts | 315 ++++++++++++++++++ packages/loot-core/src/server/main.ts | 16 +- packages/loot-core/src/server/sync/index.ts | 2 +- .../loot-core/src/server/sync/migrate.test.ts | 6 +- packages/loot-core/src/server/tools/app.ts | 2 +- 11 files changed, 422 insertions(+), 66 deletions(-) delete mode 100644 packages/loot-core/src/server/db/types.d.ts create mode 100644 packages/loot-core/src/server/db/types/index.ts diff --git a/packages/loot-core/src/server/budget/base.ts b/packages/loot-core/src/server/budget/base.ts index 56505d8d6de..a3fcb075e17 100644 --- a/packages/loot-core/src/server/budget/base.ts +++ b/packages/loot-core/src/server/budget/base.ts @@ -41,7 +41,7 @@ function createCategory(cat, sheetName, prevSheetName, start, end) { initialValue: 0, run: () => { // Making this sync is faster! - const rows = db.runQuery( + const rows = db.runQuery<{ amount: number }>( `SELECT SUM(amount) as amount FROM v_transactions_internal_alive t LEFT JOIN accounts a ON a.id = t.account WHERE t.date >= ${start} AND t.date <= ${end} @@ -86,7 +86,7 @@ function createCategoryGroup(group, sheetName) { function handleAccountChange(months, oldValue, newValue) { if (!oldValue || oldValue.offbudget !== newValue.offbudget) { - const rows = db.runQuery( + const rows = db.runQuery>( ` SELECT DISTINCT(category) as category FROM transactions WHERE acct = ? diff --git a/packages/loot-core/src/server/budget/statements.ts b/packages/loot-core/src/server/budget/statements.ts index 43a1af7d940..13e4707a5f7 100644 --- a/packages/loot-core/src/server/budget/statements.ts +++ b/packages/loot-core/src/server/budget/statements.ts @@ -1,5 +1,5 @@ import * as db from '../db'; -import { Schedule } from '../db/types'; +import { DbSchedule } from '../db'; import { GOAL_PREFIX, TEMPLATE_PREFIX } from './template-notes'; @@ -41,7 +41,7 @@ export async function getCategoriesWithTemplateNotes(): Promise< ); } -export async function getActiveSchedules(): Promise { +export async function getActiveSchedules(): Promise { return await db.all( 'SELECT id, rule, active, completed, posts_transaction, tombstone, name from schedules WHERE name NOT NULL AND tombstone = 0', ); diff --git a/packages/loot-core/src/server/budget/template-notes.test.ts b/packages/loot-core/src/server/budget/template-notes.test.ts index 1d036c92736..0170eb85fcb 100644 --- a/packages/loot-core/src/server/budget/template-notes.test.ts +++ b/packages/loot-core/src/server/budget/template-notes.test.ts @@ -1,5 +1,4 @@ import * as db from '../db'; -import { Schedule } from '../db/types'; import { CategoryWithTemplateNote, @@ -20,7 +19,7 @@ function mockGetTemplateNotesForCategories( ); } -function mockGetActiveSchedules(schedules: Schedule[]) { +function mockGetActiveSchedules(schedules: db.DbSchedule[]) { (getActiveSchedules as jest.Mock).mockResolvedValue(schedules); } @@ -245,7 +244,7 @@ describe('checkTemplates', () => { ); }); -function mockSchedules(): Schedule[] { +function mockSchedules(): db.DbSchedule[] { return [ { id: 'mock-schedule-1', diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index dc324fca85f..12e80e07a1d 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -15,11 +15,7 @@ import * as fs from '../../platform/server/fs'; import * as sqlite from '../../platform/server/sqlite'; import * as monthUtils from '../../shared/months'; import { groupById } from '../../shared/util'; -import { - CategoryEntity, - CategoryGroupEntity, - PayeeEntity, -} from '../../types/models'; +import { CategoryEntity, CategoryGroupEntity } from '../../types/models'; import { schema, schemaConfig, @@ -37,6 +33,16 @@ import { import { sendMessages, batchMessages } from '../sync'; import { shoveSortOrders, SORT_INCREMENT } from './sort'; +import { + DbAccount, + DbCategory, + DbCategoryGroup, + DbPayee, + DbTransaction, + DbViewTransaction, +} from './types'; + +export * from './types'; export { toDateRepr, fromDateRepr } from '../models'; @@ -100,17 +106,24 @@ export function runQuery( sql: string, params?: Array, fetchAll?: false, -); -export function runQuery( +): { changes: unknown }; + +export function runQuery( sql: string, params: Array | undefined, fetchAll: true, -); -export function runQuery(sql, params, fetchAll) { - // const unrecord = perf.record('sqlite'); - const result = sqlite.runQuery(db, sql, params, fetchAll); - // unrecord(); - return result; +): T[]; + +export function runQuery( + sql: string, + params: (string | number)[], + fetchAll: boolean, +) { + if (fetchAll) { + return sqlite.runQuery(db, sql, params, true); + } else { + return sqlite.runQuery(db, sql, params, false); + } } export function execQuery(sql: string) { @@ -147,19 +160,28 @@ export function asyncTransaction(fn: () => Promise) { // async. We return a promise here until we've audited all the code to // make sure nothing calls `.then` on this. export async function all(sql, params?: (string | number)[]) { - return runQuery(sql, params, true); + // TODO: In the next phase, we will make this function generic + // and pass the type of the return type to `runQuery`. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return runQuery(sql, params, true) as any[]; } export async function first(sql, params?: (string | number)[]) { const arr = await runQuery(sql, params, true); - return arr.length === 0 ? null : arr[0]; + // TODO: In the next phase, we will make this function generic + // and pass the type of the return type to `runQuery`. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return arr.length === 0 ? null : (arr[0] as any); } // The underlying sql system is now sync, but we can't update `first` yet // without auditing all uses of it export function firstSync(sql, params?: (string | number)[]) { const arr = runQuery(sql, params, true); - return arr.length === 0 ? null : arr[0]; + // TODO: In the next phase, we will make this function generic + // and pass the type of the return type to `runQuery`. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return arr.length === 0 ? null : (arr[0] as any); } // This function is marked as async because `runQuery` is no longer @@ -175,7 +197,10 @@ export async function select(table, id) { [id], true, ); - return rows[0]; + // TODO: In the next phase, we will make this function generic + // and pass the type of the return type to `runQuery`. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return rows[0] as any; } export async function update(table, params) { @@ -252,9 +277,12 @@ export async function deleteAll(table: string) { export async function selectWithSchema(table, sql, params) { const rows = await runQuery(sql, params, true); - return rows + const convertedRows = rows .map(row => convertFromSelect(schema, schemaConfig, table, row)) .filter(Boolean); + // TODO: Make convertFromSelect generic so we don't need this cast + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return convertedRows as any[]; } export async function selectFirstWithSchema(table, sql, params) { @@ -282,16 +310,18 @@ export function updateWithSchema(table, fields) { // Data-specific functions. Ideally this would be split up into // different files +// TODO: Fix return type. This should returns a DbCategory[]. export async function getCategories( - ids?: Array, + ids?: Array, ): Promise { const whereIn = ids ? `c.id IN (${toSqlQueryParameters(ids)}) AND` : ''; const query = `SELECT c.* FROM categories c WHERE ${whereIn} c.tombstone = 0 ORDER BY c.sort_order, c.id`; return ids ? await all(query, [...ids]) : await all(query); } +// TODO: Fix return type. This should returns a [DbCategoryGroup, ...DbCategory]. export async function getCategoriesGrouped( - ids?: Array, + ids?: Array, ): Promise> { const categoryGroupWhereIn = ids ? `cg.id IN (${toSqlQueryParameters(ids)}) AND` @@ -432,7 +462,11 @@ export function updateCategory(category) { return update('categories', category); } -export async function moveCategory(id, groupId, targetId?: string) { +export async function moveCategory( + id: DbCategory['id'], + groupId: DbCategoryGroup['id'], + targetId?: DbCategory['id'], +) { if (!groupId) { throw new Error('moveCategory: groupId is required'); } @@ -449,7 +483,10 @@ export async function moveCategory(id, groupId, targetId?: string) { await update('categories', { id, sort_order, cat_group: groupId }); } -export async function deleteCategory(category, transferId?: string) { +export async function deleteCategory( + category: Pick, + transferId?: DbCategory['id'], +) { if (transferId) { // We need to update all the deleted categories that currently // point to the one we're about to delete so they all are @@ -469,11 +506,11 @@ export async function deleteCategory(category, transferId?: string) { return delete_('categories', category.id); } -export async function getPayee(id) { +export async function getPayee(id: DbPayee['id']) { return first(`SELECT * FROM payees WHERE id = ?`, [id]); } -export async function getAccount(id) { +export async function getAccount(id: DbAccount['id']) { return first(`SELECT * FROM accounts WHERE id = ?`, [id]); } @@ -487,7 +524,7 @@ export async function insertPayee(payee) { return id; } -export async function deletePayee(payee) { +export async function deletePayee(payee: Pick) { const { transfer_acct } = await first('SELECT * FROM payees WHERE id = ?', [ payee.id, ]); @@ -506,7 +543,7 @@ export async function deletePayee(payee) { return delete_('payees', payee.id); } -export async function deleteTransferPayee(payee) { +export async function deleteTransferPayee(payee: Pick) { // This allows deleting transfer payees return delete_('payees', payee.id); } @@ -516,9 +553,12 @@ export function updatePayee(payee) { return update('payees', payee); } -export async function mergePayees(target: string, ids: string[]) { +export async function mergePayees( + target: DbPayee['id'], + ids: Array, +) { // Load in payees so we can check some stuff - const dbPayees: PayeeEntity[] = await all('SELECT * FROM payees'); + const dbPayees: DbPayee[] = await all('SELECT * FROM payees'); const payees = groupById(dbPayees); // Filter out any transfer payees @@ -613,7 +653,7 @@ export async function getOrphanedPayees() { return rows.map(row => row.id); } -export async function getPayeeByName(name) { +export async function getPayeeByName(name: DbPayee['name']) { return first( `SELECT * FROM payees WHERE UNICODE_LOWER(name) = ? AND tombstone = 0`, [name.toLowerCase()], @@ -632,7 +672,7 @@ export function getAccounts() { export async function insertAccount(account) { const accounts = await all( 'SELECT * FROM accounts WHERE offbudget = ? ORDER BY sort_order, name', - [account.offbudget != null ? account.offbudget : 0], + [account.offbudget ? 1 : 0], ); // Don't pass a target in, it will default to appending at the end @@ -651,7 +691,10 @@ export function deleteAccount(account) { return delete_('accounts', account.id); } -export async function moveAccount(id, targetId) { +export async function moveAccount( + id: DbAccount['id'], + targetId: DbAccount['id'], +) { const account = await first('SELECT * FROM accounts WHERE id = ?', [id]); let accounts; if (account.closed) { @@ -661,7 +704,7 @@ export async function moveAccount(id, targetId) { } else { accounts = await all( `SELECT id, sort_order FROM accounts WHERE tombstone = 0 AND offbudget = ? ORDER BY sort_order, name`, - [account.offbudget], + [account.offbudget ? 1 : 0], ); } @@ -674,7 +717,7 @@ export async function moveAccount(id, targetId) { }); } -export async function getTransaction(id) { +export async function getTransaction(id: DbViewTransaction['id']) { const rows = await selectWithSchema( 'transactions', 'SELECT * FROM v_transactions WHERE id = ?', @@ -683,7 +726,7 @@ export async function getTransaction(id) { return rows[0]; } -export async function getTransactions(accountId) { +export async function getTransactions(accountId: DbTransaction['acct']) { if (arguments.length > 1) { throw new Error( '`getTransactions` was given a second argument, it now only takes a single argument `accountId`', diff --git a/packages/loot-core/src/server/db/mappings.ts b/packages/loot-core/src/server/db/mappings.ts index 81b0c280db2..f7423cab5f0 100644 --- a/packages/loot-core/src/server/db/mappings.ts +++ b/packages/loot-core/src/server/db/mappings.ts @@ -22,14 +22,12 @@ let unlistenSync; export async function loadMappings() { // The mappings are separated into tables specific to the type of // data. But you know, we really could keep a global mapping table. - const categories = (await db.all('SELECT * FROM category_mapping')).map(r => [ - r.id, - r.transferId, - ]); - const payees = (await db.all('SELECT * FROM payee_mapping')).map(r => [ - r.id, - r.targetId, - ]); + const categories = (await db.all('SELECT * FROM category_mapping')).map( + r => [r.id, r.transferId] as const, + ); + const payees = (await db.all('SELECT * FROM payee_mapping')).map( + r => [r.id, r.targetId] as const, + ); // All ids are unique, so we can just keep a global table of mappings allMappings = new Map(categories.concat(payees)); diff --git a/packages/loot-core/src/server/db/types.d.ts b/packages/loot-core/src/server/db/types.d.ts deleted file mode 100644 index b6fdd94ec3a..00000000000 --- a/packages/loot-core/src/server/db/types.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type Schedule = { - id: string; - rule: string; - active: number; - completed: number; - posts_transaction: number; - tombstone: number; - name: string | null; -}; diff --git a/packages/loot-core/src/server/db/types/index.ts b/packages/loot-core/src/server/db/types/index.ts new file mode 100644 index 00000000000..54b42c47799 --- /dev/null +++ b/packages/loot-core/src/server/db/types/index.ts @@ -0,0 +1,315 @@ +// These are the type that exactly match the database schema. +// The `Entity` types e.g. `TransactionEntity`, `AccountEntity`, etc +// are specific to the AQL query framework and does not necessarily +// match the actual database schema. + +type JsonString = string; + +export type DbAccount = { + id: string; + name: string; + offbudget: 1 | 0; + closed: 1 | 0; + tombstone: 1 | 0; + sort_order: number; + account_id?: string | null; + balance_current?: number | null; + balance_available?: number | null; + balance_limit?: number | null; + mask?: string | null; + official_name?: string | null; + type?: string | null; + subtype?: string | null; + bank?: string | null; + account_sync_source?: 'simpleFin' | 'goCardless' | null; +}; + +export type DbBank = { + id: string; + bank_id: string; + name: string; + tombstone: 1 | 0; +}; + +export type DbCategory = { + id: string; + name: string; + is_income: 1 | 0; + cat_group: DbCategoryGroup['id']; + sort_order: number; + hidden: 1 | 0; + goal_def?: JsonString | null; + tombstone: 1 | 0; +}; + +export type DbCategoryGroup = { + id: string; + name: string; + is_income: 1 | 0; + sort_order: number; + hidden: 1 | 0; + tombstone: 1 | 0; +}; + +export type DbCategoryMapping = { + id: DbCategory['id']; + transferId: DbCategory['id']; +}; + +export type DbKvCache = { + key: string; + value: string; +}; + +export type DbKvCacheKey = { + id: number; + key: number; +}; + +export type DbClockMessage = { + id: string; + clock: string; +}; + +export type DbCrdtMessage = { + id: string; + timestamp: string; + dataset: string; + row: string; + column: string; + value: Uint8Array; +}; + +export type DbNote = { + id: string; + note: string; +}; + +export type DbPayeeMapping = { + id: DbPayee['id']; + targetId: DbPayee['id']; +}; + +export type DbPayee = { + id: string; + name: string; + transfer_acct?: DbAccount['id'] | null; + favorite: 1 | 0; + learn_categories: 1 | 0; + tombstone: 1 | 0; + // Unused in the codebase + category?: string | null; +}; + +export type DbRule = { + id: string; + stage: string; + conditions: JsonString; + actions: JsonString; + tombstone: 1 | 0; + conditions_op: string; +}; + +export type DbSchedule = { + id: string; + name: string; + rule: DbRule['id']; + active: 1 | 0; + completed: 1 | 0; + posts_transaction: 1 | 0; + tombstone: 1 | 0; +}; + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +type DbScheduleJsonPath = { + schedule_id: DbSchedule['id']; + payee: string; + account: string; + amount: string; + date: string; +}; + +export type DbScheduleNextDate = { + id: string; + schedule_id: DbSchedule['id']; + local_next_date: number; + local_next_date_ts: number; + base_next_date: number; + base_next_date_ts: number; +}; + +// This is unused in the codebase. +// eslint-disable-next-line @typescript-eslint/no-unused-vars +type DbPendingTransaction = { + id: string; + acct: number; + amount: number; + description: string; + date: string; +}; + +export type DbTransaction = { + id: string; + isParent: 1 | 0; + isChild: 1 | 0; + date: number; + acct: DbAccount['id']; + amount: number; + sort_order: number; + parent_id?: DbTransaction['id'] | null; + category?: DbCategory['id'] | null; + description?: string | null; + notes?: string | null; + financial_id?: string | null; + error?: string | null; + imported_description?: string | null; + transferred_id?: DbTransaction['id'] | null; + schedule?: DbSchedule['id'] | null; + starting_balance_flag: 1 | 0; + tombstone: 1 | 0; + cleared: 1 | 0; + reconciled: 1 | 0; + // Unused in the codebase + pending?: 1 | 0 | null; + location?: string | null; + type?: string | null; +}; + +export type DbReflectBudget = { + id: string; + month: number; + category: string; + amount: number; + carryover: number; + goal: number; + long_goal: number; +}; + +export type DbZeroBudgetMonth = { + id: string; + buffered: number; +}; + +export type DbZeroBudget = { + id: string; + month: number; + category: string; + amount: number; + carryover: number; + goal: number; + long_goal: number; +}; + +export type DbTransactionFilter = { + id: string; + name: string; + conditions: JsonString; + conditions_op: string; + tombstone: 1 | 0; +}; + +export type DbPreference = { + id: string; + value: string; +}; + +export type DbCustomReport = { + id: string; + name: string; + start_date: string; + end_date: string; + date_static: number; + date_range: string; + mode: string; + group_by: string; + balance_type: string; + show_empty: 1 | 0; + show_offbudget: 1 | 0; + show_hidden: 1 | 0; + show_uncateogorized: 1 | 0; + selected_categories: string; + graph_type: string; + conditions: JsonString; + conditions_op: string; + metadata: JsonString; + interval: string; + color_scheme: string; + include_current: 1 | 0; + sort_by: string; + tombstone: 1 | 0; +}; + +export type DbDashboard = { + id: string; + type: string; + width: number; + height: number; + x: number; + y: number; + meta: JsonString; + tombstone: 1 | 0; +}; + +export type DbViewTransactionInternal = { + id: DbTransaction['id']; + is_parent: DbTransaction['isParent']; + is_child: DbTransaction['isChild']; + date: DbTransaction['date']; + account: DbAccount['id']; + amount: DbTransaction['amount']; + parent_id: DbTransaction['parent_id'] | null; + category: DbCategory['id'] | null; + payee: DbPayee['id'] | null; + notes: DbTransaction['notes'] | null; + imported_id: DbTransaction['financial_id'] | null; + error: DbTransaction['error'] | null; + imported_payee: DbTransaction['imported_description'] | null; + starting_balance_flag: DbTransaction['starting_balance_flag'] | null; + transfer_id: DbTransaction['transferred_id'] | null; + schedule: DbSchedule['id'] | null; + sort_order: DbTransaction['sort_order']; + cleared: DbTransaction['cleared']; + tombstone: DbTransaction['tombstone']; + reconciled: DbTransaction['reconciled']; +}; + +export type DbViewTransactionInternalAlive = DbViewTransactionInternal; +export type DbViewTransaction = DbViewTransactionInternalAlive; + +export type DbViewCategory = { + id: DbCategory['id']; + name: DbCategory['name']; + is_income: DbCategory['is_income']; + hidden: DbCategory['hidden']; + group: DbCategoryGroup['id']; + sort_order: DbCategory['sort_order']; + tombstone: DbCategory['tombstone']; +}; + +export type DbViewPayee = { + id: DbPayee['id']; + name: DbAccount['name'] | DbPayee['name']; + transfer_acct: DbPayee['transfer_acct']; + tombstone: DbPayee['tombstone']; +}; + +export type DbViewSchedule = { + id: DbSchedule['id']; + name: DbSchedule['name']; + rule: DbSchedule['rule']; + next_date: + | DbScheduleNextDate['local_next_date_ts'] + | DbScheduleNextDate['local_next_date'] + | DbScheduleNextDate['base_next_date']; + active: DbSchedule['active']; + completed: DbSchedule['completed']; + posts_transaction: DbSchedule['posts_transaction']; + tombstone: DbSchedule['tombstone']; + _payee: DbPayeeMapping['targetId']; + _account: DbAccount['id']; + _amount: number; + _amountOp: string; + _date: JsonString; + _conditions: JsonString; + _actions: JsonString; +}; diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.ts index 7657dbd8289..13ced54836f 100644 --- a/packages/loot-core/src/server/main.ts +++ b/packages/loot-core/src/server/main.ts @@ -418,7 +418,7 @@ handlers['category-group-delete'] = mutator(async function ({ }); handlers['must-category-transfer'] = async function ({ id }) { - const res = await db.runQuery( + const res = await db.runQuery<{ count: number }>( `SELECT count(t.id) as count FROM transactions t LEFT JOIN category_mapping cm ON cm.id = t.category WHERE cm.transferId = ? AND t.tombstone = 0`, @@ -773,7 +773,9 @@ handlers['account-close'] = mutator(async function ({ if (numTransactions === 0) { await db.deleteAccount({ id }); } else if (forced) { - const rows = await db.runQuery( + const rows = await db.runQuery< + Pick + >( 'SELECT id, transfer_id FROM v_transactions WHERE account = ?', [id], true, @@ -1096,7 +1098,9 @@ handlers['accounts-bank-sync'] = async function ({ ids = [] }) { 'user-key', ]); - const accounts = await db.runQuery( + const accounts = await db.runQuery< + db.DbAccount & { bankId: db.DbBank['bank_id'] } + >( ` SELECT a.*, b.bank_id as bankId FROM accounts a @@ -1155,7 +1159,9 @@ handlers['accounts-bank-sync'] = async function ({ ids = [] }) { }; handlers['simplefin-batch-sync'] = async function ({ ids = [] }) { - const accounts = await db.runQuery( + const accounts = await db.runQuery< + db.DbAccount & { bankId: db.DbBank['bank_id'] } + >( `SELECT a.*, b.bank_id as bankId FROM accounts a LEFT JOIN banks b ON a.bank = b.id WHERE @@ -1214,7 +1220,7 @@ handlers['simplefin-batch-sync'] = async function ({ ids = [] }) { const errors = []; for (const account of accounts) { retVal.push({ - accountId: account.accountId, + accountId: account.id, res: { errors, newTransactions: [], diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.ts index 7c4b03adb58..5888fb768a8 100644 --- a/packages/loot-core/src/server/sync/index.ts +++ b/packages/loot-core/src/server/sync/index.ts @@ -198,7 +198,7 @@ async function compareMessages(messages: Message[]): Promise { const { dataset, row, column, timestamp } = message; const timestampStr = timestamp.toString(); - const res = db.runQuery( + const res = db.runQuery>( db.cache( 'SELECT timestamp FROM messages_crdt WHERE dataset = ? AND row = ? AND column = ? AND timestamp >= ?', ), diff --git a/packages/loot-core/src/server/sync/migrate.test.ts b/packages/loot-core/src/server/sync/migrate.test.ts index 6959766b23f..7fec886fcdf 100644 --- a/packages/loot-core/src/server/sync/migrate.test.ts +++ b/packages/loot-core/src/server/sync/migrate.test.ts @@ -79,7 +79,11 @@ describe('sync migrations', () => { tracer.expectNow('applied', ['trans1/child1']); await tracer.expectWait('applied', ['trans1/child1']); - const transactions = db.runQuery('SELECT * FROM transactions', [], true); + const transactions = db.runQuery( + 'SELECT * FROM transactions', + [], + true, + ); expect(transactions.length).toBe(1); expect(transactions[0].parent_id).toBe('trans1'); diff --git a/packages/loot-core/src/server/tools/app.ts b/packages/loot-core/src/server/tools/app.ts index bc6b0321c34..ceb535c4983 100644 --- a/packages/loot-core/src/server/tools/app.ts +++ b/packages/loot-core/src/server/tools/app.ts @@ -52,7 +52,7 @@ app.method('tools/fix-split-transactions', async () => { `); await runMutator(async () => { - const updated = deletedRows.map(row => ({ id: row.id, tombstone: 1 })); + const updated = deletedRows.map(row => ({ id: row.id, tombstone: true })); await batchUpdateTransactions({ updated }); }); From 29c817a4fe487798963532495fafcc3d46f0866f Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 16:12:42 -0800 Subject: [PATCH 02/11] Release notes --- upcoming-release-notes/4247.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 upcoming-release-notes/4247.md diff --git a/upcoming-release-notes/4247.md b/upcoming-release-notes/4247.md new file mode 100644 index 00000000000..ca01ecc5b5b --- /dev/null +++ b/upcoming-release-notes/4247.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [joel-jeremy] +--- + +[TypeScript] Make `runQuery` generic to make it easy to type DB query results. \ No newline at end of file From a4a40a6b7535ee46e01c3320b7e1336c7c249b03 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 16:17:50 -0800 Subject: [PATCH 03/11] typo --- packages/loot-core/src/server/db/types/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/loot-core/src/server/db/types/index.ts b/packages/loot-core/src/server/db/types/index.ts index 54b42c47799..c6accb44f00 100644 --- a/packages/loot-core/src/server/db/types/index.ts +++ b/packages/loot-core/src/server/db/types/index.ts @@ -1,4 +1,4 @@ -// These are the type that exactly match the database schema. +// These are the types that exactly match the database schema. // The `Entity` types e.g. `TransactionEntity`, `AccountEntity`, etc // are specific to the AQL query framework and does not necessarily // match the actual database schema. From 8d877a095c19dd480096c6941e72e2510b5a9539 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 22:47:40 -0800 Subject: [PATCH 04/11] [TypeScript] Make db.first generic --- packages/loot-core/src/mocks/budget.ts | 8 +-- .../loot-core/src/server/accounts/link.ts | 2 +- .../loot-core/src/server/accounts/payees.ts | 6 +- .../loot-core/src/server/accounts/sync.ts | 6 +- .../server/accounts/transaction-rules.test.ts | 26 +++++---- .../src/server/accounts/transaction-rules.ts | 22 ++++--- .../src/server/accounts/transfer.test.ts | 8 +-- .../loot-core/src/server/accounts/transfer.ts | 31 +++++----- packages/loot-core/src/server/api.ts | 7 ++- .../loot-core/src/server/budget/actions.ts | 4 +- packages/loot-core/src/server/budget/base.ts | 2 +- .../src/server/budget/cleanup-template.ts | 6 +- .../src/server/budget/goalsSchedule.ts | 6 +- .../loot-core/src/server/dashboard/app.ts | 4 +- packages/loot-core/src/server/db/index.ts | 40 +++++++------ packages/loot-core/src/server/filters/app.ts | 2 +- packages/loot-core/src/server/main.test.ts | 4 +- packages/loot-core/src/server/main.ts | 58 ++++++++++--------- .../src/server/migrate/migrations.test.ts | 4 +- packages/loot-core/src/server/reports/app.ts | 2 +- .../loot-core/src/server/schedules/app.ts | 6 +- .../src/server/schedules/find-schedules.ts | 2 +- packages/loot-core/src/server/update.ts | 7 ++- 23 files changed, 144 insertions(+), 119 deletions(-) diff --git a/packages/loot-core/src/mocks/budget.ts b/packages/loot-core/src/mocks/budget.ts index 303a4584346..195ea939570 100644 --- a/packages/loot-core/src/mocks/budget.ts +++ b/packages/loot-core/src/mocks/budget.ts @@ -461,14 +461,14 @@ async function fillOther(handlers, account, payees, groups) { async function createBudget(accounts, payees, groups) { const primaryAccount = accounts.find(a => (a.name = 'Bank of America')); const earliestDate = ( - await db.first( - `SELECT * FROM v_transactions t LEFT JOIN accounts a ON t.account = a.id + await db.first( + `SELECT t.date FROM v_transactions t LEFT JOIN accounts a ON t.account = a.id WHERE a.offbudget = 0 AND t.is_child = 0 ORDER BY date ASC LIMIT 1`, ) ).date; const earliestPrimaryDate = ( - await db.first( - `SELECT * FROM v_transactions t LEFT JOIN accounts a ON t.account = a.id + await db.first( + `SELECT t.date FROM v_transactions t LEFT JOIN accounts a ON t.account = a.id WHERE a.id = ? AND a.offbudget = 0 AND t.is_child = 0 ORDER BY date ASC LIMIT 1`, [primaryAccount.id], ) diff --git a/packages/loot-core/src/server/accounts/link.ts b/packages/loot-core/src/server/accounts/link.ts index db48e8c2530..655c974f963 100644 --- a/packages/loot-core/src/server/accounts/link.ts +++ b/packages/loot-core/src/server/accounts/link.ts @@ -4,7 +4,7 @@ import { v4 as uuidv4 } from 'uuid'; import * as db from '../db'; export async function findOrCreateBank(institution, requisitionId) { - const bank = await db.first( + const bank = await db.first>( 'SELECT id, bank_id, name FROM banks WHERE bank_id = ?', [requisitionId], ); diff --git a/packages/loot-core/src/server/accounts/payees.ts b/packages/loot-core/src/server/accounts/payees.ts index 30eba38b5e2..218151c4314 100644 --- a/packages/loot-core/src/server/accounts/payees.ts +++ b/packages/loot-core/src/server/accounts/payees.ts @@ -4,7 +4,7 @@ import * as db from '../db'; export async function createPayee(description) { // Check to make sure no payee already exists with exactly the same // name - const row = await db.first( + const row = await db.first>( `SELECT id FROM payees WHERE UNICODE_LOWER(name) = ? AND tombstone = 0`, [description.toLowerCase()], ); @@ -17,14 +17,14 @@ export async function createPayee(description) { } export async function getStartingBalancePayee() { - let category = await db.first(` + let category = await db.first(` SELECT * FROM categories WHERE is_income = 1 AND LOWER(name) = 'starting balances' AND tombstone = 0 `); if (category === null) { - category = await db.first( + category = await db.first( 'SELECT * FROM categories WHERE is_income = 1 AND tombstone = 0', ); } diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.ts index 5add734d796..8b34236746e 100644 --- a/packages/loot-core/src/server/accounts/sync.ts +++ b/packages/loot-core/src/server/accounts/sync.ts @@ -527,7 +527,7 @@ export async function matchTransactions( ); // The first pass runs the rules, and preps data for fuzzy matching - const accounts: AccountEntity[] = await db.getAccounts(); + const accounts: db.DbAccount[] = await db.getAccounts(); const accountsMap = new Map(accounts.map(account => [account.id, account])); const transactionsStep1 = []; @@ -546,7 +546,7 @@ export async function matchTransactions( // is the highest fidelity match and should always be attempted // first. if (trans.imported_id) { - match = await db.first( + match = await db.first( 'SELECT * FROM v_transactions WHERE imported_id = ? AND account = ?', [trans.imported_id, acctId], ); @@ -680,7 +680,7 @@ export async function addTransactions( { rawPayeeName: true }, ); - const accounts: AccountEntity[] = await db.getAccounts(); + const accounts: db.DbAccount[] = await db.getAccounts(); const accountsMap = new Map(accounts.map(account => [account.id, account])); for (const { trans: originalTrans, subtransactions } of normalized) { diff --git a/packages/loot-core/src/server/accounts/transaction-rules.test.ts b/packages/loot-core/src/server/accounts/transaction-rules.test.ts index 5004ebb5c31..13a0abf04af 100644 --- a/packages/loot-core/src/server/accounts/transaction-rules.test.ts +++ b/packages/loot-core/src/server/accounts/transaction-rules.test.ts @@ -943,11 +943,14 @@ describe('Learning categories', () => { // Internally, it should still be stored with the internal names // so that it's backwards compatible - const rawRule = await db.first('SELECT * FROM rules'); - rawRule.conditions = JSON.parse(rawRule.conditions); - rawRule.actions = JSON.parse(rawRule.actions); - expect(rawRule.conditions[0].field).toBe('imported_description'); - expect(rawRule.actions[0].field).toBe('description'); + const rawRule = await db.first('SELECT * FROM rules'); + const parsedRule = { + ...rawRule, + conditions: JSON.parse(rawRule.conditions), + actions: JSON.parse(rawRule.actions), + }; + expect(parsedRule.conditions[0].field).toBe('imported_description'); + expect(parsedRule.actions[0].field).toBe('description'); await loadRules(); @@ -973,11 +976,14 @@ describe('Learning categories', () => { // This rule internally has been stored with the public names. // Making this work now allows us to switch to it by default in // the future - const rawRule = await db.first('SELECT * FROM rules'); - rawRule.conditions = JSON.parse(rawRule.conditions); - rawRule.actions = JSON.parse(rawRule.actions); - expect(rawRule.conditions[0].field).toBe('imported_payee'); - expect(rawRule.actions[0].field).toBe('payee'); + const rawRule = await db.first('SELECT * FROM rules'); + const parsedRule = { + ...rawRule, + conditions: JSON.parse(rawRule.conditions), + actions: JSON.parse(rawRule.actions), + }; + expect(parsedRule.conditions[0].field).toBe('imported_payee'); + expect(parsedRule.actions[0].field).toBe('payee'); const [rule] = getRules(); expect(rule.conditions[0].field).toBe('imported_payee'); diff --git a/packages/loot-core/src/server/accounts/transaction-rules.ts b/packages/loot-core/src/server/accounts/transaction-rules.ts index 9d4baf72d63..feab9fb0680 100644 --- a/packages/loot-core/src/server/accounts/transaction-rules.ts +++ b/packages/loot-core/src/server/accounts/transaction-rules.ts @@ -13,7 +13,6 @@ import { type TransactionEntity, type RuleActionEntity, type RuleEntity, - AccountEntity, } from '../../types/models'; import { schemaConfig } from '../aql'; import * as db from '../db'; @@ -219,9 +218,10 @@ export async function updateRule(rule) { } export async function deleteRule(id: string) { - const schedule = await db.first('SELECT id FROM schedules WHERE rule = ?', [ - id, - ]); + const schedule = await db.first>( + 'SELECT id FROM schedules WHERE rule = ?', + [id], + ); if (schedule) { return false; @@ -277,7 +277,7 @@ function onApplySync(oldValues, newValues) { // Runner export async function runRules( trans, - accounts: Map | null = null, + accounts: Map | null = null, ) { let accountsMap = null; if (accounts === null) { @@ -627,13 +627,11 @@ export async function applyActions( return null; } - const accounts: AccountEntity[] = await db.getAccounts(); + const accounts: db.DbAccount[] = await db.getAccounts(); + const accountsMap = new Map(accounts.map(account => [account.id, account])); const transactionsForRules = await Promise.all( transactions.map(transactions => - prepareTransactionForRules( - transactions, - new Map(accounts.map(account => [account.id, account])), - ), + prepareTransactionForRules(transactions, accountsMap), ), ); @@ -866,12 +864,12 @@ export async function updateCategoryRules(transactions) { export type TransactionForRules = TransactionEntity & { payee_name?: string; - _account?: AccountEntity; + _account?: db.DbAccount; }; export async function prepareTransactionForRules( trans: TransactionEntity, - accounts: Map | null = null, + accounts: Map | null = null, ): Promise { const r: TransactionForRules = { ...trans }; if (trans.payee) { diff --git a/packages/loot-core/src/server/accounts/transfer.test.ts b/packages/loot-core/src/server/accounts/transfer.test.ts index ce77ce7e94a..cb084b0c0cf 100644 --- a/packages/loot-core/src/server/accounts/transfer.test.ts +++ b/packages/loot-core/src/server/accounts/transfer.test.ts @@ -61,10 +61,10 @@ describe('Transfer', () => { const differ = expectSnapshotWithDiffer(await getAllTransactions()); - const transferTwo = await db.first( + const transferTwo = await db.first( "SELECT * FROM payees WHERE transfer_acct = 'two'", ); - const transferThree = await db.first( + const transferThree = await db.first( "SELECT * FROM payees WHERE transfer_acct = 'three'", ); @@ -131,10 +131,10 @@ describe('Transfer', () => { test('transfers are properly de-categorized', async () => { await prepareDatabase(); - const transferTwo = await db.first( + const transferTwo = await db.first( "SELECT * FROM payees WHERE transfer_acct = 'two'", ); - const transferThree = await db.first( + const transferThree = await db.first( "SELECT * FROM payees WHERE transfer_acct = 'three'", ); diff --git a/packages/loot-core/src/server/accounts/transfer.ts b/packages/loot-core/src/server/accounts/transfer.ts index fa8969667e0..e3937e32033 100644 --- a/packages/loot-core/src/server/accounts/transfer.ts +++ b/packages/loot-core/src/server/accounts/transfer.ts @@ -2,29 +2,30 @@ import * as db from '../db'; async function getPayee(acct) { - return db.first('SELECT * FROM payees WHERE transfer_acct = ?', [acct]); + return db.first('SELECT * FROM payees WHERE transfer_acct = ?', [ + acct, + ]); } async function getTransferredAccount(transaction) { if (transaction.payee) { - const { transfer_acct } = await db.first( - 'SELECT id, transfer_acct FROM v_payees WHERE id = ?', - [transaction.payee], - ); + const { transfer_acct } = await db.first< + Pick + >('SELECT id, transfer_acct FROM v_payees WHERE id = ?', [ + transaction.payee, + ]); return transfer_acct; } return null; } async function clearCategory(transaction, transferAcct) { - const { offbudget: fromOffBudget } = await db.first( - 'SELECT offbudget FROM accounts WHERE id = ?', - [transaction.account], - ); - const { offbudget: toOffBudget } = await db.first( - 'SELECT offbudget FROM accounts WHERE id = ?', - [transferAcct], - ); + const { offbudget: fromOffBudget } = await db.first< + Pick + >('SELECT offbudget FROM accounts WHERE id = ?', [transaction.account]); + const { offbudget: toOffBudget } = await db.first< + Pick + >('SELECT offbudget FROM accounts WHERE id = ?', [transferAcct]); // If the transfer is between two on budget or two off budget accounts, // we should clear the category, because the category is not relevant @@ -50,14 +51,14 @@ export async function addTransfer(transaction, transferredAccount) { return null; } - const { id: fromPayee } = await db.first( + const { id: fromPayee } = await db.first>( 'SELECT id FROM payees WHERE transfer_acct = ?', [transaction.account], ); // We need to enforce certain constraints with child transaction transfers if (transaction.parent_id) { - const row = await db.first( + const row = await db.first>( ` SELECT p.id, p.transfer_acct FROM v_transactions t LEFT JOIN payees p ON p.id = t.payee diff --git a/packages/loot-core/src/server/api.ts b/packages/loot-core/src/server/api.ts index 72ae7b35d84..da4061f7959 100644 --- a/packages/loot-core/src/server/api.ts +++ b/packages/loot-core/src/server/api.ts @@ -94,9 +94,10 @@ async function validateExpenseCategory(debug, id) { throw APIError(`${debug}: category id is required`); } - const row = await db.first('SELECT is_income FROM categories WHERE id = ?', [ - id, - ]); + const row = await db.first>( + 'SELECT is_income FROM categories WHERE id = ?', + [id], + ); if (!row) { throw APIError(`${debug}: category “${id}” does not exist`); diff --git a/packages/loot-core/src/server/budget/actions.ts b/packages/loot-core/src/server/budget/actions.ts index e09511ca33b..f3f12f277c1 100644 --- a/packages/loot-core/src/server/budget/actions.ts +++ b/packages/loot-core/src/server/budget/actions.ts @@ -329,7 +329,7 @@ export async function setNMonthAvg({ N: number; category: string; }): Promise { - const categoryFromDb = await db.first( + const categoryFromDb = await db.first>( 'SELECT is_income FROM v_categories WHERE id = ?', [category], ); @@ -361,7 +361,7 @@ export async function holdForNextMonth({ month: string; amount: number; }): Promise { - const row = await db.first( + const row = await db.first>( 'SELECT buffered FROM zero_budget_months WHERE id = ?', [month], ); diff --git a/packages/loot-core/src/server/budget/base.ts b/packages/loot-core/src/server/budget/base.ts index a3fcb075e17..975b00ac741 100644 --- a/packages/loot-core/src/server/budget/base.ts +++ b/packages/loot-core/src/server/budget/base.ts @@ -442,7 +442,7 @@ export async function createBudget(months) { } export async function createAllBudgets() { - const earliestTransaction = await db.first( + const earliestTransaction = await db.first( 'SELECT * FROM transactions WHERE isChild=0 AND date IS NOT NULL ORDER BY date ASC LIMIT 1', ); const earliestDate = diff --git a/packages/loot-core/src/server/budget/cleanup-template.ts b/packages/loot-core/src/server/budget/cleanup-template.ts index 8c1b109212c..85d387d5abb 100644 --- a/packages/loot-core/src/server/budget/cleanup-template.ts +++ b/packages/loot-core/src/server/budget/cleanup-template.ts @@ -64,7 +64,7 @@ async function applyGroupCleanups( ); const to_budget = budgeted + Math.abs(balance); const categoryId = generalGroup[ii].category; - let carryover = await db.first( + let carryover = await db.first>( `SELECT carryover FROM zero_budgets WHERE month = ? and category = ?`, [db_month, categoryId], ); @@ -220,7 +220,7 @@ async function processCleanup(month: string): Promise { } else { warnings.push(category.name + ' does not have available funds.'); } - const carryover = await db.first( + const carryover = await db.first>( `SELECT carryover FROM zero_budgets WHERE month = ? and category = ?`, [db_month, category.id], ); @@ -249,7 +249,7 @@ async function processCleanup(month: string): Promise { const budgeted = await getSheetValue(sheetName, `budget-${category.id}`); const to_budget = budgeted + Math.abs(balance); const categoryId = category.id; - let carryover = await db.first( + let carryover = await db.first>( `SELECT carryover FROM zero_budgets WHERE month = ? and category = ?`, [db_month, categoryId], ); diff --git a/packages/loot-core/src/server/budget/goalsSchedule.ts b/packages/loot-core/src/server/budget/goalsSchedule.ts index 9eeef419113..4e0e1a625a0 100644 --- a/packages/loot-core/src/server/budget/goalsSchedule.ts +++ b/packages/loot-core/src/server/budget/goalsSchedule.ts @@ -21,8 +21,10 @@ async function createScheduleList( const errors = []; for (let ll = 0; ll < template.length; ll++) { - const { id: sid, completed: complete } = await db.first( - 'SELECT * FROM schedules WHERE TRIM(name) = ? AND tombstone = 0', + const { id: sid, completed: complete } = await db.first< + Pick + >( + 'SELECT id, completed FROM schedules WHERE TRIM(name) = ? AND tombstone = 0', [template[ll].name.trim()], ); const rule = await getRuleForSchedule(sid); diff --git a/packages/loot-core/src/server/dashboard/app.ts b/packages/loot-core/src/server/dashboard/app.ts index 8b456f280c8..d1299069b51 100644 --- a/packages/loot-core/src/server/dashboard/app.ts +++ b/packages/loot-core/src/server/dashboard/app.ts @@ -144,7 +144,9 @@ async function addDashboardWidget( // If no x & y was provided - calculate it dynamically // The new widget should be the very last one in the list of all widgets if (!('x' in widget) && !('y' in widget)) { - const data = await db.first( + const data = await db.first< + Pick + >( 'SELECT x, y, width, height FROM dashboard WHERE tombstone = 0 ORDER BY y DESC, x DESC', ); diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index 12e80e07a1d..2940c75e57e 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -37,6 +37,7 @@ import { DbAccount, DbCategory, DbCategoryGroup, + DbClockMessage, DbPayee, DbTransaction, DbViewTransaction, @@ -83,7 +84,7 @@ export function getDatabase() { } export async function loadClock() { - const row = await first('SELECT * FROM messages_clock'); + const row = await first('SELECT * FROM messages_clock'); if (row) { const clock = deserializeClock(row.clock); setClock(clock); @@ -166,12 +167,9 @@ export async function all(sql, params?: (string | number)[]) { return runQuery(sql, params, true) as any[]; } -export async function first(sql, params?: (string | number)[]) { - const arr = await runQuery(sql, params, true); - // TODO: In the next phase, we will make this function generic - // and pass the type of the return type to `runQuery`. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return arr.length === 0 ? null : (arr[0] as any); +export async function first(sql, params?: (string | number)[]) { + const arr = await runQuery(sql, params, true); + return arr.length === 0 ? null : arr[0]; } // The underlying sql system is now sync, but we can't update `first` yet @@ -353,7 +351,9 @@ export async function getCategoriesGrouped( export async function insertCategoryGroup(group) { // Don't allow duplicate group - const existingGroup = await first( + const existingGroup = await first< + Pick + >( `SELECT id, name, hidden FROM category_groups WHERE UPPER(name) = ? and tombstone = 0 LIMIT 1`, [group.name.toUpperCase()], ); @@ -363,7 +363,7 @@ export async function insertCategoryGroup(group) { ); } - const lastGroup = await first(` + const lastGroup = await first>(` SELECT sort_order FROM category_groups WHERE tombstone = 0 ORDER BY sort_order DESC, id DESC LIMIT 1 `); const sort_order = (lastGroup ? lastGroup.sort_order : 0) + SORT_INCREMENT; @@ -411,7 +411,7 @@ export async function insertCategory( let id_; await batchMessages(async () => { // Dont allow duplicated names in groups - const existingCatInGroup = await first( + const existingCatInGroup = await first>( `SELECT id FROM categories WHERE cat_group = ? and UPPER(name) = ? and tombstone = 0 LIMIT 1`, [category.cat_group, category.name.toUpperCase()], ); @@ -422,7 +422,7 @@ export async function insertCategory( } if (atEnd) { - const lastCat = await first(` + const lastCat = await first>(` SELECT sort_order FROM categories WHERE tombstone = 0 ORDER BY sort_order DESC, id DESC LIMIT 1 `); sort_order = (lastCat ? lastCat.sort_order : 0) + SORT_INCREMENT; @@ -507,11 +507,11 @@ export async function deleteCategory( } export async function getPayee(id: DbPayee['id']) { - return first(`SELECT * FROM payees WHERE id = ?`, [id]); + return first(`SELECT * FROM payees WHERE id = ?`, [id]); } export async function getAccount(id: DbAccount['id']) { - return first(`SELECT * FROM accounts WHERE id = ?`, [id]); + return first(`SELECT * FROM accounts WHERE id = ?`, [id]); } export async function insertPayee(payee) { @@ -525,9 +525,10 @@ export async function insertPayee(payee) { } export async function deletePayee(payee: Pick) { - const { transfer_acct } = await first('SELECT * FROM payees WHERE id = ?', [ - payee.id, - ]); + const { transfer_acct } = await first( + 'SELECT * FROM payees WHERE id = ?', + [payee.id], + ); if (transfer_acct) { // You should never be able to delete transfer payees return; @@ -654,7 +655,7 @@ export async function getOrphanedPayees() { } export async function getPayeeByName(name: DbPayee['name']) { - return first( + return first( `SELECT * FROM payees WHERE UNICODE_LOWER(name) = ? AND tombstone = 0`, [name.toLowerCase()], ); @@ -695,7 +696,10 @@ export async function moveAccount( id: DbAccount['id'], targetId: DbAccount['id'], ) { - const account = await first('SELECT * FROM accounts WHERE id = ?', [id]); + const account = await first( + 'SELECT * FROM accounts WHERE id = ?', + [id], + ); let accounts; if (account.closed) { accounts = await all( diff --git a/packages/loot-core/src/server/filters/app.ts b/packages/loot-core/src/server/filters/app.ts index c6b4ef69c96..03cf585b1a1 100644 --- a/packages/loot-core/src/server/filters/app.ts +++ b/packages/loot-core/src/server/filters/app.ts @@ -42,7 +42,7 @@ const filterModel = { }; async function filterNameExists(name, filterId, newItem) { - const idForName = await db.first( + const idForName = await db.first>( 'SELECT id from transaction_filters WHERE tombstone = 0 AND name = ?', [name], ); diff --git a/packages/loot-core/src/server/main.test.ts b/packages/loot-core/src/server/main.test.ts index ea4d8713007..5992de82427 100644 --- a/packages/loot-core/src/server/main.test.ts +++ b/packages/loot-core/src/server/main.test.ts @@ -67,7 +67,9 @@ describe('Budgets', () => { // Grab the clock to compare later await db.openDatabase('test-budget'); - const row = await db.first('SELECT * FROM messages_clock'); + const row = await db.first( + 'SELECT * FROM messages_clock', + ); const { error } = await runHandler(handlers['load-budget'], { id: 'test-budget', diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.ts index 13ced54836f..37db02ebf67 100644 --- a/packages/loot-core/src/server/main.ts +++ b/packages/loot-core/src/server/main.ts @@ -326,7 +326,7 @@ handlers['category-delete'] = mutator(async function ({ id, transferId }) { return withUndo(async () => { let result = {}; await batchMessages(async () => { - const row = await db.first( + const row = await db.first>( 'SELECT is_income FROM categories WHERE id = ?', [id], ); @@ -337,9 +337,10 @@ handlers['category-delete'] = mutator(async function ({ id, transferId }) { const transfer = transferId && - (await db.first('SELECT is_income FROM categories WHERE id = ?', [ - transferId, - ])); + (await db.first>( + 'SELECT is_income FROM categories WHERE id = ?', + [transferId], + )); if (!row || (transferId && !transfer)) { result = { error: 'no-categories' }; @@ -572,7 +573,7 @@ handlers['accounts-get'] = async function () { }; handlers['account-balance'] = async function ({ id, cutoff }) { - const { balance } = await db.first( + const { balance } = await db.first<{ balance: number }>( 'SELECT sum(amount) as balance FROM transactions WHERE acct = ? AND isParent = 0 AND tombstone = 0 AND date <= ?', [id, db.toDateRepr(dayFromDate(cutoff))], ); @@ -580,11 +581,11 @@ handlers['account-balance'] = async function ({ id, cutoff }) { }; handlers['account-properties'] = async function ({ id }) { - const { balance } = await db.first( + const { balance } = await db.first<{ balance: number }>( 'SELECT sum(amount) as balance FROM transactions WHERE acct = ? AND isParent = 0 AND tombstone = 0', [id], ); - const { count } = await db.first( + const { count } = await db.first<{ count: number }>( 'SELECT count(id) as count FROM transactions WHERE acct = ? AND tombstone = 0', [id], ); @@ -602,9 +603,10 @@ handlers['gocardless-accounts-link'] = async function ({ const bank = await link.findOrCreateBank(account.institution, requisitionId); if (upgradingId) { - const accRow = await db.first('SELECT * FROM accounts WHERE id = ?', [ - upgradingId, - ]); + const accRow = await db.first( + 'SELECT * FROM accounts WHERE id = ?', + [upgradingId], + ); id = accRow.id; await db.update('accounts', { id, @@ -663,9 +665,10 @@ handlers['simplefin-accounts-link'] = async function ({ ); if (upgradingId) { - const accRow = await db.first('SELECT * FROM accounts WHERE id = ?', [ - upgradingId, - ]); + const accRow = await db.first( + 'SELECT * FROM accounts WHERE id = ?', + [upgradingId], + ); id = accRow.id; await db.update('accounts', { id, @@ -754,7 +757,7 @@ handlers['account-close'] = mutator(async function ({ await handlers['account-unlink']({ id }); return withUndo(async () => { - const account = await db.first( + const account = await db.first( 'SELECT * FROM accounts WHERE id = ? AND tombstone = 0', [id], ); @@ -781,7 +784,7 @@ handlers['account-close'] = mutator(async function ({ true, ); - const { id: payeeId } = await db.first( + const { id: payeeId } = await db.first>( 'SELECT id FROM payees WHERE transfer_acct = ?', [id], ); @@ -818,7 +821,7 @@ handlers['account-close'] = mutator(async function ({ // If there is a balance we need to transfer it to the specified // account (and possibly categorize it) if (balance !== 0) { - const { id: payeeId } = await db.first( + const { id: payeeId } = await db.first>( 'SELECT id FROM payees WHERE transfer_acct = ?', [transferAccountId], ); @@ -1280,7 +1283,7 @@ handlers['transactions-import'] = mutator(function ({ }); handlers['account-unlink'] = mutator(async function ({ id }) { - const { bank: bankId } = await db.first( + const { bank: bankId } = await db.first>( 'SELECT bank FROM accounts WHERE id = ?', [id], ); @@ -1289,7 +1292,10 @@ handlers['account-unlink'] = mutator(async function ({ id }) { return 'ok'; } - const accRow = await db.first('SELECT * FROM accounts WHERE id = ?', [id]); + const accRow = await db.first( + 'SELECT * FROM accounts WHERE id = ?', + [id], + ); const isGoCardless = accRow.account_sync_source === 'goCardless'; @@ -1307,7 +1313,7 @@ handlers['account-unlink'] = mutator(async function ({ id }) { return; } - const { count } = await db.first( + const { count } = await db.first<{ count: number }>( 'SELECT COUNT(*) as count FROM accounts WHERE bank = ?', [bankId], ); @@ -1320,10 +1326,9 @@ handlers['account-unlink'] = mutator(async function ({ id }) { } if (count === 0) { - const { bank_id: requisitionId } = await db.first( - 'SELECT bank_id FROM banks WHERE id = ?', - [bankId], - ); + const { bank_id: requisitionId } = await db.first< + Pick + >('SELECT bank_id FROM banks WHERE id = ?', [bankId]); try { await post( getServer().GOCARDLESS_SERVER + '/remove-account', @@ -2345,9 +2350,10 @@ async function loadBudget(id: string) { // This is a bit leaky, but we need to set the initial budget type const { value: budgetType = 'rollover' } = - (await db.first('SELECT value from preferences WHERE id = ?', [ - 'budgetType', - ])) ?? {}; + (await db.first>( + 'SELECT value from preferences WHERE id = ?', + ['budgetType'], + )) ?? {}; sheet.get().meta().budgetType = budgetType; await budget.createAllBudgets(); diff --git a/packages/loot-core/src/server/migrate/migrations.test.ts b/packages/loot-core/src/server/migrate/migrations.test.ts index 23721fb8135..ba6f4fdd316 100644 --- a/packages/loot-core/src/server/migrate/migrations.test.ts +++ b/packages/loot-core/src/server/migrate/migrations.test.ts @@ -62,14 +62,14 @@ describe('Migrations', () => { return withMigrationsDir( __dirname + '/../../mocks/migrations', async () => { - let desc = await db.first( + let desc = await db.first<{ sql: string }>( "SELECT * FROM sqlite_master WHERE name = 'poop'", ); expect(desc).toBe(null); await migrate(db.getDatabase()); - desc = await db.first( + desc = await db.first<{ sql: string }>( "SELECT * FROM sqlite_master WHERE name = 'poop'", ); expect(desc).toBeDefined(); diff --git a/packages/loot-core/src/server/reports/app.ts b/packages/loot-core/src/server/reports/app.ts index e1356696cad..f5d3beeb71a 100644 --- a/packages/loot-core/src/server/reports/app.ts +++ b/packages/loot-core/src/server/reports/app.ts @@ -85,7 +85,7 @@ async function reportNameExists( reportId: string, newItem: boolean, ) { - const idForName: { id: string } = await db.first( + const idForName = await db.first>( 'SELECT id from custom_reports WHERE tombstone = 0 AND name = ?', [name], ); diff --git a/packages/loot-core/src/server/schedules/app.ts b/packages/loot-core/src/server/schedules/app.ts index 7016d6f627a..8939b3458f5 100644 --- a/packages/loot-core/src/server/schedules/app.ts +++ b/packages/loot-core/src/server/schedules/app.ts @@ -140,7 +140,9 @@ export async function setNextDate({ if (newNextDate !== nextDate) { // Our `update` functon requires the id of the item and we don't // have it, so we need to query it - const nd = await db.first( + const nd = await db.first< + Pick + >( 'SELECT id, base_next_date_ts FROM schedules_next_date WHERE schedule_id = ?', [id], ); @@ -166,7 +168,7 @@ export async function setNextDate({ // Methods async function checkIfScheduleExists(name, scheduleId) { - const idForName = await db.first( + const idForName = await db.first>( 'SELECT id from schedules WHERE tombstone = 0 AND name = ?', [name], ); diff --git a/packages/loot-core/src/server/schedules/find-schedules.ts b/packages/loot-core/src/server/schedules/find-schedules.ts index 8b45406138b..b52ba789ffc 100644 --- a/packages/loot-core/src/server/schedules/find-schedules.ts +++ b/packages/loot-core/src/server/schedules/find-schedules.ts @@ -337,7 +337,7 @@ export async function findSchedules() { for (const account of accounts) { // Find latest transaction-ish to start with - const latestTrans = await db.first( + const latestTrans = await db.first( 'SELECT * FROM v_transactions WHERE account = ? AND parent_id IS NULL ORDER BY date DESC LIMIT 1', [account.id], ); diff --git a/packages/loot-core/src/server/update.ts b/packages/loot-core/src/server/update.ts index 73f5a7fe62c..a8a1b1c89f5 100644 --- a/packages/loot-core/src/server/update.ts +++ b/packages/loot-core/src/server/update.ts @@ -13,9 +13,10 @@ async function runMigrations() { async function updateViews() { const hashKey = 'view-hash'; - const row = await db.first('SELECT value FROM __meta__ WHERE key = ?', [ - hashKey, - ]); + const row = await db.first<{ value: string }>( + 'SELECT value FROM __meta__ WHERE key = ?', + [hashKey], + ); const { value: hash } = row || {}; const views = makeViews(schema, schemaConfig); From bd930e7c51f460d9acaf5b88e2c90cc7d63c6987 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 22:59:07 -0800 Subject: [PATCH 05/11] Release notes --- upcoming-release-notes/4248.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 upcoming-release-notes/4248.md diff --git a/upcoming-release-notes/4248.md b/upcoming-release-notes/4248.md new file mode 100644 index 00000000000..07f329fd72e --- /dev/null +++ b/upcoming-release-notes/4248.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [joel-jeremy] +--- + +[TypeScript] Make `db.first` generic to make it easy to type DB query results. \ No newline at end of file From be4571cb56fe7fa8270e8d8fe9cf01b5584d4252 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 22:56:55 -0800 Subject: [PATCH 06/11] [TypeScript] Make db.firstSync generic --- .../loot-core/src/server/budget/actions.ts | 47 +++++++++++-------- packages/loot-core/src/server/db/index.ts | 9 ++-- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/packages/loot-core/src/server/budget/actions.ts b/packages/loot-core/src/server/budget/actions.ts index f3f12f277c1..69b5292674b 100644 --- a/packages/loot-core/src/server/budget/actions.ts +++ b/packages/loot-core/src/server/budget/actions.ts @@ -35,12 +35,14 @@ function calcBufferedAmount( return buffered + amount; } -function getBudgetTable(): string { +type BudgetTable = 'reflect_budgets' | 'zero_budgets'; + +function getBudgetTable(): BudgetTable { return isReflectBudget() ? 'reflect_budgets' : 'zero_budgets'; } export function isReflectBudget(): boolean { - const budgetType = db.firstSync( + const budgetType = db.firstSync>( `SELECT value FROM preferences WHERE id = ?`, ['budgetType'], ); @@ -91,7 +93,7 @@ export function getBudget({ month: string; }): number { const table = getBudgetTable(); - const existing = db.firstSync( + const existing = db.firstSync( `SELECT * FROM ${table} WHERE month = ? AND category = ?`, [dbMonth(month), category], ); @@ -110,10 +112,12 @@ export function setBudget({ amount = safeNumber(typeof amount === 'number' ? amount : 0); const table = getBudgetTable(); - const existing = db.firstSync( - `SELECT id FROM ${table} WHERE month = ? AND category = ?`, - [dbMonth(month), category], - ); + const existing = db.firstSync< + Pick + >(`SELECT id FROM ${table} WHERE month = ? AND category = ?`, [ + dbMonth(month), + category, + ]); if (existing) { return db.update(table, { id: existing.id, amount }); } @@ -127,10 +131,12 @@ export function setBudget({ export function setGoal({ month, category, goal, long_goal }): Promise { const table = getBudgetTable(); - const existing = db.firstSync( - `SELECT id FROM ${table} WHERE month = ? AND category = ?`, - [dbMonth(month), category], - ); + const existing = db.firstSync< + Pick + >(`SELECT id FROM ${table} WHERE month = ? AND category = ?`, [ + dbMonth(month), + category, + ]); if (existing) { return db.update(table, { id: existing.id, @@ -148,7 +154,7 @@ export function setGoal({ month, category, goal, long_goal }): Promise { } export function setBuffer(month: string, amount: unknown): Promise { - const existing = db.firstSync( + const existing = db.firstSync>( `SELECT id FROM zero_budget_months WHERE id = ?`, [month], ); @@ -167,10 +173,12 @@ function setCarryover( month: string, flag: boolean, ): Promise { - const existing = db.firstSync( - `SELECT id FROM ${table} WHERE month = ? AND category = ?`, - [month, category], - ); + const existing = db.firstSync< + Pick + >(`SELECT id FROM ${table} WHERE month = ? AND category = ?`, [ + month, + category, + ]); if (existing) { return db.update(table, { id: existing.id, carryover: flag ? 1 : 0 }); } @@ -547,9 +555,10 @@ async function addMovementNotes({ const monthBudgetNotesId = `budget-${month}`; const existingMonthBudgetNotes = addNewLine( - db.firstSync(`SELECT n.note FROM notes n WHERE n.id = ?`, [ - monthBudgetNotesId, - ])?.note, + db.firstSync>( + `SELECT n.note FROM notes n WHERE n.id = ?`, + [monthBudgetNotesId], + )?.note, ); const displayDay = monthUtils.format(monthUtils.currentDate(), 'MMMM dd'); diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index 2940c75e57e..ec8c3149518 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -174,12 +174,9 @@ export async function first(sql, params?: (string | number)[]) { // The underlying sql system is now sync, but we can't update `first` yet // without auditing all uses of it -export function firstSync(sql, params?: (string | number)[]) { - const arr = runQuery(sql, params, true); - // TODO: In the next phase, we will make this function generic - // and pass the type of the return type to `runQuery`. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return arr.length === 0 ? null : (arr[0] as any); +export function firstSync(sql, params?: (string | number)[]) { + const arr = runQuery(sql, params, true); + return arr.length === 0 ? null : arr[0]; } // This function is marked as async because `runQuery` is no longer From e634c3fdd9feb43344e6a51abbc3d113cff3ad7c Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Mon, 27 Jan 2025 23:00:12 -0800 Subject: [PATCH 07/11] Release notes --- upcoming-release-notes/4249.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 upcoming-release-notes/4249.md diff --git a/upcoming-release-notes/4249.md b/upcoming-release-notes/4249.md new file mode 100644 index 00000000000..981ff496fe5 --- /dev/null +++ b/upcoming-release-notes/4249.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [joel-jeremy] +--- + +[TypeScript] Make `db.firstSync` generic to make it easy to type DB query results. \ No newline at end of file From 89642dcde29a93fc2998eca753dbdc68ed3b0988 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Tue, 28 Jan 2025 10:39:30 -0800 Subject: [PATCH 08/11] [TypeScript] Make `db.all` generic --- .../src/server/accounts/sync.test.ts | 4 +- .../loot-core/src/server/accounts/sync.ts | 36 ++- .../server/accounts/transaction-rules.test.ts | 6 +- .../src/server/accounts/transaction-rules.ts | 4 +- .../src/server/accounts/transactions.ts | 6 +- .../src/server/accounts/transfer.test.ts | 2 +- packages/loot-core/src/server/api.ts | 7 +- .../loot-core/src/server/aql/exec.test.ts | 4 +- .../src/server/aql/schema/executors.ts | 4 +- .../src/server/aql/schema/index.test.ts | 54 +++- .../loot-core/src/server/budget/actions.ts | 18 +- packages/loot-core/src/server/budget/base.ts | 4 +- .../src/server/budget/cleanup-template.ts | 4 +- .../src/server/budget/goaltemplates.ts | 16 +- .../loot-core/src/server/budget/statements.ts | 17 +- .../loot-core/src/server/dashboard/app.ts | 3 +- packages/loot-core/src/server/db/index.ts | 82 +++--- packages/loot-core/src/server/db/mappings.ts | 12 +- packages/loot-core/src/server/main.test.ts | 10 +- packages/loot-core/src/server/main.ts | 27 +- packages/loot-core/src/server/models.ts | 242 +++++++++++++++++- .../loot-core/src/server/preferences/app.ts | 4 +- packages/loot-core/src/server/sync/index.ts | 6 +- .../loot-core/src/server/sync/migrate.test.ts | 5 +- .../loot-core/src/server/sync/sync.test.ts | 8 +- packages/loot-core/src/server/tools/app.ts | 16 +- .../loot-core/src/types/models/category.d.ts | 1 + .../loot-core/src/types/server-handlers.d.ts | 8 +- 28 files changed, 491 insertions(+), 119 deletions(-) diff --git a/packages/loot-core/src/server/accounts/sync.test.ts b/packages/loot-core/src/server/accounts/sync.test.ts index b4356875c32..5215bdc7da9 100644 --- a/packages/loot-core/src/server/accounts/sync.test.ts +++ b/packages/loot-core/src/server/accounts/sync.test.ts @@ -24,7 +24,9 @@ beforeEach(async () => { }); function getAllTransactions() { - return db.all( + return db.all< + db.DbViewTransactionInternal & { payee_name: db.DbPayee['name'] } + >( `SELECT t.*, p.name as payee_name FROM v_transactions_internal t LEFT JOIN payees p ON p.id = t.payee diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.ts index 8b34236746e..ab413e69784 100644 --- a/packages/loot-core/src/server/accounts/sync.ts +++ b/packages/loot-core/src/server/accounts/sync.ts @@ -454,7 +454,7 @@ export async function reconcileTransactions( } if (existing.is_parent && existing.cleared !== updates.cleared) { - const children = await db.all( + const children = await db.all( 'SELECT id FROM v_transactions WHERE parent_id = ?', [existing.id], ); @@ -567,7 +567,22 @@ export async function matchTransactions( // strictIdChecking has the added behaviour of only matching on transactions with no import ID // if the transaction being imported has an import ID. if (strictIdChecking) { - fuzzyDataset = await db.all( + fuzzyDataset = await db.all< + Pick< + db.DbViewTransaction, + | 'id' + | 'is_parent' + | 'date' + | 'imported_id' + | 'payee' + | 'imported_payee' + | 'category' + | 'notes' + | 'reconciled' + | 'cleared' + | 'amount' + > + >( `SELECT id, is_parent, date, imported_id, payee, imported_payee, category, notes, reconciled, cleared, amount FROM v_transactions WHERE @@ -584,7 +599,22 @@ export async function matchTransactions( ], ); } else { - fuzzyDataset = await db.all( + fuzzyDataset = await db.all< + Pick< + db.DbViewTransaction, + | 'id' + | 'is_parent' + | 'date' + | 'imported_id' + | 'payee' + | 'imported_payee' + | 'category' + | 'notes' + | 'reconciled' + | 'cleared' + | 'amount' + > + >( `SELECT id, is_parent, date, imported_id, payee, imported_payee, category, notes, reconciled, cleared, amount FROM v_transactions WHERE date >= ? AND date <= ? AND amount = ? AND account = ?`, diff --git a/packages/loot-core/src/server/accounts/transaction-rules.test.ts b/packages/loot-core/src/server/accounts/transaction-rules.test.ts index 13a0abf04af..53895eb0621 100644 --- a/packages/loot-core/src/server/accounts/transaction-rules.test.ts +++ b/packages/loot-core/src/server/accounts/transaction-rules.test.ts @@ -97,7 +97,7 @@ describe('Transaction rules', () => { conditions: [], actions: [], }); - expect((await db.all('SELECT * FROM rules')).length).toBe(1); + expect((await db.all('SELECT * FROM rules')).length).toBe(1); // Make sure it was projected expect(getRules().length).toBe(1); @@ -110,7 +110,7 @@ describe('Transaction rules', () => { { op: 'set', field: 'category', value: 'food' }, ], }); - expect((await db.all('SELECT * FROM rules')).length).toBe(2); + expect((await db.all('SELECT * FROM rules')).length).toBe(2); expect(getRules().length).toBe(2); const spy = jest.spyOn(console, 'warn').mockImplementation(); @@ -119,7 +119,7 @@ describe('Transaction rules', () => { // that will validate the input) await db.insertWithUUID('rules', { conditions: '{', actions: '}' }); // It will be in the database - expect((await db.all('SELECT * FROM rules')).length).toBe(3); + expect((await db.all('SELECT * FROM rules')).length).toBe(3); // But it will be ignored expect(getRules().length).toBe(2); diff --git a/packages/loot-core/src/server/accounts/transaction-rules.ts b/packages/loot-core/src/server/accounts/transaction-rules.ts index feab9fb0680..64056ca2c1c 100644 --- a/packages/loot-core/src/server/accounts/transaction-rules.ts +++ b/packages/loot-core/src/server/accounts/transaction-rules.ts @@ -174,7 +174,7 @@ export function makeRule(data) { export async function loadRules() { resetState(); - const rules = await db.all(` + const rules = await db.all(` SELECT * FROM rules WHERE conditions IS NOT NULL AND actions IS NOT NULL AND tombstone = 0 `); @@ -796,7 +796,7 @@ export async function updateCategoryRules(transactions) { // Also look 180 days in the future to get any future transactions // (this might change when we think about scheduled transactions) - const register: TransactionEntity[] = await db.all( + const register = await db.all( `SELECT t.* FROM v_transactions t LEFT JOIN accounts a ON a.id = t.account LEFT JOIN payees p ON p.id = t.payee diff --git a/packages/loot-core/src/server/accounts/transactions.ts b/packages/loot-core/src/server/accounts/transactions.ts index cacc9dbd4b3..2d81000b3bd 100644 --- a/packages/loot-core/src/server/accounts/transactions.ts +++ b/packages/loot-core/src/server/accounts/transactions.ts @@ -11,7 +11,7 @@ import * as transfer from './transfer'; async function idsWithChildren(ids: string[]) { const whereIds = whereIn(ids, 'parent_id'); - const rows = await db.all( + const rows = await db.all( `SELECT id FROM v_transactions_internal WHERE ${whereIds}`, ); const set = new Set(ids); @@ -56,7 +56,9 @@ export async function batchUpdateTransactions({ : []; const oldPayees = new Set(); - const accounts = await db.all('SELECT * FROM accounts WHERE tombstone = 0'); + const accounts = await db.all( + 'SELECT * FROM accounts WHERE tombstone = 0', + ); // We need to get all the payees of updated transactions _before_ // making changes diff --git a/packages/loot-core/src/server/accounts/transfer.test.ts b/packages/loot-core/src/server/accounts/transfer.test.ts index cb084b0c0cf..3e5f9f49fcf 100644 --- a/packages/loot-core/src/server/accounts/transfer.test.ts +++ b/packages/loot-core/src/server/accounts/transfer.test.ts @@ -7,7 +7,7 @@ import * as transfer from './transfer'; beforeEach(global.emptyDatabase()); function getAllTransactions() { - return db.all( + return db.all( `SELECT t.*, p.name as payee_name FROM v_transactions t LEFT JOIN payees p ON p.id = t.payee diff --git a/packages/loot-core/src/server/api.ts b/packages/loot-core/src/server/api.ts index da4061f7959..3f9bd0ffd92 100644 --- a/packages/loot-core/src/server/api.ts +++ b/packages/loot-core/src/server/api.ts @@ -53,7 +53,7 @@ function withMutation, ReturnType>( const latestTimestamp = getClock().timestamp.toString(); const result = await handler(...args); - const rows = await db.all( + const rows = await db.all>( 'SELECT DISTINCT dataset FROM messages_crdt WHERE timestamp > ?', [latestTimestamp], ); @@ -355,7 +355,8 @@ handlers['api/budget-month'] = async function ({ month }) { checkFileOpen(); await validateMonth(month); - const groups = await db.getCategoriesGrouped(); + const grouped = await db.getCategoriesGrouped(); + const groups = categoryGroupModel.fromDbArray(grouped); const sheetName = monthUtils.sheetForMonth(month); function value(name) { @@ -554,7 +555,7 @@ handlers['api/transaction-delete'] = withMutation(async function ({ id }) { handlers['api/accounts-get'] = async function () { checkFileOpen(); - const accounts = await db.getAccounts(); + const accounts = accountModel.fromDbArray(await db.getAccounts()); return accounts.map(account => accountModel.toExternal(account)); }; diff --git a/packages/loot-core/src/server/aql/exec.test.ts b/packages/loot-core/src/server/aql/exec.test.ts index 0a7bdf48cd2..c16f3af1aad 100644 --- a/packages/loot-core/src/server/aql/exec.test.ts +++ b/packages/loot-core/src/server/aql/exec.test.ts @@ -261,7 +261,9 @@ describe('runQuery', () => { it('fetches all data required for $oneof', async () => { await insertTransactions(); - const rows = await db.all('SELECT id FROM transactions WHERE amount < -50'); + const rows = await db.all>( + 'SELECT id FROM transactions WHERE amount < -50', + ); const ids = rows.slice(0, 3).map(row => row.id); ids.sort(); diff --git a/packages/loot-core/src/server/aql/schema/executors.ts b/packages/loot-core/src/server/aql/schema/executors.ts index d0180aad6fa..ae4a691908a 100644 --- a/packages/loot-core/src/server/aql/schema/executors.ts +++ b/packages/loot-core/src/server/aql/schema/executors.ts @@ -171,7 +171,9 @@ async function execTransactionsGrouped( ${sql.orderBy} `; - const allRows = await db.all(finalSql); + const allRows = await db.all< + db.DbTransaction & { _parent_id: db.DbTransaction['id'] } + >(finalSql); // Group the parents and children up const { parents, children } = allRows.reduce( diff --git a/packages/loot-core/src/server/aql/schema/index.test.ts b/packages/loot-core/src/server/aql/schema/index.test.ts index ce0e26a2c25..1b519dc28b8 100644 --- a/packages/loot-core/src/server/aql/schema/index.test.ts +++ b/packages/loot-core/src/server/aql/schema/index.test.ts @@ -9,29 +9,59 @@ beforeEach(global.emptyDatabase()); describe('schema', () => { test('never returns transactions without a date', async () => { - expect((await db.all('SELECT * FROM transactions')).length).toBe(0); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(0); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); await db.runQuery('INSERT INTO transactions (acct) VALUES (?)', ['foo']); - expect((await db.all('SELECT * FROM transactions')).length).toBe(1); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(1); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); }); test('never returns transactions without an account', async () => { - expect((await db.all('SELECT * FROM transactions')).length).toBe(0); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(0); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); await db.runQuery('INSERT INTO transactions (date) VALUES (?)', [20200101]); - expect((await db.all('SELECT * FROM transactions')).length).toBe(1); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(1); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); }); test('never returns child transactions without a parent', async () => { - expect((await db.all('SELECT * FROM transactions')).length).toBe(0); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(0); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); await db.runQuery( 'INSERT INTO transactions (date, acct, isChild) VALUES (?, ?, ?)', [20200101, 'foo', 1], ); - expect((await db.all('SELECT * FROM transactions')).length).toBe(1); - expect((await db.all('SELECT * FROM v_transactions')).length).toBe(0); + expect( + (await db.all('SELECT * FROM transactions')).length, + ).toBe(1); + expect( + (await db.all('SELECT * FROM v_transactions')) + .length, + ).toBe(0); }); }); diff --git a/packages/loot-core/src/server/budget/actions.ts b/packages/loot-core/src/server/budget/actions.ts index 69b5292674b..170cee18fc9 100644 --- a/packages/loot-core/src/server/budget/actions.ts +++ b/packages/loot-core/src/server/budget/actions.ts @@ -61,8 +61,14 @@ type BudgetData = { amount: number; }; -function getBudgetData(table: string, month: string): Promise { - return db.all( +function getBudgetData( + table: BudgetTable, + month: string, +): Promise { + return db.all< + (db.DbZeroBudget | db.DbReflectBudget) & + Pick + >( ` SELECT b.*, c.is_income FROM v_categories c LEFT JOIN ${table} b ON b.category = c.id @@ -233,7 +239,7 @@ export async function copySinglePreviousMonth({ } export async function setZero({ month }: { month: string }): Promise { - const categories = await db.all( + const categories = await db.all( 'SELECT * FROM v_categories WHERE tombstone = 0', ); @@ -252,7 +258,7 @@ export async function set3MonthAvg({ }: { month: string; }): Promise { - const categories = await db.all( + const categories = await db.all( 'SELECT * FROM v_categories WHERE tombstone = 0', ); @@ -295,7 +301,7 @@ export async function set12MonthAvg({ }: { month: string; }): Promise { - const categories = await db.all( + const categories = await db.all( 'SELECT * FROM v_categories WHERE tombstone = 0', ); @@ -314,7 +320,7 @@ export async function set6MonthAvg({ }: { month: string; }): Promise { - const categories = await db.all( + const categories = await db.all( 'SELECT * FROM v_categories WHERE tombstone = 0', ); diff --git a/packages/loot-core/src/server/budget/base.ts b/packages/loot-core/src/server/budget/base.ts index 975b00ac741..2af66c8efff 100644 --- a/packages/loot-core/src/server/budget/base.ts +++ b/packages/loot-core/src/server/budget/base.ts @@ -2,6 +2,7 @@ import * as monthUtils from '../../shared/months'; import { getChangedValues } from '../../shared/util'; import * as db from '../db'; +import { categoryGroupModel } from '../models'; import * as sheet from '../sheet'; import { resolveName } from '../spreadsheet/util'; @@ -391,7 +392,8 @@ export async function doTransfer(categoryIds, transferId) { export async function createBudget(months) { const categories = await db.getCategories(); - const groups = await db.getCategoriesGrouped(); + const grouped = await db.getCategoriesGrouped(); + const groups = categoryGroupModel.fromDbArray(grouped); sheet.startTransaction(); const meta = sheet.get().meta(); diff --git a/packages/loot-core/src/server/budget/cleanup-template.ts b/packages/loot-core/src/server/budget/cleanup-template.ts index 85d387d5abb..4c5964960f7 100644 --- a/packages/loot-core/src/server/budget/cleanup-template.ts +++ b/packages/loot-core/src/server/budget/cleanup-template.ts @@ -132,7 +132,7 @@ async function processCleanup(month: string): Promise { const db_month = parseInt(month.replace('-', '')); const category_templates = await getCategoryTemplates(); - const categories = await db.all( + const categories = await db.all( 'SELECT * FROM v_categories WHERE tombstone = 0', ); const sheetName = monthUtils.sheetForMonth(month); @@ -364,7 +364,7 @@ const TEMPLATE_PREFIX = '#cleanup '; async function getCategoryTemplates() { const templates = {}; - const notes = await db.all( + const notes = await db.all( `SELECT * FROM notes WHERE lower(note) like '%${TEMPLATE_PREFIX}%'`, ); diff --git a/packages/loot-core/src/server/budget/goaltemplates.ts b/packages/loot-core/src/server/budget/goaltemplates.ts index 7a4b00a30c1..9efe315d5d7 100644 --- a/packages/loot-core/src/server/budget/goaltemplates.ts +++ b/packages/loot-core/src/server/budget/goaltemplates.ts @@ -1,7 +1,6 @@ // @ts-strict-ignore import { Notification } from '../../client/state-types/notifications'; import * as monthUtils from '../../shared/months'; -import { CategoryEntity } from '../../types/models'; import * as db from '../db'; import { batchMessages } from '../sync'; @@ -26,7 +25,7 @@ export async function overwriteTemplate({ month }): Promise { export async function applyMultipleCategoryTemplates({ month, categoryIds }) { const placeholders = categoryIds.map(() => '?').join(', '); const query = `SELECT * FROM v_categories WHERE id IN (${placeholders})`; - const categories = await db.all(query, categoryIds); + const categories = await db.all(query, categoryIds); await storeTemplates(); const categoryTemplates = await getTemplates(categories); const ret = await processTemplate(month, true, categoryTemplates, categories); @@ -34,9 +33,10 @@ export async function applyMultipleCategoryTemplates({ month, categoryIds }) { } export async function applySingleCategoryTemplate({ month, category }) { - const categories = await db.all(`SELECT * FROM v_categories WHERE id = ?`, [ - category, - ]); + const categories = await db.all( + `SELECT * FROM v_categories WHERE id = ?`, + [category], + ); await storeTemplates(); const categoryTemplates = await getTemplates(categories[0]); const ret = await processTemplate(month, true, categoryTemplates, categories); @@ -47,8 +47,8 @@ export function runCheckTemplates() { return checkTemplates(); } -async function getCategories(): Promise { - return await db.all( +async function getCategories(): Promise { + return await db.all( ` SELECT categories.* FROM categories INNER JOIN category_groups on categories.cat_group = category_groups.id @@ -60,7 +60,7 @@ async function getCategories(): Promise { async function getTemplates(category) { //retrieves template definitions from the database - const goalDef = await db.all( + const goalDef = await db.all( 'SELECT * FROM categories WHERE goal_def IS NOT NULL', ); diff --git a/packages/loot-core/src/server/budget/statements.ts b/packages/loot-core/src/server/budget/statements.ts index 13e4707a5f7..4f6455df5dc 100644 --- a/packages/loot-core/src/server/budget/statements.ts +++ b/packages/loot-core/src/server/budget/statements.ts @@ -28,7 +28,9 @@ export type CategoryWithTemplateNote = { export async function getCategoriesWithTemplateNotes(): Promise< CategoryWithTemplateNote[] > { - return await db.all( + return await db.all< + Pick & Pick + >( ` SELECT c.id AS id, c.name as name, n.note AS note FROM notes n @@ -42,7 +44,18 @@ export async function getCategoriesWithTemplateNotes(): Promise< } export async function getActiveSchedules(): Promise { - return await db.all( + return await db.all< + Pick< + DbSchedule, + | 'id' + | 'rule' + | 'active' + | 'completed' + | 'posts_transaction' + | 'tombstone' + | 'name' + > + >( 'SELECT id, rule, active, completed, posts_transaction, tombstone, name from schedules WHERE name NOT NULL AND tombstone = 0', ); } diff --git a/packages/loot-core/src/server/dashboard/app.ts b/packages/loot-core/src/server/dashboard/app.ts index d1299069b51..096977d5c90 100644 --- a/packages/loot-core/src/server/dashboard/app.ts +++ b/packages/loot-core/src/server/dashboard/app.ts @@ -5,7 +5,6 @@ import * as fs from '../../platform/server/fs'; import { DEFAULT_DASHBOARD_STATE } from '../../shared/dashboard'; import { q } from '../../shared/query'; import { - type CustomReportEntity, type ExportImportDashboard, type ExportImportDashboardWidget, type ExportImportCustomReportWidget, @@ -178,7 +177,7 @@ async function importDashboard({ filepath }: { filepath: string }) { exportModel.validate(parsedContent); - const customReportIds: CustomReportEntity[] = await db.all( + const customReportIds = await db.all>( 'SELECT id from custom_reports', ); const customReportIdSet = new Set(customReportIds.map(({ id }) => id)); diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index ec8c3149518..2b773f49c92 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -15,7 +15,6 @@ import * as fs from '../../platform/server/fs'; import * as sqlite from '../../platform/server/sqlite'; import * as monthUtils from '../../shared/months'; import { groupById } from '../../shared/util'; -import { CategoryEntity, CategoryGroupEntity } from '../../types/models'; import { schema, schemaConfig, @@ -35,12 +34,16 @@ import { sendMessages, batchMessages } from '../sync'; import { shoveSortOrders, SORT_INCREMENT } from './sort'; import { DbAccount, + DbBank, DbCategory, DbCategoryGroup, + DbCategoryMapping, DbClockMessage, DbPayee, + DbPayeeMapping, DbTransaction, DbViewTransaction, + DbViewTransactionInternalAlive, } from './types'; export * from './types'; @@ -160,11 +163,8 @@ export function asyncTransaction(fn: () => Promise) { // This function is marked as async because `runQuery` is no longer // async. We return a promise here until we've audited all the code to // make sure nothing calls `.then` on this. -export async function all(sql, params?: (string | number)[]) { - // TODO: In the next phase, we will make this function generic - // and pass the type of the return type to `runQuery`. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return runQuery(sql, params, true) as any[]; +export async function all(sql, params?: (string | number)[]) { + return runQuery(sql, params, true); } export async function first(sql, params?: (string | number)[]) { @@ -305,19 +305,25 @@ export function updateWithSchema(table, fields) { // Data-specific functions. Ideally this would be split up into // different files -// TODO: Fix return type. This should returns a DbCategory[]. export async function getCategories( ids?: Array, -): Promise { +): Promise { const whereIn = ids ? `c.id IN (${toSqlQueryParameters(ids)}) AND` : ''; const query = `SELECT c.* FROM categories c WHERE ${whereIn} c.tombstone = 0 ORDER BY c.sort_order, c.id`; - return ids ? await all(query, [...ids]) : await all(query); + return ids + ? await all(query, [...ids]) + : await all(query); } -// TODO: Fix return type. This should returns a [DbCategoryGroup, ...DbCategory]. +/** + * Get all categories grouped by their category group. + * @param ids The IDs of the category groups to get. + * @returns The categories grouped by their category group. + * The first element of each tuple is the category group, and the rest are the categories that belong to that group. + */ export async function getCategoriesGrouped( ids?: Array, -): Promise> { +): Promise> { const categoryGroupWhereIn = ids ? `cg.id IN (${toSqlQueryParameters(ids)}) AND` : ''; @@ -331,18 +337,15 @@ export async function getCategoriesGrouped( ORDER BY c.sort_order, c.id`; const groups = ids - ? await all(categoryGroupQuery, [...ids]) - : await all(categoryGroupQuery); + ? await all(categoryGroupQuery, [...ids]) + : await all(categoryGroupQuery); const categories = ids - ? await all(categoryQuery, [...ids]) - : await all(categoryQuery); + ? await all(categoryQuery, [...ids]) + : await all(categoryQuery); return groups.map(group => { - return { - ...group, - categories: categories.filter(c => c.cat_group === group.id), - }; + return [group, ...categories.filter(c => c.cat_group === group.id)]; }); } @@ -378,7 +381,7 @@ export function updateCategoryGroup(group) { } export async function moveCategoryGroup(id, targetId) { - const groups = await all( + const groups = await all>( `SELECT id, sort_order FROM category_groups WHERE tombstone = 0 ORDER BY sort_order, id`, ); @@ -390,9 +393,10 @@ export async function moveCategoryGroup(id, targetId) { } export async function deleteCategoryGroup(group, transferId?: string) { - const categories = await all('SELECT * FROM categories WHERE cat_group = ?', [ - group.id, - ]); + const categories = await all( + 'SELECT * FROM categories WHERE cat_group = ?', + [group.id], + ); // Delete all the categories within a group await Promise.all(categories.map(cat => deleteCategory(cat, transferId))); @@ -426,7 +430,7 @@ export async function insertCategory( } else { // Unfortunately since we insert at the beginning, we need to shove // the sort orders to make sure there's room for it - const categories = await all( + const categories = await all>( `SELECT id, sort_order FROM categories WHERE cat_group = ? AND tombstone = 0 ORDER BY sort_order, id`, [category.cat_group], ); @@ -468,7 +472,7 @@ export async function moveCategory( throw new Error('moveCategory: groupId is required'); } - const categories = await all( + const categories = await all>( `SELECT id, sort_order FROM categories WHERE cat_group = ? AND tombstone = 0 ORDER BY sort_order, id`, [groupId], ); @@ -488,7 +492,7 @@ export async function deleteCategory( // We need to update all the deleted categories that currently // point to the one we're about to delete so they all are // "forwarded" to the new transferred category. - const existingTransfers = await all( + const existingTransfers = await all( 'SELECT * FROM category_mapping WHERE transferId = ?', [category.id], ); @@ -568,7 +572,7 @@ export async function mergePayees( await batchMessages(async () => { await Promise.all( ids.map(async id => { - const mappings = await all( + const mappings = await all>( 'SELECT id FROM payee_mapping WHERE targetId = ?', [id], ); @@ -592,7 +596,7 @@ export async function mergePayees( } export function getPayees() { - return all(` + return all(` SELECT p.*, COALESCE(a.name, p.name) AS name FROM payees p LEFT JOIN accounts a ON (p.transfer_acct = a.id AND a.tombstone = 0) WHERE p.tombstone = 0 AND (p.transfer_acct IS NULL OR a.id IS NOT NULL) @@ -605,7 +609,14 @@ export function getCommonPayees() { monthUtils.subWeeks(monthUtils.currentDate(), 12), ); const limit = 10; - return all(` + return all< + Pick & { + common: true; + transfer_acct: null; + c: number; + latest: DbViewTransactionInternalAlive['date']; + } + >(` SELECT p.id as id, p.name as name, p.favorite as favorite, p.category as category, TRUE as common, NULL as transfer_acct, count(*) as c, @@ -643,11 +654,11 @@ const orphanedPayeesQuery = ` /* eslint-enable rulesdir/typography */ export function syncGetOrphanedPayees() { - return all(orphanedPayeesQuery); + return all>(orphanedPayeesQuery); } export async function getOrphanedPayees() { - const rows = await all(orphanedPayeesQuery); + const rows = await all>(orphanedPayeesQuery); return rows.map(row => row.id); } @@ -659,7 +670,12 @@ export async function getPayeeByName(name: DbPayee['name']) { } export function getAccounts() { - return all( + return all< + DbAccount & { + bankName: DbBank['name']; + bankId: DbBank['id']; + } + >( `SELECT a.*, b.name as bankName, b.id as bankId FROM accounts a LEFT JOIN banks b ON a.bank = b.id WHERE a.tombstone = 0 @@ -668,7 +684,7 @@ export function getAccounts() { } export async function insertAccount(account) { - const accounts = await all( + const accounts = await all( 'SELECT * FROM accounts WHERE offbudget = ? ORDER BY sort_order, name', [account.offbudget ? 1 : 0], ); diff --git a/packages/loot-core/src/server/db/mappings.ts b/packages/loot-core/src/server/db/mappings.ts index f7423cab5f0..eaacf93db17 100644 --- a/packages/loot-core/src/server/db/mappings.ts +++ b/packages/loot-core/src/server/db/mappings.ts @@ -22,12 +22,12 @@ let unlistenSync; export async function loadMappings() { // The mappings are separated into tables specific to the type of // data. But you know, we really could keep a global mapping table. - const categories = (await db.all('SELECT * FROM category_mapping')).map( - r => [r.id, r.transferId] as const, - ); - const payees = (await db.all('SELECT * FROM payee_mapping')).map( - r => [r.id, r.targetId] as const, - ); + const categories = ( + await db.all('SELECT * FROM category_mapping') + ).map(r => [r.id, r.transferId] as const); + const payees = ( + await db.all('SELECT * FROM payee_mapping') + ).map(r => [r.id, r.targetId] as const); // All ids are unique, so we can just keep a global table of mappings allMappings = new Map(categories.concat(payees)); diff --git a/packages/loot-core/src/server/main.test.ts b/packages/loot-core/src/server/main.test.ts index 5992de82427..b5809a6ac20 100644 --- a/packages/loot-core/src/server/main.test.ts +++ b/packages/loot-core/src/server/main.test.ts @@ -135,7 +135,7 @@ describe('Accounts', () => { date: '2017-01-01', }); const differ = expectSnapshotWithDiffer( - await db.all('SELECT * FROM transactions'), + await db.all('SELECT * FROM transactions'), ); let transaction = await db.getTransaction(id); @@ -144,11 +144,15 @@ describe('Accounts', () => { payee: 'transfer-three', date: '2017-01-03', }); - differ.expectToMatchDiff(await db.all('SELECT * FROM transactions')); + differ.expectToMatchDiff( + await db.all('SELECT * FROM transactions'), + ); transaction = await db.getTransaction(id); await runHandler(handlers['transaction-delete'], transaction); - differ.expectToMatchDiff(await db.all('SELECT * FROM transactions')); + differ.expectToMatchDiff( + await db.all('SELECT * FROM transactions'), + ); }); }); diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.ts index 37db02ebf67..85e38f13971 100644 --- a/packages/loot-core/src/server/main.ts +++ b/packages/loot-core/src/server/main.ts @@ -49,6 +49,12 @@ import { APIError, TransactionError, PostError } from './errors'; import { app as filtersApp } from './filters/app'; import { handleBudgetImport } from './importers'; import { app } from './main-app'; +import { + accountModel, + categoryGroupModel, + categoryModel, + payeeModel, +} from './models'; import { mutator, runHandler } from './mutators'; import { app as notesApp } from './notes/app'; import * as Platform from './platform'; @@ -160,8 +166,8 @@ handlers['transactions-export-query'] = async function ({ query: queryState }) { handlers['get-categories'] = async function () { return { - grouped: await db.getCategoriesGrouped(), - list: await db.getCategories(), + grouped: categoryGroupModel.fromDbArray(await db.getCategoriesGrouped()), + list: categoryModel.fromDbArray(await db.getCategories()), }; }; @@ -181,7 +187,8 @@ handlers['get-budget-bounds'] = async function () { }; handlers['envelope-budget-month'] = async function ({ month }) { - const groups = await db.getCategoriesGrouped(); + const grouped = await db.getCategoriesGrouped(); + const groups = categoryGroupModel.fromDbArray(grouped); const sheetName = monthUtils.sheetForMonth(month); function value(name) { @@ -233,7 +240,8 @@ handlers['envelope-budget-month'] = async function ({ month }) { }; handlers['tracking-budget-month'] = async function ({ month }) { - const groups = await db.getCategoriesGrouped(); + const grouped = await db.getCategoriesGrouped(); + const groups = categoryGroupModel.fromDbArray(grouped); const sheetName = monthUtils.sheetForMonth(month); function value(name) { @@ -366,7 +374,8 @@ handlers['category-delete'] = mutator(async function ({ id, transferId }) { }); handlers['get-category-groups'] = async function () { - return await db.getCategoriesGrouped(); + const grouped = await db.getCategoriesGrouped(); + return categoryGroupModel.fromDbArray(grouped); }; handlers['category-group-create'] = mutator(async function ({ @@ -401,7 +410,7 @@ handlers['category-group-delete'] = mutator(async function ({ transferId, }) { return withUndo(async () => { - const groupCategories = await db.all( + const groupCategories = await db.all>( 'SELECT id FROM categories WHERE cat_group = ? AND tombstone = 0', [id], ); @@ -454,11 +463,11 @@ handlers['common-payees-get'] = async function () { }; handlers['payees-get'] = async function () { - return db.getPayees(); + return payeeModel.fromDbArray(await db.getPayees()); }; handlers['payees-get-orphaned'] = async function () { - return db.syncGetOrphanedPayees(); + return await db.syncGetOrphanedPayees(); }; handlers['payees-get-rule-counts'] = async function () { @@ -569,7 +578,7 @@ handlers['account-update'] = mutator(async function ({ id, name }) { }); handlers['accounts-get'] = async function () { - return db.getAccounts(); + return accountModel.fromDbArray(await db.getAccounts()); }; handlers['account-balance'] = async function ({ id, cutoff }) { diff --git a/packages/loot-core/src/server/models.ts b/packages/loot-core/src/server/models.ts index 772ead3f154..8ed7a4ed74e 100644 --- a/packages/loot-core/src/server/models.ts +++ b/packages/loot-core/src/server/models.ts @@ -3,8 +3,17 @@ import { CategoryEntity, CategoryGroupEntity, PayeeEntity, + TransactionEntity, } from '../types/models'; +import { + DbAccount, + DbCategory, + DbCategoryGroup, + DbPayee, + DbTransaction, + DbViewTransactionInternal, +} from './db'; import { ValidationError } from './errors'; export function requiredFields( @@ -50,7 +59,7 @@ export function fromDateRepr(number: number) { } export const accountModel = { - validate(account: AccountEntity, { update }: { update?: boolean } = {}) { + validate(account: Partial, { update }: { update?: boolean } = {}) { requiredFields( 'account', account, @@ -60,10 +69,55 @@ export const accountModel = { return account; }, + fromDbArray(accounts: DbAccount[]): AccountEntity[] { + return accounts.map(account => accountModel.fromDb(account)); + }, + fromDb(account: DbAccount): AccountEntity { + return { + id: account.id, + name: account.name, + offbudget: account.offbudget, + closed: account.closed, + sort_order: account.sort_order, + tombstone: account.tombstone, + account_id: account.account_id ?? null, + account_sync_source: account.account_sync_source ?? null, + balance_available: account.balance_available ?? null, + balance_current: account.balance_current ?? null, + balance_limit: account.balance_limit ?? null, + bank: account.bank ?? null, + mask: account.mask ?? null, + official_name: account.official_name ?? null, + } as AccountEntity; + }, + toDb(account: AccountEntity): DbAccount { + return { + id: account.id, + name: account.name, + offbudget: account.offbudget ? 1 : 0, + closed: account.closed ? 1 : 0, + tombstone: account.tombstone ? 1 : 0, + sort_order: account.sort_order, + account_sync_source: account.account_sync_source, + account_id: account.account_id, + balance_available: account.balance_available, + balance_current: account.balance_current, + balance_limit: account.balance_limit, + bank: account.bank, + mask: account.mask, + official_name: account.official_name, + // No longer used + // type, + // subtype, + }; + }, }; export const categoryModel = { - validate(category: CategoryEntity, { update }: { update?: boolean } = {}) { + validate( + category: Partial, + { update }: { update?: boolean } = {}, + ) { requiredFields( 'category', category, @@ -72,13 +126,43 @@ export const categoryModel = { ); const { sort_order, ...rest } = category; - return { ...rest, hidden: rest.hidden ? 1 : 0 }; + return { ...rest, hidden: rest.hidden ? (1 as const) : (0 as const) }; + }, + fromDbArray(categories: DbCategory[]): CategoryEntity[] { + return categories.map(category => categoryModel.fromDb(category)); + }, + fromDb(category: DbCategory): CategoryEntity { + return { + id: category.id, + name: category.name, + is_income: category.is_income === 1, + cat_group: category.cat_group, + sort_order: category.sort_order, + tombstone: category.tombstone === 1, + hidden: category.hidden === 1, + goal_def: category.goal_def ?? undefined, + }; + }, + toDb(category: CategoryEntity): DbCategory { + if (!category.cat_group) { + throw new Error('Category missing cat_group'); + } + return { + id: category.id, + name: category.name, + is_income: category.is_income ? 1 : 0, + cat_group: category.cat_group, + sort_order: category.sort_order ?? 0, + tombstone: category.tombstone ? 1 : 0, + hidden: category.hidden ? 1 : 0, + goal_def: category.goal_def, + }; }, }; export const categoryGroupModel = { validate( - categoryGroup: CategoryGroupEntity, + categoryGroup: Partial, { update }: { update?: boolean } = {}, ) { requiredFields( @@ -91,11 +175,159 @@ export const categoryGroupModel = { const { sort_order, ...rest } = categoryGroup; return { ...rest, hidden: rest.hidden ? 1 : 0 }; }, + fromDbArray( + grouped: [DbCategoryGroup, ...DbCategory[]][], + ): CategoryGroupEntity[] { + return grouped.map(([group, ...categories]) => + categoryGroupModel.fromDb(group, categories), + ); + }, + fromDb( + categoryGroup: DbCategoryGroup, + categories: DbCategory[] = [], + ): CategoryGroupEntity { + return { + id: categoryGroup.id, + name: categoryGroup.name, + is_income: categoryGroup.is_income === 1, + sort_order: categoryGroup.sort_order, + hidden: categoryGroup.hidden === 1, + tombstone: categoryGroup.tombstone === 1, + categories: categories + .filter(category => category.cat_group === categoryGroup.id) + .map(category => categoryModel.fromDb(category)), + }; + }, }; export const payeeModel = { - validate(payee: PayeeEntity, { update }: { update?: boolean } = {}) { + validate(payee: Partial, { update }: { update?: boolean } = {}) { requiredFields('payee', payee, ['name'], update); return payee; }, + fromDbArray(payees: DbPayee[]): PayeeEntity[] { + return payees.map(payee => payeeModel.fromDb(payee)); + }, + fromDb(payee: DbPayee): PayeeEntity { + return { + id: payee.id, + name: payee.name, + favorite: payee.favorite, + learn_categories: payee.learn_categories, + tombstone: payee.tombstone === 1, + transfer_acct: payee.transfer_acct ?? undefined, + }; + }, + toDb(payee: PayeeEntity): DbPayee { + return { + id: payee.id, + name: payee.name, + favorite: payee.favorite ? 1 : 0, + learn_categories: payee.learn_categories ? 1 : 0, + tombstone: payee.tombstone ? 1 : 0, + transfer_acct: payee.transfer_acct, + // No longer used + // category + }; + }, +}; + +export const transactionModel = { + validate( + transaction: Partial, + { update }: { update?: boolean } = {}, + ) { + requiredFields( + 'transaction', + transaction, + ['date', 'amount', 'acct'], + update, + ); + return transaction; + }, + fromDbView( + transaction: DbViewTransactionInternal, + subtransactions: DbViewTransactionInternal[] = [], + ): TransactionEntity { + return { + id: transaction.id, + date: fromDateRepr(transaction.date), + amount: transaction.amount, + payee: transaction.payee ?? undefined, + account: transaction.account ?? undefined, + category: transaction.category ?? undefined, + transfer_id: transaction.transfer_id ?? undefined, + imported_id: transaction.imported_id ?? undefined, + error: transaction.error ? JSON.parse(transaction.error) : undefined, + imported_payee: transaction.imported_payee ?? undefined, + starting_balance_flag: transaction.starting_balance_flag === 1, + notes: transaction.notes ?? undefined, + cleared: transaction.cleared === 1, + reconciled: transaction.reconciled === 1, + subtransactions: subtransactions.map(subtransaction => + transactionModel.fromDbView(subtransaction), + ), + schedule: transaction.schedule ?? undefined, + is_child: transaction.is_child === 1, + is_parent: transaction.is_parent === 1, + parent_id: transaction.parent_id ?? undefined, + sort_order: transaction.sort_order, + tombstone: transaction.tombstone === 1, + }; + }, + fromDb( + transaction: DbTransaction, + subtransactions: DbTransaction[] = [], + ): TransactionEntity { + return { + id: transaction.id, + date: fromDateRepr(transaction.date), + amount: transaction.amount, + // payee: transaction.payee_id, + account: transaction.acct, + category: transaction.category ?? undefined, + transfer_id: transaction.transferred_id ?? undefined, + notes: transaction.notes ?? undefined, + cleared: transaction.cleared === 1, + reconciled: transaction.reconciled === 1, + error: transaction.error ? JSON.parse(transaction.error) : undefined, + imported_id: transaction.financial_id ?? undefined, + imported_payee: transaction.imported_description ?? undefined, + starting_balance_flag: transaction.starting_balance_flag === 1, + schedule: transaction.schedule ?? undefined, + sort_order: transaction.sort_order, + tombstone: transaction.tombstone === 1, + is_child: transaction.isChild === 1, + is_parent: transaction.isParent === 1, + parent_id: transaction.parent_id ?? undefined, + subtransactions: subtransactions.map(subtransaction => + transactionModel.fromDb(subtransaction), + ), + payee: transaction.description ?? undefined, + }; + }, + toDb(transaction: TransactionEntity): DbTransaction { + return { + id: transaction.id, + date: toDateRepr(transaction.date), + amount: transaction.amount, + description: transaction.payee, + acct: transaction.account, + category: transaction.category, + transferred_id: transaction.transfer_id, + notes: transaction.notes, + error: JSON.stringify(transaction.error), + financial_id: transaction.imported_id, + imported_description: transaction.imported_payee, + schedule: transaction.schedule, + sort_order: transaction.sort_order ?? 0, + tombstone: transaction.tombstone ? 1 : 0, + isChild: transaction.is_child ? 1 : 0, + isParent: transaction.is_parent ? 1 : 0, + parent_id: transaction.parent_id, + starting_balance_flag: transaction.starting_balance_flag ? 1 : 0, + cleared: transaction.cleared ? 1 : 0, + reconciled: transaction.reconciled ? 1 : 0, + }; + }, }; diff --git a/packages/loot-core/src/server/preferences/app.ts b/packages/loot-core/src/server/preferences/app.ts index d1c73344e70..6f68c793b54 100644 --- a/packages/loot-core/src/server/preferences/app.ts +++ b/packages/loot-core/src/server/preferences/app.ts @@ -19,7 +19,9 @@ const savePreferences = async ({ }; const getPreferences = async (): Promise => { - const prefs = (await db.all('SELECT id, value FROM preferences')) as Array<{ + const prefs = (await db.all>( + 'SELECT id, value FROM preferences', + )) as Array<{ id: string; value: string; }>; diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.ts index 5888fb768a8..8fa8976790e 100644 --- a/packages/loot-core/src/server/sync/index.ts +++ b/packages/loot-core/src/server/sync/index.ts @@ -744,11 +744,13 @@ async function _fullSync( if (rebuiltMerkle.trie.hash === res.merkle.hash) { // Rebuilding the merkle worked... but why? - const clocks = await db.all('SELECT * FROM messages_clock'); + const clocks = await db.all( + 'SELECT * FROM messages_clock', + ); if (clocks.length !== 1) { console.log('Bad number of clocks:', clocks.length); } - const hash = deserializeClock(clocks[0]).merkle.hash; + const hash = deserializeClock(clocks[0].clock).merkle.hash; console.log('Merkle hash in db:', hash); } diff --git a/packages/loot-core/src/server/sync/migrate.test.ts b/packages/loot-core/src/server/sync/migrate.test.ts index 7fec886fcdf..bbdb7dd99ed 100644 --- a/packages/loot-core/src/server/sync/migrate.test.ts +++ b/packages/loot-core/src/server/sync/migrate.test.ts @@ -117,7 +117,10 @@ describe('sync migrations', () => { await sendMessages(msgs); await tracer.expect('applied'); - const transactions = await db.all('SELECT * FROM transactions', []); + const transactions = await db.all( + 'SELECT * FROM transactions', + [], + ); for (const trans of transactions) { const transMsgs = msgs .filter(msg => msg.row === trans.id) diff --git a/packages/loot-core/src/server/sync/sync.test.ts b/packages/loot-core/src/server/sync/sync.test.ts index abbde91302c..bd5adc57f41 100644 --- a/packages/loot-core/src/server/sync/sync.test.ts +++ b/packages/loot-core/src/server/sync/sync.test.ts @@ -54,8 +54,12 @@ describe('Sync', () => { expect(getClock().timestamp.toString()).toEqual(timestamp.toString()); expect(mockSyncServer.getClock().merkle).toEqual(getClock().merkle); - expect(await db.all('SELECT * FROM messages_crdt')).toMatchSnapshot(); - expect(await db.all('SELECT * FROM messages_clock')).toMatchSnapshot(); + expect( + await db.all('SELECT * FROM messages_crdt'), + ).toMatchSnapshot(); + expect( + await db.all('SELECT * FROM messages_clock'), + ).toMatchSnapshot(); }); it('should resend old messages to the server', async () => { diff --git a/packages/loot-core/src/server/tools/app.ts b/packages/loot-core/src/server/tools/app.ts index ceb535c4983..174d61ed68c 100644 --- a/packages/loot-core/src/server/tools/app.ts +++ b/packages/loot-core/src/server/tools/app.ts @@ -13,7 +13,11 @@ export const app = createApp(); app.method('tools/fix-split-transactions', async () => { // 1. Check for child transactions that have a blank payee, and set // the payee to whatever the parent has - const blankPayeeRows = await db.all(` + const blankPayeeRows = await db.all< + db.DbViewTransactionInternal & { + parentPayee: db.DbViewTransactionInternal['payee']; + } + >(` SELECT t.*, p.payee AS parentPayee FROM v_transactions_internal t LEFT JOIN v_transactions_internal p ON t.parent_id = p.id WHERE t.is_child = 1 AND t.payee IS NULL AND p.payee IS NOT NULL @@ -29,7 +33,9 @@ app.method('tools/fix-split-transactions', async () => { // 2. Make sure the "cleared" flag is synced up with the parent // transactions - const clearedRows = await db.all(` + const clearedRows = await db.all< + Pick + >(` SELECT t.id, p.cleared FROM v_transactions_internal t LEFT JOIN v_transactions_internal p ON t.parent_id = p.id WHERE t.is_child = 1 AND t.cleared != p.cleared @@ -45,7 +51,7 @@ app.method('tools/fix-split-transactions', async () => { // 3. Mark the `tombstone` field as true on any child transactions // that have a dead parent - const deletedRows = await db.all(` + const deletedRows = await db.all(` SELECT t.* FROM v_transactions_internal t LEFT JOIN v_transactions_internal p ON t.parent_id = p.id WHERE t.is_child = 1 AND t.tombstone = 0 AND (p.tombstone = 1 OR p.id IS NULL) @@ -74,7 +80,9 @@ app.method('tools/fix-split-transactions', async () => { }); // 5. Fix transfers that should not have categories - const brokenTransfers = await db.all(` + const brokenTransfers = await db.all< + Pick + >(` SELECT t1.id FROM v_transactions_internal t1 JOIN accounts a1 ON t1.account = a1.id diff --git a/packages/loot-core/src/types/models/category.d.ts b/packages/loot-core/src/types/models/category.d.ts index 80f6c121464..0a89773ee00 100644 --- a/packages/loot-core/src/types/models/category.d.ts +++ b/packages/loot-core/src/types/models/category.d.ts @@ -8,4 +8,5 @@ export interface CategoryEntity { sort_order?: number; tombstone?: boolean; hidden?: boolean; + goal_def?: string; } diff --git a/packages/loot-core/src/types/server-handlers.d.ts b/packages/loot-core/src/types/server-handlers.d.ts index 7156ce2eb6f..488e9979d78 100644 --- a/packages/loot-core/src/types/server-handlers.d.ts +++ b/packages/loot-core/src/types/server-handlers.d.ts @@ -119,10 +119,12 @@ export interface ServerHandlers { updated?; }) => Promise; - 'payees-check-orphaned': (arg: { ids }) => Promise; - 'payees-get-orphaned': () => Promise; + 'payees-check-orphaned': (arg: { + ids: Array; + }) => Promise>; + 'payees-get-orphaned': () => Promise>>; - 'payees-get-rules': (arg: { id: string }) => Promise; + 'payees-get-rules': (arg: { id: PayeeEntity['id'] }) => Promise; 'make-filters-from-conditions': (arg: { conditions: unknown; From 0a3068dbe1047295be8b1e45223867d871b4bb29 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Tue, 28 Jan 2025 10:50:38 -0800 Subject: [PATCH 09/11] Release notes --- upcoming-release-notes/4250.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 upcoming-release-notes/4250.md diff --git a/upcoming-release-notes/4250.md b/upcoming-release-notes/4250.md new file mode 100644 index 00000000000..225578c9973 --- /dev/null +++ b/upcoming-release-notes/4250.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [joel-jeremy] +--- + +[TypeScript] Make `db.all` generic to make it easy to type DB query results. \ No newline at end of file From 9884c913376faa423a9d49c887026532386021c6 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Tue, 28 Jan 2025 11:47:36 -0800 Subject: [PATCH 10/11] [TypeScript] Make `db.select` functions generic --- .../loot-core/src/server/accounts/sync.ts | 4 +- .../src/server/accounts/transactions.ts | 13 ++-- packages/loot-core/src/server/db/index.ts | 59 ++++++++++++++----- packages/loot-core/src/server/db/util.ts | 18 +++--- 4 files changed, 65 insertions(+), 29 deletions(-) diff --git a/packages/loot-core/src/server/accounts/sync.ts b/packages/loot-core/src/server/accounts/sync.ts index ab413e69784..759b5eeddf2 100644 --- a/packages/loot-core/src/server/accounts/sync.ts +++ b/packages/loot-core/src/server/accounts/sync.ts @@ -846,7 +846,7 @@ export async function syncAccount( acctId: string, bankId: string, ) { - const acctRow = await db.select('accounts', id); + const acctRow = await db.select('accounts', id); const syncStartDate = await getAccountSyncStartDate(id); const oldestTransaction = await getAccountOldestTransaction(id); @@ -893,7 +893,7 @@ export async function SimpleFinBatchSync( const account = accounts[i]; const download = res[account.accountId]; - const acctRow = await db.select('accounts', account.id); + const acctRow = await db.select('accounts', account.id); const oldestTransaction = await getAccountOldestTransaction(account.id); const newAccount = oldestTransaction == null; diff --git a/packages/loot-core/src/server/accounts/transactions.ts b/packages/loot-core/src/server/accounts/transactions.ts index 2d81000b3bd..c160a06172f 100644 --- a/packages/loot-core/src/server/accounts/transactions.ts +++ b/packages/loot-core/src/server/accounts/transactions.ts @@ -22,13 +22,18 @@ async function idsWithChildren(ids: string[]) { } async function getTransactionsByIds( - ids: string[], -): Promise { + ids: Array, +): Promise { // TODO: convert to whereIn // // or better yet, use ActualQL - return incrFetch( - (query, params) => db.selectWithSchema('transactions', query, params), + return incrFetch( + (sql, params) => + db.selectWithSchema( + 'transactions', + sql, + params, + ), ids, // eslint-disable-next-line rulesdir/typography id => `id = '${id}'`, diff --git a/packages/loot-core/src/server/db/index.ts b/packages/loot-core/src/server/db/index.ts index 2b773f49c92..7cc47b47cec 100644 --- a/packages/loot-core/src/server/db/index.ts +++ b/packages/loot-core/src/server/db/index.ts @@ -186,16 +186,16 @@ export async function run(sql, params?: (string | number)[]) { return runQuery(sql, params); } -export async function select(table, id) { - const rows = await runQuery( +export async function select( + table: string, + id: T['id'], +) { + const rows = await runQuery( 'SELECT * FROM ' + table + ' WHERE id = ?', [id], true, ); - // TODO: In the next phase, we will make this function generic - // and pass the type of the return type to `runQuery`. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return rows[0] as any; + return rows[0]; } export async function update(table, params) { @@ -270,18 +270,45 @@ export async function deleteAll(table: string) { await Promise.all(rows.map(({ id }) => delete_(table, id))); } -export async function selectWithSchema(table, sql, params) { - const rows = await runQuery(sql, params, true); +/** + * AQL-schema aware version of `select` that converts the query results + * according to the schema config. + * + * @param table The name of the table in the AQL schema. + * @param sql The SQL query. + * @param params The parameters for the SQL query. + * @returns The results of the query, converted according to the AQL schema config. + */ +export async function selectWithSchema( + table: keyof typeof schema, + sql: string, + params?: Array | undefined, +) { + const rows = await runQuery(sql, params, true); const convertedRows = rows - .map(row => convertFromSelect(schema, schemaConfig, table, row)) + .map( + row => + convertFromSelect(schema, schemaConfig, table, row) as TAfterConvert, + ) .filter(Boolean); - // TODO: Make convertFromSelect generic so we don't need this cast - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return convertedRows as any[]; + return convertedRows; } -export async function selectFirstWithSchema(table, sql, params) { - const rows = await selectWithSchema(table, sql, params); +/** + * AQL-schema aware version of `first` that converts the query results + * according to the schema config. + * + * @param table The name of the table in the AQL schema. + * @param sql The SQL query. + * @param params The parameters for the SQL query. + * @returns The first result of the query, converted according to the AQL schema config. + */ +export async function selectFirstWithSchema( + table: keyof typeof schema, + sql: string, + params?: Array | undefined, +) { + const rows = await selectWithSchema(table, sql, params); return rows.length > 0 ? rows[0] : null; } @@ -735,7 +762,7 @@ export async function moveAccount( } export async function getTransaction(id: DbViewTransaction['id']) { - const rows = await selectWithSchema( + const rows = await selectWithSchema( 'transactions', 'SELECT * FROM v_transactions WHERE id = ?', [id], @@ -750,7 +777,7 @@ export async function getTransactions(accountId: DbTransaction['acct']) { ); } - return selectWithSchema( + return selectWithSchema( 'transactions', 'SELECT * FROM v_transactions WHERE account = ?', [accountId], diff --git a/packages/loot-core/src/server/db/util.ts b/packages/loot-core/src/server/db/util.ts index c9814712e98..4bc87771783 100644 --- a/packages/loot-core/src/server/db/util.ts +++ b/packages/loot-core/src/server/db/util.ts @@ -1,13 +1,17 @@ // @ts-strict-ignore -export async function incrFetch( - runQuery, - terms, - compare, - makeQuery, - params = [], +export async function incrFetch( + runQuery: ( + query: string, + params?: Array | undefined, + fetchAll?: true, + ) => Promise, + terms: string[], + compare: (id: string) => string, + makeQuery: (sqlFilter: string) => string, + params: Array | undefined = [], ) { const pageCount = 500; - let results = []; + let results: T[] = []; let fetchedIds = new Set(); From d64214569591e8c6fa844569176716243429cc91 Mon Sep 17 00:00:00 2001 From: Joel Jeremy Marquez Date: Tue, 28 Jan 2025 11:48:19 -0800 Subject: [PATCH 11/11] Release notes --- upcoming-release-notes/4251.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 upcoming-release-notes/4251.md diff --git a/upcoming-release-notes/4251.md b/upcoming-release-notes/4251.md new file mode 100644 index 00000000000..8ad80e16f02 --- /dev/null +++ b/upcoming-release-notes/4251.md @@ -0,0 +1,6 @@ +--- +category: Maintenance +authors: [joel-jeremy] +--- + +[TypeScript] Make `db.select` functions generic to make it easy to type DB query results. \ No newline at end of file