From 2ab684a76954c8e91017d1a321972c5b0c62160c Mon Sep 17 00:00:00 2001 From: eleith Date: Tue, 10 Nov 2026 09:34:33 -0800 Subject: [PATCH] feat(db): add typed report database foundation --- .changeset/config.json | 1 + .changeset/typed-report-database.md | 4 + .github/workflows/db.yml | 49 +++++ .github/workflows/snapshots.yml | 2 + .woodpecker/db.yml | 59 +++++ .woodpecker/snapshots.yml | 2 + apps/web/docker-compose.dev.yml | 2 + packages/db/.prettierignore | 3 + packages/db/README.md | 4 + packages/db/eslint.config.js | 20 ++ packages/db/package.json | 44 ++++ packages/db/src/database.ts | 65 ++++++ packages/db/src/index.ts | 3 + packages/db/src/migrate.ts | 17 ++ packages/db/src/migrations/001_reports.ts | 60 +++++ packages/db/src/node-sqlite-dialect.ts | 36 +++ packages/db/src/reports/data.fixture.ts | 162 ++++++++++++++ packages/db/src/reports/data.ts | 191 ++++++++++++++++ packages/db/src/reports/index.ts | 13 ++ packages/db/src/reports/queries.test.ts | 116 ++++++++++ packages/db/src/reports/queries.ts | 126 +++++++++++ packages/db/src/schema.ts | 29 +++ packages/db/tsconfig.build.json | 5 + packages/db/tsconfig.json | 5 + packages/db/vitest.config.ts | 5 + packages/snapshots/package.json | 1 + packages/snapshots/src/index.ts | 5 +- .../publication.test.ts} | 168 ++------------ .../{reports.ts => reports/publication.ts} | 206 ++---------------- pnpm-lock.yaml | 43 ++++ pnpm-workspace.yaml | 1 + 31 files changed, 1106 insertions(+), 341 deletions(-) create mode 100644 .changeset/typed-report-database.md create mode 100644 .github/workflows/db.yml create mode 100644 .woodpecker/db.yml create mode 100644 packages/db/.prettierignore create mode 100644 packages/db/README.md create mode 100644 packages/db/eslint.config.js create mode 100644 packages/db/package.json create mode 100644 packages/db/src/database.ts create mode 100644 packages/db/src/index.ts create mode 100644 packages/db/src/migrate.ts create mode 100644 packages/db/src/migrations/001_reports.ts create mode 100644 packages/db/src/node-sqlite-dialect.ts create mode 100644 packages/db/src/reports/data.fixture.ts create mode 100644 packages/db/src/reports/data.ts create mode 100644 packages/db/src/reports/index.ts create mode 100644 packages/db/src/reports/queries.test.ts create mode 100644 packages/db/src/reports/queries.ts create mode 100644 packages/db/src/schema.ts create mode 100644 packages/db/tsconfig.build.json create mode 100644 packages/db/tsconfig.json create mode 100644 packages/db/vitest.config.ts rename packages/snapshots/src/{reports.test.ts => reports/publication.test.ts} (89%) rename packages/snapshots/src/{reports.ts => reports/publication.ts} (59%) diff --git a/.changeset/config.json b/.changeset/config.json index 60d1f32..286ef02 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -9,6 +9,7 @@ "updateInternalDependencies": "patch", "ignore": [ "@hledger-reports/config", + "@hledger-reports/db", "@hledger-reports/hledger", "@hledger-reports/jobs", "@hledger-reports/snapshots" diff --git a/.changeset/typed-report-database.md b/.changeset/typed-report-database.md new file mode 100644 index 0000000..d299565 --- /dev/null +++ b/.changeset/typed-report-database.md @@ -0,0 +1,4 @@ +--- +--- + +Add a typed report database package with Kysely, SQLite migrations and focused read/write functions. Existing reports continue to use JSON; saved data and financial results are unchanged. diff --git a/.github/workflows/db.yml b/.github/workflows/db.yml new file mode 100644 index 0000000..80f5ee1 --- /dev/null +++ b/.github/workflows/db.yml @@ -0,0 +1,49 @@ +name: CI (Report Database Package) + +on: + push: + branches: [main] + paths: + - '.github/workflows/db.yml' + - 'packages/db/**' + - 'packages/hledger/**' + - 'tools/**' + - 'package.json' + - 'pnpm-workspace.yaml' + - 'pnpm-lock.yaml' + - 'tsconfig.base.json' + pull_request: + branches: [main] + paths: + - '.github/workflows/db.yml' + - 'packages/db/**' + - 'packages/hledger/**' + - 'tools/**' + - 'package.json' + - 'pnpm-workspace.yaml' + - 'pnpm-lock.yaml' + - 'tsconfig.base.json' + workflow_dispatch: + +permissions: + contents: read + +jobs: + test-and-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v6 + with: + node-version: 26.5.0 + cache: pnpm + - run: pnpm install --frozen-lockfile + - name: Build package and dependencies + run: pnpm --filter '@hledger-reports/db...' build + - name: Check + run: pnpm --filter @hledger-reports/db check + - name: Lint + run: pnpm --filter @hledger-reports/db lint + - name: Test + run: pnpm --filter @hledger-reports/db test diff --git a/.github/workflows/snapshots.yml b/.github/workflows/snapshots.yml index 4797fd0..deb2862 100644 --- a/.github/workflows/snapshots.yml +++ b/.github/workflows/snapshots.yml @@ -6,6 +6,7 @@ on: paths: - '.github/workflows/snapshots.yml' - 'packages/snapshots/**' + - 'packages/db/**' - 'packages/hledger/**' - 'tools/**' - 'package.json' @@ -17,6 +18,7 @@ on: paths: - '.github/workflows/snapshots.yml' - 'packages/snapshots/**' + - 'packages/db/**' - 'packages/hledger/**' - 'tools/**' - 'package.json' diff --git a/.woodpecker/db.yml b/.woodpecker/db.yml new file mode 100644 index 0000000..eef086b --- /dev/null +++ b/.woodpecker/db.yml @@ -0,0 +1,59 @@ +workspace: + base: /workspace + path: src + +when: + - event: push + branch: main + path: + include: + - '.woodpecker/db.yml' + - 'packages/db/**' + - 'packages/hledger/**' + - 'tools/**' + - 'package.json' + - 'pnpm-workspace.yaml' + - 'pnpm-lock.yaml' + - 'tsconfig.base.json' + - event: pull_request + path: + include: + - '.woodpecker/db.yml' + - 'packages/db/**' + - 'packages/hledger/**' + - 'tools/**' + - 'package.json' + - 'pnpm-workspace.yaml' + - 'pnpm-lock.yaml' + - 'tsconfig.base.json' + - event: manual + +steps: + prepare: + image: node:26.5.0-bookworm + commands: + - npm install --global pnpm@11.9.0 + - pnpm config set store-dir .pnpm-store + - pnpm install --frozen-lockfile + + build: + image: node:26.5.0-bookworm + commands: + - npm install --global pnpm@11.9.0 + - pnpm config set store-dir .pnpm-store + - pnpm --filter '@hledger-reports/db...' build + + check: + image: node:26.5.0-bookworm + commands: + - npm install --global pnpm@11.9.0 + - pnpm config set store-dir .pnpm-store + - pnpm --filter @hledger-reports/db check + - pnpm --filter @hledger-reports/db lint + + test: + image: node:26.5.0-bookworm + commands: + - npm install --global pnpm@11.9.0 + - pnpm config set store-dir .pnpm-store + - pnpm --filter @hledger-reports/db test diff --git a/.woodpecker/snapshots.yml b/.woodpecker/snapshots.yml index 904392c..aaa93c6 100644 --- a/.woodpecker/snapshots.yml +++ b/.woodpecker/snapshots.yml @@ -9,6 +9,7 @@ when: include: - '.woodpecker/snapshots.yml' - 'packages/snapshots/**' + - 'packages/db/**' - 'packages/hledger/**' - 'tools/**' - 'package.json' @@ -20,6 +21,7 @@ when: include: - '.woodpecker/snapshots.yml' - 'packages/snapshots/**' + - 'packages/db/**' - 'packages/hledger/**' - 'tools/**' - 'package.json' diff --git a/apps/web/docker-compose.dev.yml b/apps/web/docker-compose.dev.yml index a785398..43f293f 100644 --- a/apps/web/docker-compose.dev.yml +++ b/apps/web/docker-compose.dev.yml @@ -17,6 +17,7 @@ services: - ./static:/m/apps/web/static:ro - ./vite.config.ts:/m/apps/web/vite.config.ts:ro - ../../packages/config/src:/m/packages/config/src:ro + - ../../packages/db/src:/m/packages/db/src:ro - ../../packages/jobs/src:/m/packages/jobs/src:ro - ../../packages/snapshots/src:/m/packages/snapshots/src:ro healthcheck: @@ -39,6 +40,7 @@ services: volumes: - ../worker/src:/m/apps/worker/src:ro - ../../packages/config/src:/m/packages/config/src:ro + - ../../packages/db/src:/m/packages/db/src:ro - ../../packages/hledger/src:/m/packages/hledger/src:ro - ../../packages/jobs/src:/m/packages/jobs/src:ro - ../../packages/snapshots/src:/m/packages/snapshots/src:ro diff --git a/packages/db/.prettierignore b/packages/db/.prettierignore new file mode 100644 index 0000000..e6fc353 --- /dev/null +++ b/packages/db/.prettierignore @@ -0,0 +1,3 @@ +dist/ +coverage/ +*.tsbuildinfo diff --git a/packages/db/README.md b/packages/db/README.md new file mode 100644 index 0000000..9e14707 --- /dev/null +++ b/packages/db/README.md @@ -0,0 +1,4 @@ +# Report database + +Typed report storage using Kysely and SQLite. Migrations initialize new private +generations; published databases are read-only. Snapshots owns generation publication. diff --git a/packages/db/eslint.config.js b/packages/db/eslint.config.js new file mode 100644 index 0000000..239a973 --- /dev/null +++ b/packages/db/eslint.config.js @@ -0,0 +1,20 @@ +import js from '@eslint/js'; +import ts from 'typescript-eslint'; +import prettier from 'eslint-config-prettier'; +import globals from 'globals'; + +export default ts.config( + { ignores: ['build/', 'dist/', 'coverage/'] }, + js.configs.recommended, + ...ts.configs.recommended, + prettier, + { + languageOptions: { globals: { ...globals.node } }, + rules: { + '@typescript-eslint/no-unused-vars': [ + 'error', + { argsIgnorePattern: '^_', varsIgnorePattern: '^_' } + ] + } + } +); diff --git a/packages/db/package.json b/packages/db/package.json new file mode 100644 index 0000000..a633b99 --- /dev/null +++ b/packages/db/package.json @@ -0,0 +1,44 @@ +{ + "name": "@hledger-reports/db", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": { + "development": "./src/index.ts", + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./reports": { + "development": "./src/reports/index.ts", + "types": "./dist/src/reports/index.d.ts", + "import": "./dist/src/reports/index.js" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "check": "tsc --noEmit", + "test": "vitest run", + "lint": "prettier --check . && eslint .", + "format": "prettier --write ." + }, + "dependencies": { + "@hledger-reports/hledger": "workspace:*", + "kysely": "catalog:" + }, + "devDependencies": { + "@eslint/js": "catalog:", + "@types/node": "catalog:", + "eslint": "catalog:", + "eslint-config-prettier": "catalog:", + "globals": "catalog:", + "prettier": "catalog:", + "typescript": "catalog:", + "typescript-eslint": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/db/src/database.ts b/packages/db/src/database.ts new file mode 100644 index 0000000..0e657f0 --- /dev/null +++ b/packages/db/src/database.ts @@ -0,0 +1,65 @@ +import { Kysely, sql } from 'kysely'; +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { nodeSqliteDialect } from './node-sqlite-dialect.js'; +import { reportSchemaVersion, type ReportDatabase } from './schema.js'; + +async function checkDirectory(directory: string): Promise { + const folder = await open( + directory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW + ); + await folder.close(); +} + +async function connectDatabase(path: string, readOnly: boolean): Promise> { + const native = new DatabaseSync(path, { readOnly }); + try { + if (!readOnly) native.exec('PRAGMA journal_mode = DELETE; PRAGMA foreign_keys = ON;'); + const database = new Kysely({ dialect: nodeSqliteDialect(native) }); + await database.connection().execute(async () => undefined); + return database; + } catch (error) { + native.close(); + throw error; + } +} + +export async function createReportDatabase(path: string): Promise> { + await checkDirectory(dirname(path)); + const file = await open( + path, + constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600 + ); + try { + await file.chmod(0o600); + } finally { + await file.close(); + } + return connectDatabase(path, false); +} + +export async function openReadOnlyReportDatabase( + path: string +): Promise | null> { + await checkDirectory(dirname(path)); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + if (!(await file.stat()).isFile()) throw new Error('Invalid report database file.'); + } finally { + await file.close(); + } + const database = await connectDatabase(path, true); + try { + const version = await sql<{ user_version: number }>`PRAGMA user_version`.execute(database); + if (version.rows[0]?.user_version === reportSchemaVersion) return database; + } catch (error) { + await database.destroy(); + throw error; + } + await database.destroy(); + return null; +} diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts new file mode 100644 index 0000000..cead1ce --- /dev/null +++ b/packages/db/src/index.ts @@ -0,0 +1,3 @@ +export { createReportDatabase, openReadOnlyReportDatabase } from './database.js'; +export { runMigrations } from './migrate.js'; +export type { ReportDatabase } from './schema.js'; diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts new file mode 100644 index 0000000..2b02906 --- /dev/null +++ b/packages/db/src/migrate.ts @@ -0,0 +1,17 @@ +import type { Kysely } from 'kysely'; +import { Migrator } from 'kysely/migration'; +import * as reports from './migrations/001_reports.js'; +import type { ReportDatabase } from './schema.js'; + +export async function runMigrations(database: Kysely): Promise { + const migrator = new Migrator({ + db: database, + provider: { + async getMigrations() { + return { '001_reports': reports }; + } + } + }); + const { error } = await migrator.migrateToLatest(); + if (error !== undefined) throw error; +} diff --git a/packages/db/src/migrations/001_reports.ts b/packages/db/src/migrations/001_reports.ts new file mode 100644 index 0000000..7c0aa73 --- /dev/null +++ b/packages/db/src/migrations/001_reports.ts @@ -0,0 +1,60 @@ +import { sql, type Kysely } from 'kysely'; + +export async function up(database: Kysely): Promise { + await database.schema + .createTable('publication') + .addColumn('singleton', 'integer', (column) => column.primaryKey()) + .addColumn('journal', 'text', (column) => column.notNull()) + .addColumn('overview_json', 'text', (column) => column.notNull()) + .addCheckConstraint('publication_singleton', sql`singleton = 1`) + .addCheckConstraint('publication_json', sql`json_valid(overview_json)`) + .modifyEnd(sql`STRICT`) + .execute(); + + await database.schema + .createTable('accounts') + .addColumn('id', 'text', (column) => column.notNull().primaryKey()) + .addColumn('name', 'text', (column) => + column + .modifyFront(sql`COLLATE BINARY`) + .notNull() + .unique() + ) + .addColumn('catalog_ordinal', 'integer', (column) => column.notNull().unique()) + .addColumn('access', 'text', (column) => column.notNull()) + .addColumn('row_count', 'integer') + .addColumn('summary_json', 'text', (column) => column.notNull()) + .addCheckConstraint('accounts_ordinal', sql`catalog_ordinal >= 0`) + .addCheckConstraint('accounts_access', sql`access IN ('ready', 'ambiguous')`) + .addCheckConstraint( + 'accounts_row_count', + sql`(access = 'ambiguous' AND row_count IS NULL) + OR (access = 'ready' AND row_count IS NOT NULL AND row_count >= 0)` + ) + .addCheckConstraint('accounts_summary_json', sql`json_valid(summary_json)`) + .modifyEnd(sql`STRICT`) + .execute(); + + await database.schema + .createTable('account_activity') + .addColumn('account_name', 'text', (column) => + column.modifyFront(sql`COLLATE BINARY`).notNull() + ) + .addColumn('ordinal', 'integer', (column) => column.notNull()) + .addColumn('account_date', 'text', (column) => column.notNull()) + .addColumn('row_json', 'text', (column) => column.notNull()) + .addPrimaryKeyConstraint('account_activity_key', ['account_name', 'ordinal']) + .addCheckConstraint('account_activity_ordinal', sql`ordinal >= 0`) + .addCheckConstraint('account_activity_json', sql`json_valid(row_json)`) + .addForeignKeyConstraint( + 'account_activity_account', + ['account_name'], + 'accounts', + ['name'], + (constraint) => constraint.deferrable().initiallyDeferred() + ) + .modifyEnd(sql`STRICT`) + .execute(); + + await sql`PRAGMA user_version = 1`.execute(database); +} diff --git a/packages/db/src/node-sqlite-dialect.ts b/packages/db/src/node-sqlite-dialect.ts new file mode 100644 index 0000000..0c89aa0 --- /dev/null +++ b/packages/db/src/node-sqlite-dialect.ts @@ -0,0 +1,36 @@ +import { SqliteDialect } from 'kysely'; +import type { DatabaseSync, SQLInputValue } from 'node:sqlite'; + +function sqliteParameters(values: readonly unknown[]): readonly SQLInputValue[] { + if ( + !values.every( + (value): value is SQLInputValue => + value === null || + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'bigint' || + ArrayBuffer.isView(value) + ) + ) + throw new TypeError('Unsupported SQLite parameter.'); + return values; +} + +export function nodeSqliteDialect(database: DatabaseSync): SqliteDialect { + return new SqliteDialect({ + database: { + close: () => database.close(), + prepare(query) { + const statement = database.prepare(query); + return { + get reader() { + return statement.columns().length > 0; + }, + all: (values) => statement.all(...sqliteParameters(values)), + run: (values) => statement.run(...sqliteParameters(values)), + iterate: (values) => statement.iterate(...sqliteParameters(values)) + }; + } + } + }); +} diff --git a/packages/db/src/reports/data.fixture.ts b/packages/db/src/reports/data.fixture.ts new file mode 100644 index 0000000..d2d1203 --- /dev/null +++ b/packages/db/src/reports/data.fixture.ts @@ -0,0 +1,162 @@ +import type { + DetailedAmount, + OverviewData, + PreparedAccountSummary, + PreparedAccountRow +} from '@hledger-reports/hledger'; + +const expenseAmounts = [ + { commodity: 'EUR', quantity: '0.0000000000000000000000012300' }, + { commodity: '', quantity: '-0.000' } +]; +export const range = (months: 1 | 3 | 6 | 12) => ({ + begin: { 1: '2026-04-01', 3: '2026-02-01', 6: '2025-11-01', 12: '2025-05-01' }[months], + endExclusive: '2026-04-17', + transactionCount: 1, + income: [{ commodity: '$', quantity: '9007199254740993.123456789' }], + expenses: expenseAmounts, + net: [{ commodity: '$', quantity: '9007199254740992.123456789' }], + expenseCategories: { + depth: 2 as const, + rows: [{ account: 'expenses:food', amounts: expenseAmounts }] + } +}); +export const preparedOverview = (): OverviewData => ({ + asOf: '2026-04-16', + preparedAt: '2026-05-02T12:34:56.789Z', + journalCoverage: { transactionCount: 1, firstDate: '2026-04-16', lastDate: '2026-04-16' }, + hledgerVersion: '1.99.5', + ranges: { 1: range(1), 3: range(3), 6: range(6), 12: range(12) }, + netWorth: { + currency: '$', + points: Array.from({ length: 12 }, (_, i) => ({ + date: i === 11 ? '2026-04-16' : new Date(Date.UTC(2025, i + 5, 0)).toISOString().slice(0, 10), + amounts: [{ commodity: '$', quantity: '9007199254740993.123456789' }], + unconverted: [] + })) + }, + incomeExpenses: { + months: Array.from({ length: 12 }, (_, i) => ({ + begin: new Date(Date.UTC(2025, i + 4, 1)).toISOString().slice(0, 10), + endExclusive: new Date(Date.UTC(2025, i + 5, 1)).toISOString().slice(0, 10), + income: [{ commodity: '$', quantity: '9007199254740993.123456789' }], + expenses: expenseAmounts + })) + }, + balanceSheet: { + assets: { + amounts: [{ commodity: '$', quantity: '9007199254740994.123456789' }], + unconverted: [] + }, + liabilities: { amounts: [{ commodity: '$', quantity: '-1' }], unconverted: [] } + } +}); +export const journal = '/books/main.journal'; +export const amount = (quantity: string): DetailedAmount => ({ + commodity: '$', + quantity, + cost: null, + costBasis: null, + style: null +}); +export const sourcePosition = { file: '/private/books/included.journal', line: 12, column: 1 }; +export const preparedAccountRows = ( + count = 1, + description = 'exact producer strings' +): PreparedAccountRow[] => + Array.from( + { length: count }, + (_, i): PreparedAccountRow => ({ + accountDate: '2026-04-30', + isSplit: true, + otherAccounts: ['expenses:食料', 'expenses:household'], + change: [amount('-100.00')], + balance: [amount('9007199254740993.123456789')], + accountPostingIndexes: [0], + investment: { + securities: [], + cash: { + direction: 'out', + change: [amount('-100.00')], + balance: [amount('9007199254740993.123456789')], + unclassifiedChange: [], + unclassifiedBalance: [] + }, + cashPostingIndexes: [0], + incomePostingIndexes: [], + expensePostingIndexes: [1, 2], + gainPostingIndexes: [], + issues: [] + }, + transaction: { + index: i + 1, + date: '2026-04-29', + secondaryDate: '2026-04-30', + status: 'Cleared', + code: 'REF-42', + description: `${description} ${i}`, + payee: 'Market', + note: 'supplies', + comment: 'transaction note\nproject: home\n', + precedingComment: 'retained preceding comment\n', + tags: [ + ['project', 'home'], + ['project', 'ordered'] + ], + postings: [ + ['assets:cash', '-100.00', 'bank note'], + ['expenses:食料', '75.00', 'food note'], + ['expenses:household', '25.00', 'household note'] + ].map(([account, quantity, comment], postingIndex) => { + const date = postingIndex === 0 ? '2026-04-30' : null; + const balanceAssertion = + postingIndex === 0 + ? { + amount: amount('9007199254740993.123456789'), + total: false, + inclusive: false, + sourcePosition + } + : null; + return { + account, + amounts: [amount(quantity)], + transactedAmounts: [amount(quantity)], + date, + secondaryDate: null, + status: 'Unmarked', + effectiveStatus: 'Cleared', + comment, + tags: [['receipt', 'fictional']], + kind: 'RealPosting', + balanceAssertion, + transactionIndex: String(i + 1) + }; + }), + sourcePositions: [sourcePosition, { ...sourcePosition, line: 18 }] + } + }) + ); +export const ready = (name = 'assets:cash', rowCount = 1): PreparedAccountSummary => ({ + name, + access: 'ready', + rowCount, + summary: { + status: 'ready', + balances: rowCount === 0 ? [] : [{ commodity: 'TEST', quantity: '2.000' }], + valuation: { + amounts: rowCount === 0 ? [] : [{ commodity: 'TEST', quantity: '2.000' }], + unconverted: rowCount === 0 ? [] : ['TEST'] + } + } +}); +export const ambiguous = (name = 'ambiguous'): PreparedAccountSummary => ({ + name, + access: 'ambiguous', + rowCount: null, + summary: { status: 'ambiguous' } +}); +export const undated = (name: string): PreparedAccountSummary => ({ + ...ready(name, 0), + summary: { status: 'undated' } +}); diff --git a/packages/db/src/reports/data.ts b/packages/db/src/reports/data.ts new file mode 100644 index 0000000..77c948f --- /dev/null +++ b/packages/db/src/reports/data.ts @@ -0,0 +1,191 @@ +import type { + OverviewData, + PreparedAccountSummary, + PreparedAccountRow, + PreparedBalance +} from '@hledger-reports/hledger'; + +const accountIdPattern = /^[a-z0-9]{8}(?![\s\S])/; + +export interface StoredAccount { + id: string; + name: string; + access: 'ready' | 'ambiguous'; + rowCount: number | null; + summary: PreparedBalance; +} + +export interface StoredReportData { + overview: OverviewData; + /** Private journal identity; never expose this in page data. */ + journal: string; + accounts: StoredAccount[]; +} + +export function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +export function isAccountId(value: unknown): value is string { + return typeof value === 'string' && accountIdPattern.test(value); +} + +function isNullableString(value: unknown): boolean { + return value === null || typeof value === 'string'; +} + +function hasNativeValueContainers(value: unknown): boolean { + return isObject(value) && Array.isArray(value.amounts) && Array.isArray(value.unconverted); +} + +function hasRangeContainers(value: unknown): boolean { + if (!isObject(value)) return false; + return ( + typeof value.begin === 'string' && + typeof value.endExclusive === 'string' && + typeof value.transactionCount === 'number' && + Number.isSafeInteger(value.transactionCount) && + value.transactionCount >= 0 && + Array.isArray(value.income) && + Array.isArray(value.expenses) && + Array.isArray(value.net) && + isObject(value.expenseCategories) && + value.expenseCategories.depth === 2 && + Array.isArray(value.expenseCategories.rows) + ); +} + +export function hasOverviewContainers(value: unknown): value is OverviewData { + if ( + !isObject(value) || + typeof value.preparedAt !== 'string' || + !isNullableString(value.asOf) || + !isObject(value.journalCoverage) || + typeof value.journalCoverage.transactionCount !== 'number' || + !Number.isSafeInteger(value.journalCoverage.transactionCount) || + value.journalCoverage.transactionCount < 0 || + !isNullableString(value.journalCoverage.firstDate) || + !isNullableString(value.journalCoverage.lastDate) || + typeof value.hledgerVersion !== 'string' || + !isObject(value.incomeExpenses) || + !Array.isArray(value.incomeExpenses.months) || + !isObject(value.netWorth) || + !isNullableString(value.netWorth.currency) || + !Array.isArray(value.netWorth.points) + ) + return false; + if (value.asOf === null) { + return ( + value.ranges === null && + value.balanceSheet === null && + value.incomeExpenses.months.length === 0 && + value.netWorth.currency === null && + value.netWorth.points.length === 0 + ); + } + const ranges = value.ranges; + if (!isObject(ranges) || !isObject(value.balanceSheet)) return false; + return ( + [1, 3, 6, 12].every((months) => hasRangeContainers(ranges[months])) && + value.incomeExpenses.months.length === 12 && + value.netWorth.points.length === 12 && + hasNativeValueContainers(value.balanceSheet.assets) && + hasNativeValueContainers(value.balanceSheet.liabilities) + ); +} + +export function hasAccountRowContainers(value: unknown): value is PreparedAccountRow { + if ( + !isObject(value) || + typeof value.accountDate !== 'string' || + !isObject(value.transaction) || + !Array.isArray(value.transaction.postings) || + !Array.isArray(value.accountPostingIndexes) || + !isObject(value.investment) || + !Array.isArray(value.investment.securities) || + !isObject(value.investment.cash) || + typeof value.investment.cash.direction !== 'string' || + !['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) || + !Array.isArray(value.investment.cash.change) || + !Array.isArray(value.investment.cash.balance) || + !Array.isArray(value.investment.cash.unclassifiedChange) || + !Array.isArray(value.investment.cash.unclassifiedBalance) || + !Array.isArray(value.investment.cashPostingIndexes) || + !Array.isArray(value.investment.incomePostingIndexes) || + !Array.isArray(value.investment.expensePostingIndexes) || + !Array.isArray(value.investment.gainPostingIndexes) || + !Array.isArray(value.investment.issues) || + !value.transaction.postings.every( + (posting) => + isObject(posting) && + (posting.transactedAmounts === null || Array.isArray(posting.transactedAmounts)) + ) + ) + return false; + const postingCount = value.transaction.postings.length; + return value.accountPostingIndexes.every( + (index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount + ); +} + +export function isAccountSummary(value: unknown): value is PreparedAccountSummary { + if (!isObject(value) || typeof value.name !== 'string' || !isObject(value.summary)) return false; + if (value.access === 'ambiguous') + return value.rowCount === null && value.summary.status === 'ambiguous'; + if ( + value.access !== 'ready' || + typeof value.rowCount !== 'number' || + !Number.isSafeInteger(value.rowCount) || + value.rowCount < 0 + ) + return false; + if (value.summary.status === 'undated') return true; + return ( + value.summary.status === 'ready' && + Array.isArray(value.summary.balances) && + hasNativeValueContainers(value.summary.valuation) + ); +} + +export function summaryMatchesDate(account: PreparedAccountSummary, asOf: string | null): boolean { + if (account.access === 'ambiguous') return account.summary.status === 'ambiguous'; + return account.summary.status === (asOf === null ? 'undated' : 'ready'); +} + +export function storedSummary(summary: PreparedBalance): PreparedBalance { + if (summary.status !== 'ready') return { status: summary.status }; + return { + status: 'ready', + balances: summary.balances, + valuation: { + amounts: summary.valuation.amounts, + unconverted: summary.valuation.unconverted + } + }; +} + +export function isAccountCatalog(value: unknown): value is StoredAccount[] { + if (!Array.isArray(value)) return false; + const names = new Set(); + const ids = new Set(); + for (const account of value) { + if (!isAccountSummary(account) || !('id' in account) || !isAccountId(account.id)) return false; + if (names.has(account.name) || ids.has(account.id)) return false; + names.add(account.name); + ids.add(account.id); + } + return true; +} + +export function hasStoredReportData(value: unknown): value is StoredReportData { + if ( + !isObject(value) || + !hasOverviewContainers(value.overview) || + !isAccountCatalog(value.accounts) || + typeof value.journal !== 'string' || + !value.journal.trim() + ) + return false; + const { accounts, overview } = value; + return accounts.every((account) => summaryMatchesDate(account, overview.asOf)); +} diff --git a/packages/db/src/reports/index.ts b/packages/db/src/reports/index.ts new file mode 100644 index 0000000..b5412e0 --- /dev/null +++ b/packages/db/src/reports/index.ts @@ -0,0 +1,13 @@ +export { + hasOverviewContainers, + hasAccountRowContainers, + hasStoredReportData, + isAccountCatalog, + isAccountId, + isAccountSummary, + isObject, + storedSummary, + summaryMatchesDate +} from './data.js'; +export type { StoredAccount, StoredReportData } from './data.js'; +export { writeAccountRows, writeReportData, readReportData, readAccountRows } from './queries.js'; diff --git a/packages/db/src/reports/queries.test.ts b/packages/db/src/reports/queries.test.ts new file mode 100644 index 0000000..22b58f2 --- /dev/null +++ b/packages/db/src/reports/queries.test.ts @@ -0,0 +1,116 @@ +import { mkdtemp, mkdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { sql, type Kysely } from 'kysely'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import { + createReportDatabase, + openReadOnlyReportDatabase, + runMigrations, + type ReportDatabase +} from '../index.js'; +import { readAccountRows, readReportData, writeAccountRows, writeReportData } from './queries.js'; +import type { StoredReportData } from './data.js'; +import { + ambiguous, + preparedOverview, + journal, + ready, + preparedAccountRows +} from './data.fixture.js'; + +let directory: string; +let path: string; +const databases = new Set>(); + +async function createDatabase(filename = path): Promise> { + const database = await createReportDatabase(filename); + databases.add(database); + await runMigrations(database); + return database; +} + +async function closeDatabase(database: Kysely): Promise { + await database.destroy(); + databases.delete(database); +} + +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'report-database-')); + path = join(directory, 'reports.sqlite'); +}); +afterEach(async () => { + try { + await Promise.all([...databases].map((database) => database.destroy())); + } finally { + databases.clear(); + await rm(directory, { recursive: true, force: true }); + } +}); + +it('stores exact prepared payloads and native row order using the migrated schema', async () => { + const database = await createDatabase(); + const rows = preparedAccountRows(52).reverse(); + const data: StoredReportData = { + journal, + overview: preparedOverview(), + accounts: [ + { id: 'empty001', ...ready('assets:empty', 0) }, + { id: 'cash0001', ...ready('assets:cash', rows.length) }, + { id: 'ambig001', ...ambiguous() } + ] + }; + const transaction = await database.startTransaction().execute(); + try { + await writeAccountRows(transaction, 'assets:cash', 0, rows.slice(0, 51)); + await writeAccountRows(transaction, 'assets:cash', 51, rows.slice(51)); + await writeReportData(transaction, data); + await transaction.commit().execute(); + } finally { + if (!transaction.isCommitted && !transaction.isRolledBack) + await transaction.rollback().execute(); + } + await closeDatabase(database); + + const reader = await openReadOnlyReportDatabase(path); + if (!reader) throw new Error('Expected the current report schema.'); + databases.add(reader); + expect(await readReportData(reader)).toEqual(data); + expect(await readAccountRows(reader, data.accounts[0])).toEqual([]); + expect(await readAccountRows(reader, data.accounts[1])).toEqual(rows); + expect(await readAccountRows(reader, data.accounts[2])).toBeNull(); + await expect( + writeAccountRows(reader, 'assets:cash', rows.length, rows.slice(0, 1)) + ).rejects.toThrow(); + expect((await stat(path)).mode & 0o777).toBe(0o600); +}); + +it('does not create missing storage or replace an existing report file', async () => { + const missing = join(directory, 'missing'); + await expect(openReadOnlyReportDatabase(join(missing, 'reports.sqlite'))).rejects.toThrow(); + await expect(stat(missing)).rejects.toMatchObject({ code: 'ENOENT' }); + await expect(openReadOnlyReportDatabase(path)).rejects.toThrow(); + await expect(stat(path)).rejects.toMatchObject({ code: 'ENOENT' }); + const existing = Buffer.from('Existing report data must remain untouched.'); + await writeFile(path, existing); + await expect(createReportDatabase(path)).rejects.toThrow(); + expect(await readFile(path)).toEqual(existing); +}); + +it('rejects a symlink rather than opening another report database', async () => { + const target = join(directory, 'target'); + await mkdir(target, { mode: 0o700 }); + const database = await createDatabase(join(target, 'reports.sqlite')); + await closeDatabase(database); + await symlink(join(target, 'reports.sqlite'), path); + await expect(openReadOnlyReportDatabase(path)).rejects.toThrow(); +}); + +it('withholds an unsupported schema without migrating or modifying the database', async () => { + const database = await createDatabase(); + await sql`PRAGMA user_version = 2`.execute(database); + await closeDatabase(database); + const before = await readFile(path); + expect(await openReadOnlyReportDatabase(path)).toBeNull(); + expect(await readFile(path)).toEqual(before); +}); diff --git a/packages/db/src/reports/queries.ts b/packages/db/src/reports/queries.ts new file mode 100644 index 0000000..444d0fd --- /dev/null +++ b/packages/db/src/reports/queries.ts @@ -0,0 +1,126 @@ +import type { PreparedAccountRow } from '@hledger-reports/hledger'; +import type { Kysely } from 'kysely'; +import type { ReportDatabase } from '../schema.js'; +import { + hasAccountRowContainers, + hasStoredReportData, + type StoredAccount, + type StoredReportData +} from './data.js'; + +const insertBatchSize = 50; + +export async function writeAccountRows( + database: Kysely, + name: string, + firstOrdinal: number, + rows: readonly PreparedAccountRow[] +): Promise { + if (typeof name !== 'string' || !Number.isSafeInteger(firstOrdinal) || firstOrdinal < 0) + throw new Error('Invalid account rows.'); + for (let start = 0; start < rows.length; start += insertBatchSize) { + const batch = rows.slice(start, start + insertBatchSize).map((row, index) => { + if (!hasAccountRowContainers(row)) throw new Error('Invalid account rows.'); + return { + account_name: name, + ordinal: firstOrdinal + start + index, + account_date: row.accountDate, + row_json: JSON.stringify(row) + }; + }); + await database.insertInto('account_activity').values(batch).execute(); + } +} + +export async function writeReportData( + database: Kysely, + data: StoredReportData +): Promise { + if (!hasStoredReportData(data)) throw new Error('Invalid report data.'); + await database + .insertInto('publication') + .values({ singleton: 1, journal: data.journal, overview_json: JSON.stringify(data.overview) }) + .execute(); + for (let start = 0; start < data.accounts.length; start += insertBatchSize) { + const batch = data.accounts.slice(start, start + insertBatchSize).map((account, index) => ({ + id: account.id, + name: account.name, + catalog_ordinal: start + index, + access: account.access, + row_count: account.rowCount, + summary_json: JSON.stringify(account.summary) + })); + await database.insertInto('accounts').values(batch).execute(); + } +} + +function decodePayload(value: unknown): unknown { + if (typeof value !== 'string') throw new Error('Invalid report payload.'); + return JSON.parse(value); +} + +export async function readReportData(database: Kysely): Promise { + const publication = await database + .selectFrom('publication') + .select(['journal', 'overview_json']) + .where('singleton', '=', 1) + .executeTakeFirst(); + if (!publication) throw new Error('Missing report data.'); + const records = await database + .selectFrom('accounts') + .select(['id', 'name', 'access', 'row_count', 'summary_json']) + .orderBy('catalog_ordinal', 'asc') + .execute(); + const data = { + journal: publication.journal, + overview: decodePayload(publication.overview_json), + accounts: records.map((account) => ({ + id: account.id, + name: account.name, + access: account.access, + rowCount: account.row_count, + summary: decodePayload(account.summary_json) + })) + }; + if (!hasStoredReportData(data)) throw new Error('Invalid report data.'); + return data; +} + +export async function readAccountRows( + database: Kysely, + account: StoredAccount +): Promise { + const stored = await database + .selectFrom('accounts') + .select(['name', 'access', 'row_count']) + .where('id', '=', account.id) + .executeTakeFirst(); + if ( + !stored || + stored.name !== account.name || + stored.access !== account.access || + stored.row_count !== account.rowCount + ) + throw new Error('Invalid account data.'); + const records = await database + .selectFrom('account_activity') + .select(['ordinal', 'account_date', 'row_json']) + .where('account_name', '=', account.name) + .orderBy('ordinal', 'asc') + .execute(); + if (account.access === 'ambiguous') { + if (records.length !== 0) throw new Error('Invalid account data.'); + return null; + } + if (records.length !== account.rowCount) throw new Error('Invalid account data.'); + return records.map((record, index) => { + const row = decodePayload(record.row_json); + if ( + record.ordinal !== index || + !hasAccountRowContainers(row) || + record.account_date !== row.accountDate + ) + throw new Error('Invalid account data.'); + return row; + }); +} diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts new file mode 100644 index 0000000..b8af8dc --- /dev/null +++ b/packages/db/src/schema.ts @@ -0,0 +1,29 @@ +export const reportSchemaVersion = 1; + +interface PublicationTable { + singleton: 1; + journal: string; + overview_json: string; +} + +interface AccountTable { + id: string; + name: string; + catalog_ordinal: number; + access: 'ready' | 'ambiguous'; + row_count: number | null; + summary_json: string; +} + +interface AccountActivityTable { + account_name: string; + ordinal: number; + account_date: string; + row_json: string; +} + +export interface ReportDatabase { + publication: PublicationTable; + accounts: AccountTable; + account_activity: AccountActivityTable; +} diff --git a/packages/db/tsconfig.build.json b/packages/db/tsconfig.build.json new file mode 100644 index 0000000..c2c1ecd --- /dev/null +++ b/packages/db/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "noEmit": false, "rootDir": ".", "outDir": "dist" }, + "exclude": ["dist", "**/*.test.ts", "**/*.fixture.ts", "vitest.config.ts"] +} diff --git a/packages/db/tsconfig.json b/packages/db/tsconfig.json new file mode 100644 index 0000000..f8a5241 --- /dev/null +++ b/packages/db/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { "noEmit": true }, + "include": ["src/**/*.ts", "vitest.config.ts"] +} diff --git a/packages/db/vitest.config.ts b/packages/db/vitest.config.ts new file mode 100644 index 0000000..30457ff --- /dev/null +++ b/packages/db/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { include: ['src/**/*.test.ts'], environment: 'node' } +}); diff --git a/packages/snapshots/package.json b/packages/snapshots/package.json index 71551c2..6548ba3 100644 --- a/packages/snapshots/package.json +++ b/packages/snapshots/package.json @@ -22,6 +22,7 @@ "test": "vitest run" }, "dependencies": { + "@hledger-reports/db": "workspace:*", "@hledger-reports/hledger": "workspace:*" }, "devDependencies": { diff --git a/packages/snapshots/src/index.ts b/packages/snapshots/src/index.ts index c134b29..504e3eb 100644 --- a/packages/snapshots/src/index.ts +++ b/packages/snapshots/src/index.ts @@ -3,7 +3,8 @@ export { readOverview, readReportPublication, readAccountTransactions -} from './reports.js'; -export type { ReportPublication, StoredAccount, ReportStage } from './reports.js'; +} from './reports/publication.js'; +export type { ReportPublication, ReportStage } from './reports/publication.js'; +export type { StoredAccount } from '@hledger-reports/db/reports'; export { writeValidation, readValidation } from './validation.js'; export type { JournalValidationResult, ValidationRecord } from './validation.js'; diff --git a/packages/snapshots/src/reports.test.ts b/packages/snapshots/src/reports/publication.test.ts similarity index 89% rename from packages/snapshots/src/reports.test.ts rename to packages/snapshots/src/reports/publication.test.ts index f60638e..2cb5fe7 100644 --- a/packages/snapshots/src/reports.test.ts +++ b/packages/snapshots/src/reports/publication.test.ts @@ -1,5 +1,4 @@ import type { - DetailedAmount, OverviewData, PreparedAccountSummary, PreparedAccountRow @@ -14,8 +13,19 @@ import { readAccountTransactions, readOverview, readReportPublication -} from './index.js'; -import type { StoredAccount } from './index.js'; +} from '../index.js'; +import type { StoredAccount } from '../index.js'; +import { + amount, + ambiguous, + preparedOverview as fixture, + journal, + range, + ready, + preparedAccountRows as rows, + sourcePosition, + undated +} from '../../../db/src/reports/data.fixture.js'; vi.mock('node:fs/promises', async (original) => ({ ...(await original()) @@ -29,158 +39,6 @@ let root: string; let snapshots: string; const older = { id: 'older', createdAt: new Date('2026-04-30T23:59:59.999Z') }; const newer = { id: 'newer', createdAt: new Date('2026-05-01T00:00:00.000Z') }; -const expenseAmounts = [ - { commodity: 'EUR', quantity: '0.0000000000000000000000012300' }, - { commodity: '', quantity: '-0.000' } -]; -const range = (months: 1 | 3 | 6 | 12) => ({ - begin: { 1: '2026-04-01', 3: '2026-02-01', 6: '2025-11-01', 12: '2025-05-01' }[months], - endExclusive: '2026-04-17', - transactionCount: 1, - income: [{ commodity: '$', quantity: '9007199254740993.123456789' }], - expenses: expenseAmounts, - net: [{ commodity: '$', quantity: '9007199254740992.123456789' }], - expenseCategories: { - depth: 2 as const, - rows: [{ account: 'expenses:food', amounts: expenseAmounts }] - } -}); -const fixture = (): OverviewData => ({ - asOf: '2026-04-16', - preparedAt: '2026-05-02T12:34:56.789Z', - journalCoverage: { transactionCount: 1, firstDate: '2026-04-16', lastDate: '2026-04-16' }, - hledgerVersion: '1.99.5', - ranges: { 1: range(1), 3: range(3), 6: range(6), 12: range(12) }, - netWorth: { - currency: '$', - points: Array.from({ length: 12 }, (_, i) => ({ - date: i === 11 ? '2026-04-16' : new Date(Date.UTC(2025, i + 5, 0)).toISOString().slice(0, 10), - amounts: [{ commodity: '$', quantity: '9007199254740993.123456789' }], - unconverted: [] - })) - }, - incomeExpenses: { - months: Array.from({ length: 12 }, (_, i) => ({ - begin: new Date(Date.UTC(2025, i + 4, 1)).toISOString().slice(0, 10), - endExclusive: new Date(Date.UTC(2025, i + 5, 1)).toISOString().slice(0, 10), - income: [{ commodity: '$', quantity: '9007199254740993.123456789' }], - expenses: expenseAmounts - })) - }, - balanceSheet: { - assets: { - amounts: [{ commodity: '$', quantity: '9007199254740994.123456789' }], - unconverted: [] - }, - liabilities: { amounts: [{ commodity: '$', quantity: '-1' }], unconverted: [] } - } -}); -const journal = '/books/main.journal'; -const amount = (quantity: string): DetailedAmount => ({ - commodity: '$', - quantity, - cost: null, - costBasis: null, - style: null -}); -const sourcePosition = { file: '/private/books/included.journal', line: 12, column: 1 }; -const rows = (count = 1, description = 'exact producer strings'): PreparedAccountRow[] => - Array.from( - { length: count }, - (_, i): PreparedAccountRow => ({ - accountDate: '2026-04-30', - isSplit: true, - otherAccounts: ['expenses:食料', 'expenses:household'], - change: [amount('-100.00')], - balance: [amount('9007199254740993.123456789')], - accountPostingIndexes: [0], - investment: { - securities: [], - cash: { - direction: 'out', - change: [amount('-100.00')], - balance: [amount('9007199254740993.123456789')], - unclassifiedChange: [], - unclassifiedBalance: [] - }, - cashPostingIndexes: [0], - incomePostingIndexes: [], - expensePostingIndexes: [1, 2], - gainPostingIndexes: [], - issues: [] - }, - transaction: { - index: i + 1, - date: '2026-04-29', - secondaryDate: '2026-04-30', - status: 'Cleared', - code: 'REF-42', - description: `${description} ${i}`, - payee: 'Market', - note: 'supplies', - comment: 'transaction note\nproject: home\n', - precedingComment: 'retained preceding comment\n', - tags: [ - ['project', 'home'], - ['project', 'ordered'] - ], - postings: [ - ['assets:cash', '-100.00', 'bank note'], - ['expenses:食料', '75.00', 'food note'], - ['expenses:household', '25.00', 'household note'] - ].map(([account, quantity, comment], postingIndex) => { - const date = postingIndex === 0 ? '2026-04-30' : null; - const balanceAssertion = - postingIndex === 0 - ? { - amount: amount('9007199254740993.123456789'), - total: false, - inclusive: false, - sourcePosition - } - : null; - return { - account, - amounts: [amount(quantity)], - transactedAmounts: [amount(quantity)], - date, - secondaryDate: null, - status: 'Unmarked', - effectiveStatus: 'Cleared', - comment, - tags: [['receipt', 'fictional']], - kind: 'RealPosting', - balanceAssertion, - transactionIndex: String(i + 1) - }; - }), - sourcePositions: [sourcePosition, { ...sourcePosition, line: 18 }] - } - }) - ); -const ready = (name = 'assets:cash', rowCount = 1): PreparedAccountSummary => ({ - name, - access: 'ready', - rowCount, - summary: { - status: 'ready', - balances: rowCount === 0 ? [] : [{ commodity: 'TEST', quantity: '2.000' }], - valuation: { - amounts: rowCount === 0 ? [] : [{ commodity: 'TEST', quantity: '2.000' }], - unconverted: rowCount === 0 ? [] : ['TEST'] - } - } -}); -const ambiguous = (name = 'ambiguous'): PreparedAccountSummary => ({ - name, - access: 'ambiguous', - rowCount: null, - summary: { status: 'ambiguous' } -}); -const undated = (name: string): PreparedAccountSummary => ({ - ...ready(name, 0), - summary: { status: 'undated' } -}); const filename = (run = older) => `${String(run.createdAt.getTime()).padStart(16, '0')}-${run.id}.json`; const safeError = (message: string) => (error: Error) => diff --git a/packages/snapshots/src/reports.ts b/packages/snapshots/src/reports/publication.ts similarity index 59% rename from packages/snapshots/src/reports.ts rename to packages/snapshots/src/reports/publication.ts index c30fd8f..f528eff 100644 --- a/packages/snapshots/src/reports.ts +++ b/packages/snapshots/src/reports/publication.ts @@ -1,35 +1,34 @@ import type { OverviewData, PreparedAccountSummary, - PreparedAccountRow, - PreparedBalance + PreparedAccountRow } from '@hledger-reports/hledger'; import { randomInt, randomUUID } from 'node:crypto'; import { constants } from 'node:fs'; import { mkdir, open, rename, rm, symlink, unlink } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; -import { namesIn, privateDirectory, readJson, writeJson } from './files.js'; +import { namesIn, privateDirectory, readJson, writeJson } from '../files.js'; +import { + hasOverviewContainers, + hasAccountRowContainers, + hasStoredReportData, + isAccountCatalog, + isAccountId, + isAccountSummary, + isObject, + storedSummary, + summaryMatchesDate, + type StoredAccount, + type StoredReportData +} from '@hledger-reports/db/reports'; const runIdPattern = /^[A-Za-z0-9_-]{1,128}(?![\s\S])/; const setPattern = /^(\d{16}-[A-Za-z0-9_-]{1,128})-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}(?![\s\S])/; -const idPattern = /^[a-z0-9]{8}(?![\s\S])/; const currentReportFormat = 8; -export interface StoredAccount { - id: string; - name: string; - access: 'ready' | 'ambiguous'; - rowCount: number | null; - summary: PreparedBalance; -} - -export interface ReportPublication { - overview: OverviewData; - /** Private journal identity; never expose this in page data. */ - journal: string; - accounts: StoredAccount[]; +export interface ReportPublication extends StoredReportData { /** Private, pinned completed-set directory; never expose this in financial data. */ directory: string; } @@ -44,158 +43,6 @@ export interface ReportStage { discard(): Promise; } -function isObject(value: unknown): value is Record { - return value !== null && typeof value === 'object' && !Array.isArray(value); -} - -function isNullableString(value: unknown): boolean { - return value === null || typeof value === 'string'; -} - -// Check the current header and card containers, not producer-owned financial rows. -function hasNativeValueContainers(value: unknown): boolean { - return isObject(value) && Array.isArray(value.amounts) && Array.isArray(value.unconverted); -} - -function hasRangeContainers(value: unknown): boolean { - if (!isObject(value)) return false; - return ( - typeof value.begin === 'string' && - typeof value.endExclusive === 'string' && - typeof value.transactionCount === 'number' && - Number.isSafeInteger(value.transactionCount) && - value.transactionCount >= 0 && - Array.isArray(value.income) && - Array.isArray(value.expenses) && - Array.isArray(value.net) && - isObject(value.expenseCategories) && - value.expenseCategories.depth === 2 && - Array.isArray(value.expenseCategories.rows) - ); -} - -function hasCurrentOverviewContainers(value: unknown): value is OverviewData { - if ( - !isObject(value) || - typeof value.preparedAt !== 'string' || - !isNullableString(value.asOf) || - !isObject(value.journalCoverage) || - typeof value.journalCoverage.transactionCount !== 'number' || - !Number.isSafeInteger(value.journalCoverage.transactionCount) || - value.journalCoverage.transactionCount < 0 || - !isNullableString(value.journalCoverage.firstDate) || - !isNullableString(value.journalCoverage.lastDate) || - typeof value.hledgerVersion !== 'string' || - !isObject(value.incomeExpenses) || - !Array.isArray(value.incomeExpenses.months) || - !isObject(value.netWorth) || - !isNullableString(value.netWorth.currency) || - !Array.isArray(value.netWorth.points) - ) - return false; - if (value.asOf === null) { - return ( - value.ranges === null && - value.balanceSheet === null && - value.incomeExpenses.months.length === 0 && - value.netWorth.currency === null && - value.netWorth.points.length === 0 - ); - } - const ranges = value.ranges; - if (!isObject(ranges) || !isObject(value.balanceSheet)) return false; - return ( - [1, 3, 6, 12].every((months) => hasRangeContainers(ranges[months])) && - value.incomeExpenses.months.length === 12 && - value.netWorth.points.length === 12 && - hasNativeValueContainers(value.balanceSheet.assets) && - hasNativeValueContainers(value.balanceSheet.liabilities) - ); -} - -function hasCurrentTransactionContainers(value: unknown): boolean { - if ( - !isObject(value) || - typeof value.accountDate !== 'string' || - !isObject(value.transaction) || - !Array.isArray(value.transaction.postings) || - !Array.isArray(value.accountPostingIndexes) || - !isObject(value.investment) || - !Array.isArray(value.investment.securities) || - !isObject(value.investment.cash) || - typeof value.investment.cash.direction !== 'string' || - !['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) || - !Array.isArray(value.investment.cash.change) || - !Array.isArray(value.investment.cash.balance) || - !Array.isArray(value.investment.cash.unclassifiedChange) || - !Array.isArray(value.investment.cash.unclassifiedBalance) || - !Array.isArray(value.investment.cashPostingIndexes) || - !Array.isArray(value.investment.incomePostingIndexes) || - !Array.isArray(value.investment.expensePostingIndexes) || - !Array.isArray(value.investment.gainPostingIndexes) || - !Array.isArray(value.investment.issues) || - !value.transaction.postings.every( - (posting) => - isObject(posting) && - (posting.transactedAmounts === null || Array.isArray(posting.transactedAmounts)) - ) - ) - return false; - const postingCount = value.transaction.postings.length; - return value.accountPostingIndexes.every( - (index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount - ); -} - -function isSummary(value: unknown): value is PreparedAccountSummary { - if (!isObject(value) || typeof value.name !== 'string' || !isObject(value.summary)) return false; - if (value.access === 'ambiguous') - return value.rowCount === null && value.summary.status === 'ambiguous'; - if ( - value.access !== 'ready' || - typeof value.rowCount !== 'number' || - !Number.isSafeInteger(value.rowCount) || - value.rowCount < 0 - ) - return false; - if (value.summary.status === 'undated') return true; - return ( - value.summary.status === 'ready' && - Array.isArray(value.summary.balances) && - hasNativeValueContainers(value.summary.valuation) - ); -} - -function summaryMatchesDate(account: PreparedAccountSummary, asOf: string | null): boolean { - if (account.access === 'ambiguous') return account.summary.status === 'ambiguous'; - return account.summary.status === (asOf === null ? 'undated' : 'ready'); -} - -function storedSummary(summary: PreparedBalance): PreparedBalance { - if (summary.status !== 'ready') return { status: summary.status }; - return { - status: 'ready', - balances: summary.balances, - valuation: { - amounts: summary.valuation.amounts, - unconverted: summary.valuation.unconverted - } - }; -} - -function isCatalog(value: unknown): value is StoredAccount[] { - if (!Array.isArray(value)) return false; - const names = new Set(); - const ids = new Set(); - for (const account of value) { - if (!isSummary(account) || !('id' in account) || typeof account.id !== 'string') return false; - if (!idPattern.test(account.id) || names.has(account.name) || ids.has(account.id)) return false; - names.add(account.name); - ids.add(account.id); - } - return true; -} - function requestCreationKey(token: string): string { // The separator in latestSet preserves request ordering for prefix run IDs. return setPattern.exec(token)![1]; @@ -223,16 +70,7 @@ export async function readReportPublication( const record = await readJson(directory, 'overview.json'); if (!isObject(record)) throw new Error(); if (record.format !== currentReportFormat) return null; - if ( - !hasCurrentOverviewContainers(record.overview) || - !isCatalog(record.accounts) || - typeof record.journal !== 'string' || - !record.journal.trim() - ) - throw new Error(); - const overview = record.overview; - if (!record.accounts.every((account) => summaryMatchesDate(account, overview.asOf))) - throw new Error(); + if (!hasStoredReportData(record)) throw new Error(); return { // Financial rows remain producer-owned after checking current header/containers. overview: record.overview, @@ -254,7 +92,7 @@ export async function readAccountTransactions( id: string ): Promise<{ id: string; name: string; transactions: PreparedAccountRow[] | null }> { try { - if (!idPattern.test(id)) throw new Error(); + if (!isAccountId(id)) throw new Error(); const account = publication.accounts.find((entry) => entry.id === id); if (!account) throw new Error(); const directory = publication.directory; @@ -271,7 +109,7 @@ export async function readAccountTransactions( } else if ( !Array.isArray(record.transactions) || record.transactions.length !== account.rowCount || - !record.transactions.every(hasCurrentTransactionContainers) + !record.transactions.every(hasAccountRowContainers) ) throw new Error(); return { @@ -315,7 +153,7 @@ export async function createReportStage( try { const timestamp = String(run.createdAt.getTime()).padStart(16, '0'); if (!runIdPattern.test(run.id) || !/^\d{16}$/.test(timestamp)) throw new Error(); - if (previousAccounts != null && !isCatalog(previousAccounts)) throw new Error(); + if (previousAccounts != null && !isAccountCatalog(previousAccounts)) throw new Error(); const prior = new Map((previousAccounts ?? []).map((account) => [account.name, account.id])); const used = new Set(prior.values()); function accountId(name: string): string { @@ -401,7 +239,7 @@ export async function createReportStage( publish(overview, accounts, journal) { return operation(async () => { if ( - !hasCurrentOverviewContainers(overview) || + !hasOverviewContainers(overview) || typeof journal !== 'string' || !journal.trim() || !Array.isArray(accounts) @@ -413,7 +251,7 @@ export async function createReportStage( const catalog: StoredAccount[] = []; for (const account of accounts) { if ( - !isSummary(account) || + !isAccountSummary(account) || !summaryMatchesDate(account, overview.asOf) || names.has(account.name) ) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f753ae3..a438833 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ catalogs: globals: specifier: 17.6.0 version: 17.6.0 + kysely: + specifier: 0.29.7 + version: 0.29.7 openworkflow: specifier: 0.11.0 version: 0.11.0 @@ -240,6 +243,43 @@ importers: specifier: 'catalog:' version: 4.1.7(@types/node@26.0.0)(vite@8.0.16(@types/node@26.0.0)(yaml@2.9.0)) + packages/db: + dependencies: + '@hledger-reports/hledger': + specifier: workspace:* + version: link:../hledger + kysely: + specifier: 'catalog:' + version: 0.29.7 + devDependencies: + '@eslint/js': + specifier: 'catalog:' + version: 10.0.1(eslint@10.4.1) + '@types/node': + specifier: 'catalog:' + version: 26.0.0 + eslint: + specifier: 'catalog:' + version: 10.4.1 + eslint-config-prettier: + specifier: 'catalog:' + version: 10.1.8(eslint@10.4.1) + globals: + specifier: 'catalog:' + version: 17.6.0 + prettier: + specifier: 'catalog:' + version: 3.8.3 + typescript: + specifier: 'catalog:' + version: 6.0.3 + typescript-eslint: + specifier: 'catalog:' + version: 8.60.0(eslint@10.4.1)(typescript@6.0.3) + vitest: + specifier: 'catalog:' + version: 4.1.7(@types/node@26.0.0)(vite@8.0.16(@types/node@26.0.0)(yaml@2.9.0)) + packages/hledger: devDependencies: '@eslint/js': @@ -315,6 +355,9 @@ importers: packages/snapshots: dependencies: + '@hledger-reports/db': + specifier: workspace:* + version: link:../db '@hledger-reports/hledger': specifier: workspace:* version: link:../hledger diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 2453fc1..1b75467 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -10,6 +10,7 @@ catalog: eslint: 10.4.1 eslint-config-prettier: 10.1.8 globals: 17.6.0 + kysely: 0.29.7 openworkflow: 0.11.0 prettier: 3.8.3 prettier-plugin-svelte: 3.5.2 -- 2.54.0