.changeset/publish-sqlite-report-generations.md

@@ -0,0 +1,6 @@
+---
+'@hledger-reports/web': patch
+'@hledger-reports/worker': patch
+---
+
+Store prepared financial reports in one private SQLite database per generation, keeping exact values and page behavior unchanged. After updating web and worker together, prepare reports again. Older financial JSON files are preserved, not migrated or deleted.

apps/web/src/lib/server/reports.integration.test.ts

@@ -2,8 +2,13 @@ import { mkdtemp, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { expect, it } from 'vitest';
-import type { OverviewData, PreparedAccountSummary } from '@hledger-reports/hledger';
-import { createReportStage } from '@hledger-reports/snapshots';
+import type { PreparedAccountRow, PreparedAccountSummary } from '@hledger-reports/hledger';
+import { createReportStage, readReportPublication } from '@hledger-reports/snapshots';
+import {
+	amount,
+	preparedAccountRows,
+	preparedOverview
+} from '../../../../../packages/db/src/reports/data.fixture.js';
 import { load as loadLayout } from '../../routes/(protected)/+layout.server.js';
 import { load as loadCashAccount } from '../../routes/(protected)/registers/cash/[accountId]/+page.server.js';
 
@@ -17,44 +22,85 @@ it('keeps the layout and saved account leaf on one real publication until a fres
 				journal: '/PRIVATE/fictional.journal',
 				accounts: [{ account, label: 'Cash', register: 'cash' as const }]
 			},
-			paths: { snapshotDirectory }
+			paths: { snapshotDirectory },
+			get jobsClient(): never {
+				throw new Error('Browsing must not access the queue.');
+			}
 		};
-		// An undated, empty journal is a complete current-format report, not a partial fixture.
-		const overview: OverviewData = {
-			asOf: null,
-			preparedAt: '2026-05-01T12:00:00Z',
-			journalCoverage: { transactionCount: 0, firstDate: null, lastDate: null },
-			hledgerVersion: '1.99.5',
-			ranges: null,
-			balanceSheet: null,
-			incomeExpenses: { months: [] },
-			netWorth: { currency: null, points: [] }
+		const overview = { ...preparedOverview(), preparedAt: '2026-05-01T12:00:00Z' };
+		const activity = (payee: string, balance: string) => {
+			const rows = preparedAccountRows();
+			const row = rows[0];
+			row.isSplit = false;
+			row.balance = [amount(balance)];
+			row.transaction.payee = payee;
+			row.transaction.postings = row.transaction.postings.slice(0, 2);
+			row.transaction.postings[1].amounts = [amount('100.00')];
+			row.transaction.postings[1].transactedAmounts = [amount('100.00')];
+			row.investment.expensePostingIndexes = [1];
+			return rows;
 		};
-		const accounts: PreparedAccountSummary[] = [
-			{ name: account, access: 'ready', rowCount: 0, summary: { status: 'undated' } }
+		const rowsA = activity('Market A', '9007199254740993.123456789');
+		const rowsB = activity('Market B', '9007199254740994.123456789');
+		const summary = (row: PreparedAccountRow): PreparedAccountSummary[] => [
+			{
+				name: account,
+				access: 'ready',
+				rowCount: 1,
+				summary: {
+					status: 'ready',
+					balances: row.balance,
+					valuation: { amounts: row.balance, unconverted: [] }
+				}
+			}
 		];
 		const stageA = await createReportStage(snapshotDirectory, {
 			id: 'A',
-			createdAt: new Date('2026-05-01T12:00:00Z')
+			createdAt: new Date(overview.preparedAt)
 		});
-		await stageA.publish(overview, accounts, application.config.journal);
+		await stageA.writeRows(account, rowsA);
+		await stageA.publish(overview, summary(rowsA[0]), application.config.journal);
 
 		const locals = { application };
 		const layoutA = await loadLayout({
 			locals,
 			url: new URL('http://localhost/registers/cash')
 		} as unknown as Parameters<typeof loadLayout>[0]);
-		expect(layoutA.accountSidebar.hasSavedReports).toBe(true);
 		const idA = layoutA.accountSidebar.groups[0].accounts[0].id!;
+		const publicBalanceA = { commodity: '$', quantity: '9007199254740993.123456789' };
+		const publicBalanceB = { commodity: '$', quantity: '9007199254740994.123456789' };
+		const expectedSidebarAccount = {
+			id: idA,
+			name: account,
+			label: 'Cash',
+			access: 'ready',
+			value: { status: 'ready', amounts: [publicBalanceA], unconverted: [] }
+		};
+		const expectedLayoutA = {
+			accountLabels: [{ account, label: 'Cash' }],
+			pathname: '/registers/cash',
+			accountSidebar: {
+				asOf: overview.asOf,
+				hasSavedReports: true,
+				netWorth: { amounts: [publicBalanceA], unconverted: [] },
+				readFailed: false,
+				groups: [{ kind: 'cash', accounts: [expectedSidebarAccount] }]
+			}
+		};
+		expect(layoutA).toEqual(expectedLayoutA);
 
+		const previous = await readReportPublication(snapshotDirectory);
+		if (!previous) throw new Error('Expected publication A.');
 		const preparedB = '2026-05-02T12:00:00Z';
-		const stageB = await createReportStage(snapshotDirectory, {
-			id: 'B',
-			createdAt: new Date(preparedB)
-		});
+		const stageB = await createReportStage(
+			snapshotDirectory,
+			{ id: 'B', createdAt: new Date(preparedB) },
+			previous.accounts
+		);
+		await stageB.writeRows(account, rowsB);
 		await stageB.publish(
 			{ ...overview, preparedAt: preparedB },
-			accounts,
+			summary(rowsB[0]),
 			application.config.journal
 		);
 
@@ -66,14 +112,35 @@ it('keeps the layout and saved account leaf on one real publication until a fres
 			parent: async () => layoutA,
 			setHeaders: () => {}
 		} as unknown as Parameters<typeof loadCashAccount>[0]);
-		expect(leafA).toMatchObject({
+		const publicChange = { commodity: '$', quantity: '-100.00' };
+		const expectedRowA = {
+			index: 1,
+			date: '2026-04-30',
+			payee: 'Market A',
+			categories: ['expenses:食料'],
+			notes: ['supplies', 'transaction note\nproject: home\n', 'bank note'],
+			amount: publicChange,
+			balance: publicBalanceA,
+			statuses: ['Cleared'],
+			direction: 'payment',
+			breakdown: null
+		};
+		const expectedLeafA = {
+			configured: true,
 			preparedAt: overview.preparedAt,
+			accountId: idA,
 			accountName: account,
 			access: 'ready',
-			rowCount: 0,
-			rows: [],
+			rowCount: 1,
+			page: 1,
+			pageCount: 1,
+			order: 'asc',
+			rows: [expectedRowA],
+			endingBalance: publicBalanceA,
+			unsupported: false,
 			readFailed: false
-		});
+		};
+		expect(leafA).toEqual(expectedLeafA);
 
 		const freshLocals = { application };
 		const layoutB = await loadLayout({
@@ -81,6 +148,24 @@ it('keeps the layout and saved account leaf on one real publication until a fres
 			url: new URL('http://localhost/registers/cash')
 		} as unknown as Parameters<typeof loadLayout>[0]);
 		const idB = layoutB.accountSidebar.groups[0].accounts[0].id!;
+		expect(idB).toBe(idA);
+		expect(layoutB).toEqual({
+			...expectedLayoutA,
+			accountSidebar: {
+				...expectedLayoutA.accountSidebar,
+				groups: [
+					{
+						kind: 'cash',
+						accounts: [
+							{
+								...expectedSidebarAccount,
+								value: { status: 'ready', amounts: [publicBalanceB], unconverted: [] }
+							}
+						]
+					}
+				]
+			}
+		});
 		const leafB = await loadCashAccount({
 			locals: freshLocals,
 			url: new URL(`http://localhost/registers/cash/${idB}`),
@@ -88,10 +173,20 @@ it('keeps the layout and saved account leaf on one real publication until a fres
 			parent: async () => layoutB,
 			setHeaders: () => {}
 		} as unknown as Parameters<typeof loadCashAccount>[0]);
-		expect(layoutB.accountSidebar.hasSavedReports).toBe(true);
-		expect(leafB.preparedAt).toBe(preparedB);
+		expect(leafB).toEqual({
+			...expectedLeafA,
+			preparedAt: preparedB,
+			rows: [
+				{
+					...expectedRowA,
+					payee: 'Market B',
+					balance: publicBalanceB
+				}
+			],
+			endingBalance: publicBalanceB
+		});
 		expect(JSON.stringify([layoutA, leafA, layoutB, leafB])).not.toMatch(
-			/PRIVATE|directory|journal|publication|savedReports/
+			/PRIVATE|directory|journal|publication|savedReports|sourcePosition|costBasis|style/
 		);
 	} finally {
 		await rm(root, { recursive: true, force: true });

apps/web/src/routes/(protected)/registers/cash/[accountId]/page.server.test.ts

@@ -105,6 +105,32 @@ it('loads a member from one pinned publication and delegates its requested page'
 	expect(mapCashRegisterPage).toHaveBeenCalledExactlyOnceWith(rows, 'desc', 2);
 	expect(request.setHeaders).toHaveBeenCalledWith({ 'cache-control': 'no-store' });
 });
+it('returns a safe read failure when the saved body cannot be read', async () => {
+	vi.mocked(readAccountTransactions).mockRejectedValue(
+		new Error('PRIVATE /books/included.journal:12 database diagnostic')
+	);
+	const request = event();
+	const queue = vi.spyOn(request.locals.application!, 'jobsClient', 'get');
+	const result = await load(request);
+	expect(result).toEqual({
+		configured: true,
+		readFailed: true,
+		accountId: id,
+		accountName: name,
+		access: 'ready',
+		preparedAt: publication.overview.preparedAt,
+		rowCount: 51,
+		pageCount: 2,
+		page: 2,
+		order: 'asc',
+		rows: [],
+		endingBalance: null,
+		unsupported: false
+	});
+	expect(readAccountTransactions).toHaveBeenCalledExactlyOnceWith(publication, id);
+	expect(mapCashRegisterPage).not.toHaveBeenCalled();
+	expect(queue).not.toHaveBeenCalled();
+});
 it('defaults ascending navigation to the last page', async () => {
 	await load(event());
 	expect(mapCashRegisterPage).toHaveBeenCalledExactlyOnceWith(rows, 'asc', 2);

apps/web/src/routes/(protected)/registers/investments/[accountId]/page.server.test.ts

@@ -111,6 +111,25 @@ it('loads a member from one pinned publication and delegates its requested page'
 	expect(mapInvestmentRegisterPage).toHaveBeenCalledExactlyOnceWith(rows, 'desc', 2);
 	expect(request.setHeaders).toHaveBeenCalledWith({ 'cache-control': 'no-store' });
 });
+it('returns a safe read failure when the saved body cannot be read', async () => {
+	vi.mocked(readAccountTransactions).mockRejectedValue(
+		new Error('PRIVATE /books/included.journal:12 database diagnostic')
+	);
+	const request = event();
+	const queue = vi.spyOn(request.locals.application!, 'jobsClient', 'get');
+	const result = await load(request);
+	expect(result).toEqual({
+		configured: true,
+		readFailed: true,
+		accountName: name,
+		access: 'ready',
+		preparedAt: publication.overview.preparedAt,
+		register: null
+	});
+	expect(readAccountTransactions).toHaveBeenCalledExactlyOnceWith(publication, id);
+	expect(mapInvestmentRegisterPage).not.toHaveBeenCalled();
+	expect(queue).not.toHaveBeenCalled();
+});
 it('defaults ascending navigation to the last page', async () => {
 	await load(event());
 	expect(mapInvestmentRegisterPage).toHaveBeenCalledExactlyOnceWith(rows, 'asc', 2);

apps/worker/src/workflows/prepare-reports/drain.integration.test.ts

@@ -1,106 +0,0 @@
-import { enqueuePreparation } from './test-fixtures.js';
-import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { afterEach, expect, it, vi } from 'vitest';
-import { prepareReports } from '@hledger-reports/hledger';
-import { openJobsClient, prepareReportsSpec } from '@hledger-reports/jobs';
-import { readOverview, readReportPublication } from '@hledger-reports/snapshots';
-import { lotsJournal } from '../../../fixtures/journal.js';
-import { startWorker } from '../../index.js';
-
-vi.mock('@hledger-reports/hledger', async (original) => {
-	const actual = await original<typeof import('@hledger-reports/hledger')>();
-	return { ...actual, prepareReports: vi.fn(actual.prepareReports) };
-});
-const actual = await vi.importActual<typeof import('@hledger-reports/hledger')>(
-	'@hledger-reports/hledger'
-);
-afterEach(() => {
-	vi.mocked(prepareReports).mockReset().mockImplementation(actual.prepareReports);
-});
-
-async function completeSets(snapshots: string) {
-	return (await readdir(join(snapshots, '.sets'))).filter((name) => !name.startsWith('.'));
-}
-
-it('serves health and last-good data during deferred native preparation without a partial publication', async () => {
-	const directory = await mkdtemp(join(tmpdir(), 'reports-startup-'));
-	const config = join(directory, 'config', 'hledger-reports.yaml');
-	const journal = join(directory, 'source.journal');
-	const snapshots = join(directory, 'data', 'snapshots');
-	await mkdir(join(directory, 'config'));
-	await writeFile(
-		config,
-		JSON.stringify({
-			journal,
-			url: { app: 'http://127.0.0.1:3000' },
-			auth: { credentials: { username: 'fixture', password: '${AUTH_PASSWORD}' } }
-		})
-	);
-	await writeFile(journal, lotsJournal);
-	const client = await openJobsClient(join(directory, 'data', 'jobs'));
-	let service: Awaited<ReturnType<typeof startWorker>> | undefined;
-	let release!: () => void;
-	const gate = new Promise<void>((resolve) => (release = resolve));
-	const runs = async () =>
-		(await client.backend.listWorkflowRuns({ workflowName: prepareReportsSpec.name })).data;
-	try {
-		service = await startWorker(config, { port: 0 });
-		await enqueuePreparation(client);
-		await expect.poll(async () => (await runs())[0]?.status, { timeout: 12_000 }).toBe('completed');
-		await service.stop();
-		service = undefined;
-		const publication = (await readReportPublication(snapshots))!;
-		const original = publication.overview;
-		const sets = await completeSets(snapshots);
-		expect(sets).toHaveLength(1);
-		expect(original).not.toBeNull();
-		expect(
-			JSON.parse(await readFile(join(snapshots, 'current', 'overview.json'), 'utf8'))
-		).toMatchObject({ overview: original });
-		// Edits alone do not prepare data; explicitly request the deferred run below.
-		await writeFile(
-			journal,
-			(await readFile(journal, 'utf8')).replace('Employer | Salary', 'Employer | Updated salary')
-		);
-		let captured = false;
-		vi.mocked(prepareReports).mockImplementationOnce(async (...args) => {
-			const reports = await actual.prepareReports(...args);
-			captured = true;
-			await gate;
-			return reports;
-		});
-		// Returning from start is not waiting for prepareReports or a job result.
-		service = await startWorker(config, { port: 0 });
-		await enqueuePreparation(client);
-		await expect.poll(() => captured, { timeout: 12_000 }).toBe(true);
-		expect((await fetch(`http://127.0.0.1:${service.port}/healthz`)).status).toBe(200);
-		expect(await readReportPublication(snapshots)).toEqual(publication);
-		expect(await completeSets(snapshots)).toEqual(sets);
-		expect(await runs()).toEqual([
-			expect.objectContaining({ status: 'running', output: null, attempts: 1 }),
-			expect.objectContaining({ status: 'completed', output: original })
-		]);
-		const stopping = service.stop();
-		let stopped = false;
-		void stopping.then(() => (stopped = true));
-		expect(stopped).toBe(false);
-		// Graceful stop keeps health/client alive until the active SDK handler finishes.
-		expect((await fetch(`http://127.0.0.1:${service.port}/healthz`)).status).toBe(200);
-		release();
-		await stopping;
-		service = undefined;
-		const completed = await runs();
-		expect(completed).toHaveLength(2);
-		expect(completed[0]).toMatchObject({ status: 'completed', attempts: 1 });
-		expect(await readOverview(snapshots)).toEqual(completed[0]!.output);
-		expect(await completeSets(snapshots)).toHaveLength(2);
-		expect(prepareReports).toHaveBeenCalledTimes(2);
-	} finally {
-		release();
-		await service?.stop();
-		await client.close();
-		await rm(directory, { recursive: true, force: true });
-	}
-}, 30_000);

apps/worker/src/workflows/prepare-reports/edits.integration.test.ts

@@ -1,162 +0,0 @@
-import { enqueuePreparation, financialValues } from './test-fixtures.js';
-import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { setTimeout as delay } from 'node:timers/promises';
-import { afterEach, expect, it, vi } from 'vitest';
-import { prepareReports } from '@hledger-reports/hledger';
-import { openJobsClient, prepareReportsSpec, type JobsClient } from '@hledger-reports/jobs';
-import {
-	readReportPublication,
-	readAccountTransactions,
-	type ReportPublication
-} from '@hledger-reports/snapshots';
-import { lotsJournal as original } from '../../../fixtures/journal.js';
-import { startWorker } from '../../index.js';
-
-vi.mock('@hledger-reports/hledger', async (original) => {
-	const actual = await original<typeof import('@hledger-reports/hledger')>();
-	return { ...actual, prepareReports: vi.fn(actual.prepareReports) };
-});
-const hledger = await vi.importActual<typeof import('@hledger-reports/hledger')>(
-	'@hledger-reports/hledger'
-);
-const directories: string[] = [];
-afterEach(async () => {
-	vi.mocked(prepareReports).mockReset().mockImplementation(hledger.prepareReports);
-	await Promise.all(
-		directories.splice(0).map((path) => rm(path, { recursive: true, force: true }))
-	);
-});
-async function fixture() {
-	const directory = await mkdtemp(join(tmpdir(), 'reports-source-'));
-	directories.push(directory);
-	const config = join(directory, 'config', 'hledger-reports.yaml');
-	const journal = join(directory, 'main.journal');
-	const included = join(directory, 'parts', 'base.journal');
-	const snapshots = join(directory, 'data', 'snapshots');
-	await mkdir(join(directory, 'config'));
-	await mkdir(join(directory, 'parts'));
-	await writeFile(
-		config,
-		JSON.stringify({
-			journal,
-			url: { app: 'http://127.0.0.1:3000' },
-			auth: { credentials: { username: 'fixture', password: '${AUTH_PASSWORD}' } }
-		})
-	);
-	await writeFile(journal, 'include parts/*.journal\n');
-	await writeFile(included, original);
-	return { directory, config, journal, included, snapshots, jobs: join(directory, 'data', 'jobs') };
-}
-const runs = async (client: JobsClient) =>
-	(await client.backend.listWorkflowRuns({ workflowName: prepareReportsSpec.name })).data;
-async function settled(client: JobsClient, count: number) {
-	await expect
-		.poll(
-			async () => {
-				const all = await runs(client);
-				return { count: all.length, status: all[0]?.status };
-			},
-			{ timeout: 15_000 }
-		)
-		.toEqual({ count, status: 'completed' });
-}
-function salary(value: string) {
-	return original
-		.replace('1000.00 USD\n    income:salary', `${value} USD\n    income:salary`)
-		.replace(/(income:salary\s+)-1000\.00 USD/, `$1-${value} USD`);
-}
-async function expectSalaryTransactions(publication: ReportPublication, quantity: string) {
-	expect(publication.directory).toBeTypeOf('string');
-	const salary = publication.accounts?.find((account) => account.name === 'income:salary');
-	expect(salary).toMatchObject({ access: 'ready', rowCount: 1 });
-	const { transactions: rows } = await readAccountTransactions(publication, salary!.id);
-	expect(rows).toHaveLength(1);
-	expect(financialValues(rows![0].change)).toEqual([{ commodity: 'USD', quantity }]);
-}
-function gate() {
-	let release!: () => void;
-	let released = false;
-	const waiting = new Promise<void>((resolve) => {
-		release = () => {
-			released = true;
-			resolve();
-		};
-	});
-	return {
-		waiting,
-		release,
-		get released() {
-			return released;
-		}
-	};
-}
-
-it('publishes its loaded native view despite later edits, with no automatic follow-up', async () => {
-	const f = await fixture();
-	const client = await openJobsClient(f.jobs);
-	const service = await startWorker(f.config, { port: 0 });
-	const firstRead = gate(),
-		firstRelease = gate(),
-		nextRead = gate(),
-		nextRelease = gate();
-	try {
-		await enqueuePreparation(client);
-		await settled(client, 1);
-		const initial = (await readReportPublication(f.snapshots))!;
-		await expectSalaryTransactions(initial, '-1000.00');
-		vi.mocked(prepareReports)
-			.mockImplementationOnce(async (...args) => {
-				const data = await hledger.prepareReports(...args);
-				firstRead.release();
-				await firstRelease.waiting;
-				return data;
-			})
-			.mockImplementationOnce(async (...args) => {
-				const data = await hledger.prepareReports(...args);
-				nextRead.release();
-				await nextRelease.waiting;
-				return data;
-			});
-		await writeFile(f.included, salary('1100.00'));
-		await enqueuePreparation(client);
-		await expect.poll(() => firstRead.released, { timeout: 12_000 }).toBe(true);
-		await writeFile(f.included, salary('1200.00'));
-		await writeFile(f.included, salary('1300.00'));
-		await delay(2200);
-		expect(await runs(client)).toHaveLength(2);
-		expect(await readReportPublication(f.snapshots)).toEqual(initial);
-		firstRelease.release();
-		await settled(client, 2);
-		await delay(2200);
-		expect(await runs(client)).toHaveLength(2);
-		expect(nextRead.released).toBe(false);
-		const intervening = (await readReportPublication(f.snapshots))!;
-		expect(intervening.overview.incomeExpenses.months[11].income).toEqual([
-			{ commodity: 'USD', quantity: '1100.00' }
-		]);
-		expect(intervening.directory).not.toBe(initial.directory);
-		await expectSalaryTransactions(intervening, '-1100.00');
-		await enqueuePreparation(client);
-		await expect.poll(() => nextRead.released, { timeout: 12_000 }).toBe(true);
-		expect(await runs(client)).toHaveLength(3);
-		nextRelease.release();
-		await settled(client, 3);
-		const latest = (await readReportPublication(f.snapshots))!;
-		expect(latest.overview.incomeExpenses.months[11].income).toEqual([
-			{ commodity: 'USD', quantity: '1300.00' }
-		]);
-		expect(latest.directory).not.toBe(intervening.directory);
-		await expectSalaryTransactions(latest, '-1300.00');
-		await expectSalaryTransactions(initial, '-1000.00');
-		await delay(2200);
-		expect(await runs(client)).toHaveLength(3);
-		expect(prepareReports).toHaveBeenCalledTimes(3);
-	} finally {
-		firstRelease.release();
-		nextRelease.release();
-		await service.stop();
-		await client.close();
-	}
-}, 40_000);

apps/worker/src/workflows/prepare-reports/run.integration.test.ts

@@ -1,17 +1,11 @@
-import { enqueuePreparation, readLatestPreparationState } from './test-fixtures.js';
+import { enqueuePreparation } from './test-fixtures.js';
 import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { setTimeout as delay } from 'node:timers/promises';
 import { join } from 'node:path';
 import { afterEach, expect, it } from 'vitest';
+import { openJobsClient, prepareReportsSpec, type JobsClient } from '@hledger-reports/jobs';
 import {
-	openJobsClient,
-	prepareReportsSpec,
-	liveBalanceQuerySpec,
-	type JobsClient,
-	retiredIncomePreparationSpec
-} from '@hledger-reports/jobs';
-import {
+	createReportStage,
 	readAccountTransactions,
 	readOverview,
 	readReportPublication,
@@ -45,17 +39,14 @@ async function fixture() {
 	);
 	await writeFile(journal, 'include included.journal\n');
 	await writeFile(included, source);
-	const legacy = join(snapshots, 'income-expenses', 'journal', 'old-preview.json');
-	await mkdir(join(snapshots, 'income-expenses', 'journal'), { recursive: true });
-	await writeFile(legacy, 'legacy preview unchanged');
-	return { directory, config, journal, included, jobs, snapshots, legacy };
+	return { directory, config, journal, included, jobs, snapshots };
 }
 
 async function preparations(client: JobsClient) {
 	return (await client.backend.listWorkflowRuns({ workflowName: prepareReportsSpec.name })).data;
 }
 
-async function settled(client: JobsClient, count: number, status = 'completed') {
+async function settled(client: JobsClient, status: 'completed' | 'failed' = 'completed') {
 	await expect
 		.poll(
 			async () => {
@@ -64,7 +55,7 @@ async function settled(client: JobsClient, count: number, status = 'completed')
 			},
 			{ timeout: 12_000 }
 		)
-		.toEqual({ count, status });
+		.toEqual({ count: 1, status });
 	return (await preparations(client))[0]!;
 }
 
@@ -76,7 +67,7 @@ it('explicitly prepares a saved overview through the real SDK without a browser
 		expect(await readOverview(f.snapshots)).toBeNull();
 		service = await startWorker(f.config, { port: 0 });
 		await enqueuePreparation(client);
-		const run = await settled(client, 1);
+		const run = await settled(client);
 		expect(run).toMatchObject({
 			workflowName: prepareReportsSpec.name,
 			version: '1',
@@ -122,298 +113,59 @@ it('explicitly prepares a saved overview through the real SDK without a browser
 		);
 		expect(await readFile(f.journal, 'utf8')).toBe('include included.journal\n');
 		expect(await readFile(f.included, 'utf8')).toBe(source);
-		expect(await readFile(f.legacy, 'utf8')).toBe('legacy preview unchanged');
 	} finally {
 		await service?.stop();
 		await client.close();
 	}
 }, 20_000);
 
-it('retains last-good data without retry flooding, permits an explicit SDK retry, and prepares after an explicit correction request', async () => {
+it('persists a sanitized failed SDK request and completed validation without replacing saved reports', async () => {
 	const f = await fixture();
+	const stage = await createReportStage(f.snapshots, {
+		id: 'saved',
+		createdAt: new Date('2026-05-01T00:00:00Z')
+	});
+	try {
+		await stage.publish(
+			{
+				asOf: null,
+				preparedAt: '2026-05-01T00:00:00.000Z',
+				journalCoverage: { transactionCount: 0, firstDate: null, lastDate: null },
+				hledgerVersion: '1.99.5',
+				ranges: null,
+				balanceSheet: null,
+				incomeExpenses: { months: [] },
+				netWorth: { currency: null, points: [] }
+			},
+			[],
+			f.journal
+		);
+	} finally {
+		await stage.discard();
+	}
+	const saved = await readReportPublication(f.snapshots);
+	expect(saved).not.toBeNull();
+	const savedSets = await readdir(join(f.snapshots, '.sets'));
+	await writeFile(f.included, 'BROKEN PRIVATE JOURNAL 12345.67 USD');
 	const client = await openJobsClient(f.jobs);
 	let service: Awaited<ReturnType<typeof startWorker>> | undefined;
 	try {
 		service = await startWorker(f.config, { port: 0 });
 		await enqueuePreparation(client);
-		await settled(client, 1);
-		const first = await readOverview(f.snapshots);
-		const savedSets = await readdir(join(f.snapshots, '.sets'));
-		await service.stop();
-		service = undefined;
-		await writeFile(f.included, 'BROKEN PRIVATE JOURNAL 12345.67 USD');
-		service = await startWorker(f.config, { port: 0 });
-		await enqueuePreparation(client);
-		const failed = await settled(client, 2, 'failed');
-		expect(failed.attempts).toBe(1);
-		expect(await readLatestPreparationState(client)).toBe('failed');
-		expect(await readdir(join(f.snapshots, '.sets'))).toEqual(savedSets);
+		const failed = await settled(client, 'failed');
 		expect(failed.output).toBeNull();
-		expect(failed.error?.message).toBe(
-			'hledger strict journal validation failed. Run hledger check --strict directly for diagnostics.'
-		);
+		expect(failed.error?.message).toBeTruthy();
+		expect(JSON.stringify(failed.error)).not.toMatch(/PRIVATE|12345\.67/);
+		expect(JSON.stringify(failed.error)).not.toContain(f.directory);
 		expect(await readValidation(f.snapshots)).toMatchObject({
 			journal: f.journal,
-			request: { id: failed.id },
+			request: { id: failed.id, createdAt: failed.createdAt.toISOString() },
 			result: { outcome: 'could-not-validate', completedAt: expect.any(String) }
 		});
-		expect(JSON.stringify(failed.error)).not.toContain('PRIVATE');
-		expect(JSON.stringify(failed.error)).not.toContain(f.directory);
-		expect(await readOverview(f.snapshots)).toEqual(first);
-		expect((await fetch(`http://127.0.0.1:${service.port}/healthz`)).status).toBe(200);
-		await delay(2100);
-		expect(await preparations(client)).toHaveLength(2);
-		await service.stop();
-		service = await startWorker(f.config, { port: 0 });
-		await delay(2100);
-		// Restart never requests a financial job, even after failure.
-		expect(await preparations(client)).toHaveLength(2);
-		// An explicit retry is still allowed for the same invalid input state.
-		await enqueuePreparation(client);
-		await settled(client, 3, 'failed');
-		expect(await readOverview(f.snapshots)).toEqual(first);
-		const failedValidation = await readValidation(f.snapshots);
-		expect(failedValidation).toMatchObject({ result: { outcome: 'could-not-validate' } });
-		await writeFile(f.included, changedSalary());
-		expect(
-			(
-				await (
-					await client.workflow.runWorkflow(liveBalanceQuerySpec, {
-						query: 'type:X',
-						asOf: '2026-04-30'
-					})
-				).result({ timeoutMs: 10_000 })
-			).total
-		).toEqual([{ commodity: 'USD', quantity: '100.00' }]);
-		// Correction and a live query neither request preparation nor clear a failure.
-		await delay(1500);
-		expect(await readValidation(f.snapshots)).toEqual(failedValidation);
-		expect(await readLatestPreparationState(client)).toBe('failed');
-		expect(await preparations(client)).toHaveLength(3);
-		await enqueuePreparation(client);
-		const retried = await settled(client, 4);
-		const updated = (await readOverview(f.snapshots))!;
-		expect(updated.incomeExpenses.months[11].income).toEqual([
-			{ commodity: 'USD', quantity: '1500.00' }
-		]);
-		expect(retried.output).toEqual(updated);
-		expect(await readValidation(f.snapshots)).toMatchObject({
-			request: { id: retried.id },
-			result: { outcome: 'passed' }
-		});
-		await writeFile(f.included, '');
-		await enqueuePreparation(client);
-		const emptyRun = await settled(client, 5);
-		const empty = await readOverview(f.snapshots);
-		expect(empty).toMatchObject({
-			asOf: null,
-			incomeExpenses: { months: [] },
-			ranges: null,
-			balanceSheet: null,
-			journalCoverage: { transactionCount: 0, firstDate: null, lastDate: null },
-			netWorth: { currency: null, points: [] }
-		});
-		expect(emptyRun.output).toEqual(empty);
-		expect(await readFile(f.legacy, 'utf8')).toBe('legacy preview unchanged');
+		expect(await readReportPublication(f.snapshots)).toEqual(saved);
+		expect(await readdir(join(f.snapshots, '.sets'))).toEqual(savedSets);
 	} finally {
 		await service?.stop();
 		await client.close();
 	}
-}, 30_000);
-
-it('fails old queued preview requests once without interpreting their input and prepares reports only on request', async () => {
-	const f = await fixture();
-	const client = await openJobsClient(f.jobs);
-	const handle = await client.workflow.runWorkflow(retiredIncomePreparationSpec, {
-		old: 'opaque preview'
-	});
-	const service = await startWorker(f.config, { port: 0 });
-	try {
-		await expect(handle.result({ timeoutMs: 12_000 })).rejects.toThrow();
-		const retired = await client.backend.getWorkflowRun({ workflowRunId: handle.workflowRun.id });
-		expect(retired?.attempts).toBe(1);
-		expect(retired?.error?.message).toBe(
-			'Income preparation previews are retired. Refresh the dashboard.'
-		);
-		expect(await preparations(client)).toEqual([]);
-		await enqueuePreparation(client);
-		const prepared = await settled(client, 1);
-		expect(prepared.output).toEqual(await readOverview(f.snapshots));
-		expect(await readOverview(f.snapshots)).toMatchObject({ asOf: '2026-04-16' });
-		expect(await readFile(f.legacy, 'utf8')).toBe('legacy preview unchanged');
-	} finally {
-		await service.stop();
-		await client.close();
-	}
 }, 20_000);
-
-function changedSalary() {
-	return source
-		.replace('1000.00 USD\n    income:salary', '1500.00 USD\n    income:salary')
-		.replace(/(income:salary\s+)-1000\.00 USD/, '$1-1500.00 USD');
-}
-
-it('prepares on request after a stopped-worker include edit even with saved data and an unchanged as-of date', async () => {
-	const f = await fixture();
-	const client = await openJobsClient(f.jobs);
-	let service: Awaited<ReturnType<typeof startWorker>> | undefined;
-	try {
-		service = await startWorker(f.config, { port: 0 });
-		await enqueuePreparation(client);
-		const originalRun = await settled(client, 1);
-		await service.stop();
-		service = undefined;
-		const original = (await readOverview(f.snapshots))!;
-		await writeFile(f.included, changedSalary());
-		// Last-good data is readable before the restart, with no freshness inference.
-		expect(await readOverview(f.snapshots)).toEqual(original);
-		service = await startWorker(f.config, { port: 0 });
-		await delay(1500);
-		expect(await preparations(client)).toHaveLength(1);
-		expect(await readOverview(f.snapshots)).toEqual(original);
-		await enqueuePreparation(client);
-		const refreshedRun = await settled(client, 2);
-		const refreshed = (await readOverview(f.snapshots))!;
-		expect(refreshedRun.id).not.toBe(originalRun.id);
-		expect(refreshedRun.idempotencyKey).toBeNull();
-		expect(refreshed.asOf).toBe(original.asOf);
-		expect(refreshed.journalCoverage).toEqual(original.journalCoverage);
-		expect(refreshed.incomeExpenses.months[11].income).toEqual([
-			{ commodity: 'USD', quantity: '1500.00' }
-		]);
-		expect(refreshed.netWorth?.points[11].amounts).toEqual([
-			{ commodity: 'USD', quantity: '12325.00' }
-		]);
-		expect(refreshedRun.output).toEqual(refreshed);
-		expect(refreshedRun.output).not.toEqual(originalRun.output);
-		expect(await readFile(f.journal, 'utf8')).toBe('include included.journal\n');
-		expect(await readFile(f.included, 'utf8')).toBe(changedSalary());
-	} finally {
-		await service?.stop();
-		await client.close();
-	}
-}, 30_000);
-
-it.each(['pending', 'abandoned running'] as const)(
-	'executes previously requested %s preparation on boot without creating another run',
-	async (state) => {
-		const f = await fixture();
-		const client = await openJobsClient(f.jobs);
-		let service: Awaited<ReturnType<typeof startWorker>> | undefined;
-		try {
-			service = await startWorker(f.config, { port: 0 });
-			await enqueuePreparation(client);
-			await settled(client, 1);
-			await service.stop();
-			service = undefined;
-			const original = await readOverview(f.snapshots);
-			// A previously enqueued run must execute without another boot submission.
-			const pending = await client.workflow.runWorkflow(prepareReportsSpec, undefined);
-			if (state === 'abandoned running') {
-				const claimed = await client.backend.claimWorkflowRun({
-					workerId: 'dead-worker',
-					leaseDurationMs: 100
-				});
-				expect(claimed).toMatchObject({
-					id: pending.workflowRun.id,
-					status: 'running',
-					attempts: 1
-				});
-				const step = await client.backend.createStepAttempt({
-					workflowRunId: pending.workflowRun.id,
-					workerId: 'dead-worker',
-					stepName: 'input-fingerprint',
-					stepIndex: 0,
-					kind: 'function',
-					config: {},
-					context: null
-				});
-				await client.backend.completeStepAttempt({
-					workflowRunId: pending.workflowRun.id,
-					stepAttemptId: step.id,
-					workerId: 'dead-worker',
-					output: 'legacy-input-fingerprint'
-				});
-				await expect.poll(() => Date.now() > claimed!.availableAt!.getTime()).toBe(true);
-			}
-			// An old checkpoint is ignored: recovery calculates the currently loaded journal.
-			await writeFile(f.included, changedSalary());
-			expect(await readOverview(f.snapshots)).toEqual(original);
-			service = await startWorker(f.config, { port: 0 });
-			const recovered = await settled(client, 2);
-			expect(recovered.id).toBe(pending.workflowRun.id);
-			// SDK 0.11 reclaims expired leases and executes the handler again, even
-			// with maximumAttempts=1 (that policy limits failure retries, not claims).
-			expect(recovered.attempts).toBe(state === 'pending' ? 1 : 2);
-			expect(recovered.output).toEqual(await readOverview(f.snapshots));
-			const publication = (await readReportPublication(f.snapshots))!;
-			expect(publication.journal).toBe(f.journal);
-			if (state === 'abandoned running') {
-				const persisted = await client.backend.listStepAttempts({ workflowRunId: recovered.id });
-				expect(persisted.data).toHaveLength(1);
-				expect(persisted.data[0]?.output).toBe('legacy-input-fingerprint');
-			}
-			await delay(1500);
-			expect(await preparations(client)).toHaveLength(2);
-			expect(await readOverview(f.snapshots)).toMatchObject({
-				incomeExpenses: {
-					months: expect.arrayContaining([
-						expect.objectContaining({ income: [{ commodity: 'USD', quantity: '1500.00' }] })
-					])
-				},
-				netWorth: {
-					currency: 'USD',
-					points: expect.arrayContaining([
-						expect.objectContaining({
-							date: '2026-04-16',
-							amounts: [{ commodity: 'USD', quantity: '12325.00' }]
-						})
-					])
-				}
-			});
-			expect(await readFile(f.legacy, 'utf8')).toBe('legacy preview unchanged');
-		} finally {
-			await service?.stop();
-			await client.close();
-		}
-	},
-	30_000
-);
-
-it('creates no financial work on boot, idle, edits or restart; every completed rerun needs an explicit request', async () => {
-	const f = await fixture();
-	const client = await openJobsClient(f.jobs);
-	let service: Awaited<ReturnType<typeof startWorker>> | undefined;
-	try {
-		service = await startWorker(f.config, { port: 0 });
-		await delay(1500);
-		expect(await preparations(client)).toEqual([]);
-		expect(await readLatestPreparationState(client)).toBe('idle');
-		expect(await readOverview(f.snapshots)).toBeNull();
-		expect(await readValidation(f.snapshots)).toBeNull();
-		await writeFile(f.included, changedSalary());
-		await delay(1500);
-		expect(await preparations(client)).toEqual([]);
-		await service.stop();
-		service = await startWorker(f.config, { port: 0 });
-		await delay(1500);
-		expect(await preparations(client)).toEqual([]);
-		await enqueuePreparation(client);
-		const first = await settled(client, 1);
-		await delay(1500);
-		expect(await preparations(client)).toHaveLength(1);
-		await enqueuePreparation(client);
-		const second = await settled(client, 2);
-		expect(second.id).not.toBe(first.id);
-		expect(second.idempotencyKey).toBeNull();
-		expect(await readLatestPreparationState(client)).toBe('idle');
-		expect(second.output).toEqual(await readOverview(f.snapshots));
-		await service.stop();
-		service = await startWorker(f.config, { port: 0 });
-		await delay(1500);
-		expect(await preparations(client)).toHaveLength(2);
-	} finally {
-		await service?.stop();
-		await client.close();
-	}
-}, 30_000);

apps/worker/src/workflows/prepare-reports/run.test.ts

@@ -125,7 +125,14 @@ it.each([true, false])(
 			...previous,
 			journal: same ? journal : '/other/book.journal'
 		});
-		expect(await prepare(context)).toBe(overview);
+		const result = await prepare(context);
+		expect(createReportStage).toHaveBeenCalledExactlyOnceWith(
+			directory,
+			run,
+			same ? previous.accounts : null
+		);
+		if (!same) return;
+		expect(result).toBe(overview);
 		expect(checkJournal).toHaveBeenCalledExactlyOnceWith(journal);
 		expect(writeValidation).toHaveBeenCalledExactlyOnceWith(directory, run, journal, {
 			outcome: 'passed',
@@ -134,11 +141,6 @@ it.each([true, false])(
 		expect(vi.mocked(writeValidation).mock.invocationCallOrder[0]).toBeLessThan(
 			vi.mocked(createReportStage).mock.invocationCallOrder[0]!
 		);
-		expect(createReportStage).toHaveBeenCalledExactlyOnceWith(
-			directory,
-			run,
-			same ? previous.accounts : null
-		);
 		expect(prepareReports).toHaveBeenCalledExactlyOnceWith(journal, {
 			writeRows: stage.writeRows
 		});
@@ -179,7 +181,7 @@ it.each(['producer', 'publish'] as const)('discards its stage on %s failure', as
 	if (kind !== 'publish') expect(stage.publish).not.toHaveBeenCalled();
 });
 
-it('awaits producer writes and publication before returning its own calculated result', async () => {
+it('awaits producer completion and publication before returning its own calculated result', async () => {
 	let releaseWrite!: () => void;
 	let releasePublish!: () => void;
 	stage.writeRows.mockReturnValue(new Promise<void>((resolve) => (releaseWrite = resolve)));

apps/worker/src/workflows/prepare-reports/test-fixtures.ts

@@ -1,24 +1,6 @@
-import type { ReportedAmount } from '@hledger-reports/hledger';
 import { prepareReportsSpec, type JobsClient } from '@hledger-reports/jobs';
 
-/** Compare every exact financial value independently of retained display/cost metadata. */
-export function financialValues(amounts: readonly ReportedAmount[]) {
-	return amounts.map(({ commodity, quantity }) => ({ commodity, quantity }));
-}
-
 /** Direct SDK submission only; acceptance does not wait for a gated handler. */
 export async function enqueuePreparation(client: JobsClient): Promise<void> {
 	await client.workflow.runWorkflow(prepareReportsSpec, undefined);
 }
-
-/** These isolated fixtures have one worker and explicitly sequenced preparations. */
-export async function readLatestPreparationState(client: JobsClient) {
-	const runs = await client.backend.listWorkflowRuns({
-		workflowName: prepareReportsSpec.name,
-		limit: 1
-	});
-	const latest = runs.data[0];
-	if (!latest || latest.status === 'completed') return 'idle';
-	if (latest.status === 'failed' || latest.status === 'canceled') return 'failed';
-	return latest.status;
-}

docs/architecture.md

@@ -17,9 +17,10 @@ A request keeps one saved set so navigation and page data agree. Recorded journa
 checks are separate from financial publication. Neither saved results nor a passed
 check proves that the journal has not changed.
 
-Web and worker share configuration and private storage. SQLite coordinates jobs;
-financial browsing does not depend on a running worker. Startup does not request
-preparation, though previously requested jobs can resume.
+Web and worker share configuration and private storage. The job queue and saved
+report generations use separate SQLite databases. Financial browsing does not
+depend on a running worker. Startup does not request preparation, though previously
+requested jobs can resume.
 
 See [development](development.md) for commands and [principles](philosophy.md) for
 how we change the code.

docs/development.md

@@ -60,6 +60,7 @@ cleans up fictional data; never point browser tests at a personal deployment.
 - Keep live configuration, journals and runtime data out of Git and images.
 - Compose shares `apps/web/data/` between web and worker. It contains sensitive jobs and financial results and must be writable by container UID 1000.
 - Stop both services before backing up the whole data directory. Upgrade web and worker together; restore a consistent backup rather than downgrade a migrated database.
+- Saved reports use SQLite generations. After updating web and worker together, explicitly prepare again; older financial JSON stays untouched but is no longer read.
 - Do not delete data or reset volumes to fix unavailable reports. Request preparation explicitly and reload to inspect the result.
 - Background maintenance removes eligible older balance-query jobs, not saved reports. Deletion is not secure erasure; SQLite and backups may retain data.
 

packages/db/src/database.ts

@@ -1,4 +1,4 @@
-import { Kysely, sql } from 'kysely';
+import { Kysely, sql, type ControlledTransaction } from 'kysely';
 import { constants } from 'node:fs';
 import { open } from 'node:fs/promises';
 import { dirname } from 'node:path';
@@ -6,6 +6,9 @@ import { DatabaseSync } from 'node:sqlite';
 import { nodeSqliteDialect } from './node-sqlite-dialect.js';
 import { reportSchemaVersion, type ReportDatabase } from './schema.js';
 
+export type ReportConnection = Kysely<ReportDatabase>;
+export type ReportTransaction = ControlledTransaction<ReportDatabase>;
+
 async function checkDirectory(directory: string): Promise<void> {
 	const folder = await open(
 		directory,

packages/db/src/index.ts

@@ -1,3 +1,4 @@
 export { createReportDatabase, openReadOnlyReportDatabase } from './database.js';
+export type { ReportConnection, ReportTransaction } from './database.js';
 export { runMigrations } from './migrate.js';
 export type { ReportDatabase } from './schema.js';

packages/db/src/reports/queries.test.ts

@@ -1,6 +1,7 @@
 import { mkdtemp, mkdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
 import { sql, type Kysely } from 'kysely';
 import { afterEach, beforeEach, expect, it } from 'vitest';
 import {
@@ -12,6 +13,7 @@ import {
 import { readAccountRows, readReportData, writeAccountRows, writeReportData } from './queries.js';
 import type { StoredReportData } from './data.js';
 import {
+	amount,
 	ambiguous,
 	preparedOverview,
 	journal,
@@ -51,12 +53,64 @@ afterEach(async () => {
 it('stores exact prepared payloads and native row order using the migrated schema', async () => {
 	const database = await createDatabase();
 	const rows = preparedAccountRows(52).reverse();
+	const rich = rows[0];
+	// Preserve producer-owned prices and basis, including unavailable native conversions.
+	rich.transaction.postings[1].amounts[0] = {
+		...amount('75.00'),
+		commodity: 'TEST',
+		cost: { kind: 'UnitCost', amount: amount('0.000000000000000000012300') },
+		costBasis: {
+			amount: { ...amount('9007199254740993.123456789'), commodity: 'EUR' },
+			date: '2026-04-01',
+			label: 'retained basis label'
+		},
+		style: {
+			commoditySide: 'L',
+			commoditySpaced: false,
+			decimalMark: '.',
+			digitGroups: [',', [3]],
+			precision: 2,
+			rounding: 'NoRounding'
+		}
+	};
+	rich.transaction.postings[1].transactedAmounts = [
+		{ commodity: 'EUR', quantity: '0.000000000000000000922500' }
+	];
+	rich.transaction.postings[2].tags.push(['_conversion-posting', '']);
+	rich.transaction.postings[2].transactedAmounts = null;
+	rich.investment = {
+		securities: [
+			{
+				commodity: 'TEST',
+				action: 'unknown',
+				change: [{ commodity: 'TEST', quantity: '75.00' }],
+				postingIndexes: [1],
+				prices: [amount('0.000000000000000000012300')],
+				transactedAmounts: rich.transaction.postings[1].transactedAmounts
+			}
+		],
+		cash: {
+			direction: 'unknown',
+			change: [],
+			balance: [],
+			unclassifiedChange: rich.change,
+			unclassifiedBalance: rich.balance
+		},
+		cashPostingIndexes: [],
+		incomePostingIndexes: [],
+		expensePostingIndexes: [1, 2],
+		gainPostingIndexes: [],
+		issues: ['unclassified-commodity', 'unavailable-transacted-values']
+	};
+	const otherRows = preparedAccountRows(1, 'case-distinct account');
+	otherRows[0].transaction.postings[0].account = 'assets:Cash';
 	const data: StoredReportData = {
 		journal,
 		overview: preparedOverview(),
 		accounts: [
 			{ id: 'empty001', ...ready('assets:empty', 0) },
 			{ id: 'cash0001', ...ready('assets:cash', rows.length) },
+			{ id: 'cash0002', ...ready('assets:Cash', otherRows.length) },
 			{ id: 'ambig001', ...ambiguous() }
 		]
 	};
@@ -64,6 +118,7 @@ it('stores exact prepared payloads and native row order using the migrated schem
 	try {
 		await writeAccountRows(transaction, 'assets:cash', 0, rows.slice(0, 51));
 		await writeAccountRows(transaction, 'assets:cash', 51, rows.slice(51));
+		await writeAccountRows(transaction, 'assets:Cash', 0, otherRows);
 		await writeReportData(transaction, data);
 		await transaction.commit().execute();
 	} finally {
@@ -78,13 +133,39 @@ it('stores exact prepared payloads and native row order using the migrated schem
 	expect(await readReportData(reader)).toEqual(data);
 	expect(await readAccountRows(reader, data.accounts[0])).toEqual([]);
 	expect(await readAccountRows(reader, data.accounts[1])).toEqual(rows);
-	expect(await readAccountRows(reader, data.accounts[2])).toBeNull();
+	expect(await readAccountRows(reader, data.accounts[2])).toEqual(otherRows);
+	expect(await readAccountRows(reader, data.accounts[3])).toBeNull();
 	await expect(
 		writeAccountRows(reader, 'assets:cash', rows.length, rows.slice(0, 1))
 	).rejects.toThrow();
 	expect((await stat(path)).mode & 0o777).toBe(0o600);
 });
 
+it('rejects a persisted row with an out-of-range posting reference', async () => {
+	const database = await createDatabase();
+	const account = { id: 'cash0001', ...ready() };
+	const rows = preparedAccountRows();
+	await writeReportData(database, { journal, overview: preparedOverview(), accounts: [account] });
+	await writeAccountRows(database, account.name, 0, rows);
+	await closeDatabase(database);
+
+	// Valid SQLite/JSON is not sufficient: saved posting references must still be usable.
+	const native = new DatabaseSync(path);
+	try {
+		const badRow = { ...rows[0], accountPostingIndexes: [rows[0].transaction.postings.length] };
+		native
+			.prepare('UPDATE account_activity SET row_json = ? WHERE account_name = ? AND ordinal = 0')
+			.run(JSON.stringify(badRow), account.name);
+	} finally {
+		native.close();
+	}
+	const reader = await openReadOnlyReportDatabase(path);
+	if (!reader) throw new Error('Expected the current report schema.');
+	databases.add(reader);
+	expect((await readReportData(reader)).accounts).toEqual([account]);
+	await expect(readAccountRows(reader, account)).rejects.toThrow('Invalid account data.');
+});
+
 it('does not create missing storage or replace an existing report file', async () => {
 	const missing = join(directory, 'missing');
 	await expect(openReadOnlyReportDatabase(join(missing, 'reports.sqlite'))).rejects.toThrow();

packages/snapshots/README.md

@@ -1,7 +1,7 @@
 # Saved reports
 
-Stores complete private report sets and recorded journal checks. Writes are staged;
-readers keep one selected set for a consistent request. Checks are independent of
+Stores complete private SQLite report sets. Writes are staged; readers keep one
+selected set for a consistent request. Recorded journal checks are separate from
 financial publication and do not prove freshness.
 
 Reports are derived data, not journal backups. See

packages/snapshots/src/reports/publication.ts

@@ -3,30 +3,36 @@ import type {
 	PreparedAccountSummary,
 	PreparedAccountRow
 } from '@hledger-reports/hledger';
-import { randomInt, randomUUID } from 'node:crypto';
-import { constants } from 'node:fs';
-import { mkdir, open, rename, rm, symlink, unlink } from 'node:fs/promises';
-import type { FileHandle } from 'node:fs/promises';
-import { basename, dirname, join } from 'node:path';
-import { namesIn, privateDirectory, readJson, writeJson } from '../files.js';
+import {
+	createReportDatabase,
+	openReadOnlyReportDatabase,
+	runMigrations,
+	type ReportConnection,
+	type ReportTransaction
+} from '@hledger-reports/db';
 import {
 	hasOverviewContainers,
-	hasAccountRowContainers,
-	hasStoredReportData,
 	isAccountCatalog,
 	isAccountId,
 	isAccountSummary,
-	isObject,
+	readAccountRows,
+	readReportData,
 	storedSummary,
 	summaryMatchesDate,
+	writeAccountRows,
+	writeReportData,
 	type StoredAccount,
 	type StoredReportData
 } from '@hledger-reports/db/reports';
+import { randomInt, randomUUID } from 'node:crypto';
+import { lstat, mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
+import { basename, dirname, join } from 'node:path';
+import { namesIn, privateDirectory } from '../files.js';
 
 const runIdPattern = /^[A-Za-z0-9_-]{1,128}(?![\s\S])/;
 const setPattern =
 	/^(\d{16}-[A-Za-z0-9_-]{1,128})-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}(?![\s\S])/;
-const currentReportFormat = 8;
+const databaseName = 'reports.sqlite';
 
 export interface ReportPublication extends StoredReportData {
 	/** Private, pinned completed-set directory; never expose this in financial data. */
@@ -59,6 +65,17 @@ async function latestSet(snapshotDirectory: string): Promise<string | undefined>
 		.at(-1);
 }
 
+async function openSelectedDatabase(directory: string): Promise<ReportConnection | null> {
+	try {
+		return await openReadOnlyReportDatabase(join(directory, databaseName));
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+		const legacy = await lstat(join(directory, 'overview.json'));
+		if (!legacy.isFile()) throw error;
+		return null;
+	}
+}
+
 export async function readReportPublication(
 	snapshotDirectory: string
 ): Promise<ReportPublication | null> {
@@ -67,17 +84,14 @@ export async function readReportPublication(
 		if (!latest) return null;
 		const directory = join(snapshotDirectory, '.sets', latest);
 		await privateDirectory(directory);
-		const record = await readJson(directory, 'overview.json');
-		if (!isObject(record)) throw new Error();
-		if (record.format !== currentReportFormat) return null;
-		if (!hasStoredReportData(record)) throw new Error();
-		return {
-			// Financial rows remain producer-owned after checking current header/containers.
-			overview: record.overview,
-			journal: record.journal,
-			accounts: record.accounts,
-			directory
-		};
+		const database = await openSelectedDatabase(directory);
+		if (!database) return null;
+		try {
+			const data = await readReportData(database);
+			return { ...data, directory };
+		} finally {
+			await database.destroy();
+		}
 	} catch {
 		throw new Error('Unable to read overview');
 	}
@@ -100,23 +114,17 @@ export async function readAccountTransactions(
 			throw new Error();
 		await privateDirectory(dirname(directory));
 		await privateDirectory(directory);
-		const transactionsDirectory = join(directory, 'transactions');
-		await privateDirectory(transactionsDirectory);
-		const record = await readJson(transactionsDirectory, `${id}.json`);
-		if (!isObject(record) || record.id !== id || record.name !== account.name) throw new Error();
-		if (account.access === 'ambiguous') {
-			if (record.transactions !== null) throw new Error();
-		} else if (
-			!Array.isArray(record.transactions) ||
-			record.transactions.length !== account.rowCount ||
-			!record.transactions.every(hasAccountRowContainers)
-		)
-			throw new Error();
-		return {
-			id,
-			name: account.name,
-			transactions: record.transactions as PreparedAccountRow[] | null
-		};
+		const database = await openReadOnlyReportDatabase(join(directory, databaseName));
+		if (!database) throw new Error();
+		try {
+			return {
+				id,
+				name: account.name,
+				transactions: await readAccountRows(database, account)
+			};
+		} finally {
+			await database.destroy();
+		}
 	} catch {
 		throw new Error('Unable to read account transactions');
 	}
@@ -150,6 +158,34 @@ export async function createReportStage(
 	previousAccounts?: readonly StoredAccount[] | null
 ): Promise<ReportStage> {
 	let staging: string | undefined;
+	let database: ReportConnection | undefined;
+	let transaction: ReportTransaction | undefined;
+	let databaseClosed = false;
+	let state: 'active' | 'discarding' | 'completed' | 'discarded' = 'active';
+
+	async function closeDatabase(): Promise<void> {
+		if (databaseClosed) return;
+		await database?.destroy();
+		databaseClosed = true;
+	}
+
+	async function discard(): Promise<void> {
+		if (state === 'completed' || state === 'discarded') return;
+		state = 'discarding';
+		let rollbackFailed = false;
+		if (!databaseClosed && transaction && !transaction.isCommitted && !transaction.isRolledBack) {
+			try {
+				await transaction.rollback().execute();
+			} catch {
+				rollbackFailed = true;
+			}
+		}
+		await closeDatabase();
+		if (staging) await rm(staging, { recursive: true, force: true });
+		state = 'discarded';
+		if (rollbackFailed) throw new Error('Unable to prepare reports');
+	}
+
 	try {
 		const timestamp = String(run.createdAt.getTime()).padStart(16, '0');
 		if (!runIdPattern.test(run.id) || !/^\d{16}$/.test(timestamp)) throw new Error();
@@ -177,31 +213,12 @@ export async function createReportStage(
 		const temporary = join(sets, `.${uuid}.tmp`);
 		await mkdir(temporary, { mode: 0o700 });
 		staging = temporary;
-		const transactionsDirectory = join(temporary, 'transactions');
-		await privateDirectory(transactionsDirectory, true);
-		let state: 'active' | 'completed' | 'discarded' = 'active';
+		database = await createReportDatabase(join(temporary, databaseName));
+		await runMigrations(database);
+		transaction = await database.startTransaction().execute();
+		const writer = transaction;
 		const written = new Map<string, { id: string; count: number }>();
-		let current: { name: string; file: FileHandle } | undefined;
-
-		async function closeCurrent(): Promise<void> {
-			if (!current) return;
-			const { file } = current;
-			current = undefined;
-			try {
-				await file.writeFile(']}', 'utf8');
-			} finally {
-				await file.close();
-			}
-		}
-		async function discard(): Promise<void> {
-			if (state !== 'active') return;
-			state = 'discarded';
-			try {
-				await closeCurrent();
-			} finally {
-				await rm(temporary, { recursive: true, force: true });
-			}
-		}
+		let currentName: string | undefined;
 		async function operation<T>(action: () => Promise<T>): Promise<T> {
 			try {
 				if (state !== 'active') throw new Error();
@@ -215,24 +232,13 @@ export async function createReportStage(
 			writeRows(name, rows) {
 				return operation(async () => {
 					if (typeof name !== 'string' || !Array.isArray(rows)) throw new Error();
-					if (current?.name !== name) {
-						await closeCurrent();
+					if (currentName !== name) {
 						if (written.has(name)) throw new Error();
-						const id = accountId(name);
-						const header = `{"id":${JSON.stringify(id)},"name":${JSON.stringify(name)},"transactions":[`;
-						const file = await open(
-							join(transactionsDirectory, `${id}.json`),
-							constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,
-							0o600
-						);
-						current = { name, file };
-						written.set(name, { id, count: 0 });
-						await file.writeFile(header, 'utf8');
+						currentName = name;
+						written.set(name, { id: accountId(name), count: 0 });
 					}
 					const metadata = written.get(name)!;
-					if (!rows.length) return;
-					const json = (metadata.count ? ',' : '') + JSON.stringify(rows).slice(1, -1);
-					await current!.file.writeFile(json, 'utf8');
+					await writeAccountRows(writer, name, metadata.count, rows);
 					metadata.count += rows.length;
 				});
 			},
@@ -245,7 +251,6 @@ export async function createReportStage(
 						!Array.isArray(accounts)
 					)
 						throw new Error();
-					await closeCurrent();
 					const remaining = new Set(written.keys());
 					const names = new Set<string>();
 					const catalog: StoredAccount[] = [];
@@ -264,17 +269,8 @@ export async function createReportStage(
 						} else if ((metadata?.count ?? 0) !== account.rowCount) {
 							throw new Error();
 						}
-						const id = metadata?.id ?? accountId(account.name);
-						if (!metadata) {
-							const transactions = account.access === 'ambiguous' ? null : [];
-							await writeJson(transactionsDirectory, `${id}.json`, {
-								id,
-								name: account.name,
-								transactions
-							});
-						}
 						catalog.push({
-							id,
+							id: metadata?.id ?? accountId(account.name),
 							name: account.name,
 							access: account.access,
 							rowCount: account.rowCount,
@@ -282,13 +278,9 @@ export async function createReportStage(
 						});
 					}
 					if (remaining.size) throw new Error();
-					await writeJson(temporary, 'overview.json', {
-						format: currentReportFormat,
-						journal,
-						overview,
-						accounts: catalog
-					});
-					// The directory rename is the one commit point for the entire report set.
+					await writeReportData(writer, { overview, accounts: catalog, journal });
+					await writer.commit().execute();
+					await closeDatabase();
 					await rename(temporary, join(sets, token));
 					state = 'completed';
 					await updateCurrent(snapshotDirectory);
@@ -303,7 +295,7 @@ export async function createReportStage(
 			}
 		};
 	} catch {
-		if (staging) await rm(staging, { recursive: true, force: true }).catch(() => {});
+		await discard().catch(() => {});
 		throw new Error('Unable to prepare reports');
 	}
 }